[llvm] [AMDGPU] Fold redundant inf/nan checks into frexp instructions (PR #214936)
via llvm-commits
llvm-commits at lists.llvm.org
Sat Aug 8 04:18:00 PDT 2026
https://github.com/addmisol updated https://github.com/llvm/llvm-project/pull/214936
>From d72169dac4e1a7c5899c07da7c127486240e65e4 Mon Sep 17 00:00:00 2001
From: addmisol <addmisol9 at gmail.com>
Date: Sat, 8 Aug 2026 11:57:53 +0530
Subject: [PATCH 1/4] Fix: [AMDGPU] Fold redundant inf/nan checks into frexp
instructions
Signed-off-by: addmisol <addmisol9 at gmail.com>
---
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 898 +++++++++++++-----
llvm/lib/Target/AMDGPU/SIISelLowering.h | 39 +-
.../CodeGen/AMDGPU/frexp-inf-nan-combine.ll | 528 ++++++++++
3 files changed, 1240 insertions(+), 225 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/frexp-inf-nan-combine.ll
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 8a8a3b97e8e08..8a05d8591548d 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -22,6 +22,7 @@
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
#include "SIMachineFunctionInfo.h"
#include "SIRegisterInfo.h"
+#include "llvm/ADT/APFloat.h"
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/FloatingPointMode.h"
#include "llvm/ADT/Statistic.h"
@@ -47,6 +48,7 @@
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/KnownBits.h"
#include "llvm/Support/ModRef.h"
+#include "llvm/TargetParser/AtomicScope.h"
#include "llvm/Transforms/Utils/LowerAtomic.h"
#include <optional>
@@ -67,6 +69,10 @@ static cl::opt<bool> UseDivergentRegisterIndexing(
cl::desc("Use indirect register addressing for divergent indexes"),
cl::init(false));
+static DenormalFPEnv getDenormalFPEnv(const MachineFunction &MF) {
+ return MF.getInfo<SIMachineFunctionInfo>()->getMode().getDenormalFPEnv();
+}
+
static bool denormalModeIsFlushAllF32(const MachineFunction &MF) {
const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
return Info->getMode().FP32Denormals == DenormalMode::getPreserveSign();
@@ -236,6 +242,20 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
setOperationAction(Opc, MVT::bf16, Promote);
}
+ // Only targets with packed bf16 instructions, e.g. gfx13.
+ if (Subtarget->hasBF16PackedInsts()) {
+ // Turn fsub into fadd(x, fneg y) so it reuses the packed v_pk_add_bf16
+ // path instead of promoting to f32.
+ setOperationAction(ISD::FSUB, MVT::bf16, Expand);
+ // Widen scalar fadd to a v2bf16 operation with an unused high lane.
+ setOperationAction(ISD::FADD, MVT::bf16, Custom);
+ // Widen scalar fcanonicalize to a v2bf16 operation with an unused high
+ // lane.
+ setOperationAction(ISD::FCANONICALIZE, MVT::bf16, Custom);
+ // Widen scalar fmul to a v2bf16 operation with an unused high lane.
+ setOperationAction(ISD::FMUL, MVT::bf16, Custom);
+ }
+
setOperationAction(ISD::FP_ROUND, MVT::bf16, Expand);
setOperationAction(ISD::SELECT, MVT::bf16, Promote);
@@ -343,6 +363,7 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
case ISD::BUILD_VECTOR:
case ISD::BITCAST:
case ISD::UNDEF:
+ case ISD::POISON:
case ISD::EXTRACT_VECTOR_ELT:
case ISD::INSERT_VECTOR_ELT:
case ISD::SCALAR_TO_VECTOR:
@@ -660,6 +681,7 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
case ISD::BUILD_VECTOR:
case ISD::BITCAST:
case ISD::UNDEF:
+ case ISD::POISON:
case ISD::EXTRACT_VECTOR_ELT:
case ISD::INSERT_VECTOR_ELT:
case ISD::INSERT_SUBVECTOR:
@@ -693,8 +715,8 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
// XXX - Do these do anything? Vector constants turn into build_vector.
setOperationAction(ISD::Constant, {MVT::v2i16, MVT::v2f16}, Legal);
- setOperationAction(ISD::UNDEF, {MVT::v2i16, MVT::v2f16, MVT::v2bf16},
- Legal);
+ setOperationAction({ISD::UNDEF, ISD::POISON},
+ {MVT::v2i16, MVT::v2f16, MVT::v2bf16}, Legal);
setOperationAction(ISD::STORE, MVT::v2i16, Promote);
AddPromotedToType(ISD::STORE, MVT::v2i16, MVT::i32);
@@ -897,14 +919,14 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
VT, Custom);
}
- if (Subtarget->hasPackedFP32Ops()) {
+ if (Subtarget->hasAnyPackedFP32Ops()) {
setOperationAction({ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FNEG},
MVT::v2f32, Legal);
setOperationAction({ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FNEG},
{MVT::v4f32, MVT::v8f32, MVT::v16f32, MVT::v32f32},
Custom);
}
- if (Subtarget->hasPackedFP64Ops()) {
+ if (Subtarget->hasAnyPackedFP64Ops()) {
setOperationAction({ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FNEG,
ISD::FMINNUM_IEEE, ISD::FMAXNUM_IEEE,
ISD::FCANONICALIZE, ISD::BUILD_VECTOR},
@@ -919,7 +941,7 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
{MVT::v4f64, MVT::v8f64, MVT::v16f64, MVT::v32f64}, Custom);
}
- if (Subtarget->hasPackedU64Ops()) {
+ if (Subtarget->hasAnyPackedU64Ops()) {
setOperationAction({ISD::ADD, ISD::SUB, ISD::SHL, ISD::BUILD_VECTOR},
MVT::v2i64, Legal);
setOperationAction({ISD::ADD, ISD::SUB, ISD::SHL},
@@ -935,6 +957,8 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
AddPromotedToType(ISD::SELECT, MVT::v2i16, MVT::i32);
setOperationAction(ISD::SELECT, MVT::v2f16, Promote);
AddPromotedToType(ISD::SELECT, MVT::v2f16, MVT::i32);
+ setOperationAction(ISD::SELECT, MVT::v2bf16, Promote);
+ AddPromotedToType(ISD::SELECT, MVT::v2bf16, MVT::i32);
} else {
// Legalization hack.
setOperationAction(ISD::SELECT, {MVT::v2i16, MVT::v2f16}, Custom);
@@ -1023,7 +1047,8 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
setOperationAction(ISD::MUL, MVT::i1, Promote);
if (Subtarget->hasBF16ConversionInsts()) {
- setOperationAction(ISD::FP_ROUND, {MVT::bf16, MVT::v2bf16}, Custom);
+ setOperationAction({ISD::FP_ROUND, ISD::STRICT_FP_ROUND},
+ {MVT::bf16, MVT::v2bf16}, Custom);
setOperationAction(ISD::BUILD_VECTOR, MVT::v2bf16, Legal);
}
@@ -1031,6 +1056,17 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
setOperationAction({ISD::FEXP2, ISD::FLOG2, ISD::FSQRT}, MVT::bf16, Legal);
}
+ if (Subtarget->hasOCPFP8ConversionInsts()) {
+ setOperationAction(ISD::CONVERT_FROM_ARBITRARY_FP, {MVT::f32, MVT::v2f32},
+ Custom);
+ setOperationAction(ISD::CONVERT_FROM_ARBITRARY_FP, MVT::v2i8, Custom);
+ }
+
+ if (Subtarget->hasFP8F16ConversionInsts()) {
+ setOperationAction(ISD::CONVERT_FROM_ARBITRARY_FP, {MVT::f16, MVT::v2f16},
+ Custom);
+ }
+
if (Subtarget->hasCvtPkF16F32Inst()) {
setOperationAction(ISD::FP_ROUND,
{MVT::v2f16, MVT::v4f16, MVT::v8f16, MVT::v16f16},
@@ -1417,6 +1453,7 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
if (Aux->getZExtValue() & AMDGPU::CPol::VOLATILE)
Flags |= MachineMemOperand::MOVolatile;
}
+
Flags |= MachineMemOperand::MODereferenceable;
IntrinsicInfo Info;
@@ -2379,13 +2416,16 @@ bool SITargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm,
return true;
}
-bool SITargetLowering::isExtractSubvectorCheap(EVT ResVT, EVT SrcVT,
- unsigned Index) const {
+TargetLowering::ExtractSubvectorCost
+SITargetLowering::getExtractSubvectorCost(EVT ResVT, EVT SrcVT,
+ unsigned Index) const {
if (!isOperationLegalOrCustom(ISD::EXTRACT_SUBVECTOR, ResVT))
- return false;
+ return ExtractSubvectorCost::Expensive;
// TODO: Add more cases that are cheap.
- return Index == 0;
+ if (Index == 0)
+ return ExtractSubvectorCost::Free;
+ return ExtractSubvectorCost::Expensive;
}
bool SITargetLowering::isExtractVecEltCheap(EVT VT, unsigned Index) const {
@@ -2682,7 +2722,7 @@ SDValue SITargetLowering::getPreloadedValue(
SelectionDAG &DAG, const SIMachineFunctionInfo &MFI, EVT VT,
AMDGPUFunctionArgInfo::PreloadedValue PVID) const {
const ArgDescriptor *Reg = nullptr;
- const TargetRegisterClass *RC;
+ const TargetRegisterClass *RC = nullptr;
LLT Ty;
CallingConv::ID CC = DAG.getMachineFunction().getFunction().getCallingConv();
@@ -2901,7 +2941,7 @@ void SITargetLowering::allocateSpecialEntryInputVGPRs(
// Try to allocate a VGPR at the end of the argument list, or if no argument
// VGPRs are left allocating a stack slot.
-// If \p Mask is is given it indicates bitfield position in the register.
+// If \p Mask is given it indicates bitfield position in the register.
// If \p Arg is given use it with new ]p Mask instead of allocating new.
static ArgDescriptor allocateVGPR32Input(CCState &CCInfo, unsigned Mask = ~0u,
ArgDescriptor Arg = ArgDescriptor()) {
@@ -3730,15 +3770,8 @@ SDValue SITargetLowering::LowerFormalArguments(
if (Arg.Flags.isSRet()) {
// The return object should be reasonably addressable.
-
- // FIXME: This helps when the return is a real sret. If it is a
- // automatically inserted sret (i.e. CanLowerReturn returns false), an
- // extra copy is inserted in SelectionDAGBuilder which obscures this.
- unsigned NumBits =
- 32 - getSubtarget()->getKnownHighZeroBitsForFrameIndex();
- Val = DAG.getNode(
- ISD::AssertZext, DL, VT, Val,
- DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), NumBits)));
+ Val = annotateStackObjectPointer(Val, DAG, DL,
+ Arg.Flags.getNonZeroMemAlign());
}
Val = convertABITypeToValueType(DAG, Val, VA, DL);
@@ -6021,7 +6054,8 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
Register Op1L_Op0H_Reg =
MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
- Register CarryReg = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
+ Register CarryReg =
+ MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
Register AddReg = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
Register NegatedValLo =
MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
@@ -6048,9 +6082,23 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MUL_I32), DestSub0)
.addReg(Op1L)
.addReg(LowOpcode);
- BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MUL_HI_U32), CarryReg)
- .addReg(Op1L)
- .addReg(LowOpcode);
+ if (ST.hasScalarMulHiInsts()) {
+ BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MUL_HI_U32), CarryReg)
+ .addReg(Op1L)
+ .addReg(LowOpcode);
+ } else {
+ Register VCarryReg =
+ MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
+ Register LowOpVGPR =
+ MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
+ BuildMI(BB, MI, DL, TII->get(AMDGPU::COPY), LowOpVGPR)
+ .addReg(LowOpcode);
+ BuildMI(BB, MI, DL, TII->get(AMDGPU::V_MUL_HI_U32_e64), VCarryReg)
+ .addReg(Op1L)
+ .addReg(LowOpVGPR);
+ BuildMI(BB, MI, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), CarryReg)
+ .addReg(VCarryReg);
+ }
BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MUL_I32), Op1H_Op0L_Reg)
.addReg(Op1H)
.addReg(LowOpcode);
@@ -6371,10 +6419,23 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
ActiveBits.addReg(NewActiveBitsReg).addMBB(ComputeLoop);
// Creating branching
- unsigned CMPOpc = IsWave32 ? AMDGPU::S_CMP_LG_U32 : AMDGPU::S_CMP_LG_U64;
- BuildMI(*ComputeLoop, I, DL, TII->get(CMPOpc))
- .addReg(NewActiveBitsReg)
- .addImm(0);
+ MachineInstrBuilder SetSCCInstr;
+ if (!ST.hasScalarCompareEq64()) {
+ // For targets <= gfx7, use an S_OR_B32/B64 instruction to set SCC.
+ Register LaneMaskReg = MRI.createVirtualRegister(WaveMaskRegClass);
+ unsigned CMPOpc = IsWave32 ? AMDGPU::S_OR_B32 : AMDGPU::S_OR_B64;
+ SetSCCInstr =
+ BuildMI(*ComputeLoop, I, DL, TII->get(CMPOpc), LaneMaskReg);
+ } else {
+ unsigned CMPOpc =
+ IsWave32 ? AMDGPU::S_CMP_LG_U32 : AMDGPU::S_CMP_LG_U64;
+ SetSCCInstr = BuildMI(*ComputeLoop, I, DL, TII->get(CMPOpc));
+ }
+ SetSCCInstr.addReg(NewActiveBitsReg);
+ if (ST.hasScalarCompareEq64())
+ SetSCCInstr.addImm(0);
+ else
+ SetSCCInstr.addReg(NewActiveBitsReg);
BuildMI(*ComputeLoop, I, DL, TII->get(AMDGPU::S_CBRANCH_SCC1))
.addMBB(ComputeLoop);
@@ -7320,6 +7381,14 @@ SITargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
MI.getOperand(0).setReg(OriginalExec);
return BB;
}
+ case AMDGPU::V_DOT2_F32_F16:
+ case AMDGPU::V_DOT2_F32_BF16: {
+ // Hint RA to assign dst and src2 the same physical register.
+ // For targets without VOP2, but with VOPD, variant of the instruction this
+ // is one of the conditions to attempt converting VOP3P to VOPD.
+ MRI.setSimpleHint(MI.getOperand(0).getReg(), MI.getOperand(6).getReg());
+ return BB;
+ }
default:
if (TII->isImage(MI) || TII->isMUBUF(MI)) {
if (!MI.mayStore())
@@ -7378,9 +7447,11 @@ LLT SITargetLowering::getPreferredShiftAmountTy(LLT Ty) const {
// however does not support denormals, so we do report fma as faster if we have
// a fast fma device and require denormals.
//
-bool SITargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
- EVT VT) const {
+bool SITargetLowering::isFMAFasterThanFMulAndFAdd(EVT VT,
+ DenormalFPEnv FPEnv) const {
VT = VT.getScalarType();
+ if (!VT.isSimple())
+ return false;
switch (VT.getSimpleVT().SimpleTy) {
case MVT::f32: {
@@ -7391,7 +7462,7 @@ bool SITargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
// Otherwise f32 mad is always full rate and returns the same result as
// the separate operations so should be preferred over fma.
// However does not support denormals.
- if (!denormalModeIsFlushAllF32(MF))
+ if (FPEnv.F32Mode != DenormalMode::getPreserveSign())
return Subtarget->hasFastFMAF32() || Subtarget->hasDLInsts();
// If the subtarget has v_fmac_f32, that's just as good as v_mac_f32.
@@ -7401,7 +7472,8 @@ bool SITargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
return true;
case MVT::f16:
case MVT::bf16:
- return Subtarget->has16BitInsts() && !denormalModeIsFlushAllF64F16(MF);
+ return Subtarget->has16BitInsts() &&
+ FPEnv.DefaultMode != DenormalMode::getPreserveSign();
default:
break;
}
@@ -7409,6 +7481,18 @@ bool SITargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
return false;
}
+bool SITargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
+ EVT VT) const {
+ return isFMAFasterThanFMulAndFAdd(VT, getDenormalFPEnv(MF));
+}
+
+bool SITargetLowering::isFMAFasterThanFMulAndFAdd(const Function &F,
+ Type *Ty) const {
+ return isFMAFasterThanFMulAndFAdd(
+ getValueType(F.getDataLayout(), Ty, /*AllowUnknown=*/true),
+ F.getDenormalFPEnv());
+}
+
bool SITargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
LLT Ty) const {
switch (Ty.getScalarSizeInBits()) {
@@ -7425,33 +7509,36 @@ bool SITargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
return false;
}
+bool SITargetLowering::isFMADLegal(EVT VT, DenormalFPEnv FPEnv) const {
+ // TODO: Check future ftz flag
+ // v_mad_f32/v_mac_f32 do not support denormals.
+ if (VT == MVT::f32)
+ return Subtarget->hasMadMacF32Insts() &&
+ FPEnv.F32Mode == DenormalMode::getPreserveSign();
+ if (VT == MVT::f16)
+ return Subtarget->hasMadF16() &&
+ FPEnv.DefaultMode == DenormalMode::getPreserveSign();
+
+ return false;
+}
+
bool SITargetLowering::isFMADLegal(const MachineInstr &MI, LLT Ty) const {
if (!Ty.isScalar())
return false;
+ DenormalFPEnv FPEnv = getDenormalFPEnv(*MI.getMF());
if (Ty.getScalarSizeInBits() == 16)
- return Subtarget->hasMadF16() && denormalModeIsFlushAllF64F16(*MI.getMF());
+ return isFMADLegal(MVT::f16, FPEnv);
if (Ty.getScalarSizeInBits() == 32)
- return Subtarget->hasMadMacF32Insts() &&
- denormalModeIsFlushAllF32(*MI.getMF());
+ return isFMADLegal(MVT::f32, FPEnv);
return false;
}
bool SITargetLowering::isFMADLegal(const SelectionDAG &DAG,
const SDNode *N) const {
- // TODO: Check future ftz flag
- // v_mad_f32/v_mac_f32 do not support denormals.
- EVT VT = N->getValueType(0);
- if (VT == MVT::f32)
- return Subtarget->hasMadMacF32Insts() &&
- denormalModeIsFlushAllF32(DAG.getMachineFunction());
- if (VT == MVT::f16) {
- return Subtarget->hasMadF16() &&
- denormalModeIsFlushAllF64F16(DAG.getMachineFunction());
- }
-
- return false;
+ return isFMADLegal(N->getValueType(0),
+ getDenormalFPEnv(DAG.getMachineFunction()));
}
//===----------------------------------------------------------------------===//
@@ -7592,6 +7679,8 @@ SDValue SITargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
return LowerExternalSymbol(Op, DAG);
case ISD::INTRINSIC_WO_CHAIN:
return LowerINTRINSIC_WO_CHAIN(Op, DAG);
+ case ISD::CONVERT_FROM_ARBITRARY_FP:
+ return LowerCONVERT_FROM_ARBITRARY_FP(Op, DAG);
case ISD::INTRINSIC_W_CHAIN:
return LowerINTRINSIC_W_CHAIN(Op, DAG);
case ISD::INTRINSIC_VOID:
@@ -7620,7 +7709,6 @@ SDValue SITargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
case ISD::ABS:
case ISD::FABS:
case ISD::FNEG:
- case ISD::FCANONICALIZE:
case ISD::BSWAP:
return splitUnaryVectorOp(Op, DAG);
case ISD::FP_TO_SINT_SAT:
@@ -7660,8 +7748,6 @@ SDValue SITargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
case ISD::SMAX:
case ISD::UMIN:
case ISD::UMAX:
- case ISD::FADD:
- case ISD::FMUL:
case ISD::FMINNUM_IEEE:
case ISD::FMAXNUM_IEEE:
case ISD::UADDSAT:
@@ -7669,6 +7755,15 @@ SDValue SITargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
case ISD::SADDSAT:
case ISD::SSUBSAT:
return splitBinaryVectorOp(Op, DAG);
+ case ISD::FADD:
+ case ISD::FMUL:
+ if (Op.getValueType() == MVT::bf16)
+ return lowerScalarBF16BinaryOp(Op, DAG);
+ return splitBinaryVectorOp(Op, DAG);
+ case ISD::FCANONICALIZE:
+ if (Op.getValueType() == MVT::bf16)
+ return lowerScalarBF16FCanonicalize(Op, DAG);
+ return splitUnaryVectorOp(Op, DAG);
case ISD::FCOPYSIGN:
return lowerFCOPYSIGN(Op, DAG);
case ISD::MUL:
@@ -7791,6 +7886,13 @@ SDValue SITargetLowering::lowerIntrinsicLoad(MemSDNode *M, bool IsFormat,
bool IsD16 = IsFormat && (EltType.getSizeInBits() == 16);
+ if (IsFormat && !IsD16 && EltType.getSizeInBits() < 32) {
+ DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
+ DAG.getMachineFunction().getFunction(),
+ "unsupported sub-dword format buffer load", DL.getDebugLoc()));
+ return DAG.getMergeValues({DAG.getPOISON(LoadVT), M->getOperand(0)}, DL);
+ }
+
assert(M->getNumValues() == 2 || M->getNumValues() == 3);
bool IsTFE = M->getNumValues() == 3;
@@ -7948,10 +8050,6 @@ static SDValue lowerLaneOp(const SITargetLowering &TLI, SDNode *N,
MVT IntVT = MVT::getIntegerVT(ValSize);
const GCNSubtarget *ST = TLI.getSubtarget();
- if ((IsPermLane16 && !ST->hasPermlane16Insts()) ||
- (IID == Intrinsic::amdgcn_mov_dpp8 && !ST->hasDPP8()))
- return emitRemovedIntrinsicError(DAG, SL, VT);
-
unsigned SplitSize = 32;
if (IID == Intrinsic::amdgcn_update_dpp && (ValSize % 64 == 0) &&
ST->hasDPALU_DPP() &&
@@ -8242,6 +8340,30 @@ void SITargetLowering::ReplaceNodeResults(SDNode *N,
case ISD::INTRINSIC_WO_CHAIN: {
unsigned IID = N->getConstantOperandVal(0);
switch (IID) {
+ case Intrinsic::amdgcn_wave_reduce_min:
+ case Intrinsic::amdgcn_wave_reduce_umin:
+ case Intrinsic::amdgcn_wave_reduce_max:
+ case Intrinsic::amdgcn_wave_reduce_umax:
+ case Intrinsic::amdgcn_wave_reduce_add:
+ case Intrinsic::amdgcn_wave_reduce_sub:
+ case Intrinsic::amdgcn_wave_reduce_and:
+ case Intrinsic::amdgcn_wave_reduce_or:
+ case Intrinsic::amdgcn_wave_reduce_xor: {
+ EVT VT = N->getValueType(0);
+ if (isTypeLegal(VT))
+ return;
+ SDLoc SL(N);
+ bool NeedsSignExt = IID == Intrinsic::amdgcn_wave_reduce_min ||
+ IID == Intrinsic::amdgcn_wave_reduce_max ||
+ IID == Intrinsic::amdgcn_wave_reduce_add ||
+ IID == Intrinsic::amdgcn_wave_reduce_sub;
+ unsigned ExtOpc = NeedsSignExt ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
+ SDValue ExtSrc = DAG.getNode(ExtOpc, SL, MVT::i32, N->getOperand(1));
+ SDValue Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, MVT::i32,
+ N->getOperand(0), ExtSrc, N->getOperand(2));
+ Results.push_back(DAG.getNode(ISD::TRUNCATE, SL, VT, Result));
+ return;
+ }
case Intrinsic::amdgcn_make_buffer_rsrc:
Results.push_back(lowerPointerAsRsrcIntrin(N, DAG));
return;
@@ -8307,11 +8429,12 @@ void SITargetLowering::ReplaceNodeResults(SDNode *N,
VT.getStoreSize(), Alignment);
SDValue LoadVal;
if (!Offset->isDivergent()) {
- SDValue Ops[] = {Rsrc, // source register
+ SDValue Ops[] = {DAG.getEntryNode(), // Chain
+ Rsrc, // source register
Offset, CachePolicy};
- SDValue BufferLoad =
- DAG.getMemIntrinsicNode(AMDGPUISD::SBUFFER_LOAD_UBYTE, DL,
- DAG.getVTList(MVT::i32), Ops, VT, MMO);
+ SDValue BufferLoad = DAG.getMemIntrinsicNode(
+ AMDGPUISD::SBUFFER_LOAD_UBYTE, DL,
+ DAG.getVTList(MVT::i32, MVT::Other), Ops, VT, MMO);
LoadVal = DAG.getNode(ISD::TRUNCATE, DL, VT, BufferLoad);
} else {
SDValue Ops[] = {
@@ -8651,7 +8774,8 @@ SDValue SITargetLowering::splitFP_ROUNDVectorOp(SDValue Op,
}
SDValue SITargetLowering::lowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const {
- SDValue Src = Op.getOperand(0);
+ bool IsStrict = Op->isStrictFPOpcode();
+ SDValue Src = Op.getOperand(IsStrict ? 1 : 0);
EVT SrcVT = Src.getValueType();
EVT DstVT = Op.getValueType();
@@ -8694,10 +8818,58 @@ SDValue SITargetLowering::lowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const {
// hardware f32 -> bf16 instruction.
EVT F32VT = SrcVT.changeElementType(*DAG.getContext(), MVT::f32);
SDValue Rod = expandRoundInexactToOdd(F32VT, Src, DL, DAG);
+ if (IsStrict) {
+ return DAG.getNode(
+ ISD::STRICT_FP_ROUND, DL, {DstVT, MVT::Other},
+ {Op.getOperand(0), Rod, DAG.getTargetConstant(0, DL, MVT::i32)});
+ }
return DAG.getNode(ISD::FP_ROUND, DL, DstVT, Rod,
DAG.getTargetConstant(0, DL, MVT::i32));
}
+SDValue SITargetLowering::lowerScalarBF16BinaryOp(SDValue Op,
+ SelectionDAG &DAG) const {
+ assert(Subtarget->hasBF16PackedInsts());
+
+ SDLoc DL(Op);
+
+ auto WidenOperand = [&](SDValue Src) {
+ if (Src.getOpcode() == ISD::FNEG) {
+ SDValue WideSrc = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2bf16,
+ Src.getOperand(0));
+ return DAG.getNode(ISD::FNEG, DL, MVT::v2bf16, WideSrc, Src->getFlags());
+ }
+
+ return DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2bf16, Src);
+ };
+
+ SDValue LHS = WidenOperand(Op.getOperand(0));
+ SDValue RHS = WidenOperand(Op.getOperand(1));
+ SDValue Result =
+ DAG.getNode(Op.getOpcode(), DL, MVT::v2bf16, LHS, RHS, Op->getFlags());
+
+ return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::bf16, Result,
+ DAG.getConstant(0, DL, MVT::i32));
+}
+
+SDValue
+SITargetLowering::lowerScalarBF16FCanonicalize(SDValue Op,
+ SelectionDAG &DAG) const {
+ assert(Subtarget->hasBF16PackedInsts());
+
+ SDLoc DL(Op);
+ SDValue Src = Op.getOperand(0);
+
+ // Widen to v2bf16, canonicalize with v_pk_mul_bf16, then extract.
+ SDValue WideSrc = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2bf16, Src);
+
+ SDValue Canonicalized =
+ DAG.getNode(ISD::FCANONICALIZE, DL, MVT::v2bf16, WideSrc, Op->getFlags());
+
+ return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::bf16, Canonicalized,
+ DAG.getConstant(0, DL, MVT::i32));
+}
+
SDValue SITargetLowering::lowerFMINNUM_FMAXNUM(SDValue Op,
SelectionDAG &DAG) const {
EVT VT = Op.getValueType();
@@ -8730,7 +8902,7 @@ SITargetLowering::lowerFMINIMUMNUM_FMAXIMUMNUM(SDValue Op,
return expandFMINIMUMNUM_FMAXIMUMNUM(Op.getNode(), DAG);
if (VT == MVT::v4f16 || VT == MVT::v8f16 || VT == MVT::v16f16 ||
- VT == MVT::v16bf16)
+ VT == MVT::v32f16)
return splitBinaryVectorOp(Op, DAG);
return Op;
}
@@ -10234,6 +10406,16 @@ static void packImage16bitOpsToDwords(SelectionDAG &DAG, SDValue Op,
}
}
+/// Emit a DiagnosticInfoUnsupported for an unsupported image intrinsic and
+/// return poison values of \p ResultTypes, preserving the chain if present.
+static SDValue diagnoseUnsupportedImage(SelectionDAG &DAG, SDValue Op,
+ ArrayRef<EVT> ResultTypes,
+ const SDLoc &DL, const Twine &Msg) {
+ DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
+ DAG.getMachineFunction().getFunction(), Msg, DL.getDebugLoc()));
+ return DAG.getErrorMergeValues(ResultTypes, Op.getOperand(0), DL);
+}
+
SDValue SITargetLowering::lowerImage(SDValue Op,
const AMDGPU::ImageDimIntrinsicInfo *Intr,
SelectionDAG &DAG, bool WithChain) const {
@@ -10281,6 +10463,12 @@ SDValue SITargetLowering::lowerImage(SDValue Op,
IntrOpcode == AMDGPU::IMAGE_ATOMIC_PK_ADD_BF16 ||
IntrOpcode == AMDGPU::IMAGE_ATOMIC_PK_ADD_BF16_NORTN);
+ if (!IsAtomicPacked16Bit && VData.getValueSizeInBits() != 32 &&
+ VData.getValueSizeInBits() != 64) {
+ return diagnoseUnsupportedImage(DAG, Op, OrigResultTypes, DL,
+ "unsupported image atomic data type");
+ }
+
bool Is64Bit = VData.getValueSizeInBits() == 64;
if (BaseOpcode->AtomicX2) {
SDValue VData2 = Op.getOperand(3);
@@ -10306,7 +10494,13 @@ SDValue SITargetLowering::lowerImage(SDValue Op,
VData = Op.getOperand(2);
MVT StoreVT = VData.getSimpleValueType();
- if (StoreVT.getScalarType() == MVT::f16) {
+ MVT StoreScalarVT = StoreVT.getScalarType();
+ if (StoreScalarVT != MVT::f16 && StoreScalarVT.getSizeInBits() != 32 &&
+ StoreScalarVT.getSizeInBits() != 64) {
+ return diagnoseUnsupportedImage(DAG, Op, OrigResultTypes, DL,
+ "unsupported image store data type");
+ }
+ if (StoreScalarVT == MVT::f16) {
if (!Subtarget->hasD16Images() || !BaseOpcode->HasD16)
return Op; // D16 is unsupported for this instruction
@@ -10319,7 +10513,13 @@ SDValue SITargetLowering::lowerImage(SDValue Op,
// Work out the num dwords based on the dmask popcount and underlying type
// and whether packing is supported.
MVT LoadVT = ResultTypes[0].getSimpleVT();
- if (LoadVT.getScalarType() == MVT::f16) {
+ MVT LoadScalarVT = LoadVT.getScalarType();
+ if (LoadScalarVT != MVT::f16 && LoadScalarVT.getSizeInBits() != 32 &&
+ LoadScalarVT.getSizeInBits() != 64) {
+ return diagnoseUnsupportedImage(DAG, Op, OrigResultTypes, DL,
+ "unsupported image load data type");
+ }
+ if (LoadScalarVT == MVT::f16) {
if (!Subtarget->hasD16Images() || !BaseOpcode->HasD16)
return Op; // D16 is unsupported for this instruction
@@ -10600,21 +10800,9 @@ SDValue SITargetLowering::lowerImage(SDValue Op,
Opcode = AMDGPU::getMIMGOpcode(IntrOpcode, AMDGPU::MIMGEncGfx90a,
NumVDataDwords, NumVAddrDwords);
if (Opcode == -1) {
- DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
- DAG.getMachineFunction().getFunction(),
- "requested image instruction is not supported on this GPU",
- DL.getDebugLoc()));
-
- unsigned Idx = 0;
- SmallVector<SDValue, 3> RetValues(OrigResultTypes.size());
- for (EVT VT : OrigResultTypes) {
- if (VT == MVT::Other)
- RetValues[Idx++] = Op.getOperand(0); // Chain
- else
- RetValues[Idx++] = DAG.getPOISON(VT);
- }
-
- return DAG.getMergeValues(RetValues, DL);
+ return diagnoseUnsupportedImage(
+ DAG, Op, OrigResultTypes, DL,
+ "requested image instruction is not supported on this GPU");
}
}
if (Opcode == -1 &&
@@ -10653,34 +10841,48 @@ SDValue SITargetLowering::lowerImage(SDValue Op,
NumVDataDwords, IsAtomicPacked16Bit, DL);
}
-SDValue SITargetLowering::lowerSBuffer(EVT VT, SDLoc DL, SDValue Rsrc,
+SDValue SITargetLowering::lowerSBuffer(EVT VT, EVT MemVT, SDLoc DL,
+ SDValue Chain, SDValue Rsrc,
SDValue Offset, SDValue CachePolicy,
- SelectionDAG &DAG) const {
+ SelectionDAG &DAG,
+ MachineMemOperand *MMO) const {
MachineFunction &MF = DAG.getMachineFunction();
+ bool HasChainResult = MMO != nullptr;
- const DataLayout &DataLayout = DAG.getDataLayout();
- Align Alignment =
- DataLayout.getABITypeAlign(VT.getTypeForEVT(*DAG.getContext()));
+ if (!HasChainResult) {
+ const DataLayout &DataLayout = DAG.getDataLayout();
+ Align Alignment =
+ DataLayout.getABITypeAlign(MemVT.getTypeForEVT(*DAG.getContext()));
- MachineMemOperand *MMO = MF.getMachineMemOperand(
- MachinePointerInfo(),
- MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
- MachineMemOperand::MOInvariant,
- VT.getStoreSize(), Alignment);
+ MMO = MF.getMachineMemOperand(MachinePointerInfo(),
+ MachineMemOperand::MOLoad |
+ MachineMemOperand::MODereferenceable |
+ MachineMemOperand::MOInvariant,
+ MemVT.getStoreSize(), Alignment);
+ }
if (!Offset->isDivergent()) {
- SDValue Ops[] = {Rsrc, Offset, CachePolicy};
+ SDValue Ops[] = {Chain, Rsrc, Offset, CachePolicy};
+
+ // Lower llvm.amdgcn.*s.buffer.load.{i,u}N intrinsics. First, generate
+ // s_buffer_load_u* for signed and unsigned load instructions. Next, DAG
+ // combiner tries to merge the s_buffer_load_uN with a sext instruction
+ // (performSignExtendInRegCombine()) and it replaces s_buffer_load_uN with
+ // s_buffer_load_iN.
+ auto HandleScalarSubwordLoads = [&](unsigned Opcode) -> SDValue {
+ SDValue BufferLoad = DAG.getMemIntrinsicNode(
+ Opcode, DL, DAG.getVTList(MVT::i32, MVT::Other), Ops, MemVT, MMO);
+ SDValue LoadVal = DAG.getAnyExtOrTrunc(
+ DAG.getNode(ISD::TRUNCATE, DL, MemVT, BufferLoad), DL, VT);
+ if (HasChainResult)
+ return DAG.getMergeValues({LoadVal, BufferLoad.getValue(1)}, DL);
+ return LoadVal;
+ };
+ if (MemVT == MVT::i8 && Subtarget->hasScalarSubwordLoads())
+ return HandleScalarSubwordLoads(AMDGPUISD::SBUFFER_LOAD_UBYTE);
- // Lower llvm.amdgcn.s.buffer.load.{i16, u16} intrinsics. Initially, the
- // s_buffer_load_u16 instruction is emitted for both signed and unsigned
- // loads. Later, DAG combiner tries to combine s_buffer_load_u16 with sext
- // and generates s_buffer_load_i16 (performSignExtendInRegCombine).
- if (VT == MVT::i16 && Subtarget->hasScalarSubwordLoads()) {
- SDValue BufferLoad =
- DAG.getMemIntrinsicNode(AMDGPUISD::SBUFFER_LOAD_USHORT, DL,
- DAG.getVTList(MVT::i32), Ops, VT, MMO);
- return DAG.getNode(ISD::TRUNCATE, DL, VT, BufferLoad);
- }
+ if (MemVT == MVT::i16 && Subtarget->hasScalarSubwordLoads())
+ return HandleScalarSubwordLoads(AMDGPUISD::SBUFFER_LOAD_USHORT);
// Widen vec3 load to vec4.
if (VT.isVector() && VT.getVectorNumElements() == 3 &&
@@ -10688,21 +10890,25 @@ SDValue SITargetLowering::lowerSBuffer(EVT VT, SDLoc DL, SDValue Rsrc,
EVT WidenedVT =
EVT::getVectorVT(*DAG.getContext(), VT.getVectorElementType(), 4);
auto WidenedOp = DAG.getMemIntrinsicNode(
- AMDGPUISD::SBUFFER_LOAD, DL, DAG.getVTList(WidenedVT), Ops, WidenedVT,
+ AMDGPUISD::SBUFFER_LOAD, DL, DAG.getVTList(WidenedVT, MVT::Other),
+ Ops, WidenedVT,
MF.getMachineMemOperand(MMO, 0, WidenedVT.getStoreSize()));
auto Subvector = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, WidenedOp,
DAG.getVectorIdxConstant(0, DL));
+ if (HasChainResult)
+ return DAG.getMergeValues({Subvector, WidenedOp.getValue(1)}, DL);
return Subvector;
}
return DAG.getMemIntrinsicNode(AMDGPUISD::SBUFFER_LOAD, DL,
- DAG.getVTList(VT), Ops, VT, MMO);
+ DAG.getVTList(VT, MVT::Other), Ops, MemVT,
+ MMO);
}
// We have a divergent offset. Emit a MUBUF buffer load instead. We can
// assume that the buffer is unswizzled.
SDValue Ops[] = {
- DAG.getEntryNode(), // Chain
+ Chain, // Chain
Rsrc, // rsrc
DAG.getConstant(0, DL, MVT::i32), // vindex
{}, // voffset
@@ -10711,9 +10917,14 @@ SDValue SITargetLowering::lowerSBuffer(EVT VT, SDLoc DL, SDValue Rsrc,
CachePolicy, // cachepolicy
DAG.getTargetConstant(0, DL, MVT::i1), // idxen
};
- if (VT == MVT::i16 && Subtarget->hasScalarSubwordLoads()) {
+ if ((MemVT == MVT::i8 || MemVT == MVT::i16) &&
+ Subtarget->hasScalarSubwordLoads()) {
setBufferOffsets(Offset, DAG, &Ops[3], Align(4));
- return handleByteShortBufferLoads(DAG, VT, DL, Ops, MMO);
+ SDValue Load = handleByteShortBufferLoads(DAG, MemVT, DL, Ops, MMO);
+ SDValue LoadVal = DAG.getAnyExtOrTrunc(Load.getOperand(0), DL, VT);
+ if (HasChainResult)
+ return DAG.getMergeValues({LoadVal, Load.getOperand(1)}, DL);
+ return LoadVal;
}
SmallVector<SDValue, 4> Loads;
@@ -10744,8 +10955,17 @@ SDValue SITargetLowering::lowerSBuffer(EVT VT, SDLoc DL, SDValue Rsrc,
LoadVT, LoadMMO, DAG));
}
- if (NumElts == 8 || NumElts == 16)
- return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Loads);
+ if (NumElts == 8 || NumElts == 16) {
+ SDValue LoadVal = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Loads);
+ if (HasChainResult) {
+ SmallVector<SDValue, 4> LoadChains;
+ for (SDValue Load : Loads)
+ LoadChains.push_back(Load.getValue(1));
+ SDValue Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, LoadChains);
+ return DAG.getMergeValues({LoadVal, Chain}, DL);
+ }
+ return LoadVal;
+ }
return Loads[0];
}
@@ -10804,6 +11024,61 @@ SDValue SITargetLowering::lowerWorkitemID(SelectionDAG &DAG, SDValue Op,
DAG.getValueType(SmallVT));
}
+SDValue SITargetLowering::lowerFromFP8(SDValue Op, bool IsBF8,
+ SelectionDAG &DAG) const {
+ SDLoc SL(Op);
+ SDValue Src = Op.getOperand(0);
+ EVT DstVT = Op.getValueType();
+ bool IsF16 = DstVT.getVectorElementType() == MVT::f16;
+ assert((!IsF16 || Subtarget->hasFP8F16ConversionInsts()) &&
+ "fp8/bf8 -> f16 conversion requires FP8F16ConversionInsts");
+
+ unsigned Opc;
+ if (IsF16)
+ Opc = IsBF8 ? AMDGPUISD::CVT_PK_F16_BF8 : AMDGPUISD::CVT_PK_F16_FP8;
+ else
+ Opc = IsBF8 ? AMDGPUISD::CVT_PK_F32_BF8 : AMDGPUISD::CVT_PK_F32_FP8;
+
+ // Pack the two i8 lanes into the integer type the packed HW node reads. The
+ // f16 form takes i16 and the f32 form takes i32. v2i8 bitcasts to i16
+ // directly and the f32 node reads the low half of an any-extended i32.
+ EVT PackedVT =
+ EVT::getIntegerVT(*DAG.getContext(), DstVT.getScalarSizeInBits());
+ SDValue AsI16 = DAG.getNode(ISD::BITCAST, SL, MVT::i16, Src);
+ SDValue Packed = DAG.getAnyExtOrTrunc(AsI16, SL, PackedVT);
+ return DAG.getNode(Opc, SL, DstVT, Packed);
+}
+
+SDValue
+SITargetLowering::LowerCONVERT_FROM_ARBITRARY_FP(SDValue Op,
+ SelectionDAG &DAG) const {
+ // Only handle OCP FP8 formats (E4M3FN, E5M2). FNUZ formats that are supported
+ // by gfx942 fall through to the generic expansion.
+ APFloatBase::Semantics FPSemantic =
+ static_cast<APFloatBase::Semantics>(Op.getConstantOperandVal(1));
+ if (FPSemantic != APFloatBase::S_Float8E4M3FN &&
+ FPSemantic != APFloatBase::S_Float8E5M2)
+ return SDValue();
+ const bool IsBF8 = FPSemantic == APFloatBase::S_Float8E5M2;
+
+ EVT DstVT = Op.getValueType();
+ if (!DstVT.isVector()) {
+ SDValue Src = Op.getOperand(0);
+ if (Src.getValueType() != MVT::i32) {
+ SDLoc SL(Op);
+ SDValue SrcI32 = DAG.getAnyExtOrTrunc(Src, SL, MVT::i32);
+ return DAG.getNode(ISD::CONVERT_FROM_ARBITRARY_FP, SL, DstVT, SrcI32,
+ Op.getOperand(1));
+ }
+ return Op;
+ }
+
+ EVT EltVT = DstVT.getVectorElementType();
+ if (EltVT == MVT::f16 || EltVT == MVT::f32)
+ return lowerFromFP8(Op, IsBF8, DAG);
+ return SDValue();
+}
+
SDValue SITargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
SelectionDAG &DAG) const {
MachineFunction &MF = DAG.getMachineFunction();
@@ -10904,17 +11179,9 @@ SDValue SITargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
return emitRemovedIntrinsicError(DAG, DL, VT);
return DAG.getNode(AMDGPUISD::RCP_LEGACY, DL, VT, Op.getOperand(1));
case Intrinsic::amdgcn_fma_legacy:
- if (!Subtarget->hasFmaLegacy32Insts())
- return emitRemovedIntrinsicError(DAG, DL, VT);
- return SDValue();
case Intrinsic::amdgcn_sudot4:
case Intrinsic::amdgcn_sudot8:
- if (!Subtarget->hasDot8Insts())
- return emitRemovedIntrinsicError(DAG, DL, VT);
- return SDValue();
case Intrinsic::amdgcn_tanh:
- if (!Subtarget->hasTanhInsts())
- return emitRemovedIntrinsicError(DAG, DL, VT);
return SDValue();
case Intrinsic::amdgcn_rsq_clamp: {
if (Subtarget->getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS)
@@ -11070,8 +11337,8 @@ SDValue SITargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
? AMDGPU::CPol::ALL
: AMDGPU::CPol::ALL_pregfx12))
return Op;
- return lowerSBuffer(VT, DL, Op.getOperand(1), Op.getOperand(2),
- Op.getOperand(3), DAG);
+ return lowerSBuffer(VT, VT, DL, DAG.getEntryNode(), Op.getOperand(1),
+ Op.getOperand(2), Op.getOperand(3), DAG);
}
case Intrinsic::amdgcn_fdiv_fast:
return lowerFDIV_FAST(Op, DAG);
@@ -11097,9 +11364,20 @@ SDValue SITargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
case Intrinsic::amdgcn_fract:
return DAG.getNode(AMDGPUISD::FRACT, DL, VT, Op.getOperand(1));
- case Intrinsic::amdgcn_class:
- return DAG.getNode(AMDGPUISD::FP_CLASS, DL, VT, Op.getOperand(1),
- Op.getOperand(2));
+ case Intrinsic::amdgcn_class: {
+ SDValue Src = Op.getOperand(1);
+ EVT SrcVT = Src.getValueType();
+ bool IsLegal = SrcVT == MVT::f32 || SrcVT == MVT::f64 ||
+ (SrcVT == MVT::f16 && Subtarget->has16BitInsts());
+ if (!IsLegal) {
+ DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
+ DAG.getMachineFunction().getFunction(),
+ "llvm.amdgcn.class only supports f16, f32, and f64",
+ DL.getDebugLoc()));
+ return DAG.getPOISON(VT);
+ }
+ return DAG.getNode(AMDGPUISD::FP_CLASS, DL, VT, Src, Op.getOperand(2));
+ }
case Intrinsic::amdgcn_div_fmas:
return DAG.getNode(AMDGPUISD::DIV_FMAS, DL, VT, Op.getOperand(1),
Op.getOperand(2), Op.getOperand(3), Op.getOperand(4));
@@ -11383,6 +11661,11 @@ SDValue SITargetLowering::lowerRawBufferAtomicIntrin(SDValue Op,
SDLoc DL(Op);
SDValue VData = Op.getOperand(2);
+ if (VData.getValueSizeInBits() != 32 && VData.getValueSizeInBits() != 64) {
+ SmallVector<EVT, 2> ResultTypes(Op->values());
+ return diagnoseUnsupportedImage(DAG, Op, ResultTypes, DL,
+ "unsupported buffer atomic data type");
+ }
SDValue Rsrc = bufferRsrcPtrToVector(Op.getOperand(3), DAG);
auto [VOffset, Offset] = splitBufferOffsets(Op.getOperand(4), DAG);
auto SOffset = selectSOffset(Op.getOperand(5), DAG, Subtarget);
@@ -11411,6 +11694,11 @@ SITargetLowering::lowerStructBufferAtomicIntrin(SDValue Op, SelectionDAG &DAG,
SDLoc DL(Op);
SDValue VData = Op.getOperand(2);
+ if (VData.getValueSizeInBits() != 32 && VData.getValueSizeInBits() != 64) {
+ SmallVector<EVT, 2> ResultTypes(Op->values());
+ return diagnoseUnsupportedImage(DAG, Op, ResultTypes, DL,
+ "unsupported buffer atomic data type");
+ }
SDValue Rsrc = bufferRsrcPtrToVector(Op.getOperand(3), DAG);
auto [VOffset, Offset] = splitBufferOffsets(Op.getOperand(5), DAG);
auto SOffset = selectSOffset(Op.getOperand(6), DAG, Subtarget);
@@ -11502,6 +11790,19 @@ SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
M->getVTList(), Ops, M->getMemoryVT(),
M->getMemOperand());
}
+ case Intrinsic::amdgcn_ptr_s_buffer_load: {
+ unsigned CPol = Op.getConstantOperandVal(4);
+ if (CPol & ~((Subtarget->getGeneration() >= AMDGPUSubtarget::GFX12)
+ ? AMDGPU::CPol::ALL
+ : AMDGPU::CPol::ALL_pregfx12))
+ return Op;
+
+ MemSDNode *M = cast<MemSDNode>(Op);
+ return lowerSBuffer(
+ Op.getValueType(), M->getMemoryVT(), DL, Op.getOperand(0),
+ bufferRsrcPtrToVector(Op.getOperand(2), DAG), Op.getOperand(3),
+ Op.getOperand(4), DAG, M->getMemOperand());
+ }
case Intrinsic::amdgcn_raw_buffer_load:
case Intrinsic::amdgcn_raw_ptr_buffer_load:
case Intrinsic::amdgcn_raw_atomic_buffer_load:
@@ -11575,7 +11876,7 @@ SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
DAG.getTargetConstant(0, DL, MVT::i1), // idxen
};
- if (LoadVT.getScalarType() == MVT::f16)
+ if (LoadVT.getScalarSizeInBits() == 16)
return adjustLoadValueType(AMDGPUISD::TBUFFER_LOAD_FORMAT_D16, M, DAG,
Ops);
return getMemIntrinsicNode(AMDGPUISD::TBUFFER_LOAD_FORMAT, DL,
@@ -11602,7 +11903,7 @@ SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
DAG.getTargetConstant(1, DL, MVT::i1), // idxen
};
- if (LoadVT.getScalarType() == MVT::f16)
+ if (LoadVT.getScalarSizeInBits() == 16)
return adjustLoadValueType(AMDGPUISD::TBUFFER_LOAD_FORMAT_D16, M, DAG,
Ops);
return getMemIntrinsicNode(AMDGPUISD::TBUFFER_LOAD_FORMAT, DL,
@@ -11724,6 +12025,12 @@ SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
AMDGPUISD::BUFFER_ATOMIC_COND_SUB_U32);
case Intrinsic::amdgcn_raw_buffer_atomic_cmpswap:
case Intrinsic::amdgcn_raw_ptr_buffer_atomic_cmpswap: {
+ SDValue Src = Op.getOperand(2);
+ if (Src.getValueSizeInBits() != 32 && Src.getValueSizeInBits() != 64) {
+ SmallVector<EVT, 2> ResultTypes(Op->values());
+ return diagnoseUnsupportedImage(DAG, Op, ResultTypes, DL,
+ "unsupported buffer atomic data type");
+ }
SDValue Rsrc = bufferRsrcPtrToVector(Op.getOperand(4), DAG);
auto [VOffset, Offset] = splitBufferOffsets(Op.getOperand(5), DAG);
auto SOffset = selectSOffset(Op.getOperand(6), DAG, Subtarget);
@@ -11748,6 +12055,12 @@ SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
}
case Intrinsic::amdgcn_struct_buffer_atomic_cmpswap:
case Intrinsic::amdgcn_struct_ptr_buffer_atomic_cmpswap: {
+ SDValue Src = Op.getOperand(2);
+ if (Src.getValueSizeInBits() != 32 && Src.getValueSizeInBits() != 64) {
+ SmallVector<EVT, 2> ResultTypes(Op->values());
+ return diagnoseUnsupportedImage(DAG, Op, ResultTypes, DL,
+ "unsupported buffer atomic data type");
+ }
SDValue Rsrc = bufferRsrcPtrToVector(Op->getOperand(4), DAG);
auto [VOffset, Offset] = splitBufferOffsets(Op.getOperand(6), DAG);
auto SOffset = selectSOffset(Op.getOperand(7), DAG, Subtarget);
@@ -11784,11 +12097,6 @@ SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
assert(NodePtr.getValueType() == MVT::i64);
assert(RayDir.getValueType() == MVT::v3f32);
- if (!Subtarget->hasBVHDualAndBVH8Insts()) {
- emitRemovedIntrinsicError(DAG, DL, Op.getValueType());
- return SDValue();
- }
-
bool IsBVH8 = IntrID == Intrinsic::amdgcn_image_bvh8_intersect_ray;
const unsigned NumVDataDwords = 10;
const unsigned NumVAddrDwords = IsBVH8 ? 11 : 12;
@@ -11829,11 +12137,6 @@ SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
assert(RayDir.getValueType() == MVT::v3f16 ||
RayDir.getValueType() == MVT::v3f32);
- if (!Subtarget->hasGFX10_AEncoding()) {
- emitRemovedIntrinsicError(DAG, DL, Op.getValueType());
- return SDValue();
- }
-
const bool IsGFX11 = AMDGPU::isGFX11(*Subtarget);
const bool IsGFX11Plus = AMDGPU::isGFX11Plus(*Subtarget);
const bool IsGFX12Plus = AMDGPU::isGFX12Plus(*Subtarget);
@@ -12004,10 +12307,8 @@ SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
SDValue M0Val;
M0Val = DAG.getNode(ISD::SRL, DL, MVT::i32, Op->getOperand(2),
DAG.getShiftAmountConstant(4, MVT::i32, DL));
- M0Val = SDValue(
- DAG.getMachineNode(AMDGPU::S_AND_B32, DL, MVT::i32, M0Val,
- DAG.getTargetConstant(0x3F, DL, MVT::i32)),
- 0);
+ M0Val = DAG.getNode(ISD::AND, DL, MVT::i32, M0Val,
+ DAG.getConstant(0x3F, DL, MVT::i32));
Ops.push_back(copyToM0(DAG, Chain, DL, M0Val).getValue(0));
} else
Ops.push_back(copyToM0(DAG, Chain, DL, Op->getOperand(2)).getValue(0));
@@ -12027,14 +12328,6 @@ SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
Chain, Ptr, MII->getMemOperand());
}
case Intrinsic::amdgcn_av_load_b128: {
- if (!Subtarget->hasFlatGlobalInsts()) {
- DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
- DAG.getMachineFunction().getFunction(),
- "llvm.amdgcn.av.load.b128 not supported on subtarget",
- DL.getDebugLoc()));
- return DAG.getMergeValues(
- {DAG.getPOISON(Op->getValueType(0)), Op->getOperand(0)}, DL);
- }
MemIntrinsicSDNode *MII = cast<MemIntrinsicSDNode>(Op);
SDValue Chain = Op->getOperand(0);
SDValue Ptr = Op->getOperand(2);
@@ -12224,11 +12517,6 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
switch (IntrinsicID) {
case Intrinsic::amdgcn_exp_compr: {
- if (!Subtarget->hasCompressedExport()) {
- DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
- DAG.getMachineFunction().getFunction(),
- "intrinsic not supported on subtarget", DL.getDebugLoc()));
- }
SDValue Src0 = Op.getOperand(4);
SDValue Src1 = Op.getOperand(5);
// Hack around illegal type on SI by directly selecting it.
@@ -12256,7 +12544,7 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
case Intrinsic::amdgcn_struct_tbuffer_store:
case Intrinsic::amdgcn_struct_ptr_tbuffer_store: {
SDValue VData = Op.getOperand(2);
- bool IsD16 = (VData.getValueType().getScalarType() == MVT::f16);
+ bool IsD16 = (VData.getValueType().getScalarSizeInBits() == 16);
if (IsD16)
VData = handleD16VData(VData, DAG);
SDValue Rsrc = bufferRsrcPtrToVector(Op.getOperand(3), DAG);
@@ -12284,7 +12572,7 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
case Intrinsic::amdgcn_raw_tbuffer_store:
case Intrinsic::amdgcn_raw_ptr_tbuffer_store: {
SDValue VData = Op.getOperand(2);
- bool IsD16 = (VData.getValueType().getScalarType() == MVT::f16);
+ bool IsD16 = (VData.getValueType().getScalarSizeInBits() == 16);
if (IsD16)
VData = handleD16VData(VData, DAG);
SDValue Rsrc = bufferRsrcPtrToVector(Op.getOperand(3), DAG);
@@ -12321,6 +12609,14 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
EVT VDataVT = VData.getValueType();
EVT EltType = VDataVT.getScalarType();
bool IsD16 = IsFormat && (EltType.getSizeInBits() == 16);
+
+ if (IsFormat && !IsD16 && EltType.getSizeInBits() < 32) {
+ DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
+ DAG.getMachineFunction().getFunction(),
+ "unsupported sub-dword format buffer store", DL.getDebugLoc()));
+ return Chain;
+ }
+
if (IsD16) {
VData = handleD16VData(VData, DAG);
VDataVT = VData.getValueType();
@@ -12372,6 +12668,13 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
EVT EltType = VDataVT.getScalarType();
bool IsD16 = IsFormat && (EltType.getSizeInBits() == 16);
+ if (IsFormat && !IsD16 && EltType.getSizeInBits() < 32) {
+ DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
+ DAG.getMachineFunction().getFunction(),
+ "unsupported sub-dword format buffer store", DL.getDebugLoc()));
+ return Chain;
+ }
+
if (IsD16) {
VData = handleD16VData(VData, DAG);
VDataVT = VData.getValueType();
@@ -12418,8 +12721,6 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
case Intrinsic::amdgcn_struct_buffer_load_async_lds:
case Intrinsic::amdgcn_struct_ptr_buffer_load_lds:
case Intrinsic::amdgcn_struct_ptr_buffer_load_async_lds: {
- if (!Subtarget->hasVMemToLDSLoad())
- return SDValue();
unsigned Opc;
bool HasVIndex =
IntrinsicID == Intrinsic::amdgcn_struct_buffer_load_lds ||
@@ -12644,22 +12945,17 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
SDValue BarID;
BarID = DAG.getNode(ISD::SRL, DL, MVT::i32, BarOp,
DAG.getShiftAmountConstant(4, MVT::i32, DL));
- BarID =
- SDValue(DAG.getMachineNode(AMDGPU::S_AND_B32, DL, MVT::i32, BarID,
- DAG.getTargetConstant(0x3F, DL, MVT::i32)),
- 0);
+ BarID = DAG.getNode(ISD::AND, DL, MVT::i32, BarID,
+ DAG.getConstant(0x3F, DL, MVT::i32));
// Member count should be put into M0[ShAmt:+6]
// Barrier ID should be put into M0[5:0]
- M0Val =
- SDValue(DAG.getMachineNode(AMDGPU::S_AND_B32, DL, MVT::i32, CntOp,
- DAG.getTargetConstant(0x3F, DL, MVT::i32)),
- 0);
+ SDValue MemberCnt = DAG.getNode(ISD::AND, DL, MVT::i32, CntOp,
+ DAG.getConstant(0x3F, DL, MVT::i32));
constexpr unsigned ShAmt = 16;
- M0Val = DAG.getNode(ISD::SHL, DL, MVT::i32, CntOp,
+ M0Val = DAG.getNode(ISD::SHL, DL, MVT::i32, MemberCnt,
DAG.getShiftAmountConstant(ShAmt, MVT::i32, DL));
- M0Val = SDValue(
- DAG.getMachineNode(AMDGPU::S_OR_B32, DL, MVT::i32, M0Val, BarID), 0);
+ M0Val = DAG.getNode(ISD::OR, DL, MVT::i32, M0Val, BarID);
Ops.push_back(copyToM0(DAG, Chain, DL, M0Val).getValue(0));
@@ -12710,10 +13006,8 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
SDValue M0Val;
M0Val = DAG.getNode(ISD::SRL, DL, MVT::i32, BarOp,
DAG.getShiftAmountConstant(4, MVT::i32, DL));
- M0Val =
- SDValue(DAG.getMachineNode(AMDGPU::S_AND_B32, DL, MVT::i32, M0Val,
- DAG.getTargetConstant(0x3F, DL, MVT::i32)),
- 0);
+ M0Val = DAG.getNode(ISD::AND, DL, MVT::i32, M0Val,
+ DAG.getConstant(0x3F, DL, MVT::i32));
Ops.push_back(copyToM0(DAG, Chain, DL, M0Val).getValue(0));
}
@@ -12750,13 +13044,6 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
Ptr, MII->getMemOperand());
}
case Intrinsic::amdgcn_av_store_b128: {
- if (!Subtarget->hasFlatGlobalInsts()) {
- DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
- DAG.getMachineFunction().getFunction(),
- "llvm.amdgcn.av.store.b128 not supported on subtarget",
- DL.getDebugLoc()));
- return Op->getOperand(0); // return the input chain
- }
MemIntrinsicSDNode *MII = cast<MemIntrinsicSDNode>(Op);
SDValue Chain = Op->getOperand(0);
SDValue Ptr = Op->getOperand(2);
@@ -13081,8 +13368,7 @@ SDValue SITargetLowering::widenLoad(LoadSDNode *Ld,
SDValue NewLoad = DAG.getLoad(
ISD::UNINDEXED, ISD::NON_EXTLOAD, MVT::i32, SL, Ld->getChain(), Ptr,
Ld->getOffset(), Ld->getPointerInfo(), MVT::i32, Ld->getAlign(),
- Ld->getMemOperand()->getFlags(), Ld->getAAInfo(),
- nullptr); // Drop ranges
+ Ld->getMemOperand()->getFlags(), Ld->getAAInfo()); // Drop ranges
EVT TruncVT = EVT::getIntegerVT(*DAG.getContext(), MemVT.getSizeInBits());
if (MemVT.isFloatingPoint()) {
@@ -15634,22 +15920,22 @@ SITargetLowering::performSignExtendInRegCombine(SDNode *N,
assert(Subtarget->hasScalarSubwordLoads() &&
"s_buffer_load_{u8, i8} are supported "
"in GFX12 (or newer) architectures.");
- EVT VT = Src.getValueType();
unsigned Opc = (Src.getOpcode() == AMDGPUISD::SBUFFER_LOAD_UBYTE)
? AMDGPUISD::SBUFFER_LOAD_BYTE
: AMDGPUISD::SBUFFER_LOAD_SHORT;
SDLoc DL(N);
- SDVTList ResList = DCI.DAG.getVTList(MVT::i32);
+ SDVTList ResList =
+ DCI.DAG.getVTList(MVT::i32, Src.getOperand(0).getValueType());
SDValue Ops[] = {
- Src.getOperand(0), // source register
- Src.getOperand(1), // offset
- Src.getOperand(2) // cachePolicy
+ Src.getOperand(0), // Chain
+ Src.getOperand(1), // source register
+ Src.getOperand(2), // offset
+ Src.getOperand(3) // cachePolicy
};
auto *M = cast<MemSDNode>(Src);
SDValue BufferLoad = DCI.DAG.getMemIntrinsicNode(
Opc, DL, ResList, Ops, M->getMemoryVT(), M->getMemOperand());
- SDValue LoadVal = DCI.DAG.getNode(ISD::TRUNCATE, DL, VT, BufferLoad);
- return LoadVal;
+ return DCI.DAG.getMergeValues({BufferLoad, BufferLoad.getValue(1)}, DL);
}
if (((Src.getOpcode() == AMDGPUISD::BUFFER_LOAD_UBYTE &&
VTSign->getVT() == MVT::i8) ||
@@ -15716,6 +16002,21 @@ SDValue SITargetLowering::performRcpCombine(SDNode *N,
bool SITargetLowering::isCanonicalized(SelectionDAG &DAG, SDValue Op,
SDNodeFlags UserFlags,
unsigned MaxDepth) const {
+ EVT VT = Op.getValueType();
+ assert(VT.isFloatingPoint() &&
+ "expected a floating-point value to query canonicality of");
+ return isCanonicalized(DAG, Op, VT.getScalarType(), UserFlags, MaxDepth);
+}
+
+bool SITargetLowering::isCanonicalized(SelectionDAG &DAG, SDValue Op,
+ EVT QueryVT, SDNodeFlags UserFlags,
+ unsigned MaxDepth) const {
+ assert(QueryVT.isFloatingPoint() && !QueryVT.isVector() &&
+ "QueryVT must be a floating-point scalar type");
+ EVT VT = Op.getValueType();
+ if (VT.isFloatingPoint() && VT.getScalarType() != QueryVT)
+ return false;
+
unsigned Opcode = Op.getOpcode();
if (Opcode == ISD::FCANONICALIZE)
return true;
@@ -15784,7 +16085,8 @@ bool SITargetLowering::isCanonicalized(SelectionDAG &DAG, SDValue Op,
case ISD::FNEG:
case ISD::FABS:
case ISD::FCOPYSIGN:
- return isCanonicalized(DAG, Op.getOperand(0), MaxDepth - 1);
+ return isCanonicalized(DAG, Op.getOperand(0), QueryVT, UserFlags,
+ MaxDepth - 1);
case ISD::AND:
if (Op.getValueType() == MVT::i32) {
@@ -15794,7 +16096,8 @@ bool SITargetLowering::isCanonicalized(SelectionDAG &DAG, SDValue Op,
// is valid to optimize for all types.
if (auto *RHS = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
if (RHS->getZExtValue() == 0xffff0000) {
- return isCanonicalized(DAG, Op.getOperand(0), MaxDepth - 1);
+ return isCanonicalized(DAG, Op.getOperand(0), QueryVT, UserFlags,
+ MaxDepth - 1);
}
}
}
@@ -15835,20 +16138,23 @@ bool SITargetLowering::isCanonicalized(SelectionDAG &DAG, SDValue Op,
// FIXME: Does this apply with clamp? It's implemented with max.
for (unsigned I = 0, E = Op.getNumOperands(); I != E; ++I) {
- if (!isCanonicalized(DAG, Op.getOperand(I), MaxDepth - 1))
+ if (!isCanonicalized(DAG, Op.getOperand(I), QueryVT, UserFlags,
+ MaxDepth - 1))
return false;
}
return true;
}
case ISD::SELECT: {
- return isCanonicalized(DAG, Op.getOperand(1), MaxDepth - 1) &&
- isCanonicalized(DAG, Op.getOperand(2), MaxDepth - 1);
+ return isCanonicalized(DAG, Op.getOperand(1), QueryVT, UserFlags,
+ MaxDepth - 1) &&
+ isCanonicalized(DAG, Op.getOperand(2), QueryVT, UserFlags,
+ MaxDepth - 1);
}
case ISD::BUILD_VECTOR: {
for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
SDValue SrcOp = Op.getOperand(i);
- if (!isCanonicalized(DAG, SrcOp, MaxDepth - 1))
+ if (!isCanonicalized(DAG, SrcOp, QueryVT, UserFlags, MaxDepth - 1))
return false;
}
@@ -15856,21 +16162,28 @@ bool SITargetLowering::isCanonicalized(SelectionDAG &DAG, SDValue Op,
}
case ISD::EXTRACT_VECTOR_ELT:
case ISD::EXTRACT_SUBVECTOR: {
- return isCanonicalized(DAG, Op.getOperand(0), MaxDepth - 1);
+ return isCanonicalized(DAG, Op.getOperand(0), QueryVT, UserFlags,
+ MaxDepth - 1);
}
case ISD::INSERT_VECTOR_ELT: {
- return isCanonicalized(DAG, Op.getOperand(0), MaxDepth - 1) &&
- isCanonicalized(DAG, Op.getOperand(1), MaxDepth - 1);
+ return isCanonicalized(DAG, Op.getOperand(0), QueryVT, UserFlags,
+ MaxDepth - 1) &&
+ isCanonicalized(DAG, Op.getOperand(1), QueryVT, UserFlags,
+ MaxDepth - 1);
}
+ case ISD::POISON:
+ return true;
case ISD::UNDEF:
// Could be anything.
return false;
- case ISD::BITCAST:
- // TODO: This is incorrect as it loses track of the operand's type. We may
- // end up effectively bitcasting from f32 to v2f16 or vice versa, and the
- // same bits that are canonicalized in one type need not be in the other.
- return isCanonicalized(DAG, Op.getOperand(0), MaxDepth - 1);
+ case ISD::BITCAST: {
+ // Carry QueryVT through the bitcast unchanged. The top-of-function guard
+ // rejects a source whose FP format differs from the consumed type, so a
+ // value canonical in one FP format is not assumed canonical in another.
+ SDValue Src = peekThroughBitcasts(Op.getOperand(0));
+ return isCanonicalized(DAG, Src, QueryVT, UserFlags, MaxDepth - 1);
+ }
case ISD::TRUNCATE: {
// Hack round the mess we make when legalizing extract_vector_elt
if (Op.getValueType() == MVT::i16) {
@@ -15878,7 +16191,8 @@ bool SITargetLowering::isCanonicalized(SelectionDAG &DAG, SDValue Op,
if (TruncSrc.getValueType() == MVT::i32 &&
TruncSrc.getOpcode() == ISD::BITCAST &&
TruncSrc.getOperand(0).getValueType() == MVT::v2f16) {
- return isCanonicalized(DAG, TruncSrc.getOperand(0), MaxDepth - 1);
+ return isCanonicalized(DAG, TruncSrc.getOperand(0), QueryVT, UserFlags,
+ MaxDepth - 1);
}
}
return false;
@@ -16124,14 +16438,13 @@ SITargetLowering::performFCanonicalizeCombine(SDNode *N,
}
}
- // If one half is undef, and one is constant, prefer a splat vector rather
- // than the normal qNaN. If it's a register, prefer 0.0 since that's
- // cheaper to use and may be free with a packed operation.
+ // If one half is undef, and one is constant, prefer a splat vector.
+ // Otherwise, convert the undef to 0.0 since that's cheaper to use and may
+ // be free with a packed operation.
if (NewElts[0].isUndef()) {
- if (isa<ConstantFPSDNode>(NewElts[1]))
- NewElts[0] = isa<ConstantFPSDNode>(NewElts[1])
- ? NewElts[1]
- : DAG.getConstantFP(0.0f, SL, EltVT);
+ NewElts[0] = isa<ConstantFPSDNode>(NewElts[1])
+ ? NewElts[1]
+ : DAG.getConstantFP(0.0f, SL, EltVT);
}
if (NewElts[1].isUndef()) {
@@ -17458,7 +17771,8 @@ SDValue SITargetLowering::performAddCombine(SDNode *N,
TempNode = TempNode->getOperand(AddIdx);
Src2s.push_back(TempNode);
ChainLength = I + 1;
- if (TempNode->getNumOperands() < 2)
+ // The loop body treats TempNode's operands as addends.
+ if (TempNode.getOpcode() != ISD::ADD)
break;
LHS = TempNode->getOperand(0);
RHS = TempNode->getOperand(1);
@@ -18508,11 +18822,150 @@ SDValue SITargetLowering::performClampCombine(SDNode *N,
return DCI.DAG.getConstantFP(Zero, SDLoc(N), N->getValueType(0));
}
- APFloat One(F.getSemantics(), "1.0");
+ APFloat One = APFloat::getOne(F.getSemantics());
if (F > One)
return DCI.DAG.getConstantFP(One, SDLoc(N), N->getValueType(0));
- return SDValue(CSrc, 0);
+ return getCanonicalConstantFP(DCI.DAG, SDLoc(N), N->getValueType(0), F);
+}
+
+SDValue
+SITargetLowering::performFrexpSelectCombine(SDNode *N,
+ DAGCombinerInfo &DCI) const {
+ // This optimization only applies when the hardware handles inf/nan correctly.
+ if (Subtarget->hasFractBug())
+ return SDValue();
+
+ SDValue Cond = N->getOperand(0);
+ SDValue TrueVal = N->getOperand(1);
+ SDValue FalseVal = N->getOperand(2);
+
+ // Determine which value is 0 and which might be the frexp result.
+ // Pattern 1: select cond, 0, frexp_result (cond true -> return 0)
+ // Pattern 2: select cond, frexp_result, 0 (cond false -> return 0)
+ SDValue FrexpVal;
+ bool CondSelectsZero; // If true, condition=true selects zero
+
+ auto isZero = [](SDValue V) {
+ if (auto *C = dyn_cast<ConstantSDNode>(V))
+ return C->isZero();
+ if (auto *C = dyn_cast<ConstantFPSDNode>(V))
+ return C->isZero();
+ return false;
+ };
+
+ if (isZero(TrueVal)) {
+ FrexpVal = FalseVal;
+ CondSelectsZero = true;
+ } else if (isZero(FalseVal)) {
+ FrexpVal = TrueVal;
+ CondSelectsZero = false;
+ } else {
+ return SDValue();
+ }
+
+ // Check if FrexpVal comes from amdgcn_frexp_exp or amdgcn_frexp_mant.
+ if (FrexpVal.getOpcode() != ISD::INTRINSIC_WO_CHAIN)
+ return SDValue();
+
+ unsigned IID = FrexpVal.getConstantOperandVal(0);
+ if (IID != Intrinsic::amdgcn_frexp_exp && IID != Intrinsic::amdgcn_frexp_mant)
+ return SDValue();
+
+ SDValue FrexpInput = FrexpVal.getOperand(1);
+
+ // Helper to strip fabs/fneg/fcopysign from a value.
+ auto peekFPSignOps = [](SDValue Val) {
+ if (Val.getOpcode() == ISD::FNEG)
+ Val = Val.getOperand(0);
+ if (Val.getOpcode() == ISD::FABS)
+ Val = Val.getOperand(0);
+ if (Val.getOpcode() == ISD::FCOPYSIGN)
+ Val = Val.getOperand(0);
+ return Val;
+ };
+
+ // The frexp intrinsics ignore sign, so we can strip sign ops when comparing.
+ SDValue FrexpInputStripped = peekFPSignOps(FrexpInput);
+
+ bool IsNonFiniteTest = false;
+
+ // Handle AMDGPUISD::FP_CLASS or ISD::IS_FPCLASS conditions.
+ // These test specific floating-point classes using a bitmask.
+ if (Cond.getOpcode() == AMDGPUISD::FP_CLASS ||
+ Cond.getOpcode() == ISD::IS_FPCLASS) {
+ SDValue ClassInput = Cond.getOperand(0);
+ SDValue ClassInputStripped = peekFPSignOps(ClassInput);
+
+ if (ClassInputStripped != FrexpInputStripped)
+ return SDValue();
+
+ auto *MaskNode = dyn_cast<ConstantSDNode>(Cond.getOperand(1));
+ if (!MaskNode)
+ return SDValue();
+
+ unsigned Mask = MaskNode->getZExtValue();
+
+ // fcFinite = all finite classes (not inf, not nan)
+ // If the mask tests for finite values and selects frexp when true,
+ // we can fold away the select since frexp returns 0 for non-finite.
+ constexpr unsigned fcFinite =
+ 0x1F8; // fcPosNormal|fcNegNormal|fcPosSubnormal|fcNegSubnormal|fcPosZero|fcNegZero
+ constexpr unsigned fcInfNan = 0x207; // fcPosInf|fcNegInf|fcSNan|fcQNan
+
+ if (Mask == fcFinite) {
+ // is_fpclass(x, finite) selects frexp when x is finite
+ // frexp already returns 0 for non-finite, so select frexp, 0 -> frexp
+ IsNonFiniteTest = !CondSelectsZero;
+ } else if (Mask == fcInfNan || Mask == 0x3 || Mask == 0x204) {
+ // is_fpclass(x, inf|nan) or is_fpclass(x, nan) or is_fpclass(x, inf)
+ // selects 0 when x is non-finite
+ IsNonFiniteTest = CondSelectsZero;
+ }
+ } else if (Cond.getOpcode() == ISD::SETCC) {
+ // Handle SETCC conditions for inf/nan tests.
+ ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
+ SDValue CondLHS = Cond.getOperand(0);
+ SDValue CondRHS = Cond.getOperand(1);
+ SDValue CondLHSStripped = peekFPSignOps(CondLHS);
+
+ auto isInfConstant = [](SDValue V) {
+ auto *CFP = dyn_cast<ConstantFPSDNode>(V);
+ return CFP && CFP->getValueAPF().isInfinity();
+ };
+
+ if (CC == ISD::SETUO) {
+ // fcmp uno x, y - true if either x or y is NaN
+ SDValue CondRHSStripped = peekFPSignOps(CondRHS);
+ if (CondLHSStripped == FrexpInputStripped ||
+ CondRHSStripped == FrexpInputStripped) {
+ IsNonFiniteTest = CondSelectsZero;
+ }
+ } else if ((CC == ISD::SETOEQ || CC == ISD::SETUEQ) &&
+ isInfConstant(CondRHS)) {
+ // fcmp oeq/ueq |x|, inf - true if x is inf (or inf/nan for ueq)
+ if (CondLHSStripped == FrexpInputStripped)
+ IsNonFiniteTest = CondSelectsZero;
+ } else if ((CC == ISD::SETONE || CC == ISD::SETUNE) &&
+ isInfConstant(CondRHS)) {
+ // fcmp one/une |x|, inf - true if x is NOT inf
+ if (CondLHSStripped == FrexpInputStripped)
+ IsNonFiniteTest = !CondSelectsZero;
+ } else if (CC == ISD::SETO) {
+ // fcmp ord x, y - true if both are NOT NaN
+ SDValue CondRHSStripped = peekFPSignOps(CondRHS);
+ if (CondLHSStripped == FrexpInputStripped ||
+ CondRHSStripped == FrexpInputStripped) {
+ IsNonFiniteTest = !CondSelectsZero;
+ }
+ }
+ }
+
+ if (!IsNonFiniteTest)
+ return SDValue();
+
+ // The select can be eliminated - just return the frexp result directly.
+ return FrexpVal;
}
SDValue SITargetLowering::performSelectCombine(SDNode *N,
@@ -18544,8 +18997,10 @@ SDValue SITargetLowering::performSelectCombine(SDNode *N,
if (!isFloatingPoint && !isInteger)
return SDValue();
- bool isEquality = CC == (isFloatingPoint ? ISD::SETOEQ : ISD::SETEQ);
- bool isNonEquality = CC == (isFloatingPoint ? ISD::SETONE : ISD::SETNE);
+ // Bare SETEQ/SETNE is the builder's NaN-impossible downgrade.
+ bool isEquality = CC == ISD::SETEQ || (isFloatingPoint && CC == ISD::SETOEQ);
+ bool isNonEquality =
+ CC == ISD::SETNE || (isFloatingPoint && CC == ISD::SETONE);
if (!isEquality && !isNonEquality)
return SDValue();
@@ -18581,6 +19036,11 @@ SDValue SITargetLowering::performSelectCombine(SDNode *N,
!(isNonEquality && FalseVal == ConstVal))
return SDValue();
+ // SETONE's false arm is also taken for NaN ArgVal, so require NaN excluded.
+ if (isFloatingPoint && isNonEquality && FalseVal == ConstVal &&
+ !Cond->getFlags().hasNoNaNs() && !DCI.DAG.isKnownNeverNaN(ArgVal))
+ return SDValue();
+
SDValue SelectLHS = (isEquality && TrueVal == ConstVal) ? ArgVal : TrueVal;
SDValue SelectRHS =
(isNonEquality && FalseVal == ConstVal) ? ArgVal : FalseVal;
@@ -18639,6 +19099,8 @@ SDValue SITargetLowering::PerformDAGCombine(SDNode *N,
case ISD::SETCC:
return performSetCCCombine(N, DCI);
case ISD::SELECT:
+ if (auto Res = performFrexpSelectCombine(N, DCI))
+ return Res;
if (auto Res = performSelectCombine(N, DCI))
return Res;
break;
@@ -19260,10 +19722,8 @@ MachineSDNode *SITargetLowering::buildRSRC(SelectionDAG &DAG, const SDLoc &DL,
SDValue PtrLo = DAG.getTargetExtractSubreg(AMDGPU::sub0, DL, MVT::i32, Ptr);
SDValue PtrHi = DAG.getTargetExtractSubreg(AMDGPU::sub1, DL, MVT::i32, Ptr);
if (RsrcDword1) {
- PtrHi =
- SDValue(DAG.getMachineNode(AMDGPU::S_OR_B32, DL, MVT::i32, PtrHi,
- DAG.getConstant(RsrcDword1, DL, MVT::i32)),
- 0);
+ PtrHi = DAG.getNode(ISD::OR, DL, MVT::i32, PtrHi,
+ DAG.getConstant(RsrcDword1, DL, MVT::i32));
}
SDValue DataLo =
@@ -19738,9 +20198,9 @@ void SITargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
Op, Known, DemandedElts, DAG, Depth);
}
-void SITargetLowering::computeKnownBitsForFrameIndex(
- const int FI, KnownBits &Known, const MachineFunction &MF) const {
- TargetLowering::computeKnownBitsForFrameIndex(FI, Known, MF);
+void SITargetLowering::computeKnownBitsForStackObjectPointer(
+ KnownBits &Known, const MachineFunction &MF, Align Alignment) const {
+ TargetLowering::computeKnownBitsForStackObjectPointer(Known, MF, Alignment);
// Set the high bits to zero based on the maximum allowed scratch size per
// wave. We can't use vaddr in MUBUF instructions if we don't know the address
@@ -20288,7 +20748,9 @@ SITargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *RMW) const {
auto SSID = RMW->getSyncScopeID();
bool HasSystemScope =
SSID == SyncScope::System ||
- SSID == RMW->getContext().getOrInsertSyncScopeID("one-as");
+ SSID == RMW->getContext().getOrInsertSyncScopeID(*getAtomicScopeIRString(
+ getTargetMachine().getTargetTriple(), AtomicScope::System,
+ /*OneAddressSpace=*/true));
auto Op = RMW->getOperation();
switch (Op) {
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h
index c98426cdac0b1..b5853ac1035f7 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h
@@ -17,6 +17,7 @@
#include "AMDGPUArgumentUsageInfo.h"
#include "AMDGPUISelLowering.h"
#include "SIDefines.h"
+#include "llvm/ADT/FloatingPointMode.h"
#include "llvm/CodeGen/MachineFunction.h"
namespace llvm {
@@ -85,8 +86,9 @@ class SITargetLowering final : public AMDGPUTargetLowering {
MVT VT, unsigned Offset) const;
SDValue lowerImage(SDValue Op, const AMDGPU::ImageDimIntrinsicInfo *Intr,
SelectionDAG &DAG, bool WithChain) const;
- SDValue lowerSBuffer(EVT VT, SDLoc DL, SDValue Rsrc, SDValue Offset,
- SDValue CachePolicy, SelectionDAG &DAG) const;
+ SDValue lowerSBuffer(EVT VT, EVT MemVT, SDLoc DL, SDValue Chain, SDValue Rsrc,
+ SDValue Offset, SDValue CachePolicy, SelectionDAG &DAG,
+ MachineMemOperand *MMO = nullptr) const;
SDValue lowerRawBufferAtomicIntrin(SDValue Op, SelectionDAG &DAG,
unsigned NewOpcode) const;
@@ -103,6 +105,8 @@ class SITargetLowering final : public AMDGPUTargetLowering {
SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerINTRINSIC_VOID(SDValue Op, SelectionDAG &DAG) const;
+ SDValue LowerCONVERT_FROM_ARBITRARY_FP(SDValue Op, SelectionDAG &DAG) const;
+ SDValue lowerFromFP8(SDValue Op, bool IsBF8, SelectionDAG &DAG) const;
// The raw.tbuffer and struct.tbuffer intrinsics have two offset args: offset
// (the offset that is included in bounds checking and swizzling, to be split
@@ -163,6 +167,8 @@ class SITargetLowering final : public AMDGPUTargetLowering {
/// Custom lowering for ISD::FP_ROUND for MVT::f16.
SDValue lowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const;
SDValue splitFP_ROUNDVectorOp(SDValue Op, SelectionDAG &DAG) const;
+ SDValue lowerScalarBF16BinaryOp(SDValue Op, SelectionDAG &DAG) const;
+ SDValue lowerScalarBF16FCanonicalize(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerFMINNUM_FMAXNUM(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerFMINIMUMNUM_FMAXIMUMNUM(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerFMINIMUM_FMAXIMUM(SDValue Op, SelectionDAG &DAG) const;
@@ -231,6 +237,7 @@ class SITargetLowering final : public AMDGPUTargetLowering {
SDValue performExtractVectorEltCombine(SDNode *N, DAGCombinerInfo &DCI) const;
SDValue performInsertVectorEltCombine(SDNode *N, DAGCombinerInfo &DCI) const;
SDValue performFPRoundCombine(SDNode *N, DAGCombinerInfo &DCI) const;
+ SDValue performFrexpSelectCombine(SDNode *N, DAGCombinerInfo &DCI) const;
SDValue performSelectCombine(SDNode *N, DAGCombinerInfo &DCI) const;
SDValue reassociateScalarOps(SDNode *N, SelectionDAG &DAG) const;
@@ -400,8 +407,8 @@ class SITargetLowering final : public AMDGPUTargetLowering {
bool shouldConvertConstantLoadToIntImm(const APInt &Imm,
Type *Ty) const override;
- bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT,
- unsigned Index) const override;
+ ExtractSubvectorCost getExtractSubvectorCost(EVT ResVT, EVT SrcVT,
+ unsigned Index) const override;
bool isExtractVecEltCheap(EVT VT, unsigned Index) const override;
bool isTypeDesirableForOp(unsigned Op, EVT VT) const override;
@@ -501,6 +508,15 @@ class SITargetLowering final : public AMDGPUTargetLowering {
bool isFMADLegal(const SelectionDAG &DAG, const SDNode *N) const override;
bool isFMADLegal(const MachineInstr &MI, const LLT Ty) const override;
+ /// Variants for IR level callers, which have no MachineFunction to read the
+ /// denormal mode from and must pass \p FPEnv explicitly.
+ bool isFMAFasterThanFMulAndFAdd(EVT VT, DenormalFPEnv FPEnv) const;
+
+ /// \p VT is used as written, so a vector type reports false.
+ bool isFMADLegal(EVT VT, DenormalFPEnv FPEnv) const;
+
+ bool isFMAFasterThanFMulAndFAdd(const Function &F, Type *Ty) const override;
+
SDValue splitUnaryVectorOp(SDValue Op, SelectionDAG &DAG) const;
SDValue splitBinaryVectorOp(SDValue Op, SelectionDAG &DAG) const;
SDValue splitTernaryVectorOp(SDValue Op, SelectionDAG &DAG) const;
@@ -542,9 +558,9 @@ class SITargetLowering final : public AMDGPUTargetLowering {
const APInt &DemandedElts,
const SelectionDAG &DAG,
unsigned Depth = 0) const override;
- void computeKnownBitsForFrameIndex(int FrameIdx,
- KnownBits &Known,
- const MachineFunction &MF) const override;
+ void computeKnownBitsForStackObjectPointer(KnownBits &Known,
+ const MachineFunction &MF,
+ Align Alignment) const override;
void computeKnownBitsForTargetInstr(GISelValueTracking &Analysis, Register R,
KnownBits &Known,
const APInt &DemandedElts,
@@ -568,6 +584,15 @@ class SITargetLowering final : public AMDGPUTargetLowering {
bool isCanonicalized(SelectionDAG &DAG, SDValue Op,
SDNodeFlags UserFlags = {}, unsigned MaxDepth = 5) const;
+
+ /// Returns true if \p Op is provably canonical (no FCANONICALIZE needed).
+ /// \p QueryVT is the scalar FP type being checked, threaded unchanged
+ /// through recursion since canonicality is per vector element. FP operands
+ /// whose scalar type differs from \p QueryVT are treated as non-canonical,
+ /// since canonicality does not survive a change of FP format (e.g. bitcast
+ /// v2bf16 to v2f16); non-FP operands are not checked against \p QueryVT.
+ bool isCanonicalized(SelectionDAG &DAG, SDValue Op, EVT QueryVT,
+ SDNodeFlags UserFlags, unsigned MaxDepth) const;
bool isCanonicalized(Register Reg, const MachineFunction &MF,
unsigned MaxDepth = 5) const;
bool denormalsEnabledForType(const SelectionDAG &DAG, EVT VT) const;
diff --git a/llvm/test/CodeGen/AMDGPU/frexp-inf-nan-combine.ll b/llvm/test/CodeGen/AMDGPU/frexp-inf-nan-combine.ll
new file mode 100644
index 0000000000000..c2a763e58e2c2
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/frexp-inf-nan-combine.ll
@@ -0,0 +1,528 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck -check-prefix=GFX9 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1030 < %s | FileCheck -check-prefix=GFX10 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 < %s | FileCheck -check-prefix=GFX11 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 < %s | FileCheck -check-prefix=GFX950 %s
+; RUN: llc -mtriple=amdgcn-- -mcpu=tahiti < %s | FileCheck -check-prefix=SI %s
+
+; Test that redundant inf/nan checks are folded into frexp instructions.
+; The AMDGPU frexp instructions already return 0 for inf/nan inputs.
+; This optimization should NOT apply on SI (Southern Islands) which has hasFractBug.
+
+declare {float, i32} @llvm.frexp.f32.i32(float)
+declare {double, i32} @llvm.frexp.f64.i32(double)
+declare {half, i16} @llvm.frexp.f16.i16(half)
+declare float @llvm.fabs.f32(float)
+declare double @llvm.fabs.f64(double)
+declare half @llvm.fabs.f16(half)
+
+; Pattern 1: select (fcmp uno x, 0), 0, (frexp_exp x)
+; NaN check - should fold to just frexp_exp
+define i32 @frexp_nan_clamp_exp_f32(float %x) {
+; GFX9-LABEL: frexp_nan_clamp_exp_f32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: frexp_nan_clamp_exp_f32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: frexp_nan_clamp_exp_f32:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-LABEL: frexp_nan_clamp_exp_f32:
+; GFX950: ; %bb.0:
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-LABEL: frexp_nan_clamp_exp_f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s4, 0x7f800000
+; SI-NEXT: v_cmp_lt_f32_e64 s[4:5], |v0|, s4
+; SI-NEXT: v_cmp_o_f32_e32 vcc, v0, v0
+; SI-NEXT: v_frexp_exp_i32_f32_e32 v1, v0
+; SI-NEXT: s_and_b64 vcc, vcc, s[4:5]
+; SI-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
+; SI-NEXT: s_setpc_b64 s[30:31]
+ %frexp = call {float, i32} @llvm.frexp.f32.i32(float %x)
+ %exp = extractvalue {float, i32} %frexp, 1
+ %is_nan = fcmp uno float %x, 0.0
+ %result = select i1 %is_nan, i32 0, i32 %exp
+ ret i32 %result
+}
+
+; Pattern 2: select (fcmp oeq |x|, inf), 0, (frexp_exp x)
+; Inf check - should fold to just frexp_exp
+define i32 @frexp_inf_clamp_exp_f32(float %x) {
+; GFX9-LABEL: frexp_inf_clamp_exp_f32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: frexp_inf_clamp_exp_f32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: frexp_inf_clamp_exp_f32:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-LABEL: frexp_inf_clamp_exp_f32:
+; GFX950: ; %bb.0:
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-LABEL: frexp_inf_clamp_exp_f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s4, 0x7f800000
+; SI-NEXT: v_frexp_exp_i32_f32_e32 v1, v0
+; SI-NEXT: v_cmp_lt_f32_e64 vcc, |v0|, s4
+; SI-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
+; SI-NEXT: s_setpc_b64 s[30:31]
+ %frexp = call {float, i32} @llvm.frexp.f32.i32(float %x)
+ %exp = extractvalue {float, i32} %frexp, 1
+ %abs = call float @llvm.fabs.f32(float %x)
+ %is_inf = fcmp oeq float %abs, 0x7FF0000000000000
+ %result = select i1 %is_inf, i32 0, i32 %exp
+ ret i32 %result
+}
+
+; Pattern 3: select (fcmp ueq |x|, inf), 0, (frexp_exp x)
+; Inf-or-NaN check - should fold to just frexp_exp
+define i32 @frexp_inf_or_nan_clamp_exp_f32(float %x) {
+; GFX9-LABEL: frexp_inf_or_nan_clamp_exp_f32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: frexp_inf_or_nan_clamp_exp_f32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: frexp_inf_or_nan_clamp_exp_f32:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-LABEL: frexp_inf_or_nan_clamp_exp_f32:
+; GFX950: ; %bb.0:
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-LABEL: frexp_inf_or_nan_clamp_exp_f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s4, 0x7f800000
+; SI-NEXT: s_movk_i32 s6, 0x1f8
+; SI-NEXT: v_cmp_lt_f32_e64 s[4:5], |v0|, s4
+; SI-NEXT: v_cmp_class_f32_e64 s[6:7], v0, s6
+; SI-NEXT: v_frexp_exp_i32_f32_e32 v1, v0
+; SI-NEXT: s_and_b64 vcc, s[6:7], s[4:5]
+; SI-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
+; SI-NEXT: s_setpc_b64 s[30:31]
+ %frexp = call { float, i32 } @llvm.frexp.f32.i32(float %x)
+ %exp = extractvalue { float, i32 } %frexp, 1
+ %abs = call float @llvm.fabs.f32(float %x)
+ %is_non_finite = fcmp ueq float %abs, 0x7FF0000000000000
+ %result = select i1 %is_non_finite, i32 0, i32 %exp
+ ret i32 %result
+}
+
+; Test with frexp_mant instead of frexp_exp
+define float @frexp_nan_clamp_mant_f32(float %x) {
+; GFX9-LABEL: frexp_nan_clamp_mant_f32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_frexp_mant_f32_e32 v0, v0
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: frexp_nan_clamp_mant_f32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_frexp_mant_f32_e32 v0, v0
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: frexp_nan_clamp_mant_f32:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_frexp_mant_f32_e32 v0, v0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-LABEL: frexp_nan_clamp_mant_f32:
+; GFX950: ; %bb.0:
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_frexp_mant_f32_e32 v0, v0
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-LABEL: frexp_nan_clamp_mant_f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s4, 0x7f800000
+; SI-NEXT: v_frexp_mant_f32_e32 v1, v0
+; SI-NEXT: v_cmp_lt_f32_e64 vcc, |v0|, s4
+; SI-NEXT: v_cndmask_b32_e32 v1, v0, v1, vcc
+; SI-NEXT: v_cmp_o_f32_e32 vcc, v0, v0
+; SI-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
+; SI-NEXT: s_setpc_b64 s[30:31]
+ %frexp = call {float, i32} @llvm.frexp.f32.i32(float %x)
+ %mant = extractvalue {float, i32} %frexp, 0
+ %is_nan = fcmp uno float %x, 0.0
+ %result = select i1 %is_nan, float 0.0, float %mant
+ ret float %result
+}
+
+; Test with reversed select operands: select (fcmp ord x, 0), frexp, 0
+define i32 @frexp_ord_clamp_exp_f32(float %x) {
+; GFX9-LABEL: frexp_ord_clamp_exp_f32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: frexp_ord_clamp_exp_f32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: frexp_ord_clamp_exp_f32:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-LABEL: frexp_ord_clamp_exp_f32:
+; GFX950: ; %bb.0:
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-LABEL: frexp_ord_clamp_exp_f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s4, 0x7f800000
+; SI-NEXT: v_cmp_lt_f32_e64 s[4:5], |v0|, s4
+; SI-NEXT: v_cmp_o_f32_e32 vcc, v0, v0
+; SI-NEXT: v_frexp_exp_i32_f32_e32 v1, v0
+; SI-NEXT: s_and_b64 vcc, vcc, s[4:5]
+; SI-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
+; SI-NEXT: s_setpc_b64 s[30:31]
+ %frexp = call {float, i32} @llvm.frexp.f32.i32(float %x)
+ %exp = extractvalue {float, i32} %frexp, 1
+ %is_ord = fcmp ord float %x, 0.0
+ %result = select i1 %is_ord, i32 %exp, i32 0
+ ret i32 %result
+}
+
+; Test with fcmp one |x|, inf (not inf check -> select frexp)
+define i32 @frexp_not_inf_clamp_exp_f32(float %x) {
+; GFX9-LABEL: frexp_not_inf_clamp_exp_f32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: frexp_not_inf_clamp_exp_f32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: frexp_not_inf_clamp_exp_f32:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-LABEL: frexp_not_inf_clamp_exp_f32:
+; GFX950: ; %bb.0:
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-LABEL: frexp_not_inf_clamp_exp_f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s4, 0x7f800000
+; SI-NEXT: s_movk_i32 s6, 0x1f8
+; SI-NEXT: v_cmp_lt_f32_e64 s[4:5], |v0|, s4
+; SI-NEXT: v_cmp_class_f32_e64 s[6:7], v0, s6
+; SI-NEXT: v_frexp_exp_i32_f32_e32 v1, v0
+; SI-NEXT: s_and_b64 vcc, s[6:7], s[4:5]
+; SI-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
+; SI-NEXT: s_setpc_b64 s[30:31]
+ %frexp = call {float, i32} @llvm.frexp.f32.i32(float %x)
+ %exp = extractvalue {float, i32} %frexp, 1
+ %abs = call float @llvm.fabs.f32(float %x)
+ %is_not_inf = fcmp one float %abs, 0x7FF0000000000000
+ %result = select i1 %is_not_inf, i32 %exp, i32 0
+ ret i32 %result
+}
+
+; Test f64
+define i32 @frexp_nan_clamp_exp_f64(double %x) {
+; GFX9-LABEL: frexp_nan_clamp_exp_f64:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_frexp_exp_i32_f64_e32 v0, v[0:1]
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: frexp_nan_clamp_exp_f64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_frexp_exp_i32_f64_e32 v0, v[0:1]
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: frexp_nan_clamp_exp_f64:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_frexp_exp_i32_f64_e32 v0, v[0:1]
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-LABEL: frexp_nan_clamp_exp_f64:
+; GFX950: ; %bb.0:
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_frexp_exp_i32_f64_e32 v0, v[0:1]
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-LABEL: frexp_nan_clamp_exp_f64:
+; SI: ; %bb.0:
+; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT: v_and_b32_e32 v2, 0x7fffffff, v1
+; SI-NEXT: s_mov_b32 s4, 0x7ff00000
+; SI-NEXT: v_cmp_gt_i32_e32 vcc, s4, v2
+; SI-NEXT: v_cmp_o_f64_e64 s[4:5], v[0:1], v[0:1]
+; SI-NEXT: v_frexp_exp_i32_f64_e32 v0, v[0:1]
+; SI-NEXT: s_and_b64 vcc, s[4:5], vcc
+; SI-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
+; SI-NEXT: s_setpc_b64 s[30:31]
+ %frexp = call {double, i32} @llvm.frexp.f64.i32(double %x)
+ %exp = extractvalue {double, i32} %frexp, 1
+ %is_nan = fcmp uno double %x, 0.0
+ %result = select i1 %is_nan, i32 0, i32 %exp
+ ret i32 %result
+}
+
+; Test f16
+define i16 @frexp_nan_clamp_exp_f16(half %x) {
+; GFX9-LABEL: frexp_nan_clamp_exp_f16:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_frexp_exp_i16_f16_e32 v0, v0
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: frexp_nan_clamp_exp_f16:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_frexp_exp_i16_f16_e32 v0, v0
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: frexp_nan_clamp_exp_f16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_frexp_exp_i16_f16_e32 v0.l, v0.l
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-LABEL: frexp_nan_clamp_exp_f16:
+; GFX950: ; %bb.0:
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_frexp_exp_i16_f16_e32 v0, v0
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-LABEL: frexp_nan_clamp_exp_f16:
+; SI: ; %bb.0:
+; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT: s_mov_b32 s4, 0x7f800000
+; SI-NEXT: v_cmp_lt_f32_e64 s[4:5], |v0|, s4
+; SI-NEXT: v_cmp_o_f32_e32 vcc, v0, v0
+; SI-NEXT: v_frexp_exp_i32_f32_e32 v1, v0
+; SI-NEXT: s_and_b64 vcc, vcc, s[4:5]
+; SI-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
+; SI-NEXT: s_setpc_b64 s[30:31]
+ %frexp = call {half, i16} @llvm.frexp.f16.i16(half %x)
+ %exp = extractvalue {half, i16} %frexp, 1
+ %is_nan = fcmp uno half %x, 0.0
+ %result = select i1 %is_nan, i16 0, i16 %exp
+ ret i16 %result
+}
+
+; Negative test: different input to frexp vs comparison
+define i32 @frexp_nan_different_input(float %x, float %y) {
+; GFX9-LABEL: frexp_nan_different_input:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: frexp_nan_different_input:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v1, v1
+; GFX10-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc_lo
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: frexp_nan_different_input:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v1, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc_lo
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-LABEL: frexp_nan_different_input:
+; GFX950: ; %bb.0:
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; GFX950-NEXT: v_cmp_o_f32_e32 vcc, v1, v1
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-LABEL: frexp_nan_different_input:
+; SI: ; %bb.0:
+; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s4, 0x7f800000
+; SI-NEXT: v_cmp_lt_f32_e64 s[4:5], |v0|, s4
+; SI-NEXT: v_cmp_o_f32_e32 vcc, v1, v1
+; SI-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; SI-NEXT: s_and_b64 vcc, vcc, s[4:5]
+; SI-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
+; SI-NEXT: s_setpc_b64 s[30:31]
+ %frexp = call {float, i32} @llvm.frexp.f32.i32(float %x)
+ %exp = extractvalue {float, i32} %frexp, 1
+ %is_nan = fcmp uno float %y, 0.0
+ %result = select i1 %is_nan, i32 0, i32 %exp
+ ret i32 %result
+}
+
+; Negative test: non-zero constant in select
+define i32 @frexp_nan_nonzero_const(float %x) {
+; GFX9-LABEL: frexp_nan_nonzero_const:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_frexp_exp_i32_f32_e32 v1, v0
+; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, 42, v1, vcc
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: frexp_nan_nonzero_const:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_frexp_exp_i32_f32_e32 v1, v0
+; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_cndmask_b32_e32 v0, 42, v1, vcc_lo
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: frexp_nan_nonzero_const:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_frexp_exp_i32_f32_e32 v1, v0
+; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: v_cndmask_b32_e32 v0, 42, v1, vcc_lo
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-LABEL: frexp_nan_nonzero_const:
+; GFX950: ; %bb.0:
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_frexp_exp_i32_f32_e32 v1, v0
+; GFX950-NEXT: v_cmp_o_f32_e32 vcc, v0, v0
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e32 v0, 42, v1, vcc
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-LABEL: frexp_nan_nonzero_const:
+; SI: ; %bb.0:
+; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s4, 0x7f800000
+; SI-NEXT: v_frexp_exp_i32_f32_e32 v1, v0
+; SI-NEXT: v_cmp_lt_f32_e64 vcc, |v0|, s4
+; SI-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
+; SI-NEXT: v_cmp_o_f32_e32 vcc, v0, v0
+; SI-NEXT: v_cndmask_b32_e32 v0, 42, v1, vcc
+; SI-NEXT: s_setpc_b64 s[30:31]
+ %frexp = call {float, i32} @llvm.frexp.f32.i32(float %x)
+ %exp = extractvalue {float, i32} %frexp, 1
+ %is_nan = fcmp uno float %x, 0.0
+ %result = select i1 %is_nan, i32 42, i32 %exp
+ ret i32 %result
+}
+
+; Negative test: unrelated comparison (not inf/nan test)
+define i32 @frexp_lt_zero_not_folded(float %x) {
+; GFX9-LABEL: frexp_lt_zero_not_folded:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_frexp_exp_i32_f32_e32 v1, v0
+; GFX9-NEXT: v_cmp_ngt_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: frexp_lt_zero_not_folded:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_frexp_exp_i32_f32_e32 v1, v0
+; GFX10-NEXT: v_cmp_ngt_f32_e32 vcc_lo, 0, v0
+; GFX10-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc_lo
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: frexp_lt_zero_not_folded:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_frexp_exp_i32_f32_e32 v1, v0
+; GFX11-NEXT: v_cmp_ngt_f32_e32 vcc_lo, 0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc_lo
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-LABEL: frexp_lt_zero_not_folded:
+; GFX950: ; %bb.0:
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_frexp_exp_i32_f32_e32 v1, v0
+; GFX950-NEXT: v_cmp_ngt_f32_e32 vcc, 0, v0
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-LABEL: frexp_lt_zero_not_folded:
+; SI: ; %bb.0:
+; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s4, 0x7f800000
+; SI-NEXT: v_cmp_lt_f32_e64 s[4:5], |v0|, s4
+; SI-NEXT: v_cmp_ngt_f32_e32 vcc, 0, v0
+; SI-NEXT: v_frexp_exp_i32_f32_e32 v1, v0
+; SI-NEXT: s_and_b64 vcc, vcc, s[4:5]
+; SI-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
+; SI-NEXT: s_setpc_b64 s[30:31]
+ %frexp = call {float, i32} @llvm.frexp.f32.i32(float %x)
+ %exp = extractvalue {float, i32} %frexp, 1
+ %is_lt_zero = fcmp olt float %x, 0.0
+ %result = select i1 %is_lt_zero, i32 0, i32 %exp
+ ret i32 %result
+}
>From f4a80b88d39cdace057843b7ad15d8f9bc3d2ed7 Mon Sep 17 00:00:00 2001
From: addmisol <addmisol9 at gmail.com>
Date: Sat, 8 Aug 2026 12:28:40 +0530
Subject: [PATCH 2/4] Add Changes
Signed-off-by: addmisol <addmisol9 at gmail.com>
---
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 141 ++++++++++++++++++++++
1 file changed, 141 insertions(+)
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 1d165910adeba..8a05d8591548d 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -18829,6 +18829,145 @@ SDValue SITargetLowering::performClampCombine(SDNode *N,
return getCanonicalConstantFP(DCI.DAG, SDLoc(N), N->getValueType(0), F);
}
+SDValue
+SITargetLowering::performFrexpSelectCombine(SDNode *N,
+ DAGCombinerInfo &DCI) const {
+ // This optimization only applies when the hardware handles inf/nan correctly.
+ if (Subtarget->hasFractBug())
+ return SDValue();
+
+ SDValue Cond = N->getOperand(0);
+ SDValue TrueVal = N->getOperand(1);
+ SDValue FalseVal = N->getOperand(2);
+
+ // Determine which value is 0 and which might be the frexp result.
+ // Pattern 1: select cond, 0, frexp_result (cond true -> return 0)
+ // Pattern 2: select cond, frexp_result, 0 (cond false -> return 0)
+ SDValue FrexpVal;
+ bool CondSelectsZero; // If true, condition=true selects zero
+
+ auto isZero = [](SDValue V) {
+ if (auto *C = dyn_cast<ConstantSDNode>(V))
+ return C->isZero();
+ if (auto *C = dyn_cast<ConstantFPSDNode>(V))
+ return C->isZero();
+ return false;
+ };
+
+ if (isZero(TrueVal)) {
+ FrexpVal = FalseVal;
+ CondSelectsZero = true;
+ } else if (isZero(FalseVal)) {
+ FrexpVal = TrueVal;
+ CondSelectsZero = false;
+ } else {
+ return SDValue();
+ }
+
+ // Check if FrexpVal comes from amdgcn_frexp_exp or amdgcn_frexp_mant.
+ if (FrexpVal.getOpcode() != ISD::INTRINSIC_WO_CHAIN)
+ return SDValue();
+
+ unsigned IID = FrexpVal.getConstantOperandVal(0);
+ if (IID != Intrinsic::amdgcn_frexp_exp && IID != Intrinsic::amdgcn_frexp_mant)
+ return SDValue();
+
+ SDValue FrexpInput = FrexpVal.getOperand(1);
+
+ // Helper to strip fabs/fneg/fcopysign from a value.
+ auto peekFPSignOps = [](SDValue Val) {
+ if (Val.getOpcode() == ISD::FNEG)
+ Val = Val.getOperand(0);
+ if (Val.getOpcode() == ISD::FABS)
+ Val = Val.getOperand(0);
+ if (Val.getOpcode() == ISD::FCOPYSIGN)
+ Val = Val.getOperand(0);
+ return Val;
+ };
+
+ // The frexp intrinsics ignore sign, so we can strip sign ops when comparing.
+ SDValue FrexpInputStripped = peekFPSignOps(FrexpInput);
+
+ bool IsNonFiniteTest = false;
+
+ // Handle AMDGPUISD::FP_CLASS or ISD::IS_FPCLASS conditions.
+ // These test specific floating-point classes using a bitmask.
+ if (Cond.getOpcode() == AMDGPUISD::FP_CLASS ||
+ Cond.getOpcode() == ISD::IS_FPCLASS) {
+ SDValue ClassInput = Cond.getOperand(0);
+ SDValue ClassInputStripped = peekFPSignOps(ClassInput);
+
+ if (ClassInputStripped != FrexpInputStripped)
+ return SDValue();
+
+ auto *MaskNode = dyn_cast<ConstantSDNode>(Cond.getOperand(1));
+ if (!MaskNode)
+ return SDValue();
+
+ unsigned Mask = MaskNode->getZExtValue();
+
+ // fcFinite = all finite classes (not inf, not nan)
+ // If the mask tests for finite values and selects frexp when true,
+ // we can fold away the select since frexp returns 0 for non-finite.
+ constexpr unsigned fcFinite =
+ 0x1F8; // fcPosNormal|fcNegNormal|fcPosSubnormal|fcNegSubnormal|fcPosZero|fcNegZero
+ constexpr unsigned fcInfNan = 0x207; // fcPosInf|fcNegInf|fcSNan|fcQNan
+
+ if (Mask == fcFinite) {
+ // is_fpclass(x, finite) selects frexp when x is finite
+ // frexp already returns 0 for non-finite, so select frexp, 0 -> frexp
+ IsNonFiniteTest = !CondSelectsZero;
+ } else if (Mask == fcInfNan || Mask == 0x3 || Mask == 0x204) {
+ // is_fpclass(x, inf|nan) or is_fpclass(x, nan) or is_fpclass(x, inf)
+ // selects 0 when x is non-finite
+ IsNonFiniteTest = CondSelectsZero;
+ }
+ } else if (Cond.getOpcode() == ISD::SETCC) {
+ // Handle SETCC conditions for inf/nan tests.
+ ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
+ SDValue CondLHS = Cond.getOperand(0);
+ SDValue CondRHS = Cond.getOperand(1);
+ SDValue CondLHSStripped = peekFPSignOps(CondLHS);
+
+ auto isInfConstant = [](SDValue V) {
+ auto *CFP = dyn_cast<ConstantFPSDNode>(V);
+ return CFP && CFP->getValueAPF().isInfinity();
+ };
+
+ if (CC == ISD::SETUO) {
+ // fcmp uno x, y - true if either x or y is NaN
+ SDValue CondRHSStripped = peekFPSignOps(CondRHS);
+ if (CondLHSStripped == FrexpInputStripped ||
+ CondRHSStripped == FrexpInputStripped) {
+ IsNonFiniteTest = CondSelectsZero;
+ }
+ } else if ((CC == ISD::SETOEQ || CC == ISD::SETUEQ) &&
+ isInfConstant(CondRHS)) {
+ // fcmp oeq/ueq |x|, inf - true if x is inf (or inf/nan for ueq)
+ if (CondLHSStripped == FrexpInputStripped)
+ IsNonFiniteTest = CondSelectsZero;
+ } else if ((CC == ISD::SETONE || CC == ISD::SETUNE) &&
+ isInfConstant(CondRHS)) {
+ // fcmp one/une |x|, inf - true if x is NOT inf
+ if (CondLHSStripped == FrexpInputStripped)
+ IsNonFiniteTest = !CondSelectsZero;
+ } else if (CC == ISD::SETO) {
+ // fcmp ord x, y - true if both are NOT NaN
+ SDValue CondRHSStripped = peekFPSignOps(CondRHS);
+ if (CondLHSStripped == FrexpInputStripped ||
+ CondRHSStripped == FrexpInputStripped) {
+ IsNonFiniteTest = !CondSelectsZero;
+ }
+ }
+ }
+
+ if (!IsNonFiniteTest)
+ return SDValue();
+
+ // The select can be eliminated - just return the frexp result directly.
+ return FrexpVal;
+}
+
SDValue SITargetLowering::performSelectCombine(SDNode *N,
DAGCombinerInfo &DCI) const {
@@ -18960,6 +19099,8 @@ SDValue SITargetLowering::PerformDAGCombine(SDNode *N,
case ISD::SETCC:
return performSetCCCombine(N, DCI);
case ISD::SELECT:
+ if (auto Res = performFrexpSelectCombine(N, DCI))
+ return Res;
if (auto Res = performSelectCombine(N, DCI))
return Res;
break;
>From 395d9a91061f86c67cee5ca5d69b76465c35ba27 Mon Sep 17 00:00:00 2001
From: addmisol <addmisol9 at gmail.com>
Date: Sat, 8 Aug 2026 15:29:35 +0530
Subject: [PATCH 3/4] Fix: Address comments
Signed-off-by: addmisol <addmisol9 at gmail.com>
---
llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp | 10 --
llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h | 9 +
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 56 +++---
.../CodeGen/AMDGPU/frexp-inf-nan-combine.ll | 163 +++++++++++++++++-
4 files changed, 193 insertions(+), 45 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
index 962988ff97e39..cf074af1f78d7 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -1677,16 +1677,6 @@ static SDValue peekFNeg(SDValue Val) {
return Val;
}
-static SDValue peekFPSignOps(SDValue Val) {
- if (Val.getOpcode() == ISD::FNEG)
- Val = Val.getOperand(0);
- if (Val.getOpcode() == ISD::FABS)
- Val = Val.getOperand(0);
- if (Val.getOpcode() == ISD::FCOPYSIGN)
- Val = Val.getOperand(0);
- return Val;
-}
-
SDValue AMDGPUTargetLowering::combineFMinMaxLegacyImpl(
const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, SDValue True,
SDValue False, SDValue CC, DAGCombinerInfo &DCI) const {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
index f24f33e7dcb70..ec6479c509336 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
@@ -426,6 +426,15 @@ class AMDGPUTargetLowering : public TargetLowering {
}
};
+/// Strip fabs/fneg/fcopysign from a value to get the underlying source.
+/// Useful for comparing values where sign doesn't matter (e.g., frexp).
+inline SDValue peekFPSignOps(SDValue Val) {
+ while (Val.getOpcode() == ISD::FNEG || Val.getOpcode() == ISD::FABS ||
+ Val.getOpcode() == ISD::FCOPYSIGN)
+ Val = Val.getOperand(0);
+ return Val;
+}
+
} // End namespace llvm
#endif
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 8a05d8591548d..ae550ebbec50b 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -18829,6 +18829,12 @@ SDValue SITargetLowering::performClampCombine(SDNode *N,
return getCanonicalConstantFP(DCI.DAG, SDLoc(N), N->getValueType(0), F);
}
+/// Check if a value is a positive or negative infinity constant.
+static bool isInfinityFPConstant(SDValue V) {
+ auto *CFP = dyn_cast<ConstantFPSDNode>(V);
+ return CFP && CFP->getValueAPF().isInfinity();
+}
+
SDValue
SITargetLowering::performFrexpSelectCombine(SDNode *N,
DAGCombinerInfo &DCI) const {
@@ -18843,21 +18849,19 @@ SITargetLowering::performFrexpSelectCombine(SDNode *N,
// Determine which value is 0 and which might be the frexp result.
// Pattern 1: select cond, 0, frexp_result (cond true -> return 0)
// Pattern 2: select cond, frexp_result, 0 (cond false -> return 0)
+ // Only check FP zero - frexp returns FP or integer, handle separately.
SDValue FrexpVal;
bool CondSelectsZero; // If true, condition=true selects zero
- auto isZero = [](SDValue V) {
- if (auto *C = dyn_cast<ConstantSDNode>(V))
- return C->isZero();
- if (auto *C = dyn_cast<ConstantFPSDNode>(V))
- return C->isZero();
- return false;
- };
+ bool TrueIsFPZero = isNullFPConstant(TrueVal);
+ bool FalseIsFPZero = isNullFPConstant(FalseVal);
+ bool TrueIsIntZero = isNullConstant(TrueVal);
+ bool FalseIsIntZero = isNullConstant(FalseVal);
- if (isZero(TrueVal)) {
+ if (TrueIsFPZero || TrueIsIntZero) {
FrexpVal = FalseVal;
CondSelectsZero = true;
- } else if (isZero(FalseVal)) {
+ } else if (FalseIsFPZero || FalseIsIntZero) {
FrexpVal = TrueVal;
CondSelectsZero = false;
} else {
@@ -18865,26 +18869,15 @@ SITargetLowering::performFrexpSelectCombine(SDNode *N,
}
// Check if FrexpVal comes from amdgcn_frexp_exp or amdgcn_frexp_mant.
- if (FrexpVal.getOpcode() != ISD::INTRINSIC_WO_CHAIN)
- return SDValue();
-
- unsigned IID = FrexpVal.getConstantOperandVal(0);
- if (IID != Intrinsic::amdgcn_frexp_exp && IID != Intrinsic::amdgcn_frexp_mant)
+ SDValue FrexpInput;
+ if (!sd_match(FrexpVal,
+ m_IntrinsicWOChain<Intrinsic::amdgcn_frexp_exp>(
+ m_Value(FrexpInput))) &&
+ !sd_match(FrexpVal,
+ m_IntrinsicWOChain<Intrinsic::amdgcn_frexp_mant>(
+ m_Value(FrexpInput))))
return SDValue();
- SDValue FrexpInput = FrexpVal.getOperand(1);
-
- // Helper to strip fabs/fneg/fcopysign from a value.
- auto peekFPSignOps = [](SDValue Val) {
- if (Val.getOpcode() == ISD::FNEG)
- Val = Val.getOperand(0);
- if (Val.getOpcode() == ISD::FABS)
- Val = Val.getOperand(0);
- if (Val.getOpcode() == ISD::FCOPYSIGN)
- Val = Val.getOperand(0);
- return Val;
- };
-
// The frexp intrinsics ignore sign, so we can strip sign ops when comparing.
SDValue FrexpInputStripped = peekFPSignOps(FrexpInput);
@@ -18929,11 +18922,6 @@ SITargetLowering::performFrexpSelectCombine(SDNode *N,
SDValue CondRHS = Cond.getOperand(1);
SDValue CondLHSStripped = peekFPSignOps(CondLHS);
- auto isInfConstant = [](SDValue V) {
- auto *CFP = dyn_cast<ConstantFPSDNode>(V);
- return CFP && CFP->getValueAPF().isInfinity();
- };
-
if (CC == ISD::SETUO) {
// fcmp uno x, y - true if either x or y is NaN
SDValue CondRHSStripped = peekFPSignOps(CondRHS);
@@ -18942,12 +18930,12 @@ SITargetLowering::performFrexpSelectCombine(SDNode *N,
IsNonFiniteTest = CondSelectsZero;
}
} else if ((CC == ISD::SETOEQ || CC == ISD::SETUEQ) &&
- isInfConstant(CondRHS)) {
+ isInfinityFPConstant(CondRHS)) {
// fcmp oeq/ueq |x|, inf - true if x is inf (or inf/nan for ueq)
if (CondLHSStripped == FrexpInputStripped)
IsNonFiniteTest = CondSelectsZero;
} else if ((CC == ISD::SETONE || CC == ISD::SETUNE) &&
- isInfConstant(CondRHS)) {
+ isInfinityFPConstant(CondRHS)) {
// fcmp one/une |x|, inf - true if x is NOT inf
if (CondLHSStripped == FrexpInputStripped)
IsNonFiniteTest = !CondSelectsZero;
diff --git a/llvm/test/CodeGen/AMDGPU/frexp-inf-nan-combine.ll b/llvm/test/CodeGen/AMDGPU/frexp-inf-nan-combine.ll
index c2a763e58e2c2..0cba8c03cce7c 100644
--- a/llvm/test/CodeGen/AMDGPU/frexp-inf-nan-combine.ll
+++ b/llvm/test/CodeGen/AMDGPU/frexp-inf-nan-combine.ll
@@ -3,7 +3,7 @@
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1030 < %s | FileCheck -check-prefix=GFX10 %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 < %s | FileCheck -check-prefix=GFX11 %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 < %s | FileCheck -check-prefix=GFX950 %s
-; RUN: llc -mtriple=amdgcn-- -mcpu=tahiti < %s | FileCheck -check-prefix=SI %s
+; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti < %s | FileCheck -check-prefix=SI %s
; Test that redundant inf/nan checks are folded into frexp instructions.
; The AMDGPU frexp instructions already return 0 for inf/nan inputs.
@@ -12,9 +12,14 @@
declare {float, i32} @llvm.frexp.f32.i32(float)
declare {double, i32} @llvm.frexp.f64.i32(double)
declare {half, i16} @llvm.frexp.f16.i16(half)
+declare {bfloat, i16} @llvm.frexp.bf16.i16(bfloat)
+declare {<2 x float>, <2 x i32>} @llvm.frexp.v2f32.v2i32(<2 x float>)
declare float @llvm.fabs.f32(float)
declare double @llvm.fabs.f64(double)
declare half @llvm.fabs.f16(half)
+declare bfloat @llvm.fabs.bf16(bfloat)
+declare <2 x float> @llvm.fabs.v2f32(<2 x float>)
+declare float @llvm.copysign.f32(float, float)
; Pattern 1: select (fcmp uno x, 0), 0, (frexp_exp x)
; NaN check - should fold to just frexp_exp
@@ -526,3 +531,159 @@ define i32 @frexp_lt_zero_not_folded(float %x) {
%result = select i1 %is_lt_zero, i32 0, i32 %exp
ret i32 %result
}
+
+; Test with copysign - should still fold since frexp ignores sign
+define i32 @frexp_nan_clamp_copysign_f32(float %x, float %sign) {
+; GFX9-LABEL: frexp_nan_clamp_copysign_f32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: frexp_nan_clamp_copysign_f32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: frexp_nan_clamp_copysign_f32:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-LABEL: frexp_nan_clamp_copysign_f32:
+; GFX950: ; %bb.0:
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-LABEL: frexp_nan_clamp_copysign_f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT: s_brev_b32 s4, -2
+; SI-NEXT: v_bfi_b32 v1, s4, v0, v1
+; SI-NEXT: s_mov_b32 s4, 0x7f800000
+; SI-NEXT: v_cmp_lt_f32_e64 s[4:5], |v1|, s4
+; SI-NEXT: v_cmp_o_f32_e32 vcc, v0, v0
+; SI-NEXT: v_frexp_exp_i32_f32_e32 v1, v0
+; SI-NEXT: s_and_b64 vcc, vcc, s[4:5]
+; SI-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
+; SI-NEXT: s_setpc_b64 s[30:31]
+ %copysign = call float @llvm.copysign.f32(float %x, float %sign)
+ %frexp = call {float, i32} @llvm.frexp.f32.i32(float %copysign)
+ %exp = extractvalue {float, i32} %frexp, 1
+ %is_nan = fcmp uno float %x, 0.0
+ %result = select i1 %is_nan, i32 0, i32 %exp
+ ret i32 %result
+}
+
+; Test bfloat16
+define i16 @frexp_nan_clamp_exp_bf16(bfloat %x) {
+; GFX9-LABEL: frexp_nan_clamp_exp_bf16:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_frexp_exp_i32_f32_e32 v1, v0
+; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: frexp_nan_clamp_exp_bf16:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX10-NEXT: v_frexp_exp_i32_f32_e32 v1, v0
+; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc_lo
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: frexp_nan_clamp_exp_bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_frexp_exp_i32_f32_e32 v1, v0
+; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: v_cndmask_b16 v0.l, 0, v1.l, vcc_lo
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-LABEL: frexp_nan_clamp_exp_bf16:
+; GFX950: ; %bb.0:
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX950-NEXT: v_frexp_exp_i32_f32_e32 v1, v0
+; GFX950-NEXT: v_cmp_o_f32_e32 vcc, v0, v0
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-LABEL: frexp_nan_clamp_exp_bf16:
+; SI: ; %bb.0:
+; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: s_mov_b32 s4, 0x7f800000
+; SI-NEXT: v_cmp_lt_f32_e64 s[4:5], |v0|, s4
+; SI-NEXT: v_cmp_o_f32_e32 vcc, v0, v0
+; SI-NEXT: v_frexp_exp_i32_f32_e32 v1, v0
+; SI-NEXT: s_and_b64 vcc, vcc, s[4:5]
+; SI-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
+; SI-NEXT: s_setpc_b64 s[30:31]
+ %frexp = call {bfloat, i16} @llvm.frexp.bf16.i16(bfloat %x)
+ %exp = extractvalue {bfloat, i16} %frexp, 1
+ %is_nan = fcmp uno bfloat %x, 0.0
+ %result = select i1 %is_nan, i16 0, i16 %exp
+ ret i16 %result
+}
+
+; Test v2f32 vector case
+define <2 x i32> @frexp_nan_clamp_exp_v2f32(<2 x float> %x) {
+; GFX9-LABEL: frexp_nan_clamp_exp_v2f32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; GFX9-NEXT: v_frexp_exp_i32_f32_e32 v1, v1
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: frexp_nan_clamp_exp_v2f32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; GFX10-NEXT: v_frexp_exp_i32_f32_e32 v1, v1
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: frexp_nan_clamp_exp_v2f32:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; GFX11-NEXT: v_frexp_exp_i32_f32_e32 v1, v1
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-LABEL: frexp_nan_clamp_exp_v2f32:
+; GFX950: ; %bb.0:
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; GFX950-NEXT: v_frexp_exp_i32_f32_e32 v1, v1
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-LABEL: frexp_nan_clamp_exp_v2f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s6, 0x7f800000
+; SI-NEXT: v_cmp_lt_f32_e64 s[4:5], |v1|, s6
+; SI-NEXT: v_cmp_lt_f32_e64 s[6:7], |v0|, s6
+; SI-NEXT: v_cmp_o_f32_e32 vcc, v0, v0
+; SI-NEXT: v_frexp_exp_i32_f32_e32 v3, v0
+; SI-NEXT: s_and_b64 vcc, vcc, s[6:7]
+; SI-NEXT: v_cndmask_b32_e32 v0, 0, v3, vcc
+; SI-NEXT: v_cmp_o_f32_e32 vcc, v1, v1
+; SI-NEXT: v_frexp_exp_i32_f32_e32 v2, v1
+; SI-NEXT: s_and_b64 vcc, vcc, s[4:5]
+; SI-NEXT: v_cndmask_b32_e32 v1, 0, v2, vcc
+; SI-NEXT: s_setpc_b64 s[30:31]
+ %frexp = call {<2 x float>, <2 x i32>} @llvm.frexp.v2f32.v2i32(<2 x float> %x)
+ %exp = extractvalue {<2 x float>, <2 x i32>} %frexp, 1
+ %is_nan = fcmp uno <2 x float> %x, zeroinitializer
+ %result = select <2 x i1> %is_nan, <2 x i32> zeroinitializer, <2 x i32> %exp
+ ret <2 x i32> %result
+}
>From 8c332aa998c50a52012fe57953322be27889fb12 Mon Sep 17 00:00:00 2001
From: addmisol <addmisol9 at gmail.com>
Date: Sat, 8 Aug 2026 16:47:18 +0530
Subject: [PATCH 4/4] Fix: Remove unnecessary changes for clang format | keep
only relevant
Signed-off-by: addmisol <addmisol9 at gmail.com>
---
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 10 ++++------
1 file changed, 4 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index ae550ebbec50b..f67fe52939415 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -18870,12 +18870,10 @@ SITargetLowering::performFrexpSelectCombine(SDNode *N,
// Check if FrexpVal comes from amdgcn_frexp_exp or amdgcn_frexp_mant.
SDValue FrexpInput;
- if (!sd_match(FrexpVal,
- m_IntrinsicWOChain<Intrinsic::amdgcn_frexp_exp>(
- m_Value(FrexpInput))) &&
- !sd_match(FrexpVal,
- m_IntrinsicWOChain<Intrinsic::amdgcn_frexp_mant>(
- m_Value(FrexpInput))))
+ if (!sd_match(FrexpVal, m_IntrinsicWOChain<Intrinsic::amdgcn_frexp_exp>(
+ m_Value(FrexpInput))) &&
+ !sd_match(FrexpVal, m_IntrinsicWOChain<Intrinsic::amdgcn_frexp_mant>(
+ m_Value(FrexpInput))))
return SDValue();
// The frexp intrinsics ignore sign, so we can strip sign ops when comparing.
More information about the llvm-commits
mailing list