[llvm] [AMDGPU] Fold redundant inf/nan checks into frexp instructions (PR #214936)
via llvm-commits
llvm-commits at lists.llvm.org
Sat Aug 8 03:24:53 PDT 2026
https://github.com/addmisol updated https://github.com/llvm/llvm-project/pull/214936
>From d72169dac4e1a7c5899c07da7c127486240e65e4 Mon Sep 17 00:00:00 2001
From: addmisol <addmisol9 at gmail.com>
Date: Sat, 8 Aug 2026 11:57:53 +0530
Subject: [PATCH 1/4] Fix: [AMDGPU] Fold redundant inf/nan checks into frexp
instructions
Signed-off-by: addmisol <addmisol9 at gmail.com>
---
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 898 +++++++++++++-----
llvm/lib/Target/AMDGPU/SIISelLowering.h | 39 +-
.../CodeGen/AMDGPU/frexp-inf-nan-combine.ll | 528 ++++++++++
3 files changed, 1240 insertions(+), 225 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/frexp-inf-nan-combine.ll
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 8a8a3b97e8e08..8a05d8591548d 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -22,6 +22,7 @@
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
#include "SIMachineFunctionInfo.h"
#include "SIRegisterInfo.h"
+#include "llvm/ADT/APFloat.h"
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/FloatingPointMode.h"
#include "llvm/ADT/Statistic.h"
@@ -47,6 +48,7 @@
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/KnownBits.h"
#include "llvm/Support/ModRef.h"
+#include "llvm/TargetParser/AtomicScope.h"
#include "llvm/Transforms/Utils/LowerAtomic.h"
#include <optional>
@@ -67,6 +69,10 @@ static cl::opt<bool> UseDivergentRegisterIndexing(
cl::desc("Use indirect register addressing for divergent indexes"),
cl::init(false));
+static DenormalFPEnv getDenormalFPEnv(const MachineFunction &MF) {
+ return MF.getInfo<SIMachineFunctionInfo>()->getMode().getDenormalFPEnv();
+}
+
static bool denormalModeIsFlushAllF32(const MachineFunction &MF) {
const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
return Info->getMode().FP32Denormals == DenormalMode::getPreserveSign();
@@ -236,6 +242,20 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
setOperationAction(Opc, MVT::bf16, Promote);
}
+ // Only targets with packed bf16 instructions, e.g. gfx13.
+ if (Subtarget->hasBF16PackedInsts()) {
+ // Turn fsub into fadd(x, fneg y) so it reuses the packed v_pk_add_bf16
+ // path instead of promoting to f32.
+ setOperationAction(ISD::FSUB, MVT::bf16, Expand);
+ // Widen scalar fadd to a v2bf16 operation with an unused high lane.
+ setOperationAction(ISD::FADD, MVT::bf16, Custom);
+ // Widen scalar fcanonicalize to a v2bf16 operation with an unused high
+ // lane.
+ setOperationAction(ISD::FCANONICALIZE, MVT::bf16, Custom);
+ // Widen scalar fmul to a v2bf16 operation with an unused high lane.
+ setOperationAction(ISD::FMUL, MVT::bf16, Custom);
+ }
+
setOperationAction(ISD::FP_ROUND, MVT::bf16, Expand);
setOperationAction(ISD::SELECT, MVT::bf16, Promote);
@@ -343,6 +363,7 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
case ISD::BUILD_VECTOR:
case ISD::BITCAST:
case ISD::UNDEF:
+ case ISD::POISON:
case ISD::EXTRACT_VECTOR_ELT:
case ISD::INSERT_VECTOR_ELT:
case ISD::SCALAR_TO_VECTOR:
@@ -660,6 +681,7 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
case ISD::BUILD_VECTOR:
case ISD::BITCAST:
case ISD::UNDEF:
+ case ISD::POISON:
case ISD::EXTRACT_VECTOR_ELT:
case ISD::INSERT_VECTOR_ELT:
case ISD::INSERT_SUBVECTOR:
@@ -693,8 +715,8 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
// XXX - Do these do anything? Vector constants turn into build_vector.
setOperationAction(ISD::Constant, {MVT::v2i16, MVT::v2f16}, Legal);
- setOperationAction(ISD::UNDEF, {MVT::v2i16, MVT::v2f16, MVT::v2bf16},
- Legal);
+ setOperationAction({ISD::UNDEF, ISD::POISON},
+ {MVT::v2i16, MVT::v2f16, MVT::v2bf16}, Legal);
setOperationAction(ISD::STORE, MVT::v2i16, Promote);
AddPromotedToType(ISD::STORE, MVT::v2i16, MVT::i32);
@@ -897,14 +919,14 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
VT, Custom);
}
- if (Subtarget->hasPackedFP32Ops()) {
+ if (Subtarget->hasAnyPackedFP32Ops()) {
setOperationAction({ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FNEG},
MVT::v2f32, Legal);
setOperationAction({ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FNEG},
{MVT::v4f32, MVT::v8f32, MVT::v16f32, MVT::v32f32},
Custom);
}
- if (Subtarget->hasPackedFP64Ops()) {
+ if (Subtarget->hasAnyPackedFP64Ops()) {
setOperationAction({ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FNEG,
ISD::FMINNUM_IEEE, ISD::FMAXNUM_IEEE,
ISD::FCANONICALIZE, ISD::BUILD_VECTOR},
@@ -919,7 +941,7 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
{MVT::v4f64, MVT::v8f64, MVT::v16f64, MVT::v32f64}, Custom);
}
- if (Subtarget->hasPackedU64Ops()) {
+ if (Subtarget->hasAnyPackedU64Ops()) {
setOperationAction({ISD::ADD, ISD::SUB, ISD::SHL, ISD::BUILD_VECTOR},
MVT::v2i64, Legal);
setOperationAction({ISD::ADD, ISD::SUB, ISD::SHL},
@@ -935,6 +957,8 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
AddPromotedToType(ISD::SELECT, MVT::v2i16, MVT::i32);
setOperationAction(ISD::SELECT, MVT::v2f16, Promote);
AddPromotedToType(ISD::SELECT, MVT::v2f16, MVT::i32);
+ setOperationAction(ISD::SELECT, MVT::v2bf16, Promote);
+ AddPromotedToType(ISD::SELECT, MVT::v2bf16, MVT::i32);
} else {
// Legalization hack.
setOperationAction(ISD::SELECT, {MVT::v2i16, MVT::v2f16}, Custom);
@@ -1023,7 +1047,8 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
setOperationAction(ISD::MUL, MVT::i1, Promote);
if (Subtarget->hasBF16ConversionInsts()) {
- setOperationAction(ISD::FP_ROUND, {MVT::bf16, MVT::v2bf16}, Custom);
+ setOperationAction({ISD::FP_ROUND, ISD::STRICT_FP_ROUND},
+ {MVT::bf16, MVT::v2bf16}, Custom);
setOperationAction(ISD::BUILD_VECTOR, MVT::v2bf16, Legal);
}
@@ -1031,6 +1056,17 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
setOperationAction({ISD::FEXP2, ISD::FLOG2, ISD::FSQRT}, MVT::bf16, Legal);
}
+ if (Subtarget->hasOCPFP8ConversionInsts()) {
+ setOperationAction(ISD::CONVERT_FROM_ARBITRARY_FP, {MVT::f32, MVT::v2f32},
+ Custom);
+ setOperationAction(ISD::CONVERT_FROM_ARBITRARY_FP, MVT::v2i8, Custom);
+ }
+
+ if (Subtarget->hasFP8F16ConversionInsts()) {
+ setOperationAction(ISD::CONVERT_FROM_ARBITRARY_FP, {MVT::f16, MVT::v2f16},
+ Custom);
+ }
+
if (Subtarget->hasCvtPkF16F32Inst()) {
setOperationAction(ISD::FP_ROUND,
{MVT::v2f16, MVT::v4f16, MVT::v8f16, MVT::v16f16},
@@ -1417,6 +1453,7 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
if (Aux->getZExtValue() & AMDGPU::CPol::VOLATILE)
Flags |= MachineMemOperand::MOVolatile;
}
+
Flags |= MachineMemOperand::MODereferenceable;
IntrinsicInfo Info;
@@ -2379,13 +2416,16 @@ bool SITargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm,
return true;
}
-bool SITargetLowering::isExtractSubvectorCheap(EVT ResVT, EVT SrcVT,
- unsigned Index) const {
+TargetLowering::ExtractSubvectorCost
+SITargetLowering::getExtractSubvectorCost(EVT ResVT, EVT SrcVT,
+ unsigned Index) const {
if (!isOperationLegalOrCustom(ISD::EXTRACT_SUBVECTOR, ResVT))
- return false;
+ return ExtractSubvectorCost::Expensive;
// TODO: Add more cases that are cheap.
- return Index == 0;
+ if (Index == 0)
+ return ExtractSubvectorCost::Free;
+ return ExtractSubvectorCost::Expensive;
}
bool SITargetLowering::isExtractVecEltCheap(EVT VT, unsigned Index) const {
@@ -2682,7 +2722,7 @@ SDValue SITargetLowering::getPreloadedValue(
SelectionDAG &DAG, const SIMachineFunctionInfo &MFI, EVT VT,
AMDGPUFunctionArgInfo::PreloadedValue PVID) const {
const ArgDescriptor *Reg = nullptr;
- const TargetRegisterClass *RC;
+ const TargetRegisterClass *RC = nullptr;
LLT Ty;
CallingConv::ID CC = DAG.getMachineFunction().getFunction().getCallingConv();
@@ -2901,7 +2941,7 @@ void SITargetLowering::allocateSpecialEntryInputVGPRs(
// Try to allocate a VGPR at the end of the argument list, or if no argument
// VGPRs are left allocating a stack slot.
-// If \p Mask is is given it indicates bitfield position in the register.
+// If \p Mask is given it indicates bitfield position in the register.
// If \p Arg is given use it with new ]p Mask instead of allocating new.
static ArgDescriptor allocateVGPR32Input(CCState &CCInfo, unsigned Mask = ~0u,
ArgDescriptor Arg = ArgDescriptor()) {
@@ -3730,15 +3770,8 @@ SDValue SITargetLowering::LowerFormalArguments(
if (Arg.Flags.isSRet()) {
// The return object should be reasonably addressable.
-
- // FIXME: This helps when the return is a real sret. If it is a
- // automatically inserted sret (i.e. CanLowerReturn returns false), an
- // extra copy is inserted in SelectionDAGBuilder which obscures this.
- unsigned NumBits =
- 32 - getSubtarget()->getKnownHighZeroBitsForFrameIndex();
- Val = DAG.getNode(
- ISD::AssertZext, DL, VT, Val,
- DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), NumBits)));
+ Val = annotateStackObjectPointer(Val, DAG, DL,
+ Arg.Flags.getNonZeroMemAlign());
}
Val = convertABITypeToValueType(DAG, Val, VA, DL);
@@ -6021,7 +6054,8 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
Register Op1L_Op0H_Reg =
MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
- Register CarryReg = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
+ Register CarryReg =
+ MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
Register AddReg = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
Register NegatedValLo =
MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
@@ -6048,9 +6082,23 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MUL_I32), DestSub0)
.addReg(Op1L)
.addReg(LowOpcode);
- BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MUL_HI_U32), CarryReg)
- .addReg(Op1L)
- .addReg(LowOpcode);
+ if (ST.hasScalarMulHiInsts()) {
+ BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MUL_HI_U32), CarryReg)
+ .addReg(Op1L)
+ .addReg(LowOpcode);
+ } else {
+ Register VCarryReg =
+ MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
+ Register LowOpVGPR =
+ MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
+ BuildMI(BB, MI, DL, TII->get(AMDGPU::COPY), LowOpVGPR)
+ .addReg(LowOpcode);
+ BuildMI(BB, MI, DL, TII->get(AMDGPU::V_MUL_HI_U32_e64), VCarryReg)
+ .addReg(Op1L)
+ .addReg(LowOpVGPR);
+ BuildMI(BB, MI, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), CarryReg)
+ .addReg(VCarryReg);
+ }
BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MUL_I32), Op1H_Op0L_Reg)
.addReg(Op1H)
.addReg(LowOpcode);
@@ -6371,10 +6419,23 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
ActiveBits.addReg(NewActiveBitsReg).addMBB(ComputeLoop);
// Creating branching
- unsigned CMPOpc = IsWave32 ? AMDGPU::S_CMP_LG_U32 : AMDGPU::S_CMP_LG_U64;
- BuildMI(*ComputeLoop, I, DL, TII->get(CMPOpc))
- .addReg(NewActiveBitsReg)
- .addImm(0);
+ MachineInstrBuilder SetSCCInstr;
+ if (!ST.hasScalarCompareEq64()) {
+ // For targets <= gfx7, use an S_OR_B32/B64 instruction to set SCC.
+ Register LaneMaskReg = MRI.createVirtualRegister(WaveMaskRegClass);
+ unsigned CMPOpc = IsWave32 ? AMDGPU::S_OR_B32 : AMDGPU::S_OR_B64;
+ SetSCCInstr =
+ BuildMI(*ComputeLoop, I, DL, TII->get(CMPOpc), LaneMaskReg);
+ } else {
+ unsigned CMPOpc =
+ IsWave32 ? AMDGPU::S_CMP_LG_U32 : AMDGPU::S_CMP_LG_U64;
+ SetSCCInstr = BuildMI(*ComputeLoop, I, DL, TII->get(CMPOpc));
+ }
+ SetSCCInstr.addReg(NewActiveBitsReg);
+ if (ST.hasScalarCompareEq64())
+ SetSCCInstr.addImm(0);
+ else
+ SetSCCInstr.addReg(NewActiveBitsReg);
BuildMI(*ComputeLoop, I, DL, TII->get(AMDGPU::S_CBRANCH_SCC1))
.addMBB(ComputeLoop);
@@ -7320,6 +7381,14 @@ SITargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
MI.getOperand(0).setReg(OriginalExec);
return BB;
}
+ case AMDGPU::V_DOT2_F32_F16:
+ case AMDGPU::V_DOT2_F32_BF16: {
+ // Hint RA to assign dst and src2 the same physical register.
+ // For targets without VOP2, but with VOPD, variant of the instruction this
+ // is one of the conditions to attempt converting VOP3P to VOPD.
+ MRI.setSimpleHint(MI.getOperand(0).getReg(), MI.getOperand(6).getReg());
+ return BB;
+ }
default:
if (TII->isImage(MI) || TII->isMUBUF(MI)) {
if (!MI.mayStore())
@@ -7378,9 +7447,11 @@ LLT SITargetLowering::getPreferredShiftAmountTy(LLT Ty) const {
// however does not support denormals, so we do report fma as faster if we have
// a fast fma device and require denormals.
//
-bool SITargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
- EVT VT) const {
+bool SITargetLowering::isFMAFasterThanFMulAndFAdd(EVT VT,
+ DenormalFPEnv FPEnv) const {
VT = VT.getScalarType();
+ if (!VT.isSimple())
+ return false;
switch (VT.getSimpleVT().SimpleTy) {
case MVT::f32: {
@@ -7391,7 +7462,7 @@ bool SITargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
// Otherwise f32 mad is always full rate and returns the same result as
// the separate operations so should be preferred over fma.
// However does not support denormals.
- if (!denormalModeIsFlushAllF32(MF))
+ if (FPEnv.F32Mode != DenormalMode::getPreserveSign())
return Subtarget->hasFastFMAF32() || Subtarget->hasDLInsts();
// If the subtarget has v_fmac_f32, that's just as good as v_mac_f32.
@@ -7401,7 +7472,8 @@ bool SITargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
return true;
case MVT::f16:
case MVT::bf16:
- return Subtarget->has16BitInsts() && !denormalModeIsFlushAllF64F16(MF);
+ return Subtarget->has16BitInsts() &&
+ FPEnv.DefaultMode != DenormalMode::getPreserveSign();
default:
break;
}
@@ -7409,6 +7481,18 @@ bool SITargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
return false;
}
+bool SITargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
+ EVT VT) const {
+ return isFMAFasterThanFMulAndFAdd(VT, getDenormalFPEnv(MF));
+}
+
+bool SITargetLowering::isFMAFasterThanFMulAndFAdd(const Function &F,
+ Type *Ty) const {
+ return isFMAFasterThanFMulAndFAdd(
+ getValueType(F.getDataLayout(), Ty, /*AllowUnknown=*/true),
+ F.getDenormalFPEnv());
+}
+
bool SITargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
LLT Ty) const {
switch (Ty.getScalarSizeInBits()) {
@@ -7425,33 +7509,36 @@ bool SITargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
return false;
}
+bool SITargetLowering::isFMADLegal(EVT VT, DenormalFPEnv FPEnv) const {
+ // TODO: Check future ftz flag
+ // v_mad_f32/v_mac_f32 do not support denormals.
+ if (VT == MVT::f32)
+ return Subtarget->hasMadMacF32Insts() &&
+ FPEnv.F32Mode == DenormalMode::getPreserveSign();
+ if (VT == MVT::f16)
+ return Subtarget->hasMadF16() &&
+ FPEnv.DefaultMode == DenormalMode::getPreserveSign();
+
+ return false;
+}
+
bool SITargetLowering::isFMADLegal(const MachineInstr &MI, LLT Ty) const {
if (!Ty.isScalar())
return false;
+ DenormalFPEnv FPEnv = getDenormalFPEnv(*MI.getMF());
if (Ty.getScalarSizeInBits() == 16)
- return Subtarget->hasMadF16() && denormalModeIsFlushAllF64F16(*MI.getMF());
+ return isFMADLegal(MVT::f16, FPEnv);
if (Ty.getScalarSizeInBits() == 32)
- return Subtarget->hasMadMacF32Insts() &&
- denormalModeIsFlushAllF32(*MI.getMF());
+ return isFMADLegal(MVT::f32, FPEnv);
return false;
}
bool SITargetLowering::isFMADLegal(const SelectionDAG &DAG,
const SDNode *N) const {
- // TODO: Check future ftz flag
- // v_mad_f32/v_mac_f32 do not support denormals.
- EVT VT = N->getValueType(0);
- if (VT == MVT::f32)
- return Subtarget->hasMadMacF32Insts() &&
- denormalModeIsFlushAllF32(DAG.getMachineFunction());
- if (VT == MVT::f16) {
- return Subtarget->hasMadF16() &&
- denormalModeIsFlushAllF64F16(DAG.getMachineFunction());
- }
-
- return false;
+ return isFMADLegal(N->getValueType(0),
+ getDenormalFPEnv(DAG.getMachineFunction()));
}
//===----------------------------------------------------------------------===//
@@ -7592,6 +7679,8 @@ SDValue SITargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
return LowerExternalSymbol(Op, DAG);
case ISD::INTRINSIC_WO_CHAIN:
return LowerINTRINSIC_WO_CHAIN(Op, DAG);
+ case ISD::CONVERT_FROM_ARBITRARY_FP:
+ return LowerCONVERT_FROM_ARBITRARY_FP(Op, DAG);
case ISD::INTRINSIC_W_CHAIN:
return LowerINTRINSIC_W_CHAIN(Op, DAG);
case ISD::INTRINSIC_VOID:
@@ -7620,7 +7709,6 @@ SDValue SITargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
case ISD::ABS:
case ISD::FABS:
case ISD::FNEG:
- case ISD::FCANONICALIZE:
case ISD::BSWAP:
return splitUnaryVectorOp(Op, DAG);
case ISD::FP_TO_SINT_SAT:
@@ -7660,8 +7748,6 @@ SDValue SITargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
case ISD::SMAX:
case ISD::UMIN:
case ISD::UMAX:
- case ISD::FADD:
- case ISD::FMUL:
case ISD::FMINNUM_IEEE:
case ISD::FMAXNUM_IEEE:
case ISD::UADDSAT:
@@ -7669,6 +7755,15 @@ SDValue SITargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
case ISD::SADDSAT:
case ISD::SSUBSAT:
return splitBinaryVectorOp(Op, DAG);
+ case ISD::FADD:
+ case ISD::FMUL:
+ if (Op.getValueType() == MVT::bf16)
+ return lowerScalarBF16BinaryOp(Op, DAG);
+ return splitBinaryVectorOp(Op, DAG);
+ case ISD::FCANONICALIZE:
+ if (Op.getValueType() == MVT::bf16)
+ return lowerScalarBF16FCanonicalize(Op, DAG);
+ return splitUnaryVectorOp(Op, DAG);
case ISD::FCOPYSIGN:
return lowerFCOPYSIGN(Op, DAG);
case ISD::MUL:
@@ -7791,6 +7886,13 @@ SDValue SITargetLowering::lowerIntrinsicLoad(MemSDNode *M, bool IsFormat,
bool IsD16 = IsFormat && (EltType.getSizeInBits() == 16);
+ if (IsFormat && !IsD16 && EltType.getSizeInBits() < 32) {
+ DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
+ DAG.getMachineFunction().getFunction(),
+ "unsupported sub-dword format buffer load", DL.getDebugLoc()));
+ return DAG.getMergeValues({DAG.getPOISON(LoadVT), M->getOperand(0)}, DL);
+ }
+
assert(M->getNumValues() == 2 || M->getNumValues() == 3);
bool IsTFE = M->getNumValues() == 3;
@@ -7948,10 +8050,6 @@ static SDValue lowerLaneOp(const SITargetLowering &TLI, SDNode *N,
MVT IntVT = MVT::getIntegerVT(ValSize);
const GCNSubtarget *ST = TLI.getSubtarget();
- if ((IsPermLane16 && !ST->hasPermlane16Insts()) ||
- (IID == Intrinsic::amdgcn_mov_dpp8 && !ST->hasDPP8()))
- return emitRemovedIntrinsicError(DAG, SL, VT);
-
unsigned SplitSize = 32;
if (IID == Intrinsic::amdgcn_update_dpp && (ValSize % 64 == 0) &&
ST->hasDPALU_DPP() &&
@@ -8242,6 +8340,30 @@ void SITargetLowering::ReplaceNodeResults(SDNode *N,
case ISD::INTRINSIC_WO_CHAIN: {
unsigned IID = N->getConstantOperandVal(0);
switch (IID) {
+ case Intrinsic::amdgcn_wave_reduce_min:
+ case Intrinsic::amdgcn_wave_reduce_umin:
+ case Intrinsic::amdgcn_wave_reduce_max:
+ case Intrinsic::amdgcn_wave_reduce_umax:
+ case Intrinsic::amdgcn_wave_reduce_add:
+ case Intrinsic::amdgcn_wave_reduce_sub:
+ case Intrinsic::amdgcn_wave_reduce_and:
+ case Intrinsic::amdgcn_wave_reduce_or:
+ case Intrinsic::amdgcn_wave_reduce_xor: {
+ EVT VT = N->getValueType(0);
+ if (isTypeLegal(VT))
+ return;
+ SDLoc SL(N);
+ bool NeedsSignExt = IID == Intrinsic::amdgcn_wave_reduce_min ||
+ IID == Intrinsic::amdgcn_wave_reduce_max ||
+ IID == Intrinsic::amdgcn_wave_reduce_add ||
+ IID == Intrinsic::amdgcn_wave_reduce_sub;
+ unsigned ExtOpc = NeedsSignExt ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
+ SDValue ExtSrc = DAG.getNode(ExtOpc, SL, MVT::i32, N->getOperand(1));
+ SDValue Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, MVT::i32,
+ N->getOperand(0), ExtSrc, N->getOperand(2));
+ Results.push_back(DAG.getNode(ISD::TRUNCATE, SL, VT, Result));
+ return;
+ }
case Intrinsic::amdgcn_make_buffer_rsrc:
Results.push_back(lowerPointerAsRsrcIntrin(N, DAG));
return;
@@ -8307,11 +8429,12 @@ void SITargetLowering::ReplaceNodeResults(SDNode *N,
VT.getStoreSize(), Alignment);
SDValue LoadVal;
if (!Offset->isDivergent()) {
- SDValue Ops[] = {Rsrc, // source register
+ SDValue Ops[] = {DAG.getEntryNode(), // Chain
+ Rsrc, // source register
Offset, CachePolicy};
- SDValue BufferLoad =
- DAG.getMemIntrinsicNode(AMDGPUISD::SBUFFER_LOAD_UBYTE, DL,
- DAG.getVTList(MVT::i32), Ops, VT, MMO);
+ SDValue BufferLoad = DAG.getMemIntrinsicNode(
+ AMDGPUISD::SBUFFER_LOAD_UBYTE, DL,
+ DAG.getVTList(MVT::i32, MVT::Other), Ops, VT, MMO);
LoadVal = DAG.getNode(ISD::TRUNCATE, DL, VT, BufferLoad);
} else {
SDValue Ops[] = {
@@ -8651,7 +8774,8 @@ SDValue SITargetLowering::splitFP_ROUNDVectorOp(SDValue Op,
}
SDValue SITargetLowering::lowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const {
- SDValue Src = Op.getOperand(0);
+ bool IsStrict = Op->isStrictFPOpcode();
+ SDValue Src = Op.getOperand(IsStrict ? 1 : 0);
EVT SrcVT = Src.getValueType();
EVT DstVT = Op.getValueType();
@@ -8694,10 +8818,58 @@ SDValue SITargetLowering::lowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const {
// hardware f32 -> bf16 instruction.
EVT F32VT = SrcVT.changeElementType(*DAG.getContext(), MVT::f32);
SDValue Rod = expandRoundInexactToOdd(F32VT, Src, DL, DAG);
+ if (IsStrict) {
+ return DAG.getNode(
+ ISD::STRICT_FP_ROUND, DL, {DstVT, MVT::Other},
+ {Op.getOperand(0), Rod, DAG.getTargetConstant(0, DL, MVT::i32)});
+ }
return DAG.getNode(ISD::FP_ROUND, DL, DstVT, Rod,
DAG.getTargetConstant(0, DL, MVT::i32));
}
+SDValue SITargetLowering::lowerScalarBF16BinaryOp(SDValue Op,
+ SelectionDAG &DAG) const {
+ assert(Subtarget->hasBF16PackedInsts());
+
+ SDLoc DL(Op);
+
+ auto WidenOperand = [&](SDValue Src) {
+ if (Src.getOpcode() == ISD::FNEG) {
+ SDValue WideSrc = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2bf16,
+ Src.getOperand(0));
+ return DAG.getNode(ISD::FNEG, DL, MVT::v2bf16, WideSrc, Src->getFlags());
+ }
+
+ return DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2bf16, Src);
+ };
+
+ SDValue LHS = WidenOperand(Op.getOperand(0));
+ SDValue RHS = WidenOperand(Op.getOperand(1));
+ SDValue Result =
+ DAG.getNode(Op.getOpcode(), DL, MVT::v2bf16, LHS, RHS, Op->getFlags());
+
+ return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::bf16, Result,
+ DAG.getConstant(0, DL, MVT::i32));
+}
+
+SDValue
+SITargetLowering::lowerScalarBF16FCanonicalize(SDValue Op,
+ SelectionDAG &DAG) const {
+ assert(Subtarget->hasBF16PackedInsts());
+
+ SDLoc DL(Op);
+ SDValue Src = Op.getOperand(0);
+
+ // Widen to v2bf16, canonicalize with v_pk_mul_bf16, then extract.
+ SDValue WideSrc = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2bf16, Src);
+
+ SDValue Canonicalized =
+ DAG.getNode(ISD::FCANONICALIZE, DL, MVT::v2bf16, WideSrc, Op->getFlags());
+
+ return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::bf16, Canonicalized,
+ DAG.getConstant(0, DL, MVT::i32));
+}
+
SDValue SITargetLowering::lowerFMINNUM_FMAXNUM(SDValue Op,
SelectionDAG &DAG) const {
EVT VT = Op.getValueType();
@@ -8730,7 +8902,7 @@ SITargetLowering::lowerFMINIMUMNUM_FMAXIMUMNUM(SDValue Op,
return expandFMINIMUMNUM_FMAXIMUMNUM(Op.getNode(), DAG);
if (VT == MVT::v4f16 || VT == MVT::v8f16 || VT == MVT::v16f16 ||
- VT == MVT::v16bf16)
+ VT == MVT::v32f16)
return splitBinaryVectorOp(Op, DAG);
return Op;
}
@@ -10234,6 +10406,16 @@ static void packImage16bitOpsToDwords(SelectionDAG &DAG, SDValue Op,
}
}
+/// Emit a DiagnosticInfoUnsupported for an unsupported image intrinsic and
+/// return poison values of \p ResultTypes, preserving the chain if present.
+static SDValue diagnoseUnsupportedImage(SelectionDAG &DAG, SDValue Op,
+ ArrayRef<EVT> ResultTypes,
+ const SDLoc &DL, const Twine &Msg) {
+ DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
+ DAG.getMachineFunction().getFunction(), Msg, DL.getDebugLoc()));
+ return DAG.getErrorMergeValues(ResultTypes, Op.getOperand(0), DL);
+}
+
SDValue SITargetLowering::lowerImage(SDValue Op,
const AMDGPU::ImageDimIntrinsicInfo *Intr,
SelectionDAG &DAG, bool WithChain) const {
@@ -10281,6 +10463,12 @@ SDValue SITargetLowering::lowerImage(SDValue Op,
IntrOpcode == AMDGPU::IMAGE_ATOMIC_PK_ADD_BF16 ||
IntrOpcode == AMDGPU::IMAGE_ATOMIC_PK_ADD_BF16_NORTN);
+ if (!IsAtomicPacked16Bit && VData.getValueSizeInBits() != 32 &&
+ VData.getValueSizeInBits() != 64) {
+ return diagnoseUnsupportedImage(DAG, Op, OrigResultTypes, DL,
+ "unsupported image atomic data type");
+ }
+
bool Is64Bit = VData.getValueSizeInBits() == 64;
if (BaseOpcode->AtomicX2) {
SDValue VData2 = Op.getOperand(3);
@@ -10306,7 +10494,13 @@ SDValue SITargetLowering::lowerImage(SDValue Op,
VData = Op.getOperand(2);
MVT StoreVT = VData.getSimpleValueType();
- if (StoreVT.getScalarType() == MVT::f16) {
+ MVT StoreScalarVT = StoreVT.getScalarType();
+ if (StoreScalarVT != MVT::f16 && StoreScalarVT.getSizeInBits() != 32 &&
+ StoreScalarVT.getSizeInBits() != 64) {
+ return diagnoseUnsupportedImage(DAG, Op, OrigResultTypes, DL,
+ "unsupported image store data type");
+ }
+ if (StoreScalarVT == MVT::f16) {
if (!Subtarget->hasD16Images() || !BaseOpcode->HasD16)
return Op; // D16 is unsupported for this instruction
@@ -10319,7 +10513,13 @@ SDValue SITargetLowering::lowerImage(SDValue Op,
// Work out the num dwords based on the dmask popcount and underlying type
// and whether packing is supported.
MVT LoadVT = ResultTypes[0].getSimpleVT();
- if (LoadVT.getScalarType() == MVT::f16) {
+ MVT LoadScalarVT = LoadVT.getScalarType();
+ if (LoadScalarVT != MVT::f16 && LoadScalarVT.getSizeInBits() != 32 &&
+ LoadScalarVT.getSizeInBits() != 64) {
+ return diagnoseUnsupportedImage(DAG, Op, OrigResultTypes, DL,
+ "unsupported image load data type");
+ }
+ if (LoadScalarVT == MVT::f16) {
if (!Subtarget->hasD16Images() || !BaseOpcode->HasD16)
return Op; // D16 is unsupported for this instruction
@@ -10600,21 +10800,9 @@ SDValue SITargetLowering::lowerImage(SDValue Op,
Opcode = AMDGPU::getMIMGOpcode(IntrOpcode, AMDGPU::MIMGEncGfx90a,
NumVDataDwords, NumVAddrDwords);
if (Opcode == -1) {
- DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
- DAG.getMachineFunction().getFunction(),
- "requested image instruction is not supported on this GPU",
- DL.getDebugLoc()));
-
- unsigned Idx = 0;
- SmallVector<SDValue, 3> RetValues(OrigResultTypes.size());
- for (EVT VT : OrigResultTypes) {
- if (VT == MVT::Other)
- RetValues[Idx++] = Op.getOperand(0); // Chain
- else
- RetValues[Idx++] = DAG.getPOISON(VT);
- }
-
- return DAG.getMergeValues(RetValues, DL);
+ return diagnoseUnsupportedImage(
+ DAG, Op, OrigResultTypes, DL,
+ "requested image instruction is not supported on this GPU");
}
}
if (Opcode == -1 &&
@@ -10653,34 +10841,48 @@ SDValue SITargetLowering::lowerImage(SDValue Op,
NumVDataDwords, IsAtomicPacked16Bit, DL);
}
-SDValue SITargetLowering::lowerSBuffer(EVT VT, SDLoc DL, SDValue Rsrc,
+SDValue SITargetLowering::lowerSBuffer(EVT VT, EVT MemVT, SDLoc DL,
+ SDValue Chain, SDValue Rsrc,
SDValue Offset, SDValue CachePolicy,
- SelectionDAG &DAG) const {
+ SelectionDAG &DAG,
+ MachineMemOperand *MMO) const {
MachineFunction &MF = DAG.getMachineFunction();
+ bool HasChainResult = MMO != nullptr;
- const DataLayout &DataLayout = DAG.getDataLayout();
- Align Alignment =
- DataLayout.getABITypeAlign(VT.getTypeForEVT(*DAG.getContext()));
+ if (!HasChainResult) {
+ const DataLayout &DataLayout = DAG.getDataLayout();
+ Align Alignment =
+ DataLayout.getABITypeAlign(MemVT.getTypeForEVT(*DAG.getContext()));
- MachineMemOperand *MMO = MF.getMachineMemOperand(
- MachinePointerInfo(),
- MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
- MachineMemOperand::MOInvariant,
- VT.getStoreSize(), Alignment);
+ MMO = MF.getMachineMemOperand(MachinePointerInfo(),
+ MachineMemOperand::MOLoad |
+ MachineMemOperand::MODereferenceable |
+ MachineMemOperand::MOInvariant,
+ MemVT.getStoreSize(), Alignment);
+ }
if (!Offset->isDivergent()) {
- SDValue Ops[] = {Rsrc, Offset, CachePolicy};
+ SDValue Ops[] = {Chain, Rsrc, Offset, CachePolicy};
+
+ // Lower llvm.amdgcn.*s.buffer.load.{i,u}N intrinsics. First, generate
+ // s_buffer_load_u* for signed and unsigned load instructions. Next, DAG
+ // combiner tries to merge the s_buffer_load_uN with a sext instruction
+ // (performSignExtendInRegCombine()) and it replaces s_buffer_load_uN with
+ // s_buffer_load_iN.
+ auto HandleScalarSubwordLoads = [&](unsigned Opcode) -> SDValue {
+ SDValue BufferLoad = DAG.getMemIntrinsicNode(
+ Opcode, DL, DAG.getVTList(MVT::i32, MVT::Other), Ops, MemVT, MMO);
+ SDValue LoadVal = DAG.getAnyExtOrTrunc(
+ DAG.getNode(ISD::TRUNCATE, DL, MemVT, BufferLoad), DL, VT);
+ if (HasChainResult)
+ return DAG.getMergeValues({LoadVal, BufferLoad.getValue(1)}, DL);
+ return LoadVal;
+ };
+ if (MemVT == MVT::i8 && Subtarget->hasScalarSubwordLoads())
+ return HandleScalarSubwordLoads(AMDGPUISD::SBUFFER_LOAD_UBYTE);
- // Lower llvm.amdgcn.s.buffer.load.{i16, u16} intrinsics. Initially, the
- // s_buffer_load_u16 instruction is emitted for both signed and unsigned
- // loads. Later, DAG combiner tries to combine s_buffer_load_u16 with sext
- // and generates s_buffer_load_i16 (performSignExtendInRegCombine).
- if (VT == MVT::i16 && Subtarget->hasScalarSubwordLoads()) {
- SDValue BufferLoad =
- DAG.getMemIntrinsicNode(AMDGPUISD::SBUFFER_LOAD_USHORT, DL,
- DAG.getVTList(MVT::i32), Ops, VT, MMO);
- return DAG.getNode(ISD::TRUNCATE, DL, VT, BufferLoad);
- }
+ if (MemVT == MVT::i16 && Subtarget->hasScalarSubwordLoads())
+ return HandleScalarSubwordLoads(AMDGPUISD::SBUFFER_LOAD_USHORT);
// Widen vec3 load to vec4.
if (VT.isVector() && VT.getVectorNumElements() == 3 &&
@@ -10688,21 +10890,25 @@ SDValue SITargetLowering::lowerSBuffer(EVT VT, SDLoc DL, SDValue Rsrc,
EVT WidenedVT =
EVT::getVectorVT(*DAG.getContext(), VT.getVectorElementType(), 4);
auto WidenedOp = DAG.getMemIntrinsicNode(
- AMDGPUISD::SBUFFER_LOAD, DL, DAG.getVTList(WidenedVT), Ops, WidenedVT,
+ AMDGPUISD::SBUFFER_LOAD, DL, DAG.getVTList(WidenedVT, MVT::Other),
+ Ops, WidenedVT,
MF.getMachineMemOperand(MMO, 0, WidenedVT.getStoreSize()));
auto Subvector = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, WidenedOp,
DAG.getVectorIdxConstant(0, DL));
+ if (HasChainResult)
+ return DAG.getMergeValues({Subvector, WidenedOp.getValue(1)}, DL);
return Subvector;
}
return DAG.getMemIntrinsicNode(AMDGPUISD::SBUFFER_LOAD, DL,
- DAG.getVTList(VT), Ops, VT, MMO);
+ DAG.getVTList(VT, MVT::Other), Ops, MemVT,
+ MMO);
}
// We have a divergent offset. Emit a MUBUF buffer load instead. We can
// assume that the buffer is unswizzled.
SDValue Ops[] = {
- DAG.getEntryNode(), // Chain
+ Chain, // Chain
Rsrc, // rsrc
DAG.getConstant(0, DL, MVT::i32), // vindex
{}, // voffset
@@ -10711,9 +10917,14 @@ SDValue SITargetLowering::lowerSBuffer(EVT VT, SDLoc DL, SDValue Rsrc,
CachePolicy, // cachepolicy
DAG.getTargetConstant(0, DL, MVT::i1), // idxen
};
- if (VT == MVT::i16 && Subtarget->hasScalarSubwordLoads()) {
+ if ((MemVT == MVT::i8 || MemVT == MVT::i16) &&
+ Subtarget->hasScalarSubwordLoads()) {
setBufferOffsets(Offset, DAG, &Ops[3], Align(4));
- return handleByteShortBufferLoads(DAG, VT, DL, Ops, MMO);
+ SDValue Load = handleByteShortBufferLoads(DAG, MemVT, DL, Ops, MMO);
+ SDValue LoadVal = DAG.getAnyExtOrTrunc(Load.getOperand(0), DL, VT);
+ if (HasChainResult)
+ return DAG.getMergeValues({LoadVal, Load.getOperand(1)}, DL);
+ return LoadVal;
}
SmallVector<SDValue, 4> Loads;
@@ -10744,8 +10955,17 @@ SDValue SITargetLowering::lowerSBuffer(EVT VT, SDLoc DL, SDValue Rsrc,
LoadVT, LoadMMO, DAG));
}
- if (NumElts == 8 || NumElts == 16)
- return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Loads);
+ if (NumElts == 8 || NumElts == 16) {
+ SDValue LoadVal = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Loads);
+ if (HasChainResult) {
+ SmallVector<SDValue, 4> LoadChains;
+ for (SDValue Load : Loads)
+ LoadChains.push_back(Load.getValue(1));
+ SDValue Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, LoadChains);
+ return DAG.getMergeValues({LoadVal, Chain}, DL);
+ }
+ return LoadVal;
+ }
return Loads[0];
}
@@ -10804,6 +11024,61 @@ SDValue SITargetLowering::lowerWorkitemID(SelectionDAG &DAG, SDValue Op,
DAG.getValueType(SmallVT));
}
+SDValue SITargetLowering::lowerFromFP8(SDValue Op, bool IsBF8,
+ SelectionDAG &DAG) const {
+ SDLoc SL(Op);
+ SDValue Src = Op.getOperand(0);
+ EVT DstVT = Op.getValueType();
+ bool IsF16 = DstVT.getVectorElementType() == MVT::f16;
+ assert((!IsF16 || Subtarget->hasFP8F16ConversionInsts()) &&
+ "fp8/bf8 -> f16 conversion requires FP8F16ConversionInsts");
+
+ unsigned Opc;
+ if (IsF16)
+ Opc = IsBF8 ? AMDGPUISD::CVT_PK_F16_BF8 : AMDGPUISD::CVT_PK_F16_FP8;
+ else
+ Opc = IsBF8 ? AMDGPUISD::CVT_PK_F32_BF8 : AMDGPUISD::CVT_PK_F32_FP8;
+
+ // Pack the two i8 lanes into the integer type the packed HW node reads. The
+ // f16 form takes i16 and the f32 form takes i32. v2i8 bitcasts to i16
+ // directly and the f32 node reads the low half of an any-extended i32.
+ EVT PackedVT =
+ EVT::getIntegerVT(*DAG.getContext(), DstVT.getScalarSizeInBits());
+ SDValue AsI16 = DAG.getNode(ISD::BITCAST, SL, MVT::i16, Src);
+ SDValue Packed = DAG.getAnyExtOrTrunc(AsI16, SL, PackedVT);
+ return DAG.getNode(Opc, SL, DstVT, Packed);
+}
+
+SDValue
+SITargetLowering::LowerCONVERT_FROM_ARBITRARY_FP(SDValue Op,
+ SelectionDAG &DAG) const {
+ // Only handle OCP FP8 formats (E4M3FN, E5M2). FNUZ formats that are supported
+ // by gfx942 fall through to the generic expansion.
+ APFloatBase::Semantics FPSemantic =
+ static_cast<APFloatBase::Semantics>(Op.getConstantOperandVal(1));
+ if (FPSemantic != APFloatBase::S_Float8E4M3FN &&
+ FPSemantic != APFloatBase::S_Float8E5M2)
+ return SDValue();
+ const bool IsBF8 = FPSemantic == APFloatBase::S_Float8E5M2;
+
+ EVT DstVT = Op.getValueType();
+ if (!DstVT.isVector()) {
+ SDValue Src = Op.getOperand(0);
+ if (Src.getValueType() != MVT::i32) {
+ SDLoc SL(Op);
+ SDValue SrcI32 = DAG.getAnyExtOrTrunc(Src, SL, MVT::i32);
+ return DAG.getNode(ISD::CONVERT_FROM_ARBITRARY_FP, SL, DstVT, SrcI32,
+ Op.getOperand(1));
+ }
+ return Op;
+ }
+
+ EVT EltVT = DstVT.getVectorElementType();
+ if (EltVT == MVT::f16 || EltVT == MVT::f32)
+ return lowerFromFP8(Op, IsBF8, DAG);
+ return SDValue();
+}
+
SDValue SITargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
SelectionDAG &DAG) const {
MachineFunction &MF = DAG.getMachineFunction();
@@ -10904,17 +11179,9 @@ SDValue SITargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
return emitRemovedIntrinsicError(DAG, DL, VT);
return DAG.getNode(AMDGPUISD::RCP_LEGACY, DL, VT, Op.getOperand(1));
case Intrinsic::amdgcn_fma_legacy:
- if (!Subtarget->hasFmaLegacy32Insts())
- return emitRemovedIntrinsicError(DAG, DL, VT);
- return SDValue();
case Intrinsic::amdgcn_sudot4:
case Intrinsic::amdgcn_sudot8:
- if (!Subtarget->hasDot8Insts())
- return emitRemovedIntrinsicError(DAG, DL, VT);
- return SDValue();
case Intrinsic::amdgcn_tanh:
- if (!Subtarget->hasTanhInsts())
- return emitRemovedIntrinsicError(DAG, DL, VT);
return SDValue();
case Intrinsic::amdgcn_rsq_clamp: {
if (Subtarget->getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS)
@@ -11070,8 +11337,8 @@ SDValue SITargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
? AMDGPU::CPol::ALL
: AMDGPU::CPol::ALL_pregfx12))
return Op;
- return lowerSBuffer(VT, DL, Op.getOperand(1), Op.getOperand(2),
- Op.getOperand(3), DAG);
+ return lowerSBuffer(VT, VT, DL, DAG.getEntryNode(), Op.getOperand(1),
+ Op.getOperand(2), Op.getOperand(3), DAG);
}
case Intrinsic::amdgcn_fdiv_fast:
return lowerFDIV_FAST(Op, DAG);
@@ -11097,9 +11364,20 @@ SDValue SITargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
case Intrinsic::amdgcn_fract:
return DAG.getNode(AMDGPUISD::FRACT, DL, VT, Op.getOperand(1));
- case Intrinsic::amdgcn_class:
- return DAG.getNode(AMDGPUISD::FP_CLASS, DL, VT, Op.getOperand(1),
- Op.getOperand(2));
+ case Intrinsic::amdgcn_class: {
+ SDValue Src = Op.getOperand(1);
+ EVT SrcVT = Src.getValueType();
+ bool IsLegal = SrcVT == MVT::f32 || SrcVT == MVT::f64 ||
+ (SrcVT == MVT::f16 && Subtarget->has16BitInsts());
+ if (!IsLegal) {
+ DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
+ DAG.getMachineFunction().getFunction(),
+ "llvm.amdgcn.class only supports f16, f32, and f64",
+ DL.getDebugLoc()));
+ return DAG.getPOISON(VT);
+ }
+ return DAG.getNode(AMDGPUISD::FP_CLASS, DL, VT, Src, Op.getOperand(2));
+ }
case Intrinsic::amdgcn_div_fmas:
return DAG.getNode(AMDGPUISD::DIV_FMAS, DL, VT, Op.getOperand(1),
Op.getOperand(2), Op.getOperand(3), Op.getOperand(4));
@@ -11383,6 +11661,11 @@ SDValue SITargetLowering::lowerRawBufferAtomicIntrin(SDValue Op,
SDLoc DL(Op);
SDValue VData = Op.getOperand(2);
+ if (VData.getValueSizeInBits() != 32 && VData.getValueSizeInBits() != 64) {
+ SmallVector<EVT, 2> ResultTypes(Op->values());
+ return diagnoseUnsupportedImage(DAG, Op, ResultTypes, DL,
+ "unsupported buffer atomic data type");
+ }
SDValue Rsrc = bufferRsrcPtrToVector(Op.getOperand(3), DAG);
auto [VOffset, Offset] = splitBufferOffsets(Op.getOperand(4), DAG);
auto SOffset = selectSOffset(Op.getOperand(5), DAG, Subtarget);
@@ -11411,6 +11694,11 @@ SITargetLowering::lowerStructBufferAtomicIntrin(SDValue Op, SelectionDAG &DAG,
SDLoc DL(Op);
SDValue VData = Op.getOperand(2);
+ if (VData.getValueSizeInBits() != 32 && VData.getValueSizeInBits() != 64) {
+ SmallVector<EVT, 2> ResultTypes(Op->values());
+ return diagnoseUnsupportedImage(DAG, Op, ResultTypes, DL,
+ "unsupported buffer atomic data type");
+ }
SDValue Rsrc = bufferRsrcPtrToVector(Op.getOperand(3), DAG);
auto [VOffset, Offset] = splitBufferOffsets(Op.getOperand(5), DAG);
auto SOffset = selectSOffset(Op.getOperand(6), DAG, Subtarget);
@@ -11502,6 +11790,19 @@ SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
M->getVTList(), Ops, M->getMemoryVT(),
M->getMemOperand());
}
+ case Intrinsic::amdgcn_ptr_s_buffer_load: {
+ unsigned CPol = Op.getConstantOperandVal(4);
+ if (CPol & ~((Subtarget->getGeneration() >= AMDGPUSubtarget::GFX12)
+ ? AMDGPU::CPol::ALL
+ : AMDGPU::CPol::ALL_pregfx12))
+ return Op;
+
+ MemSDNode *M = cast<MemSDNode>(Op);
+ return lowerSBuffer(
+ Op.getValueType(), M->getMemoryVT(), DL, Op.getOperand(0),
+ bufferRsrcPtrToVector(Op.getOperand(2), DAG), Op.getOperand(3),
+ Op.getOperand(4), DAG, M->getMemOperand());
+ }
case Intrinsic::amdgcn_raw_buffer_load:
case Intrinsic::amdgcn_raw_ptr_buffer_load:
case Intrinsic::amdgcn_raw_atomic_buffer_load:
@@ -11575,7 +11876,7 @@ SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
DAG.getTargetConstant(0, DL, MVT::i1), // idxen
};
- if (LoadVT.getScalarType() == MVT::f16)
+ if (LoadVT.getScalarSizeInBits() == 16)
return adjustLoadValueType(AMDGPUISD::TBUFFER_LOAD_FORMAT_D16, M, DAG,
Ops);
return getMemIntrinsicNode(AMDGPUISD::TBUFFER_LOAD_FORMAT, DL,
@@ -11602,7 +11903,7 @@ SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
DAG.getTargetConstant(1, DL, MVT::i1), // idxen
};
- if (LoadVT.getScalarType() == MVT::f16)
+ if (LoadVT.getScalarSizeInBits() == 16)
return adjustLoadValueType(AMDGPUISD::TBUFFER_LOAD_FORMAT_D16, M, DAG,
Ops);
return getMemIntrinsicNode(AMDGPUISD::TBUFFER_LOAD_FORMAT, DL,
@@ -11724,6 +12025,12 @@ SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
AMDGPUISD::BUFFER_ATOMIC_COND_SUB_U32);
case Intrinsic::amdgcn_raw_buffer_atomic_cmpswap:
case Intrinsic::amdgcn_raw_ptr_buffer_atomic_cmpswap: {
+ SDValue Src = Op.getOperand(2);
+ if (Src.getValueSizeInBits() != 32 && Src.getValueSizeInBits() != 64) {
+ SmallVector<EVT, 2> ResultTypes(Op->values());
+ return diagnoseUnsupportedImage(DAG, Op, ResultTypes, DL,
+ "unsupported buffer atomic data type");
+ }
SDValue Rsrc = bufferRsrcPtrToVector(Op.getOperand(4), DAG);
auto [VOffset, Offset] = splitBufferOffsets(Op.getOperand(5), DAG);
auto SOffset = selectSOffset(Op.getOperand(6), DAG, Subtarget);
@@ -11748,6 +12055,12 @@ SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
}
case Intrinsic::amdgcn_struct_buffer_atomic_cmpswap:
case Intrinsic::amdgcn_struct_ptr_buffer_atomic_cmpswap: {
+ SDValue Src = Op.getOperand(2);
+ if (Src.getValueSizeInBits() != 32 && Src.getValueSizeInBits() != 64) {
+ SmallVector<EVT, 2> ResultTypes(Op->values());
+ return diagnoseUnsupportedImage(DAG, Op, ResultTypes, DL,
+ "unsupported buffer atomic data type");
+ }
SDValue Rsrc = bufferRsrcPtrToVector(Op->getOperand(4), DAG);
auto [VOffset, Offset] = splitBufferOffsets(Op.getOperand(6), DAG);
auto SOffset = selectSOffset(Op.getOperand(7), DAG, Subtarget);
@@ -11784,11 +12097,6 @@ SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
assert(NodePtr.getValueType() == MVT::i64);
assert(RayDir.getValueType() == MVT::v3f32);
- if (!Subtarget->hasBVHDualAndBVH8Insts()) {
- emitRemovedIntrinsicError(DAG, DL, Op.getValueType());
- return SDValue();
- }
-
bool IsBVH8 = IntrID == Intrinsic::amdgcn_image_bvh8_intersect_ray;
const unsigned NumVDataDwords = 10;
const unsigned NumVAddrDwords = IsBVH8 ? 11 : 12;
@@ -11829,11 +12137,6 @@ SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
assert(RayDir.getValueType() == MVT::v3f16 ||
RayDir.getValueType() == MVT::v3f32);
- if (!Subtarget->hasGFX10_AEncoding()) {
- emitRemovedIntrinsicError(DAG, DL, Op.getValueType());
- return SDValue();
- }
-
const bool IsGFX11 = AMDGPU::isGFX11(*Subtarget);
const bool IsGFX11Plus = AMDGPU::isGFX11Plus(*Subtarget);
const bool IsGFX12Plus = AMDGPU::isGFX12Plus(*Subtarget);
@@ -12004,10 +12307,8 @@ SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
SDValue M0Val;
M0Val = DAG.getNode(ISD::SRL, DL, MVT::i32, Op->getOperand(2),
DAG.getShiftAmountConstant(4, MVT::i32, DL));
- M0Val = SDValue(
- DAG.getMachineNode(AMDGPU::S_AND_B32, DL, MVT::i32, M0Val,
- DAG.getTargetConstant(0x3F, DL, MVT::i32)),
- 0);
+ M0Val = DAG.getNode(ISD::AND, DL, MVT::i32, M0Val,
+ DAG.getConstant(0x3F, DL, MVT::i32));
Ops.push_back(copyToM0(DAG, Chain, DL, M0Val).getValue(0));
} else
Ops.push_back(copyToM0(DAG, Chain, DL, Op->getOperand(2)).getValue(0));
@@ -12027,14 +12328,6 @@ SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
Chain, Ptr, MII->getMemOperand());
}
case Intrinsic::amdgcn_av_load_b128: {
- if (!Subtarget->hasFlatGlobalInsts()) {
- DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
- DAG.getMachineFunction().getFunction(),
- "llvm.amdgcn.av.load.b128 not supported on subtarget",
- DL.getDebugLoc()));
- return DAG.getMergeValues(
- {DAG.getPOISON(Op->getValueType(0)), Op->getOperand(0)}, DL);
- }
MemIntrinsicSDNode *MII = cast<MemIntrinsicSDNode>(Op);
SDValue Chain = Op->getOperand(0);
SDValue Ptr = Op->getOperand(2);
@@ -12224,11 +12517,6 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
switch (IntrinsicID) {
case Intrinsic::amdgcn_exp_compr: {
- if (!Subtarget->hasCompressedExport()) {
- DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
- DAG.getMachineFunction().getFunction(),
- "intrinsic not supported on subtarget", DL.getDebugLoc()));
- }
SDValue Src0 = Op.getOperand(4);
SDValue Src1 = Op.getOperand(5);
// Hack around illegal type on SI by directly selecting it.
@@ -12256,7 +12544,7 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
case Intrinsic::amdgcn_struct_tbuffer_store:
case Intrinsic::amdgcn_struct_ptr_tbuffer_store: {
SDValue VData = Op.getOperand(2);
- bool IsD16 = (VData.getValueType().getScalarType() == MVT::f16);
+ bool IsD16 = (VData.getValueType().getScalarSizeInBits() == 16);
if (IsD16)
VData = handleD16VData(VData, DAG);
SDValue Rsrc = bufferRsrcPtrToVector(Op.getOperand(3), DAG);
@@ -12284,7 +12572,7 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
case Intrinsic::amdgcn_raw_tbuffer_store:
case Intrinsic::amdgcn_raw_ptr_tbuffer_store: {
SDValue VData = Op.getOperand(2);
- bool IsD16 = (VData.getValueType().getScalarType() == MVT::f16);
+ bool IsD16 = (VData.getValueType().getScalarSizeInBits() == 16);
if (IsD16)
VData = handleD16VData(VData, DAG);
SDValue Rsrc = bufferRsrcPtrToVector(Op.getOperand(3), DAG);
@@ -12321,6 +12609,14 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
EVT VDataVT = VData.getValueType();
EVT EltType = VDataVT.getScalarType();
bool IsD16 = IsFormat && (EltType.getSizeInBits() == 16);
+
+ if (IsFormat && !IsD16 && EltType.getSizeInBits() < 32) {
+ DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
+ DAG.getMachineFunction().getFunction(),
+ "unsupported sub-dword format buffer store", DL.getDebugLoc()));
+ return Chain;
+ }
+
if (IsD16) {
VData = handleD16VData(VData, DAG);
VDataVT = VData.getValueType();
@@ -12372,6 +12668,13 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
EVT EltType = VDataVT.getScalarType();
bool IsD16 = IsFormat && (EltType.getSizeInBits() == 16);
+ if (IsFormat && !IsD16 && EltType.getSizeInBits() < 32) {
+ DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
+ DAG.getMachineFunction().getFunction(),
+ "unsupported sub-dword format buffer store", DL.getDebugLoc()));
+ return Chain;
+ }
+
if (IsD16) {
VData = handleD16VData(VData, DAG);
VDataVT = VData.getValueType();
@@ -12418,8 +12721,6 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
case Intrinsic::amdgcn_struct_buffer_load_async_lds:
case Intrinsic::amdgcn_struct_ptr_buffer_load_lds:
case Intrinsic::amdgcn_struct_ptr_buffer_load_async_lds: {
- if (!Subtarget->hasVMemToLDSLoad())
- return SDValue();
unsigned Opc;
bool HasVIndex =
IntrinsicID == Intrinsic::amdgcn_struct_buffer_load_lds ||
@@ -12644,22 +12945,17 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
SDValue BarID;
BarID = DAG.getNode(ISD::SRL, DL, MVT::i32, BarOp,
DAG.getShiftAmountConstant(4, MVT::i32, DL));
- BarID =
- SDValue(DAG.getMachineNode(AMDGPU::S_AND_B32, DL, MVT::i32, BarID,
- DAG.getTargetConstant(0x3F, DL, MVT::i32)),
- 0);
+ BarID = DAG.getNode(ISD::AND, DL, MVT::i32, BarID,
+ DAG.getConstant(0x3F, DL, MVT::i32));
// Member count should be put into M0[ShAmt:+6]
// Barrier ID should be put into M0[5:0]
- M0Val =
- SDValue(DAG.getMachineNode(AMDGPU::S_AND_B32, DL, MVT::i32, CntOp,
- DAG.getTargetConstant(0x3F, DL, MVT::i32)),
- 0);
+ SDValue MemberCnt = DAG.getNode(ISD::AND, DL, MVT::i32, CntOp,
+ DAG.getConstant(0x3F, DL, MVT::i32));
constexpr unsigned ShAmt = 16;
- M0Val = DAG.getNode(ISD::SHL, DL, MVT::i32, CntOp,
+ M0Val = DAG.getNode(ISD::SHL, DL, MVT::i32, MemberCnt,
DAG.getShiftAmountConstant(ShAmt, MVT::i32, DL));
- M0Val = SDValue(
- DAG.getMachineNode(AMDGPU::S_OR_B32, DL, MVT::i32, M0Val, BarID), 0);
+ M0Val = DAG.getNode(ISD::OR, DL, MVT::i32, M0Val, BarID);
Ops.push_back(copyToM0(DAG, Chain, DL, M0Val).getValue(0));
@@ -12710,10 +13006,8 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
SDValue M0Val;
M0Val = DAG.getNode(ISD::SRL, DL, MVT::i32, BarOp,
DAG.getShiftAmountConstant(4, MVT::i32, DL));
- M0Val =
- SDValue(DAG.getMachineNode(AMDGPU::S_AND_B32, DL, MVT::i32, M0Val,
- DAG.getTargetConstant(0x3F, DL, MVT::i32)),
- 0);
+ M0Val = DAG.getNode(ISD::AND, DL, MVT::i32, M0Val,
+ DAG.getConstant(0x3F, DL, MVT::i32));
Ops.push_back(copyToM0(DAG, Chain, DL, M0Val).getValue(0));
}
@@ -12750,13 +13044,6 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
Ptr, MII->getMemOperand());
}
case Intrinsic::amdgcn_av_store_b128: {
- if (!Subtarget->hasFlatGlobalInsts()) {
- DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
- DAG.getMachineFunction().getFunction(),
- "llvm.amdgcn.av.store.b128 not supported on subtarget",
- DL.getDebugLoc()));
- return Op->getOperand(0); // return the input chain
- }
MemIntrinsicSDNode *MII = cast<MemIntrinsicSDNode>(Op);
SDValue Chain = Op->getOperand(0);
SDValue Ptr = Op->getOperand(2);
@@ -13081,8 +13368,7 @@ SDValue SITargetLowering::widenLoad(LoadSDNode *Ld,
SDValue NewLoad = DAG.getLoad(
ISD::UNINDEXED, ISD::NON_EXTLOAD, MVT::i32, SL, Ld->getChain(), Ptr,
Ld->getOffset(), Ld->getPointerInfo(), MVT::i32, Ld->getAlign(),
- Ld->getMemOperand()->getFlags(), Ld->getAAInfo(),
- nullptr); // Drop ranges
+ Ld->getMemOperand()->getFlags(), Ld->getAAInfo()); // Drop ranges
EVT TruncVT = EVT::getIntegerVT(*DAG.getContext(), MemVT.getSizeInBits());
if (MemVT.isFloatingPoint()) {
@@ -15634,22 +15920,22 @@ SITargetLowering::performSignExtendInRegCombine(SDNode *N,
assert(Subtarget->hasScalarSubwordLoads() &&
"s_buffer_load_{u8, i8} are supported "
"in GFX12 (or newer) architectures.");
- EVT VT = Src.getValueType();
unsigned Opc = (Src.getOpcode() == AMDGPUISD::SBUFFER_LOAD_UBYTE)
? AMDGPUISD::SBUFFER_LOAD_BYTE
: AMDGPUISD::SBUFFER_LOAD_SHORT;
SDLoc DL(N);
- SDVTList ResList = DCI.DAG.getVTList(MVT::i32);
+ SDVTList ResList =
+ DCI.DAG.getVTList(MVT::i32, Src.getOperand(0).getValueType());
SDValue Ops[] = {
- Src.getOperand(0), // source register
- Src.getOperand(1), // offset
- Src.getOperand(2) // cachePolicy
+ Src.getOperand(0), // Chain
+ Src.getOperand(1), // source register
+ Src.getOperand(2), // offset
+ Src.getOperand(3) // cachePolicy
};
auto *M = cast<MemSDNode>(Src);
SDValue BufferLoad = DCI.DAG.getMemIntrinsicNode(
Opc, DL, ResList, Ops, M->getMemoryVT(), M->getMemOperand());
- SDValue LoadVal = DCI.DAG.getNode(ISD::TRUNCATE, DL, VT, BufferLoad);
- return LoadVal;
+ return DCI.DAG.getMergeValues({BufferLoad, BufferLoad.getValue(1)}, DL);
}
if (((Src.getOpcode() == AMDGPUISD::BUFFER_LOAD_UBYTE &&
VTSign->getVT() == MVT::i8) ||
@@ -15716,6 +16002,21 @@ SDValue SITargetLowering::performRcpCombine(SDNode *N,
bool SITargetLowering::isCanonicalized(SelectionDAG &DAG, SDValue Op,
SDNodeFlags UserFlags,
unsigned MaxDepth) const {
+ EVT VT = Op.getValueType();
+ assert(VT.isFloatingPoint() &&
+ "expected a floating-point value to query canonicality of");
+ return isCanonicalized(DAG, Op, VT.getScalarType(), UserFlags, MaxDepth);
+}
+
+bool SITargetLowering::isCanonicalized(SelectionDAG &DAG, SDValue Op,
+ EVT QueryVT, SDNodeFlags UserFlags,
+ unsigned MaxDepth) const {
+ assert(QueryVT.isFloatingPoint() && !QueryVT.isVector() &&
+ "QueryVT must be a floating-point scalar type");
+ EVT VT = Op.getValueType();
+ if (VT.isFloatingPoint() && VT.getScalarType() != QueryVT)
+ return false;
+
unsigned Opcode = Op.getOpcode();
if (Opcode == ISD::FCANONICALIZE)
return true;
@@ -15784,7 +16085,8 @@ bool SITargetLowering::isCanonicalized(SelectionDAG &DAG, SDValue Op,
case ISD::FNEG:
case ISD::FABS:
case ISD::FCOPYSIGN:
- return isCanonicalized(DAG, Op.getOperand(0), MaxDepth - 1);
+ return isCanonicalized(DAG, Op.getOperand(0), QueryVT, UserFlags,
+ MaxDepth - 1);
case ISD::AND:
if (Op.getValueType() == MVT::i32) {
@@ -15794,7 +16096,8 @@ bool SITargetLowering::isCanonicalized(SelectionDAG &DAG, SDValue Op,
// is valid to optimize for all types.
if (auto *RHS = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
if (RHS->getZExtValue() == 0xffff0000) {
- return isCanonicalized(DAG, Op.getOperand(0), MaxDepth - 1);
+ return isCanonicalized(DAG, Op.getOperand(0), QueryVT, UserFlags,
+ MaxDepth - 1);
}
}
}
@@ -15835,20 +16138,23 @@ bool SITargetLowering::isCanonicalized(SelectionDAG &DAG, SDValue Op,
// FIXME: Does this apply with clamp? It's implemented with max.
for (unsigned I = 0, E = Op.getNumOperands(); I != E; ++I) {
- if (!isCanonicalized(DAG, Op.getOperand(I), MaxDepth - 1))
+ if (!isCanonicalized(DAG, Op.getOperand(I), QueryVT, UserFlags,
+ MaxDepth - 1))
return false;
}
return true;
}
case ISD::SELECT: {
- return isCanonicalized(DAG, Op.getOperand(1), MaxDepth - 1) &&
- isCanonicalized(DAG, Op.getOperand(2), MaxDepth - 1);
+ return isCanonicalized(DAG, Op.getOperand(1), QueryVT, UserFlags,
+ MaxDepth - 1) &&
+ isCanonicalized(DAG, Op.getOperand(2), QueryVT, UserFlags,
+ MaxDepth - 1);
}
case ISD::BUILD_VECTOR: {
for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
SDValue SrcOp = Op.getOperand(i);
- if (!isCanonicalized(DAG, SrcOp, MaxDepth - 1))
+ if (!isCanonicalized(DAG, SrcOp, QueryVT, UserFlags, MaxDepth - 1))
return false;
}
@@ -15856,21 +16162,28 @@ bool SITargetLowering::isCanonicalized(SelectionDAG &DAG, SDValue Op,
}
case ISD::EXTRACT_VECTOR_ELT:
case ISD::EXTRACT_SUBVECTOR: {
- return isCanonicalized(DAG, Op.getOperand(0), MaxDepth - 1);
+ return isCanonicalized(DAG, Op.getOperand(0), QueryVT, UserFlags,
+ MaxDepth - 1);
}
case ISD::INSERT_VECTOR_ELT: {
- return isCanonicalized(DAG, Op.getOperand(0), MaxDepth - 1) &&
- isCanonicalized(DAG, Op.getOperand(1), MaxDepth - 1);
+ return isCanonicalized(DAG, Op.getOperand(0), QueryVT, UserFlags,
+ MaxDepth - 1) &&
+ isCanonicalized(DAG, Op.getOperand(1), QueryVT, UserFlags,
+ MaxDepth - 1);
}
+ case ISD::POISON:
+ return true;
case ISD::UNDEF:
// Could be anything.
return false;
- case ISD::BITCAST:
- // TODO: This is incorrect as it loses track of the operand's type. We may
- // end up effectively bitcasting from f32 to v2f16 or vice versa, and the
- // same bits that are canonicalized in one type need not be in the other.
- return isCanonicalized(DAG, Op.getOperand(0), MaxDepth - 1);
+ case ISD::BITCAST: {
+ // Carry QueryVT through the bitcast unchanged. The top-of-function guard
+ // rejects a source whose FP format differs from the consumed type, so a
+ // value canonical in one FP format is not assumed canonical in another.
+ SDValue Src = peekThroughBitcasts(Op.getOperand(0));
+ return isCanonicalized(DAG, Src, QueryVT, UserFlags, MaxDepth - 1);
+ }
case ISD::TRUNCATE: {
// Hack round the mess we make when legalizing extract_vector_elt
if (Op.getValueType() == MVT::i16) {
@@ -15878,7 +16191,8 @@ bool SITargetLowering::isCanonicalized(SelectionDAG &DAG, SDValue Op,
if (TruncSrc.getValueType() == MVT::i32 &&
TruncSrc.getOpcode() == ISD::BITCAST &&
TruncSrc.getOperand(0).getValueType() == MVT::v2f16) {
- return isCanonicalized(DAG, TruncSrc.getOperand(0), MaxDepth - 1);
+ return isCanonicalized(DAG, TruncSrc.getOperand(0), QueryVT, UserFlags,
+ MaxDepth - 1);
}
}
return false;
@@ -16124,14 +16438,13 @@ SITargetLowering::performFCanonicalizeCombine(SDNode *N,
}
}
- // If one half is undef, and one is constant, prefer a splat vector rather
- // than the normal qNaN. If it's a register, prefer 0.0 since that's
- // cheaper to use and may be free with a packed operation.
+ // If one half is undef, and one is constant, prefer a splat vector.
+ // Otherwise, convert the undef to 0.0 since that's cheaper to use and may
+ // be free with a packed operation.
if (NewElts[0].isUndef()) {
- if (isa<ConstantFPSDNode>(NewElts[1]))
- NewElts[0] = isa<ConstantFPSDNode>(NewElts[1])
- ? NewElts[1]
- : DAG.getConstantFP(0.0f, SL, EltVT);
+ NewElts[0] = isa<ConstantFPSDNode>(NewElts[1])
+ ? NewElts[1]
+ : DAG.getConstantFP(0.0f, SL, EltVT);
}
if (NewElts[1].isUndef()) {
@@ -17458,7 +17771,8 @@ SDValue SITargetLowering::performAddCombine(SDNode *N,
TempNode = TempNode->getOperand(AddIdx);
Src2s.push_back(TempNode);
ChainLength = I + 1;
- if (TempNode->getNumOperands() < 2)
+ // The loop body treats TempNode's operands as addends.
+ if (TempNode.getOpcode() != ISD::ADD)
break;
LHS = TempNode->getOperand(0);
RHS = TempNode->getOperand(1);
@@ -18508,11 +18822,150 @@ SDValue SITargetLowering::performClampCombine(SDNode *N,
return DCI.DAG.getConstantFP(Zero, SDLoc(N), N->getValueType(0));
}
- APFloat One(F.getSemantics(), "1.0");
+ APFloat One = APFloat::getOne(F.getSemantics());
if (F > One)
return DCI.DAG.getConstantFP(One, SDLoc(N), N->getValueType(0));
- return SDValue(CSrc, 0);
+ return getCanonicalConstantFP(DCI.DAG, SDLoc(N), N->getValueType(0), F);
+}
+
+SDValue
+SITargetLowering::performFrexpSelectCombine(SDNode *N,
+ DAGCombinerInfo &DCI) const {
+ // This optimization only applies when the hardware handles inf/nan correctly.
+ if (Subtarget->hasFractBug())
+ return SDValue();
+
+ SDValue Cond = N->getOperand(0);
+ SDValue TrueVal = N->getOperand(1);
+ SDValue FalseVal = N->getOperand(2);
+
+ // Determine which value is 0 and which might be the frexp result.
+ // Pattern 1: select cond, 0, frexp_result (cond true -> return 0)
+ // Pattern 2: select cond, frexp_result, 0 (cond false -> return 0)
+ SDValue FrexpVal;
+ bool CondSelectsZero; // If true, condition=true selects zero
+
+ auto isZero = [](SDValue V) {
+ if (auto *C = dyn_cast<ConstantSDNode>(V))
+ return C->isZero();
+ if (auto *C = dyn_cast<ConstantFPSDNode>(V))
+ return C->isZero();
+ return false;
+ };
+
+ if (isZero(TrueVal)) {
+ FrexpVal = FalseVal;
+ CondSelectsZero = true;
+ } else if (isZero(FalseVal)) {
+ FrexpVal = TrueVal;
+ CondSelectsZero = false;
+ } else {
+ return SDValue();
+ }
+
+ // Check if FrexpVal comes from amdgcn_frexp_exp or amdgcn_frexp_mant.
+ if (FrexpVal.getOpcode() != ISD::INTRINSIC_WO_CHAIN)
+ return SDValue();
+
+ unsigned IID = FrexpVal.getConstantOperandVal(0);
+ if (IID != Intrinsic::amdgcn_frexp_exp && IID != Intrinsic::amdgcn_frexp_mant)
+ return SDValue();
+
+ SDValue FrexpInput = FrexpVal.getOperand(1);
+
+ // Helper to strip fabs/fneg/fcopysign from a value.
+ auto peekFPSignOps = [](SDValue Val) {
+ if (Val.getOpcode() == ISD::FNEG)
+ Val = Val.getOperand(0);
+ if (Val.getOpcode() == ISD::FABS)
+ Val = Val.getOperand(0);
+ if (Val.getOpcode() == ISD::FCOPYSIGN)
+ Val = Val.getOperand(0);
+ return Val;
+ };
+
+ // The frexp intrinsics ignore sign, so we can strip sign ops when comparing.
+ SDValue FrexpInputStripped = peekFPSignOps(FrexpInput);
+
+ bool IsNonFiniteTest = false;
+
+ // Handle AMDGPUISD::FP_CLASS or ISD::IS_FPCLASS conditions.
+ // These test specific floating-point classes using a bitmask.
+ if (Cond.getOpcode() == AMDGPUISD::FP_CLASS ||
+ Cond.getOpcode() == ISD::IS_FPCLASS) {
+ SDValue ClassInput = Cond.getOperand(0);
+ SDValue ClassInputStripped = peekFPSignOps(ClassInput);
+
+ if (ClassInputStripped != FrexpInputStripped)
+ return SDValue();
+
+ auto *MaskNode = dyn_cast<ConstantSDNode>(Cond.getOperand(1));
+ if (!MaskNode)
+ return SDValue();
+
+ unsigned Mask = MaskNode->getZExtValue();
+
+ // fcFinite = all finite classes (not inf, not nan)
+ // If the mask tests for finite values and selects frexp when true,
+ // we can fold away the select since frexp returns 0 for non-finite.
+ constexpr unsigned fcFinite =
+ 0x1F8; // fcPosNormal|fcNegNormal|fcPosSubnormal|fcNegSubnormal|fcPosZero|fcNegZero
+ constexpr unsigned fcInfNan = 0x207; // fcPosInf|fcNegInf|fcSNan|fcQNan
+
+ if (Mask == fcFinite) {
+ // is_fpclass(x, finite) selects frexp when x is finite
+ // frexp already returns 0 for non-finite, so select frexp, 0 -> frexp
+ IsNonFiniteTest = !CondSelectsZero;
+ } else if (Mask == fcInfNan || Mask == 0x3 || Mask == 0x204) {
+ // is_fpclass(x, inf|nan) or is_fpclass(x, nan) or is_fpclass(x, inf)
+ // selects 0 when x is non-finite
+ IsNonFiniteTest = CondSelectsZero;
+ }
+ } else if (Cond.getOpcode() == ISD::SETCC) {
+ // Handle SETCC conditions for inf/nan tests.
+ ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
+ SDValue CondLHS = Cond.getOperand(0);
+ SDValue CondRHS = Cond.getOperand(1);
+ SDValue CondLHSStripped = peekFPSignOps(CondLHS);
+
+ auto isInfConstant = [](SDValue V) {
+ auto *CFP = dyn_cast<ConstantFPSDNode>(V);
+ return CFP && CFP->getValueAPF().isInfinity();
+ };
+
+ if (CC == ISD::SETUO) {
+ // fcmp uno x, y - true if either x or y is NaN
+ SDValue CondRHSStripped = peekFPSignOps(CondRHS);
+ if (CondLHSStripped == FrexpInputStripped ||
+ CondRHSStripped == FrexpInputStripped) {
+ IsNonFiniteTest = CondSelectsZero;
+ }
+ } else if ((CC == ISD::SETOEQ || CC == ISD::SETUEQ) &&
+ isInfConstant(CondRHS)) {
+ // fcmp oeq/ueq |x|, inf - true if x is inf (or inf/nan for ueq)
+ if (CondLHSStripped == FrexpInputStripped)
+ IsNonFiniteTest = CondSelectsZero;
+ } else if ((CC == ISD::SETONE || CC == ISD::SETUNE) &&
+ isInfConstant(CondRHS)) {
+ // fcmp one/une |x|, inf - true if x is NOT inf
+ if (CondLHSStripped == FrexpInputStripped)
+ IsNonFiniteTest = !CondSelectsZero;
+ } else if (CC == ISD::SETO) {
+ // fcmp ord x, y - true if both are NOT NaN
+ SDValue CondRHSStripped = peekFPSignOps(CondRHS);
+ if (CondLHSStripped == FrexpInputStripped ||
+ CondRHSStripped == FrexpInputStripped) {
+ IsNonFiniteTest = !CondSelectsZero;
+ }
+ }
+ }
+
+ if (!IsNonFiniteTest)
+ return SDValue();
+
+ // The select can be eliminated - just return the frexp result directly.
+ return FrexpVal;
}
SDValue SITargetLowering::performSelectCombine(SDNode *N,
@@ -18544,8 +18997,10 @@ SDValue SITargetLowering::performSelectCombine(SDNode *N,
if (!isFloatingPoint && !isInteger)
return SDValue();
- bool isEquality = CC == (isFloatingPoint ? ISD::SETOEQ : ISD::SETEQ);
- bool isNonEquality = CC == (isFloatingPoint ? ISD::SETONE : ISD::SETNE);
+ // Bare SETEQ/SETNE is the builder's NaN-impossible downgrade.
+ bool isEquality = CC == ISD::SETEQ || (isFloatingPoint && CC == ISD::SETOEQ);
+ bool isNonEquality =
+ CC == ISD::SETNE || (isFloatingPoint && CC == ISD::SETONE);
if (!isEquality && !isNonEquality)
return SDValue();
@@ -18581,6 +19036,11 @@ SDValue SITargetLowering::performSelectCombine(SDNode *N,
!(isNonEquality && FalseVal == ConstVal))
return SDValue();
+ // SETONE's false arm is also taken for NaN ArgVal, so require NaN excluded.
+ if (isFloatingPoint && isNonEquality && FalseVal == ConstVal &&
+ !Cond->getFlags().hasNoNaNs() && !DCI.DAG.isKnownNeverNaN(ArgVal))
+ return SDValue();
+
SDValue SelectLHS = (isEquality && TrueVal == ConstVal) ? ArgVal : TrueVal;
SDValue SelectRHS =
(isNonEquality && FalseVal == ConstVal) ? ArgVal : FalseVal;
@@ -18639,6 +19099,8 @@ SDValue SITargetLowering::PerformDAGCombine(SDNode *N,
case ISD::SETCC:
return performSetCCCombine(N, DCI);
case ISD::SELECT:
+ if (auto Res = performFrexpSelectCombine(N, DCI))
+ return Res;
if (auto Res = performSelectCombine(N, DCI))
return Res;
break;
@@ -19260,10 +19722,8 @@ MachineSDNode *SITargetLowering::buildRSRC(SelectionDAG &DAG, const SDLoc &DL,
SDValue PtrLo = DAG.getTargetExtractSubreg(AMDGPU::sub0, DL, MVT::i32, Ptr);
SDValue PtrHi = DAG.getTargetExtractSubreg(AMDGPU::sub1, DL, MVT::i32, Ptr);
if (RsrcDword1) {
- PtrHi =
- SDValue(DAG.getMachineNode(AMDGPU::S_OR_B32, DL, MVT::i32, PtrHi,
- DAG.getConstant(RsrcDword1, DL, MVT::i32)),
- 0);
+ PtrHi = DAG.getNode(ISD::OR, DL, MVT::i32, PtrHi,
+ DAG.getConstant(RsrcDword1, DL, MVT::i32));
}
SDValue DataLo =
@@ -19738,9 +20198,9 @@ void SITargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
Op, Known, DemandedElts, DAG, Depth);
}
-void SITargetLowering::computeKnownBitsForFrameIndex(
- const int FI, KnownBits &Known, const MachineFunction &MF) const {
- TargetLowering::computeKnownBitsForFrameIndex(FI, Known, MF);
+void SITargetLowering::computeKnownBitsForStackObjectPointer(
+ KnownBits &Known, const MachineFunction &MF, Align Alignment) const {
+ TargetLowering::computeKnownBitsForStackObjectPointer(Known, MF, Alignment);
// Set the high bits to zero based on the maximum allowed scratch size per
// wave. We can't use vaddr in MUBUF instructions if we don't know the address
@@ -20288,7 +20748,9 @@ SITargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *RMW) const {
auto SSID = RMW->getSyncScopeID();
bool HasSystemScope =
SSID == SyncScope::System ||
- SSID == RMW->getContext().getOrInsertSyncScopeID("one-as");
+ SSID == RMW->getContext().getOrInsertSyncScopeID(*getAtomicScopeIRString(
+ getTargetMachine().getTargetTriple(), AtomicScope::System,
+ /*OneAddressSpace=*/true));
auto Op = RMW->getOperation();
switch (Op) {
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h
index c98426cdac0b1..b5853ac1035f7 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h
@@ -17,6 +17,7 @@
#include "AMDGPUArgumentUsageInfo.h"
#include "AMDGPUISelLowering.h"
#include "SIDefines.h"
+#include "llvm/ADT/FloatingPointMode.h"
#include "llvm/CodeGen/MachineFunction.h"
namespace llvm {
@@ -85,8 +86,9 @@ class SITargetLowering final : public AMDGPUTargetLowering {
MVT VT, unsigned Offset) const;
SDValue lowerImage(SDValue Op, const AMDGPU::ImageDimIntrinsicInfo *Intr,
SelectionDAG &DAG, bool WithChain) const;
- SDValue lowerSBuffer(EVT VT, SDLoc DL, SDValue Rsrc, SDValue Offset,
- SDValue CachePolicy, SelectionDAG &DAG) const;
+ SDValue lowerSBuffer(EVT VT, EVT MemVT, SDLoc DL, SDValue Chain, SDValue Rsrc,
+ SDValue Offset, SDValue CachePolicy, SelectionDAG &DAG,
+ MachineMemOperand *MMO = nullptr) const;
SDValue lowerRawBufferAtomicIntrin(SDValue Op, SelectionDAG &DAG,
unsigned NewOpcode) const;
@@ -103,6 +105,8 @@ class SITargetLowering final : public AMDGPUTargetLowering {
SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerINTRINSIC_VOID(SDValue Op, SelectionDAG &DAG) const;
+ SDValue LowerCONVERT_FROM_ARBITRARY_FP(SDValue Op, SelectionDAG &DAG) const;
+ SDValue lowerFromFP8(SDValue Op, bool IsBF8, SelectionDAG &DAG) const;
// The raw.tbuffer and struct.tbuffer intrinsics have two offset args: offset
// (the offset that is included in bounds checking and swizzling, to be split
@@ -163,6 +167,8 @@ class SITargetLowering final : public AMDGPUTargetLowering {
/// Custom lowering for ISD::FP_ROUND for MVT::f16.
SDValue lowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const;
SDValue splitFP_ROUNDVectorOp(SDValue Op, SelectionDAG &DAG) const;
+ SDValue lowerScalarBF16BinaryOp(SDValue Op, SelectionDAG &DAG) const;
+ SDValue lowerScalarBF16FCanonicalize(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerFMINNUM_FMAXNUM(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerFMINIMUMNUM_FMAXIMUMNUM(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerFMINIMUM_FMAXIMUM(SDValue Op, SelectionDAG &DAG) const;
@@ -231,6 +237,7 @@ class SITargetLowering final : public AMDGPUTargetLowering {
SDValue performExtractVectorEltCombine(SDNode *N, DAGCombinerInfo &DCI) const;
SDValue performInsertVectorEltCombine(SDNode *N, DAGCombinerInfo &DCI) const;
SDValue performFPRoundCombine(SDNode *N, DAGCombinerInfo &DCI) const;
+ SDValue performFrexpSelectCombine(SDNode *N, DAGCombinerInfo &DCI) const;
SDValue performSelectCombine(SDNode *N, DAGCombinerInfo &DCI) const;
SDValue reassociateScalarOps(SDNode *N, SelectionDAG &DAG) const;
@@ -400,8 +407,8 @@ class SITargetLowering final : public AMDGPUTargetLowering {
bool shouldConvertConstantLoadToIntImm(const APInt &Imm,
Type *Ty) const override;
- bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT,
- unsigned Index) const override;
+ ExtractSubvectorCost getExtractSubvectorCost(EVT ResVT, EVT SrcVT,
+ unsigned Index) const override;
bool isExtractVecEltCheap(EVT VT, unsigned Index) const override;
bool isTypeDesirableForOp(unsigned Op, EVT VT) const override;
@@ -501,6 +508,15 @@ class SITargetLowering final : public AMDGPUTargetLowering {
bool isFMADLegal(const SelectionDAG &DAG, const SDNode *N) const override;
bool isFMADLegal(const MachineInstr &MI, const LLT Ty) const override;
+ /// Variants for IR level callers, which have no MachineFunction to read the
+ /// denormal mode from and must pass \p FPEnv explicitly.
+ bool isFMAFasterThanFMulAndFAdd(EVT VT, DenormalFPEnv FPEnv) const;
+
+ /// \p VT is used as written, so a vector type reports false.
+ bool isFMADLegal(EVT VT, DenormalFPEnv FPEnv) const;
+
+ bool isFMAFasterThanFMulAndFAdd(const Function &F, Type *Ty) const override;
+
SDValue splitUnaryVectorOp(SDValue Op, SelectionDAG &DAG) const;
SDValue splitBinaryVectorOp(SDValue Op, SelectionDAG &DAG) const;
SDValue splitTernaryVectorOp(SDValue Op, SelectionDAG &DAG) const;
@@ -542,9 +558,9 @@ class SITargetLowering final : public AMDGPUTargetLowering {
const APInt &DemandedElts,
const SelectionDAG &DAG,
unsigned Depth = 0) const override;
- void computeKnownBitsForFrameIndex(int FrameIdx,
- KnownBits &Known,
- const MachineFunction &MF) const override;
+ void computeKnownBitsForStackObjectPointer(KnownBits &Known,
+ const MachineFunction &MF,
+ Align Alignment) const override;
void computeKnownBitsForTargetInstr(GISelValueTracking &Analysis, Register R,
KnownBits &Known,
const APInt &DemandedElts,
@@ -568,6 +584,15 @@ class SITargetLowering final : public AMDGPUTargetLowering {
bool isCanonicalized(SelectionDAG &DAG, SDValue Op,
SDNodeFlags UserFlags = {}, unsigned MaxDepth = 5) const;
+
+ /// Returns true if \p Op is provably canonical (no FCANONICALIZE needed).
+ /// \p QueryVT is the scalar FP type being checked, threaded unchanged
+ /// through recursion since canonicality is per vector element. FP operands
+ /// whose scalar type differs from \p QueryVT are treated as non-canonical,
+ /// since canonicality does not survive a change of FP format (e.g. bitcast
+ /// v2bf16 to v2f16); non-FP operands are not checked against \p QueryVT.
+ bool isCanonicalized(SelectionDAG &DAG, SDValue Op, EVT QueryVT,
+ SDNodeFlags UserFlags, unsigned MaxDepth) const;
bool isCanonicalized(Register Reg, const MachineFunction &MF,
unsigned MaxDepth = 5) const;
bool denormalsEnabledForType(const SelectionDAG &DAG, EVT VT) const;
diff --git a/llvm/test/CodeGen/AMDGPU/frexp-inf-nan-combine.ll b/llvm/test/CodeGen/AMDGPU/frexp-inf-nan-combine.ll
new file mode 100644
index 0000000000000..c2a763e58e2c2
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/frexp-inf-nan-combine.ll
@@ -0,0 +1,528 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck -check-prefix=GFX9 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1030 < %s | FileCheck -check-prefix=GFX10 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 < %s | FileCheck -check-prefix=GFX11 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 < %s | FileCheck -check-prefix=GFX950 %s
+; RUN: llc -mtriple=amdgcn-- -mcpu=tahiti < %s | FileCheck -check-prefix=SI %s
+
+; Test that redundant inf/nan checks are folded into frexp instructions.
+; The AMDGPU frexp instructions already return 0 for inf/nan inputs.
+; This optimization should NOT apply on SI (Southern Islands) which has hasFractBug.
+
+declare {float, i32} @llvm.frexp.f32.i32(float)
+declare {double, i32} @llvm.frexp.f64.i32(double)
+declare {half, i16} @llvm.frexp.f16.i16(half)
+declare float @llvm.fabs.f32(float)
+declare double @llvm.fabs.f64(double)
+declare half @llvm.fabs.f16(half)
+
+; Pattern 1: select (fcmp uno x, 0), 0, (frexp_exp x)
+; NaN check - should fold to just frexp_exp
+define i32 @frexp_nan_clamp_exp_f32(float %x) {
+; GFX9-LABEL: frexp_nan_clamp_exp_f32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: frexp_nan_clamp_exp_f32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: frexp_nan_clamp_exp_f32:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-LABEL: frexp_nan_clamp_exp_f32:
+; GFX950: ; %bb.0:
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-LABEL: frexp_nan_clamp_exp_f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s4, 0x7f800000
+; SI-NEXT: v_cmp_lt_f32_e64 s[4:5], |v0|, s4
+; SI-NEXT: v_cmp_o_f32_e32 vcc, v0, v0
+; SI-NEXT: v_frexp_exp_i32_f32_e32 v1, v0
+; SI-NEXT: s_and_b64 vcc, vcc, s[4:5]
+; SI-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
+; SI-NEXT: s_setpc_b64 s[30:31]
+ %frexp = call {float, i32} @llvm.frexp.f32.i32(float %x)
+ %exp = extractvalue {float, i32} %frexp, 1
+ %is_nan = fcmp uno float %x, 0.0
+ %result = select i1 %is_nan, i32 0, i32 %exp
+ ret i32 %result
+}
+
+; Pattern 2: select (fcmp oeq |x|, inf), 0, (frexp_exp x)
+; Inf check - should fold to just frexp_exp
+define i32 @frexp_inf_clamp_exp_f32(float %x) {
+; GFX9-LABEL: frexp_inf_clamp_exp_f32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: frexp_inf_clamp_exp_f32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: frexp_inf_clamp_exp_f32:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-LABEL: frexp_inf_clamp_exp_f32:
+; GFX950: ; %bb.0:
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-LABEL: frexp_inf_clamp_exp_f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s4, 0x7f800000
+; SI-NEXT: v_frexp_exp_i32_f32_e32 v1, v0
+; SI-NEXT: v_cmp_lt_f32_e64 vcc, |v0|, s4
+; SI-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
+; SI-NEXT: s_setpc_b64 s[30:31]
+ %frexp = call {float, i32} @llvm.frexp.f32.i32(float %x)
+ %exp = extractvalue {float, i32} %frexp, 1
+ %abs = call float @llvm.fabs.f32(float %x)
+ %is_inf = fcmp oeq float %abs, 0x7FF0000000000000
+ %result = select i1 %is_inf, i32 0, i32 %exp
+ ret i32 %result
+}
+
+; Pattern 3: select (fcmp ueq |x|, inf), 0, (frexp_exp x)
+; Inf-or-NaN check - should fold to just frexp_exp
+define i32 @frexp_inf_or_nan_clamp_exp_f32(float %x) {
+; GFX9-LABEL: frexp_inf_or_nan_clamp_exp_f32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: frexp_inf_or_nan_clamp_exp_f32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: frexp_inf_or_nan_clamp_exp_f32:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-LABEL: frexp_inf_or_nan_clamp_exp_f32:
+; GFX950: ; %bb.0:
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-LABEL: frexp_inf_or_nan_clamp_exp_f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s4, 0x7f800000
+; SI-NEXT: s_movk_i32 s6, 0x1f8
+; SI-NEXT: v_cmp_lt_f32_e64 s[4:5], |v0|, s4
+; SI-NEXT: v_cmp_class_f32_e64 s[6:7], v0, s6
+; SI-NEXT: v_frexp_exp_i32_f32_e32 v1, v0
+; SI-NEXT: s_and_b64 vcc, s[6:7], s[4:5]
+; SI-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
+; SI-NEXT: s_setpc_b64 s[30:31]
+ %frexp = call { float, i32 } @llvm.frexp.f32.i32(float %x)
+ %exp = extractvalue { float, i32 } %frexp, 1
+ %abs = call float @llvm.fabs.f32(float %x)
+ %is_non_finite = fcmp ueq float %abs, 0x7FF0000000000000
+ %result = select i1 %is_non_finite, i32 0, i32 %exp
+ ret i32 %result
+}
+
+; Test with frexp_mant instead of frexp_exp
+define float @frexp_nan_clamp_mant_f32(float %x) {
+; GFX9-LABEL: frexp_nan_clamp_mant_f32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_frexp_mant_f32_e32 v0, v0
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: frexp_nan_clamp_mant_f32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_frexp_mant_f32_e32 v0, v0
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: frexp_nan_clamp_mant_f32:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_frexp_mant_f32_e32 v0, v0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-LABEL: frexp_nan_clamp_mant_f32:
+; GFX950: ; %bb.0:
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_frexp_mant_f32_e32 v0, v0
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-LABEL: frexp_nan_clamp_mant_f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s4, 0x7f800000
+; SI-NEXT: v_frexp_mant_f32_e32 v1, v0
+; SI-NEXT: v_cmp_lt_f32_e64 vcc, |v0|, s4
+; SI-NEXT: v_cndmask_b32_e32 v1, v0, v1, vcc
+; SI-NEXT: v_cmp_o_f32_e32 vcc, v0, v0
+; SI-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
+; SI-NEXT: s_setpc_b64 s[30:31]
+ %frexp = call {float, i32} @llvm.frexp.f32.i32(float %x)
+ %mant = extractvalue {float, i32} %frexp, 0
+ %is_nan = fcmp uno float %x, 0.0
+ %result = select i1 %is_nan, float 0.0, float %mant
+ ret float %result
+}
+
+; Test with reversed select operands: select (fcmp ord x, 0), frexp, 0
+define i32 @frexp_ord_clamp_exp_f32(float %x) {
+; GFX9-LABEL: frexp_ord_clamp_exp_f32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: frexp_ord_clamp_exp_f32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: frexp_ord_clamp_exp_f32:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-LABEL: frexp_ord_clamp_exp_f32:
+; GFX950: ; %bb.0:
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-LABEL: frexp_ord_clamp_exp_f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s4, 0x7f800000
+; SI-NEXT: v_cmp_lt_f32_e64 s[4:5], |v0|, s4
+; SI-NEXT: v_cmp_o_f32_e32 vcc, v0, v0
+; SI-NEXT: v_frexp_exp_i32_f32_e32 v1, v0
+; SI-NEXT: s_and_b64 vcc, vcc, s[4:5]
+; SI-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
+; SI-NEXT: s_setpc_b64 s[30:31]
+ %frexp = call {float, i32} @llvm.frexp.f32.i32(float %x)
+ %exp = extractvalue {float, i32} %frexp, 1
+ %is_ord = fcmp ord float %x, 0.0
+ %result = select i1 %is_ord, i32 %exp, i32 0
+ ret i32 %result
+}
+
+; Test with fcmp one |x|, inf (not inf check -> select frexp)
+define i32 @frexp_not_inf_clamp_exp_f32(float %x) {
+; GFX9-LABEL: frexp_not_inf_clamp_exp_f32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: frexp_not_inf_clamp_exp_f32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: frexp_not_inf_clamp_exp_f32:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-LABEL: frexp_not_inf_clamp_exp_f32:
+; GFX950: ; %bb.0:
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-LABEL: frexp_not_inf_clamp_exp_f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s4, 0x7f800000
+; SI-NEXT: s_movk_i32 s6, 0x1f8
+; SI-NEXT: v_cmp_lt_f32_e64 s[4:5], |v0|, s4
+; SI-NEXT: v_cmp_class_f32_e64 s[6:7], v0, s6
+; SI-NEXT: v_frexp_exp_i32_f32_e32 v1, v0
+; SI-NEXT: s_and_b64 vcc, s[6:7], s[4:5]
+; SI-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
+; SI-NEXT: s_setpc_b64 s[30:31]
+ %frexp = call {float, i32} @llvm.frexp.f32.i32(float %x)
+ %exp = extractvalue {float, i32} %frexp, 1
+ %abs = call float @llvm.fabs.f32(float %x)
+ %is_not_inf = fcmp one float %abs, 0x7FF0000000000000
+ %result = select i1 %is_not_inf, i32 %exp, i32 0
+ ret i32 %result
+}
+
+; Test f64
+define i32 @frexp_nan_clamp_exp_f64(double %x) {
+; GFX9-LABEL: frexp_nan_clamp_exp_f64:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_frexp_exp_i32_f64_e32 v0, v[0:1]
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: frexp_nan_clamp_exp_f64:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_frexp_exp_i32_f64_e32 v0, v[0:1]
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: frexp_nan_clamp_exp_f64:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_frexp_exp_i32_f64_e32 v0, v[0:1]
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-LABEL: frexp_nan_clamp_exp_f64:
+; GFX950: ; %bb.0:
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_frexp_exp_i32_f64_e32 v0, v[0:1]
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-LABEL: frexp_nan_clamp_exp_f64:
+; SI: ; %bb.0:
+; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT: v_and_b32_e32 v2, 0x7fffffff, v1
+; SI-NEXT: s_mov_b32 s4, 0x7ff00000
+; SI-NEXT: v_cmp_gt_i32_e32 vcc, s4, v2
+; SI-NEXT: v_cmp_o_f64_e64 s[4:5], v[0:1], v[0:1]
+; SI-NEXT: v_frexp_exp_i32_f64_e32 v0, v[0:1]
+; SI-NEXT: s_and_b64 vcc, s[4:5], vcc
+; SI-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
+; SI-NEXT: s_setpc_b64 s[30:31]
+ %frexp = call {double, i32} @llvm.frexp.f64.i32(double %x)
+ %exp = extractvalue {double, i32} %frexp, 1
+ %is_nan = fcmp uno double %x, 0.0
+ %result = select i1 %is_nan, i32 0, i32 %exp
+ ret i32 %result
+}
+
+; Test f16
+define i16 @frexp_nan_clamp_exp_f16(half %x) {
+; GFX9-LABEL: frexp_nan_clamp_exp_f16:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_frexp_exp_i16_f16_e32 v0, v0
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: frexp_nan_clamp_exp_f16:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_frexp_exp_i16_f16_e32 v0, v0
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: frexp_nan_clamp_exp_f16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_frexp_exp_i16_f16_e32 v0.l, v0.l
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-LABEL: frexp_nan_clamp_exp_f16:
+; GFX950: ; %bb.0:
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_frexp_exp_i16_f16_e32 v0, v0
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-LABEL: frexp_nan_clamp_exp_f16:
+; SI: ; %bb.0:
+; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT: s_mov_b32 s4, 0x7f800000
+; SI-NEXT: v_cmp_lt_f32_e64 s[4:5], |v0|, s4
+; SI-NEXT: v_cmp_o_f32_e32 vcc, v0, v0
+; SI-NEXT: v_frexp_exp_i32_f32_e32 v1, v0
+; SI-NEXT: s_and_b64 vcc, vcc, s[4:5]
+; SI-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
+; SI-NEXT: s_setpc_b64 s[30:31]
+ %frexp = call {half, i16} @llvm.frexp.f16.i16(half %x)
+ %exp = extractvalue {half, i16} %frexp, 1
+ %is_nan = fcmp uno half %x, 0.0
+ %result = select i1 %is_nan, i16 0, i16 %exp
+ ret i16 %result
+}
+
+; Negative test: different input to frexp vs comparison
+define i32 @frexp_nan_different_input(float %x, float %y) {
+; GFX9-LABEL: frexp_nan_different_input:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: frexp_nan_different_input:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v1, v1
+; GFX10-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc_lo
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: frexp_nan_different_input:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v1, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc_lo
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-LABEL: frexp_nan_different_input:
+; GFX950: ; %bb.0:
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; GFX950-NEXT: v_cmp_o_f32_e32 vcc, v1, v1
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-LABEL: frexp_nan_different_input:
+; SI: ; %bb.0:
+; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s4, 0x7f800000
+; SI-NEXT: v_cmp_lt_f32_e64 s[4:5], |v0|, s4
+; SI-NEXT: v_cmp_o_f32_e32 vcc, v1, v1
+; SI-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; SI-NEXT: s_and_b64 vcc, vcc, s[4:5]
+; SI-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
+; SI-NEXT: s_setpc_b64 s[30:31]
+ %frexp = call {float, i32} @llvm.frexp.f32.i32(float %x)
+ %exp = extractvalue {float, i32} %frexp, 1
+ %is_nan = fcmp uno float %y, 0.0
+ %result = select i1 %is_nan, i32 0, i32 %exp
+ ret i32 %result
+}
+
+; Negative test: non-zero constant in select
+define i32 @frexp_nan_nonzero_const(float %x) {
+; GFX9-LABEL: frexp_nan_nonzero_const:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_frexp_exp_i32_f32_e32 v1, v0
+; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, 42, v1, vcc
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: frexp_nan_nonzero_const:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_frexp_exp_i32_f32_e32 v1, v0
+; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_cndmask_b32_e32 v0, 42, v1, vcc_lo
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: frexp_nan_nonzero_const:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_frexp_exp_i32_f32_e32 v1, v0
+; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: v_cndmask_b32_e32 v0, 42, v1, vcc_lo
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-LABEL: frexp_nan_nonzero_const:
+; GFX950: ; %bb.0:
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_frexp_exp_i32_f32_e32 v1, v0
+; GFX950-NEXT: v_cmp_o_f32_e32 vcc, v0, v0
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e32 v0, 42, v1, vcc
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-LABEL: frexp_nan_nonzero_const:
+; SI: ; %bb.0:
+; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s4, 0x7f800000
+; SI-NEXT: v_frexp_exp_i32_f32_e32 v1, v0
+; SI-NEXT: v_cmp_lt_f32_e64 vcc, |v0|, s4
+; SI-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
+; SI-NEXT: v_cmp_o_f32_e32 vcc, v0, v0
+; SI-NEXT: v_cndmask_b32_e32 v0, 42, v1, vcc
+; SI-NEXT: s_setpc_b64 s[30:31]
+ %frexp = call {float, i32} @llvm.frexp.f32.i32(float %x)
+ %exp = extractvalue {float, i32} %frexp, 1
+ %is_nan = fcmp uno float %x, 0.0
+ %result = select i1 %is_nan, i32 42, i32 %exp
+ ret i32 %result
+}
+
+; Negative test: unrelated comparison (not inf/nan test)
+define i32 @frexp_lt_zero_not_folded(float %x) {
+; GFX9-LABEL: frexp_lt_zero_not_folded:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_frexp_exp_i32_f32_e32 v1, v0
+; GFX9-NEXT: v_cmp_ngt_f32_e32 vcc, 0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: frexp_lt_zero_not_folded:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_frexp_exp_i32_f32_e32 v1, v0
+; GFX10-NEXT: v_cmp_ngt_f32_e32 vcc_lo, 0, v0
+; GFX10-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc_lo
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: frexp_lt_zero_not_folded:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_frexp_exp_i32_f32_e32 v1, v0
+; GFX11-NEXT: v_cmp_ngt_f32_e32 vcc_lo, 0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc_lo
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-LABEL: frexp_lt_zero_not_folded:
+; GFX950: ; %bb.0:
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_frexp_exp_i32_f32_e32 v1, v0
+; GFX950-NEXT: v_cmp_ngt_f32_e32 vcc, 0, v0
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-LABEL: frexp_lt_zero_not_folded:
+; SI: ; %bb.0:
+; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s4, 0x7f800000
+; SI-NEXT: v_cmp_lt_f32_e64 s[4:5], |v0|, s4
+; SI-NEXT: v_cmp_ngt_f32_e32 vcc, 0, v0
+; SI-NEXT: v_frexp_exp_i32_f32_e32 v1, v0
+; SI-NEXT: s_and_b64 vcc, vcc, s[4:5]
+; SI-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
+; SI-NEXT: s_setpc_b64 s[30:31]
+ %frexp = call {float, i32} @llvm.frexp.f32.i32(float %x)
+ %exp = extractvalue {float, i32} %frexp, 1
+ %is_lt_zero = fcmp olt float %x, 0.0
+ %result = select i1 %is_lt_zero, i32 0, i32 %exp
+ ret i32 %result
+}
>From f4a80b88d39cdace057843b7ad15d8f9bc3d2ed7 Mon Sep 17 00:00:00 2001
From: addmisol <addmisol9 at gmail.com>
Date: Sat, 8 Aug 2026 12:28:40 +0530
Subject: [PATCH 2/4] Add Changes
Signed-off-by: addmisol <addmisol9 at gmail.com>
---
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 141 ++++++++++++++++++++++
1 file changed, 141 insertions(+)
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 1d165910adeba..8a05d8591548d 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -18829,6 +18829,145 @@ SDValue SITargetLowering::performClampCombine(SDNode *N,
return getCanonicalConstantFP(DCI.DAG, SDLoc(N), N->getValueType(0), F);
}
+SDValue
+SITargetLowering::performFrexpSelectCombine(SDNode *N,
+ DAGCombinerInfo &DCI) const {
+ // This optimization only applies when the hardware handles inf/nan correctly.
+ if (Subtarget->hasFractBug())
+ return SDValue();
+
+ SDValue Cond = N->getOperand(0);
+ SDValue TrueVal = N->getOperand(1);
+ SDValue FalseVal = N->getOperand(2);
+
+ // Determine which value is 0 and which might be the frexp result.
+ // Pattern 1: select cond, 0, frexp_result (cond true -> return 0)
+ // Pattern 2: select cond, frexp_result, 0 (cond false -> return 0)
+ SDValue FrexpVal;
+ bool CondSelectsZero; // If true, condition=true selects zero
+
+ auto isZero = [](SDValue V) {
+ if (auto *C = dyn_cast<ConstantSDNode>(V))
+ return C->isZero();
+ if (auto *C = dyn_cast<ConstantFPSDNode>(V))
+ return C->isZero();
+ return false;
+ };
+
+ if (isZero(TrueVal)) {
+ FrexpVal = FalseVal;
+ CondSelectsZero = true;
+ } else if (isZero(FalseVal)) {
+ FrexpVal = TrueVal;
+ CondSelectsZero = false;
+ } else {
+ return SDValue();
+ }
+
+ // Check if FrexpVal comes from amdgcn_frexp_exp or amdgcn_frexp_mant.
+ if (FrexpVal.getOpcode() != ISD::INTRINSIC_WO_CHAIN)
+ return SDValue();
+
+ unsigned IID = FrexpVal.getConstantOperandVal(0);
+ if (IID != Intrinsic::amdgcn_frexp_exp && IID != Intrinsic::amdgcn_frexp_mant)
+ return SDValue();
+
+ SDValue FrexpInput = FrexpVal.getOperand(1);
+
+ // Helper to strip fabs/fneg/fcopysign from a value.
+ auto peekFPSignOps = [](SDValue Val) {
+ if (Val.getOpcode() == ISD::FNEG)
+ Val = Val.getOperand(0);
+ if (Val.getOpcode() == ISD::FABS)
+ Val = Val.getOperand(0);
+ if (Val.getOpcode() == ISD::FCOPYSIGN)
+ Val = Val.getOperand(0);
+ return Val;
+ };
+
+ // The frexp intrinsics ignore sign, so we can strip sign ops when comparing.
+ SDValue FrexpInputStripped = peekFPSignOps(FrexpInput);
+
+ bool IsNonFiniteTest = false;
+
+ // Handle AMDGPUISD::FP_CLASS or ISD::IS_FPCLASS conditions.
+ // These test specific floating-point classes using a bitmask.
+ if (Cond.getOpcode() == AMDGPUISD::FP_CLASS ||
+ Cond.getOpcode() == ISD::IS_FPCLASS) {
+ SDValue ClassInput = Cond.getOperand(0);
+ SDValue ClassInputStripped = peekFPSignOps(ClassInput);
+
+ if (ClassInputStripped != FrexpInputStripped)
+ return SDValue();
+
+ auto *MaskNode = dyn_cast<ConstantSDNode>(Cond.getOperand(1));
+ if (!MaskNode)
+ return SDValue();
+
+ unsigned Mask = MaskNode->getZExtValue();
+
+ // fcFinite = all finite classes (not inf, not nan)
+ // If the mask tests for finite values and selects frexp when true,
+ // we can fold away the select since frexp returns 0 for non-finite.
+ constexpr unsigned fcFinite =
+ 0x1F8; // fcPosNormal|fcNegNormal|fcPosSubnormal|fcNegSubnormal|fcPosZero|fcNegZero
+ constexpr unsigned fcInfNan = 0x207; // fcPosInf|fcNegInf|fcSNan|fcQNan
+
+ if (Mask == fcFinite) {
+ // is_fpclass(x, finite) selects frexp when x is finite
+ // frexp already returns 0 for non-finite, so select frexp, 0 -> frexp
+ IsNonFiniteTest = !CondSelectsZero;
+ } else if (Mask == fcInfNan || Mask == 0x3 || Mask == 0x204) {
+ // is_fpclass(x, inf|nan) or is_fpclass(x, nan) or is_fpclass(x, inf)
+ // selects 0 when x is non-finite
+ IsNonFiniteTest = CondSelectsZero;
+ }
+ } else if (Cond.getOpcode() == ISD::SETCC) {
+ // Handle SETCC conditions for inf/nan tests.
+ ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
+ SDValue CondLHS = Cond.getOperand(0);
+ SDValue CondRHS = Cond.getOperand(1);
+ SDValue CondLHSStripped = peekFPSignOps(CondLHS);
+
+ auto isInfConstant = [](SDValue V) {
+ auto *CFP = dyn_cast<ConstantFPSDNode>(V);
+ return CFP && CFP->getValueAPF().isInfinity();
+ };
+
+ if (CC == ISD::SETUO) {
+ // fcmp uno x, y - true if either x or y is NaN
+ SDValue CondRHSStripped = peekFPSignOps(CondRHS);
+ if (CondLHSStripped == FrexpInputStripped ||
+ CondRHSStripped == FrexpInputStripped) {
+ IsNonFiniteTest = CondSelectsZero;
+ }
+ } else if ((CC == ISD::SETOEQ || CC == ISD::SETUEQ) &&
+ isInfConstant(CondRHS)) {
+ // fcmp oeq/ueq |x|, inf - true if x is inf (or inf/nan for ueq)
+ if (CondLHSStripped == FrexpInputStripped)
+ IsNonFiniteTest = CondSelectsZero;
+ } else if ((CC == ISD::SETONE || CC == ISD::SETUNE) &&
+ isInfConstant(CondRHS)) {
+ // fcmp one/une |x|, inf - true if x is NOT inf
+ if (CondLHSStripped == FrexpInputStripped)
+ IsNonFiniteTest = !CondSelectsZero;
+ } else if (CC == ISD::SETO) {
+ // fcmp ord x, y - true if both are NOT NaN
+ SDValue CondRHSStripped = peekFPSignOps(CondRHS);
+ if (CondLHSStripped == FrexpInputStripped ||
+ CondRHSStripped == FrexpInputStripped) {
+ IsNonFiniteTest = !CondSelectsZero;
+ }
+ }
+ }
+
+ if (!IsNonFiniteTest)
+ return SDValue();
+
+ // The select can be eliminated - just return the frexp result directly.
+ return FrexpVal;
+}
+
SDValue SITargetLowering::performSelectCombine(SDNode *N,
DAGCombinerInfo &DCI) const {
@@ -18960,6 +19099,8 @@ SDValue SITargetLowering::PerformDAGCombine(SDNode *N,
case ISD::SETCC:
return performSetCCCombine(N, DCI);
case ISD::SELECT:
+ if (auto Res = performFrexpSelectCombine(N, DCI))
+ return Res;
if (auto Res = performSelectCombine(N, DCI))
return Res;
break;
>From 395d9a91061f86c67cee5ca5d69b76465c35ba27 Mon Sep 17 00:00:00 2001
From: addmisol <addmisol9 at gmail.com>
Date: Sat, 8 Aug 2026 15:29:35 +0530
Subject: [PATCH 3/4] Fix: Address comments
Signed-off-by: addmisol <addmisol9 at gmail.com>
---
llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp | 10 --
llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h | 9 +
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 56 +++---
.../CodeGen/AMDGPU/frexp-inf-nan-combine.ll | 163 +++++++++++++++++-
4 files changed, 193 insertions(+), 45 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
index 962988ff97e39..cf074af1f78d7 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -1677,16 +1677,6 @@ static SDValue peekFNeg(SDValue Val) {
return Val;
}
-static SDValue peekFPSignOps(SDValue Val) {
- if (Val.getOpcode() == ISD::FNEG)
- Val = Val.getOperand(0);
- if (Val.getOpcode() == ISD::FABS)
- Val = Val.getOperand(0);
- if (Val.getOpcode() == ISD::FCOPYSIGN)
- Val = Val.getOperand(0);
- return Val;
-}
-
SDValue AMDGPUTargetLowering::combineFMinMaxLegacyImpl(
const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, SDValue True,
SDValue False, SDValue CC, DAGCombinerInfo &DCI) const {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
index f24f33e7dcb70..ec6479c509336 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
@@ -426,6 +426,15 @@ class AMDGPUTargetLowering : public TargetLowering {
}
};
+/// Strip fabs/fneg/fcopysign from a value to get the underlying source.
+/// Useful for comparing values where sign doesn't matter (e.g., frexp).
+inline SDValue peekFPSignOps(SDValue Val) {
+ while (Val.getOpcode() == ISD::FNEG || Val.getOpcode() == ISD::FABS ||
+ Val.getOpcode() == ISD::FCOPYSIGN)
+ Val = Val.getOperand(0);
+ return Val;
+}
+
} // End namespace llvm
#endif
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 8a05d8591548d..ae550ebbec50b 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -18829,6 +18829,12 @@ SDValue SITargetLowering::performClampCombine(SDNode *N,
return getCanonicalConstantFP(DCI.DAG, SDLoc(N), N->getValueType(0), F);
}
+/// Check if a value is a positive or negative infinity constant.
+static bool isInfinityFPConstant(SDValue V) {
+ auto *CFP = dyn_cast<ConstantFPSDNode>(V);
+ return CFP && CFP->getValueAPF().isInfinity();
+}
+
SDValue
SITargetLowering::performFrexpSelectCombine(SDNode *N,
DAGCombinerInfo &DCI) const {
@@ -18843,21 +18849,19 @@ SITargetLowering::performFrexpSelectCombine(SDNode *N,
// Determine which value is 0 and which might be the frexp result.
// Pattern 1: select cond, 0, frexp_result (cond true -> return 0)
// Pattern 2: select cond, frexp_result, 0 (cond false -> return 0)
+ // Only check FP zero - frexp returns FP or integer, handle separately.
SDValue FrexpVal;
bool CondSelectsZero; // If true, condition=true selects zero
- auto isZero = [](SDValue V) {
- if (auto *C = dyn_cast<ConstantSDNode>(V))
- return C->isZero();
- if (auto *C = dyn_cast<ConstantFPSDNode>(V))
- return C->isZero();
- return false;
- };
+ bool TrueIsFPZero = isNullFPConstant(TrueVal);
+ bool FalseIsFPZero = isNullFPConstant(FalseVal);
+ bool TrueIsIntZero = isNullConstant(TrueVal);
+ bool FalseIsIntZero = isNullConstant(FalseVal);
- if (isZero(TrueVal)) {
+ if (TrueIsFPZero || TrueIsIntZero) {
FrexpVal = FalseVal;
CondSelectsZero = true;
- } else if (isZero(FalseVal)) {
+ } else if (FalseIsFPZero || FalseIsIntZero) {
FrexpVal = TrueVal;
CondSelectsZero = false;
} else {
@@ -18865,26 +18869,15 @@ SITargetLowering::performFrexpSelectCombine(SDNode *N,
}
// Check if FrexpVal comes from amdgcn_frexp_exp or amdgcn_frexp_mant.
- if (FrexpVal.getOpcode() != ISD::INTRINSIC_WO_CHAIN)
- return SDValue();
-
- unsigned IID = FrexpVal.getConstantOperandVal(0);
- if (IID != Intrinsic::amdgcn_frexp_exp && IID != Intrinsic::amdgcn_frexp_mant)
+ SDValue FrexpInput;
+ if (!sd_match(FrexpVal,
+ m_IntrinsicWOChain<Intrinsic::amdgcn_frexp_exp>(
+ m_Value(FrexpInput))) &&
+ !sd_match(FrexpVal,
+ m_IntrinsicWOChain<Intrinsic::amdgcn_frexp_mant>(
+ m_Value(FrexpInput))))
return SDValue();
- SDValue FrexpInput = FrexpVal.getOperand(1);
-
- // Helper to strip fabs/fneg/fcopysign from a value.
- auto peekFPSignOps = [](SDValue Val) {
- if (Val.getOpcode() == ISD::FNEG)
- Val = Val.getOperand(0);
- if (Val.getOpcode() == ISD::FABS)
- Val = Val.getOperand(0);
- if (Val.getOpcode() == ISD::FCOPYSIGN)
- Val = Val.getOperand(0);
- return Val;
- };
-
// The frexp intrinsics ignore sign, so we can strip sign ops when comparing.
SDValue FrexpInputStripped = peekFPSignOps(FrexpInput);
@@ -18929,11 +18922,6 @@ SITargetLowering::performFrexpSelectCombine(SDNode *N,
SDValue CondRHS = Cond.getOperand(1);
SDValue CondLHSStripped = peekFPSignOps(CondLHS);
- auto isInfConstant = [](SDValue V) {
- auto *CFP = dyn_cast<ConstantFPSDNode>(V);
- return CFP && CFP->getValueAPF().isInfinity();
- };
-
if (CC == ISD::SETUO) {
// fcmp uno x, y - true if either x or y is NaN
SDValue CondRHSStripped = peekFPSignOps(CondRHS);
@@ -18942,12 +18930,12 @@ SITargetLowering::performFrexpSelectCombine(SDNode *N,
IsNonFiniteTest = CondSelectsZero;
}
} else if ((CC == ISD::SETOEQ || CC == ISD::SETUEQ) &&
- isInfConstant(CondRHS)) {
+ isInfinityFPConstant(CondRHS)) {
// fcmp oeq/ueq |x|, inf - true if x is inf (or inf/nan for ueq)
if (CondLHSStripped == FrexpInputStripped)
IsNonFiniteTest = CondSelectsZero;
} else if ((CC == ISD::SETONE || CC == ISD::SETUNE) &&
- isInfConstant(CondRHS)) {
+ isInfinityFPConstant(CondRHS)) {
// fcmp one/une |x|, inf - true if x is NOT inf
if (CondLHSStripped == FrexpInputStripped)
IsNonFiniteTest = !CondSelectsZero;
diff --git a/llvm/test/CodeGen/AMDGPU/frexp-inf-nan-combine.ll b/llvm/test/CodeGen/AMDGPU/frexp-inf-nan-combine.ll
index c2a763e58e2c2..0cba8c03cce7c 100644
--- a/llvm/test/CodeGen/AMDGPU/frexp-inf-nan-combine.ll
+++ b/llvm/test/CodeGen/AMDGPU/frexp-inf-nan-combine.ll
@@ -3,7 +3,7 @@
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1030 < %s | FileCheck -check-prefix=GFX10 %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 < %s | FileCheck -check-prefix=GFX11 %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 < %s | FileCheck -check-prefix=GFX950 %s
-; RUN: llc -mtriple=amdgcn-- -mcpu=tahiti < %s | FileCheck -check-prefix=SI %s
+; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti < %s | FileCheck -check-prefix=SI %s
; Test that redundant inf/nan checks are folded into frexp instructions.
; The AMDGPU frexp instructions already return 0 for inf/nan inputs.
@@ -12,9 +12,14 @@
declare {float, i32} @llvm.frexp.f32.i32(float)
declare {double, i32} @llvm.frexp.f64.i32(double)
declare {half, i16} @llvm.frexp.f16.i16(half)
+declare {bfloat, i16} @llvm.frexp.bf16.i16(bfloat)
+declare {<2 x float>, <2 x i32>} @llvm.frexp.v2f32.v2i32(<2 x float>)
declare float @llvm.fabs.f32(float)
declare double @llvm.fabs.f64(double)
declare half @llvm.fabs.f16(half)
+declare bfloat @llvm.fabs.bf16(bfloat)
+declare <2 x float> @llvm.fabs.v2f32(<2 x float>)
+declare float @llvm.copysign.f32(float, float)
; Pattern 1: select (fcmp uno x, 0), 0, (frexp_exp x)
; NaN check - should fold to just frexp_exp
@@ -526,3 +531,159 @@ define i32 @frexp_lt_zero_not_folded(float %x) {
%result = select i1 %is_lt_zero, i32 0, i32 %exp
ret i32 %result
}
+
+; Test with copysign - should still fold since frexp ignores sign
+define i32 @frexp_nan_clamp_copysign_f32(float %x, float %sign) {
+; GFX9-LABEL: frexp_nan_clamp_copysign_f32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: frexp_nan_clamp_copysign_f32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: frexp_nan_clamp_copysign_f32:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-LABEL: frexp_nan_clamp_copysign_f32:
+; GFX950: ; %bb.0:
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-LABEL: frexp_nan_clamp_copysign_f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT: s_brev_b32 s4, -2
+; SI-NEXT: v_bfi_b32 v1, s4, v0, v1
+; SI-NEXT: s_mov_b32 s4, 0x7f800000
+; SI-NEXT: v_cmp_lt_f32_e64 s[4:5], |v1|, s4
+; SI-NEXT: v_cmp_o_f32_e32 vcc, v0, v0
+; SI-NEXT: v_frexp_exp_i32_f32_e32 v1, v0
+; SI-NEXT: s_and_b64 vcc, vcc, s[4:5]
+; SI-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
+; SI-NEXT: s_setpc_b64 s[30:31]
+ %copysign = call float @llvm.copysign.f32(float %x, float %sign)
+ %frexp = call {float, i32} @llvm.frexp.f32.i32(float %copysign)
+ %exp = extractvalue {float, i32} %frexp, 1
+ %is_nan = fcmp uno float %x, 0.0
+ %result = select i1 %is_nan, i32 0, i32 %exp
+ ret i32 %result
+}
+
+; Test bfloat16
+define i16 @frexp_nan_clamp_exp_bf16(bfloat %x) {
+; GFX9-LABEL: frexp_nan_clamp_exp_bf16:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_frexp_exp_i32_f32_e32 v1, v0
+; GFX9-NEXT: v_cmp_o_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: frexp_nan_clamp_exp_bf16:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX10-NEXT: v_frexp_exp_i32_f32_e32 v1, v0
+; GFX10-NEXT: v_cmp_o_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc_lo
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: frexp_nan_clamp_exp_bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_frexp_exp_i32_f32_e32 v1, v0
+; GFX11-NEXT: v_cmp_o_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: v_cndmask_b16 v0.l, 0, v1.l, vcc_lo
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-LABEL: frexp_nan_clamp_exp_bf16:
+; GFX950: ; %bb.0:
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX950-NEXT: v_frexp_exp_i32_f32_e32 v1, v0
+; GFX950-NEXT: v_cmp_o_f32_e32 vcc, v0, v0
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-LABEL: frexp_nan_clamp_exp_bf16:
+; SI: ; %bb.0:
+; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: s_mov_b32 s4, 0x7f800000
+; SI-NEXT: v_cmp_lt_f32_e64 s[4:5], |v0|, s4
+; SI-NEXT: v_cmp_o_f32_e32 vcc, v0, v0
+; SI-NEXT: v_frexp_exp_i32_f32_e32 v1, v0
+; SI-NEXT: s_and_b64 vcc, vcc, s[4:5]
+; SI-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
+; SI-NEXT: s_setpc_b64 s[30:31]
+ %frexp = call {bfloat, i16} @llvm.frexp.bf16.i16(bfloat %x)
+ %exp = extractvalue {bfloat, i16} %frexp, 1
+ %is_nan = fcmp uno bfloat %x, 0.0
+ %result = select i1 %is_nan, i16 0, i16 %exp
+ ret i16 %result
+}
+
+; Test v2f32 vector case
+define <2 x i32> @frexp_nan_clamp_exp_v2f32(<2 x float> %x) {
+; GFX9-LABEL: frexp_nan_clamp_exp_v2f32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; GFX9-NEXT: v_frexp_exp_i32_f32_e32 v1, v1
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: frexp_nan_clamp_exp_v2f32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; GFX10-NEXT: v_frexp_exp_i32_f32_e32 v1, v1
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: frexp_nan_clamp_exp_v2f32:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; GFX11-NEXT: v_frexp_exp_i32_f32_e32 v1, v1
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-LABEL: frexp_nan_clamp_exp_v2f32:
+; GFX950: ; %bb.0:
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_frexp_exp_i32_f32_e32 v0, v0
+; GFX950-NEXT: v_frexp_exp_i32_f32_e32 v1, v1
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-LABEL: frexp_nan_clamp_exp_v2f32:
+; SI: ; %bb.0:
+; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s6, 0x7f800000
+; SI-NEXT: v_cmp_lt_f32_e64 s[4:5], |v1|, s6
+; SI-NEXT: v_cmp_lt_f32_e64 s[6:7], |v0|, s6
+; SI-NEXT: v_cmp_o_f32_e32 vcc, v0, v0
+; SI-NEXT: v_frexp_exp_i32_f32_e32 v3, v0
+; SI-NEXT: s_and_b64 vcc, vcc, s[6:7]
+; SI-NEXT: v_cndmask_b32_e32 v0, 0, v3, vcc
+; SI-NEXT: v_cmp_o_f32_e32 vcc, v1, v1
+; SI-NEXT: v_frexp_exp_i32_f32_e32 v2, v1
+; SI-NEXT: s_and_b64 vcc, vcc, s[4:5]
+; SI-NEXT: v_cndmask_b32_e32 v1, 0, v2, vcc
+; SI-NEXT: s_setpc_b64 s[30:31]
+ %frexp = call {<2 x float>, <2 x i32>} @llvm.frexp.v2f32.v2i32(<2 x float> %x)
+ %exp = extractvalue {<2 x float>, <2 x i32>} %frexp, 1
+ %is_nan = fcmp uno <2 x float> %x, zeroinitializer
+ %result = select <2 x i1> %is_nan, <2 x i32> zeroinitializer, <2 x i32> %exp
+ ret <2 x i32> %result
+}
>From ab71e80f4912104727b03eac3fea0fa2854fb6f8 Mon Sep 17 00:00:00 2001
From: addmisol <addmisol9 at gmail.com>
Date: Sat, 8 Aug 2026 15:54:29 +0530
Subject: [PATCH 4/4] Fix: clang formatting
Signed-off-by: addmisol <addmisol9 at gmail.com>
---
llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp | 475 +++++++++---------
llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h | 61 +--
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 36 +-
llvm/lib/Target/AMDGPU/SIISelLowering.h | 109 ++--
4 files changed, 326 insertions(+), 355 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
index cf074af1f78d7..c680b10fe4fc3 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -34,9 +34,8 @@ using namespace llvm;
#include "AMDGPUGenCallingConv.inc"
static cl::opt<bool> AMDGPUBypassSlowDiv(
- "amdgpu-bypass-slow-div",
- cl::desc("Skip 64-bit divide for dynamic 32-bit values"),
- cl::init(true));
+ "amdgpu-bypass-slow-div",
+ cl::desc("Skip 64-bit divide for dynamic 32-bit values"), cl::init(true));
// Find a larger type to do a load / store of a vector with.
EVT AMDGPUTargetLowering::getEquivalentMemType(LLVMContext &Ctx, EVT VT) {
@@ -479,7 +478,7 @@ AMDGPUTargetLowering::AMDGPUTargetLowering(const TargetMachine &TM,
Expand);
setOperationAction(ISD::FP_TO_FP16, {MVT::f64, MVT::f32}, Custom);
- const MVT ScalarIntVTs[] = { MVT::i32, MVT::i64 };
+ const MVT ScalarIntVTs[] = {MVT::i32, MVT::i64};
for (MVT VT : ScalarIntVTs) {
// These should use [SU]DIVREM, so set them to expand
setOperationAction({ISD::SDIV, ISD::UDIV, ISD::SREM, ISD::UREM}, VT,
@@ -522,8 +521,8 @@ AMDGPUTargetLowering::AMDGPUTargetLowering(const TargetMachine &TM,
setOperationAction({ISD::CTLZ, ISD::CTLZ_ZERO_POISON}, VT, Custom);
static const MVT::SimpleValueType VectorIntTypes[] = {
- MVT::v2i32, MVT::v3i32, MVT::v4i32, MVT::v5i32, MVT::v6i32, MVT::v7i32,
- MVT::v9i32, MVT::v10i32, MVT::v11i32, MVT::v12i32};
+ MVT::v2i32, MVT::v3i32, MVT::v4i32, MVT::v5i32, MVT::v6i32,
+ MVT::v7i32, MVT::v9i32, MVT::v10i32, MVT::v11i32, MVT::v12i32};
for (MVT VT : VectorIntTypes) {
// Expand the following operations for the current type by default.
@@ -552,8 +551,8 @@ AMDGPUTargetLowering::AMDGPUTargetLowering(const TargetMachine &TM,
}
static const MVT::SimpleValueType FloatVectorTypes[] = {
- MVT::v2f32, MVT::v3f32, MVT::v4f32, MVT::v5f32, MVT::v6f32, MVT::v7f32,
- MVT::v9f32, MVT::v10f32, MVT::v11f32, MVT::v12f32};
+ MVT::v2f32, MVT::v3f32, MVT::v4f32, MVT::v5f32, MVT::v6f32,
+ MVT::v7f32, MVT::v9f32, MVT::v10f32, MVT::v11f32, MVT::v12f32};
for (MVT VT : FloatVectorTypes) {
setOperationAction(
@@ -624,9 +623,9 @@ AMDGPUTargetLowering::AMDGPUTargetLowering(const TargetMachine &TM,
// memcpy/memmove/memset are expanded in the IR, so we shouldn't need to worry
// about these during lowering.
- MaxStoresPerMemcpy = 0xffffffff;
+ MaxStoresPerMemcpy = 0xffffffff;
MaxStoresPerMemmove = 0xffffffff;
- MaxStoresPerMemset = 0xffffffff;
+ MaxStoresPerMemset = 0xffffffff;
// The expansion for 64-bit division is enormous.
if (AMDGPUBypassSlowDiv)
@@ -872,9 +871,9 @@ bool AMDGPUTargetLowering::shouldReduceLoadWidth(
return (OldSize < 32);
}
-bool AMDGPUTargetLowering::isLoadBitCastBeneficial(EVT LoadTy, EVT CastTy,
- const SelectionDAG &DAG,
- const MachineMemOperand &MMO) const {
+bool AMDGPUTargetLowering::isLoadBitCastBeneficial(
+ EVT LoadTy, EVT CastTy, const SelectionDAG &DAG,
+ const MachineMemOperand &MMO) const {
assert(LoadTy.getSizeInBits() == CastTy.getSizeInBits());
@@ -978,13 +977,14 @@ bool AMDGPUTargetLowering::isFNegFree(EVT VT) const {
return VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f16 || VT == MVT::bf16;
}
-bool AMDGPUTargetLowering:: storeOfVectorConstantIsCheap(bool IsZero, EVT MemVT,
- unsigned NumElem,
- unsigned AS) const {
+bool AMDGPUTargetLowering::storeOfVectorConstantIsCheap(bool IsZero, EVT MemVT,
+ unsigned NumElem,
+ unsigned AS) const {
return true;
}
-bool AMDGPUTargetLowering::aggressivelyPreferBuildVectorSources(EVT VecVT) const {
+bool AMDGPUTargetLowering::aggressivelyPreferBuildVectorSources(
+ EVT VecVT) const {
// There are few operations which truly have vector input operands. Any vector
// operation is going to involve operations on each component, and a
// build_vector will be a copy per element, so it always makes sense to use a
@@ -1002,7 +1002,7 @@ bool AMDGPUTargetLowering::isTruncateFree(EVT Source, EVT Dest) const {
unsigned SrcSize = Source.getSizeInBits();
unsigned DestSize = Dest.getSizeInBits();
- return DestSize < SrcSize && DestSize % 32 == 0 ;
+ return DestSize < SrcSize && DestSize % 32 == 0;
}
bool AMDGPUTargetLowering::isTruncateFree(Type *Source, Type *Dest) const {
@@ -1011,7 +1011,7 @@ bool AMDGPUTargetLowering::isTruncateFree(Type *Source, Type *Dest) const {
unsigned SrcSize = Source->getScalarSizeInBits();
unsigned DestSize = Dest->getScalarSizeInBits();
- if (DestSize== 16 && Subtarget->has16BitInsts())
+ if (DestSize == 16 && Subtarget->has16BitInsts())
return SrcSize >= 32;
return DestSize < SrcSize && DestSize % 32 == 0;
@@ -1034,7 +1034,7 @@ bool AMDGPUTargetLowering::isZExtFree(EVT Src, EVT Dest) const {
// good.
if (Src == MVT::i16)
- return Dest == MVT::i32 ||Dest == MVT::i64 ;
+ return Dest == MVT::i32 || Dest == MVT::i64;
return Src == MVT::i32 && Dest == MVT::i64;
}
@@ -1088,7 +1088,7 @@ bool AMDGPUTargetLowering::isNarrowingProfitable(SDNode *N, EVT SrcVT,
}
bool AMDGPUTargetLowering::isDesirableToCommuteWithShift(
- const SDNode* N, CombineLevel Level) const {
+ const SDNode *N, CombineLevel Level) const {
assert((N->getOpcode() == ISD::SHL || N->getOpcode() == ISD::SRA ||
N->getOpcode() == ISD::SRL) &&
"Expected shift op");
@@ -1103,8 +1103,8 @@ bool AMDGPUTargetLowering::isDesirableToCommuteWithShift(
// Always commute pre-type legalization and right shifts.
// We're looking for shl(or(x,y),z) patterns.
- if (Level < CombineLevel::AfterLegalizeTypes ||
- N->getOpcode() != ISD::SHL || N->getOperand(0).getOpcode() != ISD::OR)
+ if (Level < CombineLevel::AfterLegalizeTypes || N->getOpcode() != ISD::SHL ||
+ N->getOperand(0).getOpcode() != ISD::OR)
return true;
// If only user is a i32 right-shift, then don't destroy a BFE pattern.
@@ -1216,8 +1216,7 @@ CCAssignFn *AMDGPUCallLowering::CCAssignFnForReturn(CallingConv::ID CC,
/// calling convention analysis function and the register type (Ins[x].VT) as
/// the ValVT.
void AMDGPUTargetLowering::analyzeFormalArgumentsCompute(
- CCState &State,
- const SmallVectorImpl<ISD::InputArg> &Ins) const {
+ CCState &State, const SmallVectorImpl<ISD::InputArg> &Ins) const {
const MachineFunction &MF = State.getMachineFunction();
const Function &Fn = MF.getFunction();
LLVMContext &Ctx = Fn.getContext();
@@ -1254,8 +1253,8 @@ void AMDGPUTargetLowering::analyzeFormalArgumentsCompute(
ComputeValueVTs(*this, DL, BaseArgTy, ValueVTs, /*MemVTs=*/nullptr,
&Offsets, ArgOffset);
- for (unsigned Value = 0, NumValues = ValueVTs.size();
- Value != NumValues; ++Value) {
+ for (unsigned Value = 0, NumValues = ValueVTs.size(); Value != NumValues;
+ ++Value) {
uint64_t BasePartOffset = Offsets[Value];
EVT ArgVT = ValueVTs[Value];
@@ -1274,13 +1273,13 @@ void AMDGPUTargetLowering::analyzeFormalArgumentsCompute(
}
} else if (ArgVT.isVector() && RegisterVT.isVector() &&
ArgVT.getScalarType() == RegisterVT.getScalarType()) {
- assert(ArgVT.getVectorNumElements() > RegisterVT.getVectorNumElements());
+ assert(ArgVT.getVectorNumElements() >
+ RegisterVT.getVectorNumElements());
// We have a vector value which has been split into a vector with
// the same scalar type, but fewer elements. This should handle
// all the floating-point vector types.
MemVT = RegisterVT;
- } else if (ArgVT.isVector() &&
- ArgVT.getVectorNumElements() == NumRegs) {
+ } else if (ArgVT.isVector() && ArgVT.getVectorNumElements() == NumRegs) {
// This arg has been split so that each element is stored in a separate
// register.
MemVT = ArgVT.getScalarType();
@@ -1298,8 +1297,8 @@ void AMDGPUTargetLowering::analyzeFormalArgumentsCompute(
assert(MemoryBits % NumElements == 0);
// This vector type has been split into another vector type with
// a different elements size.
- EVT ScalarVT = EVT::getIntegerVT(State.getContext(),
- MemoryBits / NumElements);
+ EVT ScalarVT =
+ EVT::getIntegerVT(State.getContext(), MemoryBits / NumElements);
MemVT = EVT::getVectorVT(State.getContext(), ScalarVT, NumElements);
} else {
llvm_unreachable("cannot deduce memory type.");
@@ -1319,24 +1318,23 @@ void AMDGPUTargetLowering::analyzeFormalArgumentsCompute(
unsigned PartOffset = 0;
for (unsigned i = 0; i != NumRegs; ++i) {
- State.addLoc(CCValAssign::getCustomMem(InIndex++, RegisterVT,
- BasePartOffset + PartOffset,
- MemVT.getSimpleVT(),
- CCValAssign::Full));
+ State.addLoc(CCValAssign::getCustomMem(
+ InIndex++, RegisterVT, BasePartOffset + PartOffset,
+ MemVT.getSimpleVT(), CCValAssign::Full));
PartOffset += MemVT.getStoreSize();
}
}
}
}
-SDValue AMDGPUTargetLowering::LowerReturn(
- SDValue Chain, CallingConv::ID CallConv,
- bool isVarArg,
- const SmallVectorImpl<ISD::OutputArg> &Outs,
- const SmallVectorImpl<SDValue> &OutVals,
- const SDLoc &DL, SelectionDAG &DAG) const {
+SDValue
+AMDGPUTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
+ bool isVarArg,
+ const SmallVectorImpl<ISD::OutputArg> &Outs,
+ const SmallVectorImpl<SDValue> &OutVals,
+ const SDLoc &DL, SelectionDAG &DAG) const {
// FIXME: Fails for r600 tests
- //assert(!isVarArg && Outs.empty() && OutVals.empty() &&
+ // assert(!isVarArg && Outs.empty() && OutVals.empty() &&
// "wave terminate should not have return values");
return DAG.getNode(AMDGPUISD::ENDPGM, DL, MVT::Other, Chain);
}
@@ -1390,9 +1388,10 @@ SDValue AMDGPUTargetLowering::addTokenForArgument(SDValue Chain,
return DAG.getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ArgChains);
}
-SDValue AMDGPUTargetLowering::lowerUnhandledCall(CallLoweringInfo &CLI,
- SmallVectorImpl<SDValue> &InVals,
- StringRef Reason) const {
+SDValue
+AMDGPUTargetLowering::lowerUnhandledCall(CallLoweringInfo &CLI,
+ SmallVectorImpl<SDValue> &InVals,
+ StringRef Reason) const {
SDValue Callee = CLI.Callee;
SelectionDAG &DAG = CLI.DAG;
@@ -1421,8 +1420,9 @@ SDValue AMDGPUTargetLowering::lowerUnhandledCall(CallLoweringInfo &CLI,
return DAG.getCALLSEQ_END(Chain, 0, 0, /*InGlue=*/SDValue(), CLI.DL);
}
-SDValue AMDGPUTargetLowering::LowerCall(CallLoweringInfo &CLI,
- SmallVectorImpl<SDValue> &InVals) const {
+SDValue
+AMDGPUTargetLowering::LowerCall(CallLoweringInfo &CLI,
+ SmallVectorImpl<SDValue> &InVals) const {
return lowerUnhandledCall(CLI, InVals, "unsupported call to function ");
}
@@ -1444,20 +1444,30 @@ SDValue AMDGPUTargetLowering::LowerOperation(SDValue Op,
llvm_unreachable("Custom lowering code for this "
"instruction is not implemented yet!");
break;
- case ISD::SIGN_EXTEND_INREG: return LowerSIGN_EXTEND_INREG(Op, DAG);
- case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, DAG);
- case ISD::EXTRACT_SUBVECTOR: return LowerEXTRACT_SUBVECTOR(Op, DAG);
- case ISD::UDIVREM: return LowerUDIVREM(Op, DAG);
+ case ISD::SIGN_EXTEND_INREG:
+ return LowerSIGN_EXTEND_INREG(Op, DAG);
+ case ISD::CONCAT_VECTORS:
+ return LowerCONCAT_VECTORS(Op, DAG);
+ case ISD::EXTRACT_SUBVECTOR:
+ return LowerEXTRACT_SUBVECTOR(Op, DAG);
+ case ISD::UDIVREM:
+ return LowerUDIVREM(Op, DAG);
case ISD::SDIVREM:
return LowerSDIVREM(Op, DAG);
- case ISD::FCEIL: return LowerFCEIL(Op, DAG);
- case ISD::FTRUNC: return LowerFTRUNC(Op, DAG);
- case ISD::FRINT: return LowerFRINT(Op, DAG);
- case ISD::FNEARBYINT: return LowerFNEARBYINT(Op, DAG);
+ case ISD::FCEIL:
+ return LowerFCEIL(Op, DAG);
+ case ISD::FTRUNC:
+ return LowerFTRUNC(Op, DAG);
+ case ISD::FRINT:
+ return LowerFRINT(Op, DAG);
+ case ISD::FNEARBYINT:
+ return LowerFNEARBYINT(Op, DAG);
case ISD::FROUNDEVEN:
return LowerFROUNDEVEN(Op, DAG);
- case ISD::FROUND: return LowerFROUND(Op, DAG);
- case ISD::FFLOOR: return LowerFFLOOR(Op, DAG);
+ case ISD::FROUND:
+ return LowerFROUND(Op, DAG);
+ case ISD::FFLOOR:
+ return LowerFFLOOR(Op, DAG);
case ISD::FLOG2:
return LowerFLOG2(Op, DAG);
case ISD::FLOG:
@@ -1468,9 +1478,12 @@ SDValue AMDGPUTargetLowering::LowerOperation(SDValue Op,
return lowerFEXP(Op, DAG);
case ISD::FEXP2:
return lowerFEXP2(Op, DAG);
- case ISD::SINT_TO_FP: return LowerSINT_TO_FP(Op, DAG);
- case ISD::UINT_TO_FP: return LowerUINT_TO_FP(Op, DAG);
- case ISD::FP_TO_FP16: return LowerFP_TO_FP16(Op, DAG);
+ case ISD::SINT_TO_FP:
+ return LowerSINT_TO_FP(Op, DAG);
+ case ISD::UINT_TO_FP:
+ return LowerUINT_TO_FP(Op, DAG);
+ case ISD::FP_TO_FP16:
+ return LowerFP_TO_FP16(Op, DAG);
case ISD::FP_TO_SINT:
case ISD::FP_TO_UINT:
return LowerFP_TO_INT(Op, DAG);
@@ -1484,7 +1497,8 @@ SDValue AMDGPUTargetLowering::LowerOperation(SDValue Op,
return LowerCTLZ_CTTZ(Op, DAG);
case ISD::CTLS:
return LowerCTLS(Op, DAG);
- case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG);
+ case ISD::DYNAMIC_STACKALLOC:
+ return LowerDYNAMIC_STACKALLOC(Op, DAG);
}
return Op;
}
@@ -1581,8 +1595,8 @@ SDValue AMDGPUTargetLowering::LowerGlobalAddress(AMDGPUMachineFunctionInfo *MFI,
// not eliminated, we don't want a compile time error. Just emit a warning
// and a trap, since there should be no callable path here.
SDValue Trap = DAG.getNode(ISD::TRAP, DL, MVT::Other, DAG.getEntryNode());
- SDValue OutputChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
- Trap, DAG.getRoot());
+ SDValue OutputChain =
+ DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Trap, DAG.getRoot());
DAG.setRoot(OutputChain);
return DAG.getPOISON(Op.getValueType());
}
@@ -1875,7 +1889,6 @@ SDValue AMDGPUTargetLowering::SplitVectorLoad(const SDValue Op,
EVT VT = Op.getValueType();
SDLoc SL(Op);
-
// If this is a 2 element vector, we really want to scalarize and not create
// weird 1 element vectors.
if (VT.getVectorNumElements() == 2) {
@@ -2075,8 +2088,8 @@ SDValue AMDGPUTargetLowering::LowerDIVREMToFloat(SDValue Op, SelectionDAG &DAG,
// float fb = (float)ib;
SDValue fb = DAG.getNode(ToFp, DL, FltVT, ib);
- SDValue fq = DAG.getNode(ISD::FMUL, DL, FltVT,
- fa, DAG.getNode(AMDGPUISD::RCP, DL, FltVT, fb));
+ SDValue fq = DAG.getNode(ISD::FMUL, DL, FltVT, fa,
+ DAG.getNode(AMDGPUISD::RCP, DL, FltVT, fb));
// fq = trunc(fq);
fq = DAG.getNode(ISD::FTRUNC, DL, FltVT, fq);
@@ -2088,12 +2101,11 @@ SDValue AMDGPUTargetLowering::LowerDIVREMToFloat(SDValue Op, SelectionDAG &DAG,
SDValue Rem = DAG.getNode(ISD::MUL, DL, VT, Div, RHS);
Rem = DAG.getNode(ISD::SUB, DL, VT, LHS, Rem);
- return DAG.getMergeValues({ Div, Rem }, DL);
+ return DAG.getMergeValues({Div, Rem}, DL);
}
-void AMDGPUTargetLowering::LowerUDIVREM64(SDValue Op,
- SelectionDAG &DAG,
- SmallVectorImpl<SDValue> &Results) const {
+void AMDGPUTargetLowering::LowerUDIVREM64(
+ SDValue Op, SelectionDAG &DAG, SmallVectorImpl<SDValue> &Results) const {
SDLoc DL(Op);
EVT VT = Op.getValueType();
@@ -2104,7 +2116,7 @@ void AMDGPUTargetLowering::LowerUDIVREM64(SDValue Op,
SDValue One = DAG.getConstant(1, DL, HalfVT);
SDValue Zero = DAG.getConstant(0, DL, HalfVT);
- //HiLo split
+ // HiLo split
SDValue LHS_Lo, LHS_Hi;
SDValue LHS = Op.getOperand(0);
std::tie(LHS_Lo, LHS_Hi) = DAG.SplitScalar(LHS, DL, HalfVT, HalfVT);
@@ -2143,25 +2155,29 @@ void AMDGPUTargetLowering::LowerUDIVREM64(SDValue Op,
SDValue Cvt_Lo = DAG.getNode(ISD::UINT_TO_FP, DL, MVT::f32, RHS_Lo);
SDValue Cvt_Hi = DAG.getNode(ISD::UINT_TO_FP, DL, MVT::f32, RHS_Hi);
- SDValue Mad1 = DAG.getNode(FMAD, DL, MVT::f32, Cvt_Hi,
- DAG.getConstantFP(APInt(32, 0x4f800000).bitsToFloat(), DL, MVT::f32),
- Cvt_Lo);
+ SDValue Mad1 = DAG.getNode(
+ FMAD, DL, MVT::f32, Cvt_Hi,
+ DAG.getConstantFP(APInt(32, 0x4f800000).bitsToFloat(), DL, MVT::f32),
+ Cvt_Lo);
SDValue Rcp = DAG.getNode(AMDGPUISD::RCP, DL, MVT::f32, Mad1);
- SDValue Mul1 = DAG.getNode(ISD::FMUL, DL, MVT::f32, Rcp,
- DAG.getConstantFP(APInt(32, 0x5f7ffffc).bitsToFloat(), DL, MVT::f32));
- SDValue Mul2 = DAG.getNode(ISD::FMUL, DL, MVT::f32, Mul1,
- DAG.getConstantFP(APInt(32, 0x2f800000).bitsToFloat(), DL, MVT::f32));
+ SDValue Mul1 = DAG.getNode(
+ ISD::FMUL, DL, MVT::f32, Rcp,
+ DAG.getConstantFP(APInt(32, 0x5f7ffffc).bitsToFloat(), DL, MVT::f32));
+ SDValue Mul2 = DAG.getNode(
+ ISD::FMUL, DL, MVT::f32, Mul1,
+ DAG.getConstantFP(APInt(32, 0x2f800000).bitsToFloat(), DL, MVT::f32));
SDValue Trunc = DAG.getNode(ISD::FTRUNC, DL, MVT::f32, Mul2);
- SDValue Mad2 = DAG.getNode(FMAD, DL, MVT::f32, Trunc,
- DAG.getConstantFP(APInt(32, 0xcf800000).bitsToFloat(), DL, MVT::f32),
- Mul1);
+ SDValue Mad2 = DAG.getNode(
+ FMAD, DL, MVT::f32, Trunc,
+ DAG.getConstantFP(APInt(32, 0xcf800000).bitsToFloat(), DL, MVT::f32),
+ Mul1);
SDValue Rcp_Lo = DAG.getNode(ISD::FP_TO_UINT, DL, HalfVT, Mad2);
SDValue Rcp_Hi = DAG.getNode(ISD::FP_TO_UINT, DL, HalfVT, Trunc);
- SDValue Rcp64 = DAG.getBitcast(VT,
- DAG.getBuildVector(MVT::v2i32, DL, {Rcp_Lo, Rcp_Hi}));
+ SDValue Rcp64 = DAG.getBitcast(
+ VT, DAG.getBuildVector(MVT::v2i32, DL, {Rcp_Lo, Rcp_Hi}));
SDValue Zero64 = DAG.getConstant(0, DL, VT);
- SDValue One64 = DAG.getConstant(1, DL, VT);
+ SDValue One64 = DAG.getConstant(1, DL, VT);
SDValue Zero1 = DAG.getConstant(0, DL, MVT::i1);
SDVTList HalfCarryVT = DAG.getVTList(HalfVT, MVT::i1);
@@ -2176,8 +2192,8 @@ void AMDGPUTargetLowering::LowerUDIVREM64(SDValue Op,
Mulhi1_Lo, Zero1);
SDValue Add1_Hi = DAG.getNode(ISD::UADDO_CARRY, DL, HalfCarryVT, Rcp_Hi,
Mulhi1_Hi, Add1_Lo.getValue(1));
- SDValue Add1 = DAG.getBitcast(VT,
- DAG.getBuildVector(MVT::v2i32, DL, {Add1_Lo, Add1_Hi}));
+ SDValue Add1 = DAG.getBitcast(
+ VT, DAG.getBuildVector(MVT::v2i32, DL, {Add1_Lo, Add1_Hi}));
// Second round of UNR.
SDValue Mullo2 = DAG.getNode(ISD::MUL, DL, VT, Neg_RHS, Add1);
@@ -2189,8 +2205,8 @@ void AMDGPUTargetLowering::LowerUDIVREM64(SDValue Op,
Mulhi2_Lo, Zero1);
SDValue Add2_Hi = DAG.getNode(ISD::UADDO_CARRY, DL, HalfCarryVT, Add1_Hi,
Mulhi2_Hi, Add2_Lo.getValue(1));
- SDValue Add2 = DAG.getBitcast(VT,
- DAG.getBuildVector(MVT::v2i32, DL, {Add2_Lo, Add2_Hi}));
+ SDValue Add2 = DAG.getBitcast(
+ VT, DAG.getBuildVector(MVT::v2i32, DL, {Add2_Lo, Add2_Hi}));
SDValue Mulhi3 = DAG.getNode(ISD::MULHU, DL, VT, LHS, Add2);
@@ -2198,19 +2214,19 @@ void AMDGPUTargetLowering::LowerUDIVREM64(SDValue Op,
SDValue Mul3_Lo, Mul3_Hi;
std::tie(Mul3_Lo, Mul3_Hi) = DAG.SplitScalar(Mul3, DL, HalfVT, HalfVT);
- SDValue Sub1_Lo = DAG.getNode(ISD::USUBO_CARRY, DL, HalfCarryVT, LHS_Lo,
- Mul3_Lo, Zero1);
+ SDValue Sub1_Lo =
+ DAG.getNode(ISD::USUBO_CARRY, DL, HalfCarryVT, LHS_Lo, Mul3_Lo, Zero1);
SDValue Sub1_Hi = DAG.getNode(ISD::USUBO_CARRY, DL, HalfCarryVT, LHS_Hi,
Mul3_Hi, Sub1_Lo.getValue(1));
SDValue Sub1_Mi = DAG.getNode(ISD::SUB, DL, HalfVT, LHS_Hi, Mul3_Hi);
- SDValue Sub1 = DAG.getBitcast(VT,
- DAG.getBuildVector(MVT::v2i32, DL, {Sub1_Lo, Sub1_Hi}));
+ SDValue Sub1 = DAG.getBitcast(
+ VT, DAG.getBuildVector(MVT::v2i32, DL, {Sub1_Lo, Sub1_Hi}));
SDValue MinusOne = DAG.getConstant(0xffffffffu, DL, HalfVT);
- SDValue C1 = DAG.getSelectCC(DL, Sub1_Hi, RHS_Hi, MinusOne, Zero,
- ISD::SETUGE);
- SDValue C2 = DAG.getSelectCC(DL, Sub1_Lo, RHS_Lo, MinusOne, Zero,
- ISD::SETUGE);
+ SDValue C1 =
+ DAG.getSelectCC(DL, Sub1_Hi, RHS_Hi, MinusOne, Zero, ISD::SETUGE);
+ SDValue C2 =
+ DAG.getSelectCC(DL, Sub1_Lo, RHS_Lo, MinusOne, Zero, ISD::SETUGE);
SDValue C3 = DAG.getSelectCC(DL, Sub1_Hi, RHS_Hi, C2, C1, ISD::SETEQ);
// TODO: Here and below portions of the code can be enclosed into if/endif.
@@ -2218,43 +2234,43 @@ void AMDGPUTargetLowering::LowerUDIVREM64(SDValue Op,
// potential endif to substitute PHIs.
// if C3 != 0 ...
- SDValue Sub2_Lo = DAG.getNode(ISD::USUBO_CARRY, DL, HalfCarryVT, Sub1_Lo,
- RHS_Lo, Zero1);
+ SDValue Sub2_Lo =
+ DAG.getNode(ISD::USUBO_CARRY, DL, HalfCarryVT, Sub1_Lo, RHS_Lo, Zero1);
SDValue Sub2_Mi = DAG.getNode(ISD::USUBO_CARRY, DL, HalfCarryVT, Sub1_Mi,
RHS_Hi, Sub1_Lo.getValue(1));
SDValue Sub2_Hi = DAG.getNode(ISD::USUBO_CARRY, DL, HalfCarryVT, Sub2_Mi,
Zero, Sub2_Lo.getValue(1));
- SDValue Sub2 = DAG.getBitcast(VT,
- DAG.getBuildVector(MVT::v2i32, DL, {Sub2_Lo, Sub2_Hi}));
+ SDValue Sub2 = DAG.getBitcast(
+ VT, DAG.getBuildVector(MVT::v2i32, DL, {Sub2_Lo, Sub2_Hi}));
SDValue Add3 = DAG.getNode(ISD::ADD, DL, VT, Mulhi3, One64);
- SDValue C4 = DAG.getSelectCC(DL, Sub2_Hi, RHS_Hi, MinusOne, Zero,
- ISD::SETUGE);
- SDValue C5 = DAG.getSelectCC(DL, Sub2_Lo, RHS_Lo, MinusOne, Zero,
- ISD::SETUGE);
+ SDValue C4 =
+ DAG.getSelectCC(DL, Sub2_Hi, RHS_Hi, MinusOne, Zero, ISD::SETUGE);
+ SDValue C5 =
+ DAG.getSelectCC(DL, Sub2_Lo, RHS_Lo, MinusOne, Zero, ISD::SETUGE);
SDValue C6 = DAG.getSelectCC(DL, Sub2_Hi, RHS_Hi, C5, C4, ISD::SETEQ);
// if (C6 != 0)
SDValue Add4 = DAG.getNode(ISD::ADD, DL, VT, Add3, One64);
- SDValue Sub3_Lo = DAG.getNode(ISD::USUBO_CARRY, DL, HalfCarryVT, Sub2_Lo,
- RHS_Lo, Zero1);
+ SDValue Sub3_Lo =
+ DAG.getNode(ISD::USUBO_CARRY, DL, HalfCarryVT, Sub2_Lo, RHS_Lo, Zero1);
SDValue Sub3_Mi = DAG.getNode(ISD::USUBO_CARRY, DL, HalfCarryVT, Sub2_Mi,
RHS_Hi, Sub2_Lo.getValue(1));
SDValue Sub3_Hi = DAG.getNode(ISD::USUBO_CARRY, DL, HalfCarryVT, Sub3_Mi,
Zero, Sub3_Lo.getValue(1));
- SDValue Sub3 = DAG.getBitcast(VT,
- DAG.getBuildVector(MVT::v2i32, DL, {Sub3_Lo, Sub3_Hi}));
+ SDValue Sub3 = DAG.getBitcast(
+ VT, DAG.getBuildVector(MVT::v2i32, DL, {Sub3_Lo, Sub3_Hi}));
// endif C6
// endif C3
SDValue Sel1 = DAG.getSelectCC(DL, C6, Zero, Add4, Add3, ISD::SETNE);
- SDValue Div = DAG.getSelectCC(DL, C3, Zero, Sel1, Mulhi3, ISD::SETNE);
+ SDValue Div = DAG.getSelectCC(DL, C3, Zero, Sel1, Mulhi3, ISD::SETNE);
SDValue Sel2 = DAG.getSelectCC(DL, C6, Zero, Sub3, Sub2, ISD::SETNE);
- SDValue Rem = DAG.getSelectCC(DL, C3, Zero, Sel2, Sub1, ISD::SETNE);
+ SDValue Rem = DAG.getSelectCC(DL, C3, Zero, Sel2, Sub1, ISD::SETNE);
Results.push_back(Div);
Results.push_back(Rem);
@@ -2267,11 +2283,13 @@ void AMDGPUTargetLowering::LowerUDIVREM64(SDValue Op,
SDValue DIV_Part = DAG.getNode(ISD::UDIV, DL, HalfVT, LHS_Hi, RHS_Lo);
SDValue REM_Part = DAG.getNode(ISD::UREM, DL, HalfVT, LHS_Hi, RHS_Lo);
- SDValue REM_Lo = DAG.getSelectCC(DL, RHS_Hi, Zero, REM_Part, LHS_Hi, ISD::SETEQ);
+ SDValue REM_Lo =
+ DAG.getSelectCC(DL, RHS_Hi, Zero, REM_Part, LHS_Hi, ISD::SETEQ);
SDValue REM = DAG.getBuildVector(MVT::v2i32, DL, {REM_Lo, Zero});
REM = DAG.getNode(ISD::BITCAST, DL, MVT::i64, REM);
- SDValue DIV_Hi = DAG.getSelectCC(DL, RHS_Hi, Zero, DIV_Part, Zero, ISD::SETEQ);
+ SDValue DIV_Hi =
+ DAG.getSelectCC(DL, RHS_Hi, Zero, DIV_Part, Zero, ISD::SETEQ);
SDValue DIV_Lo = Zero;
const unsigned halfBitWidth = HalfVT.getSizeInBits();
@@ -2384,15 +2402,14 @@ SDValue AMDGPUTargetLowering::LowerSDIVREM(SDValue Op,
DAG.ComputeNumSignBits(RHS) > 32) {
EVT HalfVT = VT.getHalfSizedIntegerVT(*DAG.getContext());
- //HiLo split
+ // HiLo split
SDValue LHS_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, LHS, Zero);
SDValue RHS_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, RHS, Zero);
- SDValue DIVREM = DAG.getNode(ISD::SDIVREM, DL, DAG.getVTList(HalfVT, HalfVT),
- LHS_Lo, RHS_Lo);
+ SDValue DIVREM = DAG.getNode(ISD::SDIVREM, DL,
+ DAG.getVTList(HalfVT, HalfVT), LHS_Lo, RHS_Lo);
SDValue Res[2] = {
- DAG.getNode(ISD::SIGN_EXTEND, DL, VT, DIVREM.getValue(0)),
- DAG.getNode(ISD::SIGN_EXTEND, DL, VT, DIVREM.getValue(1))
- };
+ DAG.getNode(ISD::SIGN_EXTEND, DL, VT, DIVREM.getValue(0)),
+ DAG.getNode(ISD::SIGN_EXTEND, DL, VT, DIVREM.getValue(1))};
return DAG.getMergeValues(Res, DL);
}
@@ -2416,10 +2433,7 @@ SDValue AMDGPUTargetLowering::LowerSDIVREM(SDValue Op,
Div = DAG.getNode(ISD::SUB, DL, VT, Div, DSign);
Rem = DAG.getNode(ISD::SUB, DL, VT, Rem, RSign);
- SDValue Res[2] = {
- Div,
- Rem
- };
+ SDValue Res[2] = {Div, Rem};
return DAG.getMergeValues(Res, DL);
}
@@ -2453,8 +2467,7 @@ static SDValue extractF64Exponent(SDValue Hi, const SDLoc &SL,
const unsigned FractBits = 52;
const unsigned ExpBits = 11;
- SDValue ExpPart = DAG.getNode(AMDGPUISD::BFE_U32, SL, MVT::i32,
- Hi,
+ SDValue ExpPart = DAG.getNode(AMDGPUISD::BFE_U32, SL, MVT::i32, Hi,
DAG.getConstant(FractBits - 32, SL, MVT::i32),
DAG.getConstant(ExpBits, SL, MVT::i32));
SDValue Exp = DAG.getNode(ISD::SUB, SL, MVT::i32, ExpPart,
@@ -2488,8 +2501,8 @@ SDValue AMDGPUTargetLowering::LowerFTRUNC(SDValue Op, SelectionDAG &DAG) const {
SignBit64 = DAG.getNode(ISD::BITCAST, SL, MVT::i64, SignBit64);
SDValue BcInt = DAG.getNode(ISD::BITCAST, SL, MVT::i64, Src);
- const SDValue FractMask
- = DAG.getConstant((UINT64_C(1) << FractBits) - 1, SL, MVT::i64);
+ const SDValue FractMask =
+ DAG.getConstant((UINT64_C(1) << FractBits) - 1, SL, MVT::i64);
SDValue Shr = DAG.getNode(ISD::SRA, SL, MVT::i64, FractMask, Exp);
SDValue Not = DAG.getNOT(SL, Shr, MVT::i64);
@@ -2503,7 +2516,8 @@ SDValue AMDGPUTargetLowering::LowerFTRUNC(SDValue Op, SelectionDAG &DAG) const {
SDValue ExpLt0 = DAG.getSetCC(SL, SetCCVT, Exp, Zero, ISD::SETLT);
SDValue ExpGt51 = DAG.getSetCC(SL, SetCCVT, Exp, FiftyOne, ISD::SETGT);
- SDValue Tmp1 = DAG.getNode(ISD::SELECT, SL, MVT::i64, ExpLt0, SignBit64, Tmp0);
+ SDValue Tmp1 =
+ DAG.getNode(ISD::SELECT, SL, MVT::i64, ExpLt0, SignBit64, Tmp0);
SDValue Tmp2 = DAG.getNode(ISD::SELECT, SL, MVT::i64, ExpGt51, BcInt, Tmp1);
return DAG.getNode(ISD::BITCAST, SL, MVT::f64, Tmp2);
@@ -2573,8 +2587,7 @@ SDValue AMDGPUTargetLowering::LowerFROUND(SDValue Op, SelectionDAG &DAG) const {
const SDValue Zero = DAG.getConstantFP(0.0, SL, VT);
const SDValue One = DAG.getConstantFP(1.0, SL, VT);
- EVT SetCCVT =
- getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
+ EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
const SDValue Half = DAG.getConstantFP(0.5, SL, VT);
SDValue Cmp = DAG.getSetCC(SL, SetCCVT, AbsDiff, Half, ISD::SETOGE);
@@ -3343,7 +3356,8 @@ SDValue AMDGPUTargetLowering::lowerCTLZResults(SDValue Op,
return DAG.getNode(ISD::TRUNCATE, SL, ResultVT, NewOp);
}
-SDValue AMDGPUTargetLowering::LowerCTLZ_CTTZ(SDValue Op, SelectionDAG &DAG) const {
+SDValue AMDGPUTargetLowering::LowerCTLZ_CTTZ(SDValue Op,
+ SelectionDAG &DAG) const {
SDLoc SL(Op);
SDValue Src = Op.getOperand(0);
@@ -3563,8 +3577,8 @@ SDValue AMDGPUTargetLowering::LowerINT_TO_FP64(SDValue Op, SelectionDAG &DAG,
SDValue Lo, Hi;
std::tie(Lo, Hi) = split64BitValue(Src, DAG);
- SDValue CvtHi = DAG.getNode(Signed ? ISD::SINT_TO_FP : ISD::UINT_TO_FP,
- SL, MVT::f64, Hi);
+ SDValue CvtHi =
+ DAG.getNode(Signed ? ISD::SINT_TO_FP : ISD::UINT_TO_FP, SL, MVT::f64, Hi);
SDValue CvtLo = DAG.getNode(ISD::UINT_TO_FP, SL, MVT::f64, Lo);
@@ -3575,7 +3589,7 @@ SDValue AMDGPUTargetLowering::LowerINT_TO_FP64(SDValue Op, SelectionDAG &DAG,
}
SDValue AMDGPUTargetLowering::LowerUINT_TO_FP(SDValue Op,
- SelectionDAG &DAG) const {
+ SelectionDAG &DAG) const {
// TODO: Factor out code common with LowerSINT_TO_FP.
EVT DestVT = Op.getValueType();
SDValue Src = Op.getOperand(0);
@@ -3711,7 +3725,8 @@ SDValue AMDGPUTargetLowering::LowerFP_TO_INT64(SDValue Op, SelectionDAG &DAG,
return Result;
}
-SDValue AMDGPUTargetLowering::LowerFP_TO_FP16(SDValue Op, SelectionDAG &DAG) const {
+SDValue AMDGPUTargetLowering::LowerFP_TO_FP16(SDValue Op,
+ SelectionDAG &DAG) const {
SDLoc DL(Op);
SDValue N0 = Op.getOperand(0);
@@ -3740,8 +3755,8 @@ SDValue AMDGPUTargetLowering::LowerF64ToF16Safe(SDValue Src, const SDLoc &DL,
SDValue Zero = DAG.getConstant(0, DL, MVT::i32);
SDValue One = DAG.getConstant(1, DL, MVT::i32);
SDValue U = DAG.getNode(ISD::BITCAST, DL, MVT::i64, Src);
- SDValue UH = DAG.getNode(ISD::SRL, DL, MVT::i64, U,
- DAG.getConstant(32, DL, MVT::i64));
+ SDValue UH =
+ DAG.getNode(ISD::SRL, DL, MVT::i64, U, DAG.getConstant(32, DL, MVT::i64));
UH = DAG.getZExtOrTrunc(UH, DL, MVT::i32);
U = DAG.getZExtOrTrunc(U, DL, MVT::i32);
SDValue E = DAG.getNode(ISD::SRL, DL, MVT::i32, UH,
@@ -3753,8 +3768,8 @@ SDValue AMDGPUTargetLowering::LowerF64ToF16Safe(SDValue Src, const SDLoc &DL,
E = DAG.getNode(ISD::ADD, DL, MVT::i32, E,
DAG.getConstant(-ExpBiasf64 + ExpBiasf16, DL, MVT::i32));
- SDValue M = DAG.getNode(ISD::SRL, DL, MVT::i32, UH,
- DAG.getConstant(8, DL, MVT::i32));
+ SDValue M =
+ DAG.getNode(ISD::SRL, DL, MVT::i32, UH, DAG.getConstant(8, DL, MVT::i32));
M = DAG.getNode(ISD::AND, DL, MVT::i32, M,
DAG.getConstant(0xffe, DL, MVT::i32));
@@ -3767,17 +3782,18 @@ SDValue AMDGPUTargetLowering::LowerF64ToF16Safe(SDValue Src, const SDLoc &DL,
// (M != 0 ? 0x0200 : 0) | 0x7c00;
SDValue I = DAG.getNode(ISD::OR, DL, MVT::i32,
- DAG.getSelectCC(DL, M, Zero, DAG.getConstant(0x0200, DL, MVT::i32),
- Zero, ISD::SETNE), DAG.getConstant(0x7c00, DL, MVT::i32));
+ DAG.getSelectCC(DL, M, Zero,
+ DAG.getConstant(0x0200, DL, MVT::i32),
+ Zero, ISD::SETNE),
+ DAG.getConstant(0x7c00, DL, MVT::i32));
// N = M | (E << 12);
SDValue N = DAG.getNode(ISD::OR, DL, MVT::i32, M,
- DAG.getNode(ISD::SHL, DL, MVT::i32, E,
- DAG.getConstant(12, DL, MVT::i32)));
+ DAG.getNode(ISD::SHL, DL, MVT::i32, E,
+ DAG.getConstant(12, DL, MVT::i32)));
// B = clamp(1-E, 0, 13);
- SDValue OneSubExp = DAG.getNode(ISD::SUB, DL, MVT::i32,
- One, E);
+ SDValue OneSubExp = DAG.getNode(ISD::SUB, DL, MVT::i32, One, E);
SDValue B = DAG.getNode(ISD::SMAX, DL, MVT::i32, OneSubExp, Zero);
B = DAG.getNode(ISD::SMIN, DL, MVT::i32, B,
DAG.getConstant(13, DL, MVT::i32));
@@ -3793,23 +3809,22 @@ SDValue AMDGPUTargetLowering::LowerF64ToF16Safe(SDValue Src, const SDLoc &DL,
SDValue V = DAG.getSelectCC(DL, E, One, D, N, ISD::SETLT);
SDValue VLow3 = DAG.getNode(ISD::AND, DL, MVT::i32, V,
DAG.getConstant(0x7, DL, MVT::i32));
- V = DAG.getNode(ISD::SRL, DL, MVT::i32, V,
- DAG.getConstant(2, DL, MVT::i32));
- SDValue V0 = DAG.getSelectCC(DL, VLow3, DAG.getConstant(3, DL, MVT::i32),
- One, Zero, ISD::SETEQ);
- SDValue V1 = DAG.getSelectCC(DL, VLow3, DAG.getConstant(5, DL, MVT::i32),
- One, Zero, ISD::SETGT);
+ V = DAG.getNode(ISD::SRL, DL, MVT::i32, V, DAG.getConstant(2, DL, MVT::i32));
+ SDValue V0 = DAG.getSelectCC(DL, VLow3, DAG.getConstant(3, DL, MVT::i32), One,
+ Zero, ISD::SETEQ);
+ SDValue V1 = DAG.getSelectCC(DL, VLow3, DAG.getConstant(5, DL, MVT::i32), One,
+ Zero, ISD::SETGT);
V1 = DAG.getNode(ISD::OR, DL, MVT::i32, V0, V1);
V = DAG.getNode(ISD::ADD, DL, MVT::i32, V, V1);
V = DAG.getSelectCC(DL, E, DAG.getConstant(30, DL, MVT::i32),
DAG.getConstant(0x7c00, DL, MVT::i32), V, ISD::SETGT);
- V = DAG.getSelectCC(DL, E, DAG.getConstant(1039, DL, MVT::i32),
- I, V, ISD::SETEQ);
+ V = DAG.getSelectCC(DL, E, DAG.getConstant(1039, DL, MVT::i32), I, V,
+ ISD::SETEQ);
// Extract the sign bit.
SDValue Sign = DAG.getNode(ISD::SRL, DL, MVT::i32, UH,
- DAG.getConstant(16, DL, MVT::i32));
+ DAG.getConstant(16, DL, MVT::i32));
Sign = DAG.getNode(ISD::AND, DL, MVT::i32, Sign,
DAG.getConstant(0x8000, DL, MVT::i32));
@@ -4136,9 +4151,8 @@ SDValue AMDGPUTargetLowering::performLoadCombine(SDNode *N,
EVT NewVT = getEquivalentMemType(*DAG.getContext(), VT);
- SDValue NewLoad
- = DAG.getLoad(NewVT, SL, LN->getChain(),
- LN->getBasePtr(), LN->getMemOperand());
+ SDValue NewLoad = DAG.getLoad(NewVT, SL, LN->getChain(), LN->getBasePtr(),
+ LN->getMemOperand());
SDValue BC = DAG.getNode(ISD::BITCAST, SL, VT, NewLoad);
DCI.CombineTo(N, BC, NewLoad.getValue(1));
@@ -4197,15 +4211,16 @@ SDValue AMDGPUTargetLowering::performStoreCombine(SDNode *N,
DAG.ReplaceAllUsesOfValueWith(Val, CastBack);
}
- return DAG.getStore(SN->getChain(), SL, CastVal,
- SN->getBasePtr(), SN->getMemOperand());
+ return DAG.getStore(SN->getChain(), SL, CastVal, SN->getBasePtr(),
+ SN->getMemOperand());
}
// FIXME: This should go in generic DAG combiner with an isTruncateFree check,
// but isTruncateFree is inaccurate for i16 now because of SALU vs. VALU
// issues.
-SDValue AMDGPUTargetLowering::performAssertSZExtCombine(SDNode *N,
- DAGCombinerInfo &DCI) const {
+SDValue
+AMDGPUTargetLowering::performAssertSZExtCombine(SDNode *N,
+ DAGCombinerInfo &DCI) const {
SelectionDAG &DAG = DCI.DAG;
SDValue N0 = N->getOperand(0);
@@ -4228,7 +4243,7 @@ SDValue AMDGPUTargetLowering::performAssertSZExtCombine(SDNode *N,
}
SDValue AMDGPUTargetLowering::performIntrinsicWOChainCombine(
- SDNode *N, DAGCombinerInfo &DCI) const {
+ SDNode *N, DAGCombinerInfo &DCI) const {
unsigned IID = N->getConstantOperandVal(0);
switch (IID) {
case Intrinsic::amdgcn_mul_i24:
@@ -4266,9 +4281,8 @@ SDValue AMDGPUTargetLowering::performIntrinsicWOChainCombine(
/// Split the 64-bit value \p LHS into two 32-bit components, and perform the
/// binary operation \p Opc to it with the corresponding constant operands.
SDValue AMDGPUTargetLowering::splitBinaryBitConstantOpImpl(
- DAGCombinerInfo &DCI, const SDLoc &SL,
- unsigned Opc, SDValue LHS,
- uint32_t ValLo, uint32_t ValHi) const {
+ DAGCombinerInfo &DCI, const SDLoc &SL, unsigned Opc, SDValue LHS,
+ uint32_t ValLo, uint32_t ValHi) const {
SelectionDAG &DAG = DCI.DAG;
SDValue Lo, Hi;
std::tie(Lo, Hi) = split64BitValue(LHS, DAG);
@@ -4596,8 +4610,9 @@ SDValue AMDGPUTargetLowering::performSrlCombine(SDNode *N,
return DAG.getNode(ISD::BITCAST, SL, VT, Vec);
}
-SDValue AMDGPUTargetLowering::performTruncateCombine(
- SDNode *N, DAGCombinerInfo &DCI) const {
+SDValue
+AMDGPUTargetLowering::performTruncateCombine(SDNode *N,
+ DAGCombinerInfo &DCI) const {
SDLoc SL(N);
SelectionDAG &DAG = DCI.DAG;
EVT VT = N->getValueType(0);
@@ -4611,8 +4626,8 @@ SDValue AMDGPUTargetLowering::performTruncateCombine(
EVT EltVT = Elt0.getValueType();
if (VT.getFixedSizeInBits() <= EltVT.getFixedSizeInBits()) {
if (EltVT.isFloatingPoint()) {
- Elt0 = DAG.getNode(ISD::BITCAST, SL,
- EltVT.changeTypeToInteger(), Elt0);
+ Elt0 =
+ DAG.getNode(ISD::BITCAST, SL, EltVT.changeTypeToInteger(), Elt0);
}
return DAG.getNode(ISD::TRUNCATE, SL, VT, Elt0);
@@ -4652,8 +4667,7 @@ SDValue AMDGPUTargetLowering::performTruncateCombine(
if (VT.getScalarSizeInBits() < 32) {
EVT SrcVT = Src.getValueType();
if (SrcVT.getScalarSizeInBits() > 32 &&
- (Src.getOpcode() == ISD::SRL ||
- Src.getOpcode() == ISD::SRA ||
+ (Src.getOpcode() == ISD::SRL || Src.getOpcode() == ISD::SRA ||
Src.getOpcode() == ISD::SHL)) {
SDValue Amt = Src.getOperand(1);
KnownBits Known = DAG.computeKnownBits(Amt);
@@ -4665,13 +4679,14 @@ SDValue AMDGPUTargetLowering::performTruncateCombine(
const unsigned MaxCstSize =
(Src.getOpcode() == ISD::SHL) ? 31 : (32 - VT.getScalarSizeInBits());
if (Known.getMaxValue().ule(MaxCstSize)) {
- EVT MidVT = VT.isVector() ?
- EVT::getVectorVT(*DAG.getContext(), MVT::i32,
- VT.getVectorNumElements()) : MVT::i32;
+ EVT MidVT = VT.isVector()
+ ? EVT::getVectorVT(*DAG.getContext(), MVT::i32,
+ VT.getVectorNumElements())
+ : MVT::i32;
EVT NewShiftVT = getShiftAmountTy(MidVT, DAG.getDataLayout());
- SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, MidVT,
- Src.getOperand(0));
+ SDValue Trunc =
+ DAG.getNode(ISD::TRUNCATE, SL, MidVT, Src.getOperand(0));
DCI.AddToWorklist(Trunc.getNode());
if (Amt.getValueType() != NewShiftVT) {
@@ -4679,8 +4694,8 @@ SDValue AMDGPUTargetLowering::performTruncateCombine(
DCI.AddToWorklist(Amt.getNode());
}
- SDValue ShrunkShift = DAG.getNode(Src.getOpcode(), SL, MidVT,
- Trunc, Amt);
+ SDValue ShrunkShift =
+ DAG.getNode(Src.getOpcode(), SL, MidVT, Trunc, Amt);
return DAG.getNode(ISD::TRUNCATE, SL, VT, ShrunkShift);
}
}
@@ -4693,8 +4708,8 @@ SDValue AMDGPUTargetLowering::performTruncateCombine(
// instructions. If we only match on the legalized i64 mul expansion,
// SimplifyDemandedBits will be unable to remove them because there will be
// multiple uses due to the separate mul + mulh[su].
-static SDValue getMul24(SelectionDAG &DAG, const SDLoc &SL,
- SDValue N0, SDValue N1, unsigned Size, bool Signed) {
+static SDValue getMul24(SelectionDAG &DAG, const SDLoc &SL, SDValue N0,
+ SDValue N1, unsigned Size, bool Signed) {
if (Size <= 32) {
unsigned MulOpc = Signed ? AMDGPUISD::MUL_I24 : AMDGPUISD::MUL_U24;
return DAG.getNode(MulOpc, SL, MVT::i32, N0, N1);
@@ -4918,10 +4933,8 @@ SDValue AMDGPUTargetLowering::performMulhuCombine(SDNode *N,
return DAG.getZExtOrTrunc(Mulhi, DL, VT);
}
-SDValue AMDGPUTargetLowering::getFFBX_U32(SelectionDAG &DAG,
- SDValue Op,
- const SDLoc &DL,
- unsigned Opc) const {
+SDValue AMDGPUTargetLowering::getFFBX_U32(SelectionDAG &DAG, SDValue Op,
+ const SDLoc &DL, unsigned Opc) const {
EVT VT = Op.getValueType();
if (VT.bitsGT(MVT::i32))
return SDValue();
@@ -4943,9 +4956,10 @@ SDValue AMDGPUTargetLowering::getFFBX_U32(SelectionDAG &DAG,
// against the bitwidth.
//
// TODO: Should probably combine against FFBH_U32 instead of ctlz directly.
-SDValue AMDGPUTargetLowering::performCtlz_CttzCombine(const SDLoc &SL, SDValue Cond,
- SDValue LHS, SDValue RHS,
- DAGCombinerInfo &DCI) const {
+SDValue
+AMDGPUTargetLowering::performCtlz_CttzCombine(const SDLoc &SL, SDValue Cond,
+ SDValue LHS, SDValue RHS,
+ DAGCombinerInfo &DCI) const {
if (!isNullConstant(Cond.getOperand(1)))
return SDValue();
@@ -4978,16 +4992,13 @@ SDValue AMDGPUTargetLowering::performCtlz_CttzCombine(const SDLoc &SL, SDValue C
}
static SDValue distributeOpThroughSelect(TargetLowering::DAGCombinerInfo &DCI,
- unsigned Op,
- const SDLoc &SL,
- SDValue Cond,
- SDValue N1,
- SDValue N2) {
+ unsigned Op, const SDLoc &SL,
+ SDValue Cond, SDValue N1, SDValue N2) {
SelectionDAG &DAG = DCI.DAG;
EVT VT = N1.getValueType();
- SDValue NewSelect = DAG.getNode(ISD::SELECT, SL, VT, Cond,
- N1.getOperand(0), N2.getOperand(0));
+ SDValue NewSelect = DAG.getNode(ISD::SELECT, SL, VT, Cond, N1.getOperand(0),
+ N2.getOperand(0));
DCI.AddToWorklist(NewSelect.getNode());
return DAG.getNode(Op, SL, VT, NewSelect);
}
@@ -5013,8 +5024,8 @@ AMDGPUTargetLowering::foldFreeOpFromSelect(TargetLowering::DAGCombinerInfo &DCI,
if (!AMDGPUTargetLowering::allUsesHaveSourceMods(N.getNode()))
return SDValue();
- return distributeOpThroughSelect(DCI, LHS.getOpcode(),
- SDLoc(N), Cond, LHS, RHS);
+ return distributeOpThroughSelect(DCI, LHS.getOpcode(), SDLoc(N), Cond, LHS,
+ RHS);
}
bool Inv = false;
@@ -5067,8 +5078,8 @@ AMDGPUTargetLowering::foldFreeOpFromSelect(TargetLowering::DAGCombinerInfo &DCI,
if (Inv)
std::swap(NewLHS, NewRHS);
- SDValue NewSelect = DAG.getNode(ISD::SELECT, SL, VT,
- Cond, NewLHS, NewRHS);
+ SDValue NewSelect =
+ DAG.getNode(ISD::SELECT, SL, VT, Cond, NewLHS, NewRHS);
DCI.AddToWorklist(NewSelect.getNode());
return DAG.getNode(LHS.getOpcode(), SL, VT, NewSelect);
}
@@ -5111,10 +5122,10 @@ SDValue AMDGPUTargetLowering::performSelectCombine(SDNode *N,
}
if (VT == MVT::f32 && Subtarget->hasFminFmaxLegacy()) {
- SDValue MinMax
- = combineFMinMaxLegacy(SDLoc(N), VT, LHS, RHS, True, False, CC, DCI);
+ SDValue MinMax =
+ combineFMinMaxLegacy(SDLoc(N), VT, LHS, RHS, True, False, CC, DCI);
// Revisit this node so we can catch min3/max3/med3 patterns.
- //DCI.AddToWorklist(MinMax.getNode());
+ // DCI.AddToWorklist(MinMax.getNode());
return MinMax;
}
}
@@ -5126,10 +5137,10 @@ SDValue AMDGPUTargetLowering::performSelectCombine(SDNode *N,
static bool isInv2Pi(const APFloat &APF) {
static const APFloat KF16(APFloat::IEEEhalf(), APInt(16, 0x3118));
static const APFloat KF32(APFloat::IEEEsingle(), APInt(32, 0x3e22f983));
- static const APFloat KF64(APFloat::IEEEdouble(), APInt(64, 0x3fc45f306dc9c882));
+ static const APFloat KF64(APFloat::IEEEdouble(),
+ APInt(64, 0x3fc45f306dc9c882));
- return APF.bitwiseIsEqual(KF16) ||
- APF.bitwiseIsEqual(KF32) ||
+ return APF.bitwiseIsEqual(KF16) || APF.bitwiseIsEqual(KF32) ||
APF.bitwiseIsEqual(KF64);
}
@@ -5338,7 +5349,8 @@ SDValue AMDGPUTargetLowering::performFNegCombine(SDNode *N,
SDValue Ops[3];
for (unsigned I = 0; I < 3; ++I)
- Ops[I] = DAG.getNode(ISD::FNEG, SL, VT, N0->getOperand(I), N0->getFlags());
+ Ops[I] =
+ DAG.getNode(ISD::FNEG, SL, VT, N0->getOperand(I), N0->getFlags());
SDValue Res = DAG.getNode(AMDGPUISD::FMED3, SL, VT, Ops, N0->getFlags());
if (Res.getOpcode() != AMDGPUISD::FMED3)
@@ -5385,8 +5397,8 @@ SDValue AMDGPUTargetLowering::performFNegCombine(SDNode *N,
if (CvtSrc.getOpcode() == ISD::FNEG) {
// (fneg (fp_round (fneg x))) -> (fp_round x)
- return DAG.getNode(ISD::FP_ROUND, SL, VT,
- CvtSrc.getOperand(0), N0.getOperand(1));
+ return DAG.getNode(ISD::FP_ROUND, SL, VT, CvtSrc.getOperand(0),
+ N0.getOperand(1));
}
if (!N0.hasOneUse())
@@ -5553,7 +5565,7 @@ SDValue AMDGPUTargetLowering::PerformDAGCombine(SDNode *N,
SelectionDAG &DAG = DCI.DAG;
SDLoc DL(N);
- switch(N->getOpcode()) {
+ switch (N->getOpcode()) {
default:
break;
case ISD::BITCAST: {
@@ -5687,7 +5699,7 @@ SDValue AMDGPUTargetLowering::PerformDAGCombine(SDNode *N,
if (OffsetVal == 0) {
// This is already sign / zero extended, so try to fold away extra BFEs.
- unsigned SignBits = Signed ? (32 - WidthVal + 1) : (32 - WidthVal);
+ unsigned SignBits = Signed ? (32 - WidthVal + 1) : (32 - WidthVal);
unsigned OpSignBits = DAG.ComputeNumSignBits(BitsFrom);
if (OpSignBits >= SignBits)
@@ -5710,31 +5722,23 @@ SDValue AMDGPUTargetLowering::PerformDAGCombine(SDNode *N,
if (ConstantSDNode *CVal = dyn_cast<ConstantSDNode>(BitsFrom)) {
if (Signed) {
- return constantFoldBFE<int32_t>(DAG,
- CVal->getSExtValue(),
- OffsetVal,
- WidthVal,
- DL);
+ return constantFoldBFE<int32_t>(DAG, CVal->getSExtValue(), OffsetVal,
+ WidthVal, DL);
}
- return constantFoldBFE<uint32_t>(DAG,
- CVal->getZExtValue(),
- OffsetVal,
- WidthVal,
- DL);
+ return constantFoldBFE<uint32_t>(DAG, CVal->getZExtValue(), OffsetVal,
+ WidthVal, DL);
}
if ((OffsetVal + WidthVal) >= 32 &&
!(OffsetVal == 16 && WidthVal == 16 && Subtarget->hasSDWA())) {
SDValue ShiftVal = DAG.getConstant(OffsetVal, DL, MVT::i32);
- return DAG.getNode(Signed ? ISD::SRA : ISD::SRL, DL, MVT::i32,
- BitsFrom, ShiftVal);
+ return DAG.getNode(Signed ? ISD::SRA : ISD::SRL, DL, MVT::i32, BitsFrom,
+ ShiftVal);
}
if (BitsFrom.hasOneUse()) {
- APInt Demanded = APInt::getBitsSet(32,
- OffsetVal,
- OffsetVal + WidthVal);
+ APInt Demanded = APInt::getBitsSet(32, OffsetVal, OffsetVal + WidthVal);
KnownBits Known;
TargetLowering::TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
@@ -5841,11 +5845,9 @@ bool AMDGPUTargetLowering::SimplifyDemandedBitsForTargetNode(
// Helper functions
//===----------------------------------------------------------------------===//
-SDValue AMDGPUTargetLowering::CreateLiveInRegister(SelectionDAG &DAG,
- const TargetRegisterClass *RC,
- Register Reg, EVT VT,
- const SDLoc &SL,
- bool RawReg) const {
+SDValue AMDGPUTargetLowering::CreateLiveInRegister(
+ SelectionDAG &DAG, const TargetRegisterClass *RC, Register Reg, EVT VT,
+ const SDLoc &SL, bool RawReg) const {
MachineFunction &MF = DAG.getMachineFunction();
MachineRegisterInfo &MRI = MF.getRegInfo();
Register VReg;
@@ -5877,8 +5879,7 @@ static int getOrCreateFixedStackObject(MachineFrameInfo &MFI, unsigned Size,
return MFI.CreateFixedObject(Size, Offset, true);
}
-SDValue AMDGPUTargetLowering::loadStackInputValue(SelectionDAG &DAG,
- EVT VT,
+SDValue AMDGPUTargetLowering::loadStackInputValue(SelectionDAG &DAG, EVT VT,
const SDLoc &SL,
int64_t Offset) const {
MachineFunction &MF = DAG.getMachineFunction();
@@ -5918,9 +5919,9 @@ SDValue AMDGPUTargetLowering::loadInputValue(SelectionDAG &DAG,
const ArgDescriptor &Arg) const {
assert(Arg && "Attempting to load missing argument");
- SDValue V = Arg.isRegister() ?
- CreateLiveInRegister(DAG, RC, Arg.getRegister(), VT, SL) :
- loadStackInputValue(DAG, VT, SL, Arg.getStackOffset());
+ SDValue V = Arg.isRegister()
+ ? CreateLiveInRegister(DAG, RC, Arg.getRegister(), VT, SL)
+ : loadStackInputValue(DAG, VT, SL, Arg.getStackOffset());
if (!Arg.isMasked())
return V;
@@ -6012,8 +6013,8 @@ static unsigned workitemIntrinsicDim(unsigned ID) {
}
void AMDGPUTargetLowering::computeKnownBitsForTargetNode(
- const SDValue Op, KnownBits &Known,
- const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const {
+ const SDValue Op, KnownBits &Known, const APInt &DemandedElts,
+ const SelectionDAG &DAG, unsigned Depth) const {
Known.resetAll(); // Don't know anything.
@@ -6101,7 +6102,7 @@ void AMDGPUTargetLowering::computeKnownBitsForTargetNode(
}
break;
}
- case AMDGPUISD::BUFFER_LOAD_UBYTE: {
+ case AMDGPUISD::BUFFER_LOAD_UBYTE: {
Known.Zero.setHighBits(24);
break;
}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
index ec6479c509336..e277643a2c33f 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
@@ -32,7 +32,8 @@ class AMDGPUTargetLowering : public TargetLowering {
/// legalized from a smaller type VT. Need to match pre-legalized type because
/// the generic legalization inserts the add/sub between the select and
/// compare.
- SDValue getFFBX_U32(SelectionDAG &DAG, SDValue Op, const SDLoc &DL, unsigned Opc) const;
+ SDValue getFFBX_U32(SelectionDAG &DAG, SDValue Op, const SDLoc &DL,
+ unsigned Opc) const;
public:
/// \returns The minimum number of bits needed to store the value of \Op as an
@@ -130,7 +131,7 @@ class AMDGPUTargetLowering : public TargetLowering {
SDValue performMulhsCombine(SDNode *N, DAGCombinerInfo &DCI) const;
SDValue performMulhuCombine(SDNode *N, DAGCombinerInfo &DCI) const;
SDValue performCtlz_CttzCombine(const SDLoc &SL, SDValue Cond, SDValue LHS,
- SDValue RHS, DAGCombinerInfo &DCI) const;
+ SDValue RHS, DAGCombinerInfo &DCI) const;
SDValue foldFreeOpFromSelect(TargetLowering::DAGCombinerInfo &DCI,
SDValue N) const;
@@ -184,11 +185,10 @@ class AMDGPUTargetLowering : public TargetLowering {
SDValue LowerUDIVREM(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerDIVREMToFloat(SDValue Op, SelectionDAG &DAG, bool sign) const;
void LowerUDIVREM64(SDValue Op, SelectionDAG &DAG,
- SmallVectorImpl<SDValue> &Results) const;
+ SmallVectorImpl<SDValue> &Results) const;
void analyzeFormalArgumentsCompute(
- CCState &State,
- const SmallVectorImpl<ISD::InputArg> &Ins) const;
+ CCState &State, const SmallVectorImpl<ISD::InputArg> &Ins) const;
public:
AMDGPUTargetLowering(const TargetMachine &TM, const TargetSubtargetInfo &STI,
@@ -236,8 +236,7 @@ class AMDGPUTargetLowering : public TargetLowering {
bool isLoadBitCastBeneficial(EVT, EVT, const SelectionDAG &DAG,
const MachineMemOperand &MMO) const final;
- bool storeOfVectorConstantIsCheap(bool IsZero, EVT MemVT,
- unsigned NumElem,
+ bool storeOfVectorConstantIsCheap(bool IsZero, EVT MemVT, unsigned NumElem,
unsigned AS) const override;
bool aggressivelyPreferBuildVectorSources(EVT VecVT) const override;
bool isCheapToSpeculateCttz(Type *Ty) const override;
@@ -270,10 +269,8 @@ class AMDGPUTargetLowering : public TargetLowering {
const SmallVectorImpl<SDValue> &OutVals, const SDLoc &DL,
SelectionDAG &DAG) const override;
- SDValue addTokenForArgument(SDValue Chain,
- SelectionDAG &DAG,
- MachineFrameInfo &MFI,
- int ClobberedFI) const;
+ SDValue addTokenForArgument(SDValue Chain, SelectionDAG &DAG,
+ MachineFrameInfo &MFI, int ClobberedFI) const;
SDValue lowerUnhandledCall(CallLoweringInfo &CLI,
SmallVectorImpl<SDValue> &InVals,
@@ -292,8 +289,7 @@ class AMDGPUTargetLowering : public TargetLowering {
TargetLoweringOpt &TLO,
unsigned Depth) const override;
- void ReplaceNodeResults(SDNode * N,
- SmallVectorImpl<SDValue> &Results,
+ void ReplaceNodeResults(SDNode *N, SmallVectorImpl<SDValue> &Results,
SelectionDAG &DAG) const override;
SDValue combineFMinMaxLegacyImpl(const SDLoc &DL, EVT VT, SDValue LHS,
@@ -318,8 +314,8 @@ class AMDGPUTargetLowering : public TargetLowering {
return true;
}
SDValue getSqrtEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled,
- int &RefinementSteps, bool &UseOneConstNR,
- bool Reciprocal) const override;
+ int &RefinementSteps, bool &UseOneConstNR,
+ bool Reciprocal) const override;
SDValue getRecipEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled,
int &RefinementSteps) const override;
@@ -329,13 +325,13 @@ class AMDGPUTargetLowering : public TargetLowering {
/// Determine which of the bits specified in \p Mask are known to be
/// either zero or one and return them in the \p KnownZero and \p KnownOne
/// bitsets.
- void computeKnownBitsForTargetNode(const SDValue Op,
- KnownBits &Known,
+ void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known,
const APInt &DemandedElts,
const SelectionDAG &DAG,
unsigned Depth = 0) const override;
- unsigned ComputeNumSignBitsForTargetNode(SDValue Op, const APInt &DemandedElts,
+ unsigned ComputeNumSignBitsForTargetNode(SDValue Op,
+ const APInt &DemandedElts,
const SelectionDAG &DAG,
unsigned Depth = 0) const override;
@@ -361,39 +357,32 @@ class AMDGPUTargetLowering : public TargetLowering {
///
/// \returns a RegisterSDNode representing Reg if \p RawReg is true, otherwise
/// a copy from the register.
- SDValue CreateLiveInRegister(SelectionDAG &DAG,
- const TargetRegisterClass *RC,
- Register Reg, EVT VT,
- const SDLoc &SL,
+ SDValue CreateLiveInRegister(SelectionDAG &DAG, const TargetRegisterClass *RC,
+ Register Reg, EVT VT, const SDLoc &SL,
bool RawReg = false) const;
- SDValue CreateLiveInRegister(SelectionDAG &DAG,
- const TargetRegisterClass *RC,
+ SDValue CreateLiveInRegister(SelectionDAG &DAG, const TargetRegisterClass *RC,
Register Reg, EVT VT) const {
return CreateLiveInRegister(DAG, RC, Reg, VT, SDLoc(DAG.getEntryNode()));
}
// Returns the raw live in register rather than a copy from it.
SDValue CreateLiveInRegisterRaw(SelectionDAG &DAG,
- const TargetRegisterClass *RC,
- Register Reg, EVT VT) const {
- return CreateLiveInRegister(DAG, RC, Reg, VT, SDLoc(DAG.getEntryNode()), true);
+ const TargetRegisterClass *RC, Register Reg,
+ EVT VT) const {
+ return CreateLiveInRegister(DAG, RC, Reg, VT, SDLoc(DAG.getEntryNode()),
+ true);
}
/// Similar to CreateLiveInRegister, except value maybe loaded from a stack
/// slot rather than passed in a register.
- SDValue loadStackInputValue(SelectionDAG &DAG,
- EVT VT,
- const SDLoc &SL,
+ SDValue loadStackInputValue(SelectionDAG &DAG, EVT VT, const SDLoc &SL,
int64_t Offset) const;
- SDValue storeStackInputValue(SelectionDAG &DAG,
- const SDLoc &SL,
- SDValue Chain,
- SDValue ArgVal,
+ SDValue storeStackInputValue(SelectionDAG &DAG, const SDLoc &SL,
+ SDValue Chain, SDValue ArgVal,
int64_t Offset) const;
- SDValue loadInputValue(SelectionDAG &DAG,
- const TargetRegisterClass *RC,
+ SDValue loadInputValue(SelectionDAG &DAG, const TargetRegisterClass *RC,
EVT VT, const SDLoc &SL,
const ArgDescriptor &Arg) const;
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index ae550ebbec50b..44f0bf7f56f42 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -6520,7 +6520,7 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
else
ClampInstr.addReg(CarryOutReg, RegState::Define); // carry-out reg
}
- ClampInstr.addReg(Src0); // src0
+ ClampInstr.addReg(Src0); // src0
if (isFPOp)
ClampInstr.addImm(SISrcMods::NONE); // src1 mod
ClampInstr.addReg(Src1); // src1
@@ -12756,18 +12756,18 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
case 12:
if (!Subtarget->hasLDSLoadB96_B128())
return SDValue();
- Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX3_LDS_BOTHEN
- : AMDGPU::BUFFER_LOAD_DWORDX3_LDS_IDXEN
- : HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX3_LDS_OFFEN
- : AMDGPU::BUFFER_LOAD_DWORDX3_LDS_OFFSET;
+ Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX3_LDS_BOTHEN
+ : AMDGPU::BUFFER_LOAD_DWORDX3_LDS_IDXEN
+ : HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX3_LDS_OFFEN
+ : AMDGPU::BUFFER_LOAD_DWORDX3_LDS_OFFSET;
break;
case 16:
if (!Subtarget->hasLDSLoadB96_B128())
return SDValue();
- Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX4_LDS_BOTHEN
- : AMDGPU::BUFFER_LOAD_DWORDX4_LDS_IDXEN
- : HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX4_LDS_OFFEN
- : AMDGPU::BUFFER_LOAD_DWORDX4_LDS_OFFSET;
+ Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX4_LDS_BOTHEN
+ : AMDGPU::BUFFER_LOAD_DWORDX4_LDS_IDXEN
+ : HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX4_LDS_OFFEN
+ : AMDGPU::BUFFER_LOAD_DWORDX4_LDS_OFFSET;
break;
}
@@ -12797,11 +12797,11 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
Aux & (IsGFX12Plus ? AMDGPU::CPol::SWZ : AMDGPU::CPol::SWZ_pregfx12)
? 1
: 0,
- DL, MVT::i8)); // swz
+ DL, MVT::i8)); // swz
Ops.push_back(
DAG.getTargetConstant(isAsyncLDSDMA(IntrinsicID), DL, MVT::i8));
- Ops.push_back(M0Val.getValue(0)); // Chain
- Ops.push_back(M0Val.getValue(1)); // Glue
+ Ops.push_back(M0Val.getValue(0)); // Chain
+ Ops.push_back(M0Val.getValue(1)); // Glue
auto *M = cast<MemSDNode>(Op);
auto *Load = DAG.getMachineNode(Opc, DL, M->getVTList(), Ops);
@@ -12879,7 +12879,7 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
Ops.push_back(VOffset);
}
- Ops.push_back(Op.getOperand(5)); // Offset
+ Ops.push_back(Op.getOperand(5)); // Offset
unsigned Aux = Op.getConstantOperandVal(6);
Ops.push_back(DAG.getTargetConstant(Aux & ~AMDGPU::CPol::VIRTUAL_BITS, DL,
@@ -18870,12 +18870,10 @@ SITargetLowering::performFrexpSelectCombine(SDNode *N,
// Check if FrexpVal comes from amdgcn_frexp_exp or amdgcn_frexp_mant.
SDValue FrexpInput;
- if (!sd_match(FrexpVal,
- m_IntrinsicWOChain<Intrinsic::amdgcn_frexp_exp>(
- m_Value(FrexpInput))) &&
- !sd_match(FrexpVal,
- m_IntrinsicWOChain<Intrinsic::amdgcn_frexp_mant>(
- m_Value(FrexpInput))))
+ if (!sd_match(FrexpVal, m_IntrinsicWOChain<Intrinsic::amdgcn_frexp_exp>(
+ m_Value(FrexpInput))) &&
+ !sd_match(FrexpVal, m_IntrinsicWOChain<Intrinsic::amdgcn_frexp_mant>(
+ m_Value(FrexpInput))))
return SDValue();
// The frexp intrinsics ignore sign, so we can strip sign ops when comparing.
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h
index b5853ac1035f7..f0de7e745dae3 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h
@@ -35,16 +35,17 @@ class SITargetLowering final : public AMDGPUTargetLowering {
const GCNSubtarget *Subtarget;
public:
- MVT getRegisterTypeForCallingConv(LLVMContext &Context,
- CallingConv::ID CC,
+ MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC,
EVT VT) const override;
unsigned getNumRegistersForCallingConv(LLVMContext &Context,
CallingConv::ID CC,
EVT VT) const override;
- unsigned getVectorTypeBreakdownForCallingConv(
- LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT,
- unsigned &NumIntermediates, MVT &RegisterVT) const override;
+ unsigned getVectorTypeBreakdownForCallingConv(LLVMContext &Context,
+ CallingConv::ID CC, EVT VT,
+ EVT &IntermediateVT,
+ unsigned &NumIntermediates,
+ MVT &RegisterVT) const override;
MachinePointerInfo getKernargSegmentPtrInfo(MachineFunction &MF) const;
@@ -73,8 +74,7 @@ class SITargetLowering final : public AMDGPUTargetLowering {
AMDGPUFunctionArgInfo::PreloadedValue ClusterIdPV,
AMDGPUFunctionArgInfo::PreloadedValue ClusterMaxIdPV,
AMDGPUFunctionArgInfo::PreloadedValue ClusterWorkGroupIdPV) const;
- SDValue getPreloadedValue(SelectionDAG &DAG,
- const SIMachineFunctionInfo &MFI,
+ SDValue getPreloadedValue(SelectionDAG &DAG, const SIMachineFunctionInfo &MFI,
EVT VT,
AMDGPUFunctionArgInfo::PreloadedValue) const;
@@ -82,8 +82,8 @@ class SITargetLowering final : public AMDGPUTargetLowering {
SelectionDAG &DAG) const override;
SDValue LowerExternalSymbol(SDValue Op, SelectionDAG &DAG) const;
- SDValue lowerImplicitZextParam(SelectionDAG &DAG, SDValue Op,
- MVT VT, unsigned Offset) const;
+ SDValue lowerImplicitZextParam(SelectionDAG &DAG, SDValue Op, MVT VT,
+ unsigned Offset) const;
SDValue lowerImage(SDValue Op, const AMDGPU::ImageDimIntrinsicInfo *Intr,
SelectionDAG &DAG, bool WithChain) const;
SDValue lowerSBuffer(EVT VT, EVT MemVT, SDLoc DL, SDValue Chain, SDValue Rsrc,
@@ -137,8 +137,8 @@ class SITargetLowering final : public AMDGPUTargetLowering {
SDValue LowerBRCOND(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerSPONENTRY(SDValue Op, SelectionDAG &DAG) const;
- SDValue adjustLoadValueType(unsigned Opcode, MemSDNode *M,
- SelectionDAG &DAG, ArrayRef<SDValue> Ops,
+ SDValue adjustLoadValueType(unsigned Opcode, MemSDNode *M, SelectionDAG &DAG,
+ ArrayRef<SDValue> Ops,
bool IsIntrinsic = false) const;
SDValue lowerIntrinsicLoad(MemSDNode *M, bool IsFormat, SelectionDAG &DAG,
@@ -155,14 +155,12 @@ class SITargetLowering final : public AMDGPUTargetLowering {
/// Converts \p Op, which must be of floating point type, to the
/// floating point type \p VT, by either extending or truncating it.
- SDValue getFPExtOrFPRound(SelectionDAG &DAG,
- SDValue Op,
- const SDLoc &DL,
+ SDValue getFPExtOrFPRound(SelectionDAG &DAG, SDValue Op, const SDLoc &DL,
EVT VT) const;
- SDValue convertArgType(
- SelectionDAG &DAG, EVT VT, EVT MemVT, const SDLoc &SL, SDValue Val,
- bool Signed, const ISD::InputArg *Arg = nullptr) const;
+ SDValue convertArgType(SelectionDAG &DAG, EVT VT, EVT MemVT, const SDLoc &SL,
+ SDValue Val, bool Signed,
+ const ISD::InputArg *Arg = nullptr) const;
/// Custom lowering for ISD::FP_ROUND for MVT::f16.
SDValue lowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const;
@@ -200,13 +198,10 @@ class SITargetLowering final : public AMDGPUTargetLowering {
SDNode *adjustWritemask(MachineSDNode *&N, SelectionDAG &DAG) const;
- SDValue performUCharToFloatCombine(SDNode *N,
- DAGCombinerInfo &DCI) const;
+ SDValue performUCharToFloatCombine(SDNode *N, DAGCombinerInfo &DCI) const;
SDValue performFCopySignCombine(SDNode *N, DAGCombinerInfo &DCI) const;
- SDValue performSHLPtrCombine(SDNode *N,
- unsigned AS,
- EVT MemVT,
+ SDValue performSHLPtrCombine(SDNode *N, unsigned AS, EVT MemVT,
DAGCombinerInfo &DCI) const;
SDValue performMemSDNodeCombine(MemSDNode *N, DAGCombinerInfo &DCI) const;
@@ -241,8 +236,8 @@ class SITargetLowering final : public AMDGPUTargetLowering {
SDValue performSelectCombine(SDNode *N, DAGCombinerInfo &DCI) const;
SDValue reassociateScalarOps(SDNode *N, SelectionDAG &DAG) const;
- unsigned getFusedOpcode(const SelectionDAG &DAG,
- const SDNode *N0, const SDNode *N1) const;
+ unsigned getFusedOpcode(const SelectionDAG &DAG, const SDNode *N0,
+ const SDNode *N1) const;
SDValue tryFoldToMad64_32(SDNode *N, DAGCombinerInfo &DCI) const;
SDValue foldAddSub64WithZeroLowBitsTo32(SDNode *N,
DAGCombinerInfo &DCI) const;
@@ -405,7 +400,7 @@ class SITargetLowering final : public AMDGPUTargetLowering {
getPreferredVectorAction(MVT VT) const override;
bool shouldConvertConstantLoadToIntImm(const APInt &Imm,
- Type *Ty) const override;
+ Type *Ty) const override;
ExtractSubvectorCost getExtractSubvectorCost(EVT ResVT, EVT SrcVT,
unsigned Index) const override;
@@ -424,8 +419,8 @@ class SITargetLowering final : public AMDGPUTargetLowering {
bool supportSplitCSR(MachineFunction *MF) const override;
void initializeSplitCSR(MachineBasicBlock *Entry) const override;
void insertCopiesSplitCSR(
- MachineBasicBlock *Entry,
- const SmallVectorImpl<MachineBasicBlock *> &Exits) const override;
+ MachineBasicBlock *Entry,
+ const SmallVectorImpl<MachineBasicBlock *> &Exits) const override;
SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv,
bool isVarArg,
@@ -433,8 +428,8 @@ class SITargetLowering final : public AMDGPUTargetLowering {
const SDLoc &DL, SelectionDAG &DAG,
SmallVectorImpl<SDValue> &InVals) const override;
- bool CanLowerReturn(CallingConv::ID CallConv,
- MachineFunction &MF, bool isVarArg,
+ bool CanLowerReturn(CallingConv::ID CallConv, MachineFunction &MF,
+ bool isVarArg,
const SmallVectorImpl<ISD::OutputArg> &Outs,
LLVMContext &Context, const Type *RetTy) const override;
@@ -443,13 +438,11 @@ class SITargetLowering final : public AMDGPUTargetLowering {
const SmallVectorImpl<SDValue> &OutVals, const SDLoc &DL,
SelectionDAG &DAG) const override;
- void passSpecialInputs(
- CallLoweringInfo &CLI,
- CCState &CCInfo,
- const SIMachineFunctionInfo &Info,
- SmallVectorImpl<std::pair<unsigned, SDValue>> &RegsToPass,
- SmallVectorImpl<SDValue> &MemOpChains,
- SDValue Chain) const;
+ void
+ passSpecialInputs(CallLoweringInfo &CLI, CCState &CCInfo,
+ const SIMachineFunctionInfo &Info,
+ SmallVectorImpl<std::pair<unsigned, SDValue>> &RegsToPass,
+ SmallVectorImpl<SDValue> &MemOpChains, SDValue Chain) const;
SDValue LowerCallResult(SDValue Chain, SDValue InGlue,
CallingConv::ID CallConv, bool isVarArg,
@@ -461,10 +454,10 @@ class SITargetLowering final : public AMDGPUTargetLowering {
bool mayBeEmittedAsTailCall(const CallInst *) const override;
bool isEligibleForTailCallOptimization(
- SDValue Callee, CallingConv::ID CalleeCC, bool isVarArg,
- const SmallVectorImpl<ISD::OutputArg> &Outs,
- const SmallVectorImpl<SDValue> &OutVals,
- const SmallVectorImpl<ISD::InputArg> &Ins, SelectionDAG &DAG) const;
+ SDValue Callee, CallingConv::ID CalleeCC, bool isVarArg,
+ const SmallVectorImpl<ISD::OutputArg> &Outs,
+ const SmallVectorImpl<SDValue> &OutVals,
+ const SmallVectorImpl<ISD::InputArg> &Ins, SelectionDAG &DAG) const;
SDValue LowerCall(CallLoweringInfo &CLI,
SmallVectorImpl<SDValue> &InVals) const override;
@@ -480,7 +473,7 @@ class SITargetLowering final : public AMDGPUTargetLowering {
SDValue lowerSET_FPENV(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerROTR(SDValue Op, SelectionDAG &DAG) const;
- Register getRegisterByName(const char* RegName, LLT VT,
+ Register getRegisterByName(const char *RegName, LLT VT,
const MachineFunction &MF) const override;
MachineBasicBlock *splitKillBlock(MachineInstr &MI,
@@ -546,8 +539,7 @@ class SITargetLowering final : public AMDGPUTargetLowering {
bool getAsmOperandConstVal(SDValue Op, uint64_t &Val) const;
bool checkAsmConstraintVal(SDValue Op, StringRef Constraint,
uint64_t Val) const;
- bool checkAsmConstraintValA(SDValue Op,
- uint64_t Val,
+ bool checkAsmConstraintValA(SDValue Op, uint64_t Val,
unsigned MaxSize = 64) const;
SDValue copyToM0(SelectionDAG &DAG, SDValue Chain, const SDLoc &DL,
SDValue V) const;
@@ -614,8 +606,7 @@ class SITargetLowering final : public AMDGPUTargetLowering {
void emitExpandAtomicLoad(LoadInst *LI) const override;
void emitExpandAtomicStore(StoreInst *SI) const override;
- LoadInst *
- lowerIdempotentRMWIntoFencedLoad(AtomicRMWInst *AI) const override;
+ LoadInst *lowerIdempotentRMWIntoFencedLoad(AtomicRMWInst *AI) const override;
const TargetRegisterClass *getRegClassFor(MVT VT,
bool isDivergent) const override;
@@ -625,8 +616,7 @@ class SITargetLowering final : public AMDGPUTargetLowering {
unsigned
getMaxPermittedBytesForAlignment(MachineBasicBlock *MBB) const override;
- void allocateHSAUserSGPRs(CCState &CCInfo,
- MachineFunction &MF,
+ void allocateHSAUserSGPRs(CCState &CCInfo, MachineFunction &MF,
const SIRegisterInfo &TRI,
SIMachineFunctionInfo &Info) const;
@@ -641,28 +631,21 @@ class SITargetLowering final : public AMDGPUTargetLowering {
const SIRegisterInfo &TRI,
SIMachineFunctionInfo &Info) const;
- void allocateSystemSGPRs(CCState &CCInfo,
- MachineFunction &MF,
+ void allocateSystemSGPRs(CCState &CCInfo, MachineFunction &MF,
SIMachineFunctionInfo &Info,
- CallingConv::ID CallConv,
- bool IsShader) const;
+ CallingConv::ID CallConv, bool IsShader) const;
- void allocateSpecialEntryInputVGPRs(CCState &CCInfo,
- MachineFunction &MF,
+ void allocateSpecialEntryInputVGPRs(CCState &CCInfo, MachineFunction &MF,
const SIRegisterInfo &TRI,
SIMachineFunctionInfo &Info) const;
- void allocateSpecialInputSGPRs(
- CCState &CCInfo,
- MachineFunction &MF,
- const SIRegisterInfo &TRI,
- SIMachineFunctionInfo &Info) const;
-
- void allocateSpecialInputVGPRs(CCState &CCInfo,
- MachineFunction &MF,
+ void allocateSpecialInputSGPRs(CCState &CCInfo, MachineFunction &MF,
const SIRegisterInfo &TRI,
SIMachineFunctionInfo &Info) const;
- void allocateSpecialInputVGPRsFixed(CCState &CCInfo,
- MachineFunction &MF,
+
+ void allocateSpecialInputVGPRs(CCState &CCInfo, MachineFunction &MF,
+ const SIRegisterInfo &TRI,
+ SIMachineFunctionInfo &Info) const;
+ void allocateSpecialInputVGPRsFixed(CCState &CCInfo, MachineFunction &MF,
const SIRegisterInfo &TRI,
SIMachineFunctionInfo &Info) const;
More information about the llvm-commits
mailing list