[llvm] AMDGPU/GlobalISel: Switch to extended LLTs (PR #196522)
via llvm-commits
llvm-commits at lists.llvm.org
Fri May 8 05:58:12 PDT 2026
github-actions[bot] wrote:
<!--LLVM CODE FORMAT COMMENT: {clang-format}-->
:warning: C/C++ code formatter, clang-format found issues in your code. :warning:
<details>
<summary>
You can test this locally with the following command:
</summary>
``````````bash
git-clang-format --diff origin/main HEAD --extensions cpp,h -- llvm/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h llvm/include/llvm/CodeGen/MachineFunction.h llvm/lib/CodeGen/GlobalISel/CallLowering.cpp llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp llvm/lib/Target/AMDGPU/AMDGPUCombinerHelper.cpp llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp llvm/lib/Target/AMDGPU/AMDGPUPreLegalizerCombiner.cpp llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.h llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp --diff_from_common_commit
``````````
:warning:
The reproduction instructions above might return results for more than one PR
in a stack if you are using a stacked PR workflow. You can limit the results by
changing `origin/main` to the base branch/commit you want to compare against.
:warning:
</details>
<details>
<summary>
View the diff from clang-format here.
</summary>
``````````diff
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
index 35b05c581..67f136ecc 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
@@ -193,7 +193,8 @@ void CombinerHelper::replaceRegWith(MachineRegisterInfo &MRI, Register FromReg,
if (MRI.constrainRegAttrs(ToReg, FromReg)) {
// Notify about ToReg's def so CSE re-hashes it after constrainRegAttrs
// may have changed its type in MRI.
- MachineInstr *ToRegDef = ToReg.isVirtual() ? MRI.getUniqueVRegDef(ToReg) : nullptr;
+ MachineInstr *ToRegDef =
+ ToReg.isVirtual() ? MRI.getUniqueVRegDef(ToReg) : nullptr;
if (ToRegDef)
Observer.changingInstr(*ToRegDef);
// Notify the observer about each use instruction that is about to have its
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index aaefb3134..ee390f8a3 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -2146,7 +2146,7 @@ Register LegalizerHelper::coerceToScalar(Register Val) {
const DataLayout &DL = MIRBuilder.getDataLayout();
LLT EltTy = Ty.getScalarType();
LLT NewTy = EltTy.isFloat() ? EltTy.changeElementSize(Ty.getSizeInBits())
- : LLT::integer(Ty.getSizeInBits());
+ : LLT::integer(Ty.getSizeInBits());
if (Ty.isPointer()) {
if (DL.isNonIntegralAddressSpace(Ty.getAddressSpace()))
return Register();
@@ -7484,7 +7484,8 @@ LegalizerHelper::narrowScalarInsert(MachineInstr &MI, unsigned TypeIdx,
uint64_t WideSize = DstRegs.size() * NarrowSize;
Register DstReg = MI.getOperand(0).getReg();
if (WideSize > RegTy.getSizeInBits()) {
- Register MergeReg = MRI.createGenericVirtualRegister(LLT::integer(WideSize));
+ Register MergeReg =
+ MRI.createGenericVirtualRegister(LLT::integer(WideSize));
MIRBuilder.buildMergeLikeInstr(MergeReg, DstRegs);
MIRBuilder.buildTrunc(DstReg, MergeReg);
} else
@@ -9281,9 +9282,9 @@ LegalizerHelper::lowerMergeValues(MachineInstr &MI) {
auto [DstReg, DstTy, Src0Reg, Src0Ty] = MI.getFirst2RegLLTs();
unsigned PartSize = Src0Ty.getSizeInBits();
- LLT WideTy = DstTy.isPointer() ? LLT::scalar(DstTy.getSizeInBits())
- : DstTy.isScalar() ? DstTy
- : LLT::integer(DstTy.getSizeInBits());
+ LLT WideTy = DstTy.isPointer() ? LLT::scalar(DstTy.getSizeInBits())
+ : DstTy.isScalar() ? DstTy
+ : LLT::integer(DstTy.getSizeInBits());
Register ResultReg = MIRBuilder.buildZExt(WideTy, Src0Reg).getReg(0);
for (unsigned I = 2; I != NumOps; ++I) {
@@ -9335,14 +9336,16 @@ LegalizerHelper::lowerUnmergeValues(MachineInstr &MI) {
LLT IntTy = LLT::integer(SrcTy.getSizeInBits());
LLT IntDstTy = LLT::integer(DstTy.getSizeInBits());
- // Bitcast source to integer if needed (e.g. coerceToScalar on <2 x f16> gives f32).
+ // Bitcast source to integer if needed (e.g. coerceToScalar on <2 x f16> gives
+ // f32).
Register IntSrcReg = SrcReg;
if (!SrcTy.isInteger())
IntSrcReg = MIRBuilder.buildBitcast(IntTy, SrcReg).getReg(0);
const unsigned DstSize = DstTy.getSizeInBits();
- // Build a destination value: truncate integer bits, then bitcast to DstTy if float.
+ // Build a destination value: truncate integer bits, then bitcast to DstTy if
+ // float.
auto buildDst = [&](Register Dst, Register Src) {
if (!DstTy.isFloat())
MIRBuilder.buildTrunc(Dst, Src);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index e3e316fe3..db6e3c274 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -208,9 +208,9 @@ static LegalizeMutation bitcastToVectorElement32(unsigned TypeIdx) {
const LLT Ty = Query.Types[TypeIdx];
unsigned Size = Ty.getSizeInBits();
assert(Size % 32 == 0);
- return std::pair(
- TypeIdx, LLT::scalarOrVector(ElementCount::getFixed(Size / 32),
- LLT::integer(32)));
+ return std::pair(TypeIdx,
+ LLT::scalarOrVector(ElementCount::getFixed(Size / 32),
+ LLT::integer(32)));
};
}
@@ -732,17 +732,11 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
const LLT I64 = LLT::integer(64);
const LLT V2I16 = LLT::fixed_vector(2, I16);
- const std::initializer_list<LLT> FPTypesBase = {
- F32, F64
- };
+ const std::initializer_list<LLT> FPTypesBase = {F32, F64};
- const std::initializer_list<LLT> FPTypes16 = {
- F32, F64, F16
- };
+ const std::initializer_list<LLT> FPTypes16 = {F32, F64, F16};
- const std::initializer_list<LLT> FPTypesPK16 = {
- F32, F64, F16, V2F16
- };
+ const std::initializer_list<LLT> FPTypesPK16 = {F32, F64, F16, V2F16};
const LLT MinScalarFPTy = ST.has16BitInsts() ? F16 : F32;
@@ -805,36 +799,36 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
assert(ST.hasMad64_32());
getActionDefinitionsBuilder({G_UADDSAT, G_USUBSAT, G_SADDSAT, G_SSUBSAT})
- .legalFor({I32, I16, V2S16}) // Clamp modifier
- .minScalarOrElt(0, S16)
- .clampMaxNumElementsStrict(0, S16, 2)
- .scalarize(0)
- .widenScalarToNextPow2(0, 32)
- .lower();
+ .legalFor({I32, I16, V2S16}) // Clamp modifier
+ .minScalarOrElt(0, S16)
+ .clampMaxNumElementsStrict(0, S16, 2)
+ .scalarize(0)
+ .widenScalarToNextPow2(0, 32)
+ .lower();
} else if (ST.has16BitInsts()) {
getActionDefinitionsBuilder({G_ADD, G_SUB})
- .legalFor({I32, I16})
- .minScalar(0, S16)
- .widenScalarToNextMultipleOf(0, 32)
- .maxScalar(0, S32)
- .scalarize(0);
+ .legalFor({I32, I16})
+ .minScalar(0, S16)
+ .widenScalarToNextMultipleOf(0, 32)
+ .maxScalar(0, S32)
+ .scalarize(0);
getActionDefinitionsBuilder(G_MUL)
- .legalFor({I32, I16})
- .scalarize(0)
- .minScalar(0, S16)
- .widenScalarToNextMultipleOf(0, 32)
- .custom();
+ .legalFor({I32, I16})
+ .scalarize(0)
+ .minScalar(0, S16)
+ .widenScalarToNextMultipleOf(0, 32)
+ .custom();
assert(ST.hasMad64_32());
// Technically the saturating operations require clamp bit support, but this
// was introduced at the same time as 16-bit operations.
getActionDefinitionsBuilder({G_UADDSAT, G_USUBSAT})
- .legalFor({I32, I16}) // Clamp modifier
- .minScalar(0, S16)
- .scalarize(0)
- .widenScalarToNextPow2(0, 16)
- .lower();
+ .legalFor({I32, I16}) // Clamp modifier
+ .minScalar(0, S16)
+ .scalarize(0)
+ .widenScalarToNextPow2(0, 16)
+ .lower();
// We're just lowering this, but it helps get a better result to try to
// coerce to the desired type first.
@@ -844,16 +838,16 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
.lower();
} else {
getActionDefinitionsBuilder({G_ADD, G_SUB})
- .legalFor({I32})
- .widenScalarToNextMultipleOf(0, 32)
- .clampScalar(0, S32, S32)
- .scalarize(0);
+ .legalFor({I32})
+ .widenScalarToNextMultipleOf(0, 32)
+ .clampScalar(0, S32, S32)
+ .scalarize(0);
auto &Mul = getActionDefinitionsBuilder(G_MUL)
- .legalFor({I32})
- .scalarize(0)
- .minScalar(0, S32)
- .widenScalarToNextMultipleOf(0, 32);
+ .legalFor({I32})
+ .scalarize(0)
+ .minScalar(0, S32)
+ .widenScalarToNextMultipleOf(0, 32);
if (ST.hasMad64_32())
Mul.custom();
@@ -862,10 +856,10 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
if (ST.hasIntClamp()) {
getActionDefinitionsBuilder({G_UADDSAT, G_USUBSAT})
- .legalFor({I32}) // Clamp modifier.
- .scalarize(0)
- .minScalarOrElt(0, S32)
- .lower();
+ .legalFor({I32}) // Clamp modifier.
+ .scalarize(0)
+ .minScalarOrElt(0, S32)
+ .lower();
} else {
// Clamp bit support was added in VI, along with 16-bit operations.
getActionDefinitionsBuilder({G_UADDSAT, G_USUBSAT})
@@ -932,15 +926,15 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
.lower();
getActionDefinitionsBuilder(G_CONSTANT)
- .legalFor({S1, I32, I64, I16, GlobalPtr,
- LocalPtr, ConstantPtr, PrivatePtr, FlatPtr })
- .legalIf(isPointer(0))
- .clampScalar(0, S32, S64)
- .widenScalarToNextPow2(0);
+ .legalFor({S1, I32, I64, I16, GlobalPtr, LocalPtr, ConstantPtr,
+ PrivatePtr, FlatPtr})
+ .legalIf(isPointer(0))
+ .clampScalar(0, S32, S64)
+ .widenScalarToNextPow2(0);
getActionDefinitionsBuilder(G_FCONSTANT)
- .legalFor({F32, F64, F16, BF16})
- .clampScalar(0, F16, F64);
+ .legalFor({F32, F64, F16, BF16})
+ .clampScalar(0, F16, F64);
getActionDefinitionsBuilder({G_IMPLICIT_DEF, G_FREEZE})
.legalIf(isRegisterClassType(ST, 0))
@@ -972,14 +966,13 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
getActionDefinitionsBuilder(G_BLOCK_ADDR).legalFor({CodePtr});
- auto &FPOpActions = getActionDefinitionsBuilder(
- { G_FADD, G_FMUL, G_FMA, G_FCANONICALIZE,
- G_STRICT_FADD, G_STRICT_FMUL, G_STRICT_FMA})
- .legalFor({F32, F64});
- auto &TrigActions = getActionDefinitionsBuilder({G_FSIN, G_FCOS})
- .customFor({F32, F64});
- auto &FDIVActions = getActionDefinitionsBuilder(G_FDIV)
- .customFor({F32, F64});
+ auto &FPOpActions =
+ getActionDefinitionsBuilder({G_FADD, G_FMUL, G_FMA, G_FCANONICALIZE,
+ G_STRICT_FADD, G_STRICT_FMUL, G_STRICT_FMA})
+ .legalFor({F32, F64});
+ auto &TrigActions =
+ getActionDefinitionsBuilder({G_FSIN, G_FCOS}).customFor({F32, F64});
+ auto &FDIVActions = getActionDefinitionsBuilder(G_FDIV).customFor({F32, F64});
if (ST.has16BitInsts()) {
if (ST.hasVOP3PInsts())
@@ -1018,34 +1011,24 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
if (ST.hasVOP3PInsts()) {
MinNumMaxNum.customFor(FPTypesPK16)
- .moreElementsIf(isSmallOddVector(0), oneMoreElement(0))
- .clampMaxNumElements(0, F16, 2)
- .clampScalar(0, F16, F64)
- .scalarize(0);
+ .moreElementsIf(isSmallOddVector(0), oneMoreElement(0))
+ .clampMaxNumElements(0, F16, 2)
+ .clampScalar(0, F16, F64)
+ .scalarize(0);
} else if (ST.has16BitInsts()) {
- MinNumMaxNum.customFor(FPTypes16)
- .clampScalar(0, F16, F64)
- .scalarize(0);
+ MinNumMaxNum.customFor(FPTypes16).clampScalar(0, F16, F64).scalarize(0);
} else {
- MinNumMaxNum.customFor(FPTypesBase)
- .clampScalar(0, F32, F64)
- .scalarize(0);
+ MinNumMaxNum.customFor(FPTypesBase).clampScalar(0, F32, F64).scalarize(0);
}
if (ST.hasVOP3PInsts())
FPOpActions.clampMaxNumElementsStrict(0, F16, 2);
- FPOpActions
- .scalarize(0)
- .clampScalar(0, ST.has16BitInsts() ? F16 : F32, F64);
+ FPOpActions.scalarize(0).clampScalar(0, ST.has16BitInsts() ? F16 : F32, F64);
- TrigActions
- .scalarize(0)
- .clampScalar(0, ST.has16BitInsts() ? F16 : F32, F64);
+ TrigActions.scalarize(0).clampScalar(0, ST.has16BitInsts() ? F16 : F32, F64);
- FDIVActions
- .scalarize(0)
- .clampScalar(0, ST.has16BitInsts() ? F16 : F32, F64);
+ FDIVActions.scalarize(0).clampScalar(0, ST.has16BitInsts() ? F16 : F32, F64);
auto &FNegAbs = getActionDefinitionsBuilder({G_FNEG, G_FABS});
FNegAbs.legalFor(FPTypesPK16)
@@ -1057,26 +1040,26 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
if (ST.has16BitInsts()) {
getActionDefinitionsBuilder(G_FSQRT)
- .legalFor({F16})
- .customFor({F32, F64})
- .scalarize(0)
- .unsupported();
+ .legalFor({F16})
+ .customFor({F32, F64})
+ .scalarize(0)
+ .unsupported();
getActionDefinitionsBuilder(G_FFLOOR)
- .legalFor({F32, F64, F16})
- .scalarize(0)
- .clampScalar(0, F16, F64);
+ .legalFor({F32, F64, F16})
+ .scalarize(0)
+ .clampScalar(0, F16, F64);
getActionDefinitionsBuilder({G_FLDEXP, G_STRICT_FLDEXP})
- .legalFor({{F32, S32}, {F64, S32}, {F16, S16}})
- .scalarize(0)
- .maxScalarIf(typeIs(0, F16), 1, S16)
- .clampScalar(1, S32, S32)
- .lower();
+ .legalFor({{F32, S32}, {F64, S32}, {F16, S16}})
+ .scalarize(0)
+ .maxScalarIf(typeIs(0, F16), 1, S16)
+ .clampScalar(1, S32, S32)
+ .lower();
getActionDefinitionsBuilder(G_FFREXP)
- .customFor({{F32, S32}, {F64, S32}, {F16, S16}, {F16, S32}})
- .scalarize(0)
- .lower();
+ .customFor({{F32, S32}, {F64, S32}, {F16, S16}, {F16, S32}})
+ .scalarize(0)
+ .lower();
getActionDefinitionsBuilder(G_FMODF)
.lowerFor({F16, F32, F64})
@@ -1084,37 +1067,36 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
.lower();
} else {
getActionDefinitionsBuilder(G_FSQRT)
- .customFor({F32, F64, F16})
- .scalarize(0)
- .unsupported();
-
+ .customFor({F32, F64, F16})
+ .scalarize(0)
+ .unsupported();
if (ST.hasFractBug()) {
getActionDefinitionsBuilder(G_FFLOOR)
- .customFor({F64})
- .legalFor({F32, F64})
- .scalarize(0)
- .clampScalar(0, F32, F64);
+ .customFor({F64})
+ .legalFor({F32, F64})
+ .scalarize(0)
+ .clampScalar(0, F32, F64);
} else {
getActionDefinitionsBuilder(G_FFLOOR)
- .legalFor({F32, F64})
- .scalarize(0)
- .clampScalar(0, F32, F64);
+ .legalFor({F32, F64})
+ .scalarize(0)
+ .clampScalar(0, F32, F64);
}
getActionDefinitionsBuilder({G_FLDEXP, G_STRICT_FLDEXP})
- .legalFor({{F32, S32}, {F64, S32}})
- .scalarize(0)
- .clampScalar(0, F32, F64)
- .clampScalar(1, S32, S32)
- .lower();
+ .legalFor({{F32, S32}, {F64, S32}})
+ .scalarize(0)
+ .clampScalar(0, F32, F64)
+ .clampScalar(1, S32, S32)
+ .lower();
getActionDefinitionsBuilder(G_FFREXP)
- .customFor({{F32, S32}, {F64, S32}})
- .scalarize(0)
- .minScalar(0, F32)
- .clampScalar(1, S32, S32)
- .lower();
+ .customFor({{F32, S32}, {F64, S32}})
+ .scalarize(0)
+ .minScalar(0, F32)
+ .clampScalar(1, S32, S32)
+ .lower();
getActionDefinitionsBuilder(G_FMODF)
.lowerFor({F32, F64})
@@ -1132,31 +1114,29 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
FPTruncActions.scalarize(0).lower();
getActionDefinitionsBuilder(G_FPEXT)
- .legalFor({{F64, F32}, {F32, F16}})
- .narrowScalarFor({{F64, F16}}, changeTo(0, F32))
- .scalarize(0);
+ .legalFor({{F64, F32}, {F32, F16}})
+ .narrowScalarFor({{F64, F16}}, changeTo(0, F32))
+ .scalarize(0);
auto &FSubActions = getActionDefinitionsBuilder({G_FSUB, G_STRICT_FSUB});
if (ST.has16BitInsts()) {
FSubActions
- // Use actual fsub instruction
- .legalFor({F32, F16})
- // Must use fadd + fneg
- .lowerFor({F64, V2F16});
+ // Use actual fsub instruction
+ .legalFor({F32, F16})
+ // Must use fadd + fneg
+ .lowerFor({F64, V2F16});
} else {
FSubActions
- // Use actual fsub instruction
- .legalFor({F32})
- // Must use fadd + fneg
- .lowerFor({F64, F16, V2F16});
+ // Use actual fsub instruction
+ .legalFor({F32})
+ // Must use fadd + fneg
+ .lowerFor({F64, F16, V2F16});
}
if (ST.hasPackedFP32Ops())
FSubActions.lowerFor({V2S32}).clampMaxNumElements(0, S32, 2);
- FSubActions
- .scalarize(0)
- .clampScalar(0, F32, F64);
+ FSubActions.scalarize(0).clampScalar(0, F32, F64);
// Whether this is legal depends on the floating point mode for the function.
auto &FMad = getActionDefinitionsBuilder(G_FMAD);
@@ -1173,8 +1153,7 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
if (ST.has16BitInsts()) {
FRem.customFor({F16, F32, F64});
} else {
- FRem.minScalar(0, F32)
- .customFor({F32, F64});
+ FRem.minScalar(0, F32).customFor({F32, F64});
}
FRem.scalarize(0);
@@ -1190,13 +1169,19 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
.alwaysLegal();
getActionDefinitionsBuilder({G_SEXT, G_ZEXT, G_ANYEXT})
- .legalFor({{I64, I32}, {I32, I16}, {I64, I16},
- {I32, S1}, {I64, S1}, {I16, S1},
- {I32, F16}, {F32, F16},
- {I32, BF16}, {F32, BF16}})
- .scalarize(0)
- .clampScalar(0, S32, S64)
- .widenScalarToNextPow2(1, 32);
+ .legalFor({{I64, I32},
+ {I32, I16},
+ {I64, I16},
+ {I32, S1},
+ {I64, S1},
+ {I16, S1},
+ {I32, F16},
+ {F32, F16},
+ {I32, BF16},
+ {F32, BF16}})
+ .scalarize(0)
+ .clampScalar(0, S32, S64)
+ .widenScalarToNextPow2(1, 32);
// TODO: Split s1->s64 during regbankselect for VALU.
auto &IToFP = getActionDefinitionsBuilder({G_SITOFP, G_UITOFP})
@@ -1206,14 +1191,14 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
if (ST.has16BitInsts())
IToFP.legalFor({{F16, I16}});
IToFP.clampScalar(1, S32, S64)
- .minScalar(0, F32)
- .scalarize(0)
- .widenScalarToNextPow2(1);
+ .minScalar(0, F32)
+ .scalarize(0)
+ .widenScalarToNextPow2(1);
auto &FPToI = getActionDefinitionsBuilder({G_FPTOSI, G_FPTOUI})
- .legalFor({{I32, F32}, {I32, F64}, {I32, F16}})
- .customFor({{I64, F32}, {I64, F64}})
- .narrowScalarFor({{I64, F16}}, changeTo(0, I32));
+ .legalFor({{I32, F32}, {I32, F64}, {I32, F16}})
+ .customFor({{I64, F32}, {I64, F64}})
+ .narrowScalarFor({{I64, F16}}, changeTo(0, I32));
if (ST.has16BitInsts())
FPToI.legalFor({{I16, F16}});
else
@@ -1323,10 +1308,7 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
if (ST.hasSALUFloatInsts())
FCmpBuilder.legalForCartesianProduct({S32}, {F16, F32});
- FCmpBuilder
- .widenScalarToNextPow2(1)
- .clampScalar(1, F32, F64)
- .scalarize(0);
+ FCmpBuilder.widenScalarToNextPow2(1).clampScalar(1, F32, F64).scalarize(0);
// FIXME: fpow has a selection pattern that should move to custom lowering.
auto &ExpOps = getActionDefinitionsBuilder(G_FPOW);
@@ -1334,12 +1316,11 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
ExpOps.customFor({{F32}, {F16}});
else
ExpOps.customFor({F32});
- ExpOps.clampScalar(0, MinScalarFPTy, F32)
- .scalarize(0);
+ ExpOps.clampScalar(0, MinScalarFPTy, F32).scalarize(0);
getActionDefinitionsBuilder(G_FPOWI)
- .clampScalar(0, MinScalarFPTy, F32)
- .lower();
+ .clampScalar(0, MinScalarFPTy, F32)
+ .lower();
getActionDefinitionsBuilder(G_FLOG2)
.legalFor(ST.has16BitInsts(), {F16})
@@ -1356,17 +1337,16 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
auto &LogOps =
getActionDefinitionsBuilder({G_FLOG, G_FLOG10, G_FEXP, G_FEXP10});
LogOps.customFor({F32, F16, F64});
- LogOps.clampScalar(0, MinScalarFPTy, F32)
- .scalarize(0);
+ LogOps.clampScalar(0, MinScalarFPTy, F32).scalarize(0);
// The 64-bit versions produce 32-bit results, but only on the SALU.
getActionDefinitionsBuilder(G_CTPOP)
- .legalFor({{I32, I32}, {I32, I64}})
- .clampScalar(0, S32, S32)
- .widenScalarToNextPow2(1, 32)
- .clampScalar(1, S32, S64)
- .scalarize(0)
- .widenScalarToNextPow2(0, 32);
+ .legalFor({{I32, I32}, {I32, I64}})
+ .clampScalar(0, S32, S32)
+ .widenScalarToNextPow2(1, 32)
+ .clampScalar(1, S32, S64)
+ .scalarize(0)
+ .widenScalarToNextPow2(0, 32);
// If no 16 bit instr is available, lower into different instructions.
if (ST.has16BitInsts())
@@ -1421,20 +1401,20 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
// S64 is only legal on SALU, and needs to be broken into 32-bit elements in
// RegBankSelect.
getActionDefinitionsBuilder(G_BITREVERSE)
- .legalFor({I32, I64})
- .clampScalar(0, S32, S64)
- .scalarize(0)
- .widenScalarToNextPow2(0);
+ .legalFor({I32, I64})
+ .clampScalar(0, S32, S64)
+ .scalarize(0)
+ .widenScalarToNextPow2(0);
if (ST.has16BitInsts()) {
getActionDefinitionsBuilder(G_BSWAP)
- .legalFor({I16, I32, V2S16})
- .clampMaxNumElementsStrict(0, S16, 2)
- // FIXME: Fixing non-power-of-2 before clamp is workaround for
- // narrowScalar limitation.
- .widenScalarToNextPow2(0)
- .clampScalar(0, S16, S32)
- .scalarize(0);
+ .legalFor({I16, I32, V2S16})
+ .clampMaxNumElementsStrict(0, S16, 2)
+ // FIXME: Fixing non-power-of-2 before clamp is workaround for
+ // narrowScalar limitation.
+ .widenScalarToNextPow2(0)
+ .clampScalar(0, S16, S32)
+ .scalarize(0);
if (ST.hasVOP3PInsts()) {
getActionDefinitionsBuilder(G_ABS)
@@ -1463,30 +1443,30 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
}
} else {
getActionDefinitionsBuilder({G_SMIN, G_SMAX, G_UMIN, G_UMAX, G_ABS})
- .legalFor({I32, I16})
- .widenScalarToNextPow2(0)
- .minScalar(0, S16)
- .scalarize(0)
- .lower();
+ .legalFor({I32, I16})
+ .widenScalarToNextPow2(0)
+ .minScalar(0, S16)
+ .scalarize(0)
+ .lower();
}
} else {
// TODO: Should have same legality without v_perm_b32
getActionDefinitionsBuilder(G_BSWAP)
- .legalFor({I32})
- .lowerIf(scalarNarrowerThan(0, 32))
- // FIXME: Fixing non-power-of-2 before clamp is workaround for
- // narrowScalar limitation.
- .widenScalarToNextPow2(0)
- .maxScalar(0, S32)
- .scalarize(0)
- .lower();
+ .legalFor({I32})
+ .lowerIf(scalarNarrowerThan(0, 32))
+ // FIXME: Fixing non-power-of-2 before clamp is workaround for
+ // narrowScalar limitation.
+ .widenScalarToNextPow2(0)
+ .maxScalar(0, S32)
+ .scalarize(0)
+ .lower();
getActionDefinitionsBuilder({G_SMIN, G_SMAX, G_UMIN, G_UMAX, G_ABS})
- .legalFor({I32})
- .minScalar(0, S32)
- .widenScalarToNextPow2(0)
- .scalarize(0)
- .lower();
+ .legalFor({I32})
+ .minScalar(0, S32)
+ .widenScalarToNextPow2(0)
+ .scalarize(0)
+ .lower();
}
getActionDefinitionsBuilder(G_INTTOPTR)
@@ -1765,18 +1745,20 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
// inserting addrspacecasts.
ExtLoads.customIf(typeIs(1, Constant32Ptr));
- ExtLoads.clampScalar(0, I32, I32)
- .widenScalarToNextPow2(0)
- .lower();
-
- auto &Atomics = getActionDefinitionsBuilder(
- {G_ATOMICRMW_XCHG, G_ATOMICRMW_ADD, G_ATOMICRMW_SUB,
- G_ATOMICRMW_AND, G_ATOMICRMW_OR, G_ATOMICRMW_XOR,
- G_ATOMICRMW_MAX, G_ATOMICRMW_MIN, G_ATOMICRMW_UMAX,
- G_ATOMICRMW_UMIN, G_ATOMICRMW_UINC_WRAP, G_ATOMICRMW_UDEC_WRAP})
- .legalFor({{I32, GlobalPtr}, {I32, LocalPtr},
- {I64, GlobalPtr}, {I64, LocalPtr},
- {I32, RegionPtr}, {I64, RegionPtr}});
+ ExtLoads.clampScalar(0, I32, I32).widenScalarToNextPow2(0).lower();
+
+ auto &Atomics =
+ getActionDefinitionsBuilder(
+ {G_ATOMICRMW_XCHG, G_ATOMICRMW_ADD, G_ATOMICRMW_SUB, G_ATOMICRMW_AND,
+ G_ATOMICRMW_OR, G_ATOMICRMW_XOR, G_ATOMICRMW_MAX, G_ATOMICRMW_MIN,
+ G_ATOMICRMW_UMAX, G_ATOMICRMW_UMIN, G_ATOMICRMW_UINC_WRAP,
+ G_ATOMICRMW_UDEC_WRAP})
+ .legalFor({{I32, GlobalPtr},
+ {I32, LocalPtr},
+ {I64, GlobalPtr},
+ {I64, LocalPtr},
+ {I32, RegionPtr},
+ {I64, RegionPtr}});
if (ST.hasFlatAddressSpace()) {
Atomics.legalFor({{I32, FlatPtr}, {I64, FlatPtr}});
}
@@ -1806,11 +1788,7 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
// These are legal with some caveats, and should have undergone expansion in
// the IR in most situations
// TODO: Move atomic expansion into legalizer
- Atomic.legalFor({
- {F32, GlobalPtr},
- {F64, GlobalPtr},
- {F64, FlatPtr}
- });
+ Atomic.legalFor({{F32, GlobalPtr}, {F64, GlobalPtr}, {F64, FlatPtr}});
}
if (ST.hasAtomicBufferGlobalPkAddF16NoRtnInsts() ||
@@ -1840,10 +1818,12 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
// BUFFER/FLAT_ATOMIC_CMP_SWAP on GCN GPUs needs input marshalling, and output
// demarshalling
getActionDefinitionsBuilder(G_ATOMIC_CMPXCHG)
- .customFor({{I32, GlobalPtr}, {I64, GlobalPtr},
- {I32, FlatPtr}, {I64, FlatPtr}})
- .legalFor({{I32, LocalPtr}, {I64, LocalPtr},
- {I32, RegionPtr}, {I64, RegionPtr}});
+ .customFor(
+ {{I32, GlobalPtr}, {I64, GlobalPtr}, {I32, FlatPtr}, {I64, FlatPtr}})
+ .legalFor({{I32, LocalPtr},
+ {I64, LocalPtr},
+ {I32, RegionPtr},
+ {I64, RegionPtr}});
// TODO: Pointer types, any 32-bit or 64-bit vector
// Condition should be s32 for scalar, s1 for vector.
@@ -1867,33 +1847,34 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
// TODO: Only the low 4/5/6 bits of the shift amount are observed, so we can
// be more flexible with the shift amount type.
auto &Shifts = getActionDefinitionsBuilder({G_SHL, G_LSHR, G_ASHR})
- .legalFor({{I32, I32}, {I64, I32}});
+ .legalFor({{I32, I32}, {I64, I32}});
if (ST.has16BitInsts()) {
if (ST.hasVOP3PInsts()) {
Shifts.legalFor({{I16, I16}, {V2I16, V2I16}})
- .clampMaxNumElements(0, S16, 2);
+ .clampMaxNumElements(0, S16, 2);
} else
Shifts.legalFor({{I16, I16}});
// TODO: Support 16-bit shift amounts for all types
Shifts.widenScalarIf(
- [=](const LegalityQuery &Query) {
- // Use 16-bit shift amounts for any 16-bit shift. Otherwise we want a
- // 32-bit amount.
- const LLT ValTy = Query.Types[0];
- const LLT AmountTy = Query.Types[1];
- return ValTy.isScalar() && ValTy.getSizeInBits() <= 16 &&
- AmountTy.getSizeInBits() < 16;
- }, changeElementSizeTo(1, I16));
+ [=](const LegalityQuery &Query) {
+ // Use 16-bit shift amounts for any 16-bit shift. Otherwise we want a
+ // 32-bit amount.
+ const LLT ValTy = Query.Types[0];
+ const LLT AmountTy = Query.Types[1];
+ return ValTy.isScalar() && ValTy.getSizeInBits() <= 16 &&
+ AmountTy.getSizeInBits() < 16;
+ },
+ changeElementSizeTo(1, I16));
Shifts.maxScalarIf(typeIs(0, I16), 1, I16);
Shifts.clampScalar(1, I32, I32);
Shifts.widenScalarToNextPow2(0, 16);
Shifts.clampScalar(0, I16, I64);
getActionDefinitionsBuilder({G_SSHLSAT, G_USHLSAT})
- .minScalar(0, I16)
- .scalarize(0)
- .lower();
+ .minScalar(0, I16)
+ .scalarize(0)
+ .lower();
} else {
// Make sure we legalize the shift amount type first, as the general
// expansion for the shifted type will produce much worse code if it hasn't
@@ -1903,9 +1884,9 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
Shifts.clampScalar(0, I32, I64);
getActionDefinitionsBuilder({G_SSHLSAT, G_USHLSAT})
- .minScalar(0, I32)
- .scalarize(0)
- .lower();
+ .minScalar(0, I32)
+ .scalarize(0)
+ .lower();
}
Shifts.scalarize(0);
@@ -2014,10 +1995,12 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
.legalForCartesianProduct(AllS64Vectors, {S64})
.clampNumElements(0, V16S32, V32S32)
.clampNumElements(0, V2S64, V16S64)
- .fewerElementsIf(isWideVec16(0), [](const LegalityQuery &Query) {
- LLT EltTy = Query.Types[0].getElementType();
- return std::make_pair(0, LLT::fixed_vector(2, EltTy));
- })
+ .fewerElementsIf(isWideVec16(0),
+ [](const LegalityQuery &Query) {
+ LLT EltTy = Query.Types[0].getElementType();
+ return std::make_pair(0,
+ LLT::fixed_vector(2, EltTy));
+ })
.moreElementsIf(isIllegalRegisterType(ST, 0),
moreElementsToNextExistingRegClass(0));
@@ -2085,7 +2068,8 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
elementTypeIs(1, S16)),
[](const LegalityQuery &Query) {
LLT EltTy = Query.Types[1].getElementType();
- return std::make_pair(1, LLT::fixed_vector(2, EltTy));
+ return std::make_pair(
+ 1, LLT::fixed_vector(2, EltTy));
})
// Clamp the little scalar to s8-s256 and make it a power of 2. It's
// not worth considering the multiples of 64 since 2*192 and 2*384
@@ -2107,12 +2091,12 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
if (Op == G_MERGE_VALUES) {
Builder.widenScalarIf(
- // TODO: Use 16-bit shifts if legal for 8-bit values?
- [=](const LegalityQuery &Query) {
- const LLT Ty = Query.Types[LitTyIdx];
- return Ty.getSizeInBits() < 32;
- },
- changeTo(LitTyIdx, LLT::integer(32)));
+ // TODO: Use 16-bit shifts if legal for 8-bit values?
+ [=](const LegalityQuery &Query) {
+ const LLT Ty = Query.Types[LitTyIdx];
+ return Ty.getSizeInBits() < 32;
+ },
+ changeTo(LitTyIdx, LLT::integer(32)));
}
Builder.widenScalarIf(
@@ -2167,8 +2151,7 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
.lower();
auto &FSHRActionDefs = getActionDefinitionsBuilder(G_FSHR);
- FSHRActionDefs.legalFor({{I32, I32}})
- .clampMaxNumElementsStrict(0, S16, 2);
+ FSHRActionDefs.legalFor({{I32, I32}}).clampMaxNumElementsStrict(0, S16, 2);
if (ST.hasVOP3PInsts())
FSHRActionDefs.lowerFor({{V2S16, V2S16}});
FSHRActionDefs.scalarize(0).lower();
@@ -2185,8 +2168,7 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
.lower();
}
- getActionDefinitionsBuilder(G_READCYCLECOUNTER)
- .legalFor({I64});
+ getActionDefinitionsBuilder(G_READCYCLECOUNTER).legalFor({I64});
getActionDefinitionsBuilder(G_READSTEADYCOUNTER).legalFor({I64});
@@ -2831,7 +2813,8 @@ bool AMDGPULegalizerInfo::legalizeITOFP(
return true;
}
- assert(MRI.getType(Dst) == S64 || MRI.getType(Dst).getScalarSizeInBits() == 32);
+ assert(MRI.getType(Dst) == S64 ||
+ MRI.getType(Dst).getScalarSizeInBits() == 32);
auto One = B.buildConstant(I32, 1);
@@ -2851,7 +2834,8 @@ bool AMDGPULegalizerInfo::legalizeITOFP(
auto Unmerge2 = B.buildUnmerge({I32, I32}, Norm);
auto Adjust = B.buildUMin(I32, One, Unmerge2.getReg(0));
auto Norm2 = B.buildOr(I32, Unmerge2.getReg(1), Adjust);
- auto FVal = Signed ? B.buildSITOFP(DstTy, Norm2) : B.buildUITOFP(DstTy, Norm2);
+ auto FVal =
+ Signed ? B.buildSITOFP(DstTy, Norm2) : B.buildUITOFP(DstTy, Norm2);
auto Scale = B.buildSub(I32, ThirtyTwo, ShAmt);
B.buildFLdexp(Dst, FVal, Scale);
MI.eraseFromParent();
@@ -5153,7 +5137,7 @@ void AMDGPULegalizerInfo::legalizeUnsignedDIV_REM32Impl(MachineIRBuilder &B,
MachineRegisterInfo &MRI = *B.getMRI();
const LLT S1 = LLT::scalar(1);
const LLT FloatTy = LLT::float32(); // float type for FP operations
- const LLT Ty = MRI.getType(X); // integer working type (preserves kind)
+ const LLT Ty = MRI.getType(X); // integer working type (preserves kind)
// See AMDGPUCodeGenPrepare::expandDivRem32 for a description of the
// algorithm used here.
@@ -5945,24 +5929,28 @@ bool AMDGPULegalizerInfo::legalizeFSQRTF32(MachineInstr &MI,
auto NegOne = B.buildConstant(I32, -1);
auto SqrtSNextDown = B.buildAdd(I32, SqrtSInt, NegOne);
- auto NegSqrtSNextDown = B.buildFNeg(F32, B.buildBitcast(F32, SqrtSNextDown), Flags);
+ auto NegSqrtSNextDown =
+ B.buildFNeg(F32, B.buildBitcast(F32, SqrtSNextDown), Flags);
auto SqrtVP = B.buildFMA(F32, NegSqrtSNextDown, SqrtS, SqrtX, Flags);
auto PosOne = B.buildConstant(I32, 1);
auto SqrtSNextUp = B.buildAdd(I32, SqrtSInt, PosOne);
- auto NegSqrtSNextUp = B.buildFNeg(F32, B.buildBitcast(F32, SqrtSNextUp), Flags);
+ auto NegSqrtSNextUp =
+ B.buildFNeg(F32, B.buildBitcast(F32, SqrtSNextUp), Flags);
auto SqrtVS = B.buildFMA(F32, NegSqrtSNextUp, SqrtS, SqrtX, Flags);
auto Zero = B.buildFConstant(F32, 0.0f);
auto SqrtVPLE0 = B.buildFCmp(CmpInst::FCMP_OLE, S1, SqrtVP, Zero, Flags);
SqrtS = B.buildSelect(F32, SqrtVPLE0, B.buildBitcast(F32, SqrtSNextDown),
- SqrtS, Flags).getReg(0);
+ SqrtS, Flags)
+ .getReg(0);
auto SqrtVPVSGT0 = B.buildFCmp(CmpInst::FCMP_OGT, S1, SqrtVS, Zero, Flags);
SqrtS = B.buildSelect(F32, SqrtVPVSGT0, B.buildBitcast(F32, SqrtSNextUp),
- SqrtS, Flags).getReg(0);
+ SqrtS, Flags)
+ .getReg(0);
} else {
auto SqrtR =
B.buildIntrinsic(Intrinsic::amdgcn_rsq, {F32}).addReg(SqrtX.getReg(0));
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp
index 5cb1b648c..c8d452fbc 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp
@@ -645,8 +645,9 @@ bool RegBankLegalizeHelper::lowerS_BFE(MachineInstr &MI) {
// Select machine instruction, because of reg class constraining, insert
// copies from reg class to reg bank.
- auto S_BFE = B.buildInstr(Opc, {{SgprRB, Ty}},
- {B.buildCopy(Ty, Src), B.buildCopy(LLT::integer(32), Src1)});
+ auto S_BFE =
+ B.buildInstr(Opc, {{SgprRB, Ty}},
+ {B.buildCopy(Ty, Src), B.buildCopy(LLT::integer(32), Src1)});
constrainSelectedInstRegOperands(*S_BFE, *ST.getInstrInfo(),
*ST.getRegisterInfo(), RBI);
@@ -792,7 +793,8 @@ bool RegBankLegalizeHelper::lowerSplitTo32Select(MachineInstr &MI) {
LLT DstTy = MRI.getType(Dst);
assert(DstTy == V4S16 || DstTy == V2S32 || DstTy == S64 ||
(DstTy.isPointer() && DstTy.getSizeInBits() == 64));
- LLT Ty = DstTy == V4S16 ? V2S16 : (DstTy.isInteger() ? LLT::integer(32) : S32);
+ LLT Ty =
+ DstTy == V4S16 ? V2S16 : (DstTy.isInteger() ? LLT::integer(32) : S32);
auto Op2 = B.buildUnmerge({VgprRB, Ty}, MI.getOperand(2).getReg());
auto Op3 = B.buildUnmerge({VgprRB, Ty}, MI.getOperand(3).getReg());
Register Cond = MI.getOperand(1).getReg();
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
index 2e50fd370..c0d65c70a 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
@@ -1197,8 +1197,8 @@ bool AMDGPURegisterBankInfo::applyMappingDynStackAlloc(
auto OldSP = B.buildCopy(PtrTy, SPReg);
if (Alignment > TFI.getStackAlign()) {
auto StackAlignMask = (Alignment.value() << ST.getWavefrontSizeLog2()) - 1;
- auto Tmp1 = B.buildPtrAdd(PtrTy, OldSP,
- B.buildConstant(LLT::integer(32), StackAlignMask));
+ auto Tmp1 = B.buildPtrAdd(
+ PtrTy, OldSP, B.buildConstant(LLT::integer(32), StackAlignMask));
B.buildMaskLowPtrBits(Dst, Tmp1,
Log2(Alignment) + ST.getWavefrontSizeLog2());
} else {
``````````
</details>
https://github.com/llvm/llvm-project/pull/196522
More information about the llvm-commits
mailing list