[llvm] 8433cf6 - [AMDGPU] Make v2f64 fneg legal on gfx1251 (#203427)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Jun 11 19:18:28 PDT 2026
Author: Stanislav Mekhanoshin
Date: 2026-06-11T19:18:24-07:00
New Revision: 8433cf6d6ccc49a0fd99ed86c7d94a1fbb2fd1be
URL: https://github.com/llvm/llvm-project/commit/8433cf6d6ccc49a0fd99ed86c7d94a1fbb2fd1be
DIFF: https://github.com/llvm/llvm-project/commit/8433cf6d6ccc49a0fd99ed86c7d94a1fbb2fd1be.diff
LOG: [AMDGPU] Make v2f64 fneg legal on gfx1251 (#203427)
Added:
Modified:
llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
llvm/lib/Target/AMDGPU/SIISelLowering.cpp
llvm/lib/Target/AMDGPU/SIInstructions.td
llvm/test/CodeGen/AMDGPU/packed-fp64.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
index 78fcd0fd24b6f..278c2dc94c56b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -5476,7 +5476,7 @@ SDValue AMDGPUTargetLowering::performFNegCombine(SDNode *N,
SDValue BCSrc = N0.getOperand(0);
if (BCSrc.getOpcode() == ISD::BUILD_VECTOR) {
SDValue HighBits = BCSrc.getOperand(BCSrc.getNumOperands() - 1);
- if (HighBits.getValueType().getSizeInBits() != 32 ||
+ if (VT != MVT::f64 || HighBits.getValueType().getSizeInBits() != 32 ||
!fnegFoldsIntoOp(HighBits.getNode()))
return SDValue();
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
index 07d83b1e05ab1..77ef0eca23743 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
@@ -5359,6 +5359,13 @@ std::pair<Register, unsigned> AMDGPUInstructionSelector::selectVOP3PModsImpl(
else if (Stat.second == SrcStatus::IS_LO_NEG)
Mods ^= SISrcMods::NEG;
+ // 64-bit VOP3P instructions do not have OPSEL or ABS.
+ // TODO: Select NEG_LO and NEG_HI modifiers from BUILD_VECTOR.
+ if (MRI.getType(RootReg).getSizeInBits() == 128) {
+ Mods |= SISrcMods::OP_SEL_1; // Just the default, OPSEL unsupported.
+ return {Stat.first, Mods};
+ }
+
MachineInstr *MI = MRI.getVRegDef(Stat.first);
if (MI->getOpcode() != AMDGPU::G_BUILD_VECTOR || MI->getNumOperands() != 3 ||
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index dbb64391ce7eb..0ad3fbe9400ad 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -732,6 +732,9 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
S32, S64, S16, V2S16
};
+ const std::initializer_list<LLT> FPTypesPK16_64 = {S32, S64, S16, V2S16,
+ V2S64};
+
const LLT MinScalarFPTy = ST.has16BitInsts() ? S16 : S32;
getActionDefinitionsBuilder(G_BR).alwaysLegal();
@@ -986,6 +989,11 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
FPOpActions.clampMaxNumElementsStrict(0, S64, 2);
}
+ if (ST.hasPackedFP64Ops()) {
+ FPOpActions.legalFor({V2S64});
+ FPOpActions.clampMaxNumElementsStrict(0, S64, 2);
+ }
+
auto &MinNumMaxNumIeee =
getActionDefinitionsBuilder({G_FMINNUM_IEEE, G_FMAXNUM_IEEE});
@@ -1006,7 +1014,14 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
auto &MinNumMaxNum = getActionDefinitionsBuilder(
{G_FMINNUM, G_FMAXNUM, G_FMINIMUMNUM, G_FMAXIMUMNUM});
- if (ST.hasVOP3PInsts()) {
+ if (ST.hasPackedFP64Ops()) {
+ MinNumMaxNum.customFor(FPTypesPK16_64)
+ .moreElementsIf(isSmallOddVector(0), oneMoreElement(0))
+ .clampMaxNumElements(0, S16, 2)
+ .clampMaxNumElements(0, S64, 2)
+ .clampScalar(0, S16, S64)
+ .scalarize(0);
+ } else if (ST.hasVOP3PInsts()) {
MinNumMaxNum.customFor(FPTypesPK16)
.moreElementsIf(isSmallOddVector(0), oneMoreElement(0))
.clampMaxNumElements(0, S16, 2)
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 9013087a75332..3026d0dfa1dab 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -905,8 +905,9 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
Custom);
}
if (Subtarget->hasPackedFP64Ops()) {
- setOperationAction({ISD::FADD, ISD::FMUL, ISD::FMA}, MVT::v2f64, Legal);
- setOperationAction({ISD::FADD, ISD::FMUL, ISD::FMA},
+ setOperationAction({ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FNEG},
+ MVT::v2f64, Legal);
+ setOperationAction({ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FNEG},
{MVT::v4f64, MVT::v8f64, MVT::v16f64, MVT::v32f64},
Custom);
}
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index 272b84f1c665c..6b09faba5ddcc 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -2216,6 +2216,19 @@ def : GCNPat <
SReg_32)), sub1))
>;
+def : GCNPat <
+ (UniformUnaryFrag<fneg> (v2f64 SGPR_128:$src)),
+ (v2f64 (REG_SEQUENCE SGPR_128,
+ (i32 (EXTRACT_SUBREG $src, sub0)), sub0,
+ (i32 (COPY_TO_REGCLASS (S_XOR_B32 (i32 (EXTRACT_SUBREG $src, sub1)),
+ (i32 (S_MOV_B32 (i32 0x80000000)))),
+ SReg_32)), sub1,
+ (i32 (EXTRACT_SUBREG $src, sub2)), sub2,
+ (i32 (COPY_TO_REGCLASS (S_XOR_B32 (i32 (EXTRACT_SUBREG $src, sub3)),
+ (i32 (S_MOV_B32 (i32 0x80000000)))),
+ SReg_32)), sub3))
+>;
+
// FIXME: Use S_BITSET0_B32/B64?
def : GCNPat <
(UniformUnaryFrag<fabs> (f64 SReg_64:$src)),
@@ -2357,6 +2370,14 @@ def : GCNPat <
let SubtargetPredicate = HasPackedFP32Ops;
}
+def : GCNPat <
+ (DivergentUnaryFrag<fneg> (v2f64 VReg_128:$src)),
+ (V_PK_ADD_F64 !or(SRCMODS.NEG, SRCMODS.NEG_HI), VReg_128:$src,
+ !or(SRCMODS.NEG, SRCMODS.NEG_HI), (v2i64 (as_i64imm (i32 0))))
+> {
+ let SubtargetPredicate = HasPackedFP64Ops;
+}
+
foreach fp16vt = [f16, bf16] in {
let True16Predicate = NotUseRealTrue16Insts in {
def : GCNPat <
diff --git a/llvm/test/CodeGen/AMDGPU/packed-fp64.ll b/llvm/test/CodeGen/AMDGPU/packed-fp64.ll
index cec24d57a3d37..fecf30d8d8f63 100644
--- a/llvm/test/CodeGen/AMDGPU/packed-fp64.ll
+++ b/llvm/test/CodeGen/AMDGPU/packed-fp64.ll
@@ -1819,3 +1819,51 @@ define amdgpu_kernel void @fma_v32_vs(ptr addrspace(1) %a, <32 x double> %x) {
store <32 x double> %fma, ptr addrspace(1) %gep, align 128
ret void
}
+
+define amdgpu_kernel void @fneg_v2f64_pkfma(ptr addrspace(1) %out) {
+; GFX1251-SDAG-LABEL: fneg_v2f64_pkfma:
+; GFX1251-SDAG: ; %bb.0: ; %entry
+; GFX1251-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1251-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX1251-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1251-SDAG-NEXT: s_mov_b32 s2, 0
+; GFX1251-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1251-SDAG-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v2, s2
+; GFX1251-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1251-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX1251-SDAG-NEXT: v_cndmask_b32_e64 v1, 0x3ff00000, 0, vcc_lo
+; GFX1251-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1251-SDAG-NEXT: v_mov_b32_e32 v3, v1
+; GFX1251-SDAG-NEXT: v_pk_fma_f64 v[0:3], v[0:3], 0, v[0:3] neg_lo:[0,0,1] neg_hi:[0,0,1]
+; GFX1251-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1251-SDAG-NEXT: global_store_b128 v4, v[0:3], s[0:1]
+; GFX1251-SDAG-NEXT: s_endpgm
+;
+; GFX1251-GISEL-LABEL: fneg_v2f64_pkfma:
+; GFX1251-GISEL: ; %bb.0: ; %entry
+; GFX1251-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1251-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX1251-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1251-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1251-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1251-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX1251-GISEL-NEXT: v_cndmask_b32_e64 v1, 0x3ff00000, 0, vcc_lo
+; GFX1251-GISEL-NEXT: v_mov_b32_e32 v4, v0
+; GFX1251-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1251-GISEL-NEXT: v_xor_b32_e32 v5, 0x80000000, v1
+; GFX1251-GISEL-NEXT: v_mov_b64_e32 v[2:3], v[0:1]
+; GFX1251-GISEL-NEXT: v_mov_b64_e32 v[6:7], v[4:5]
+; GFX1251-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1251-GISEL-NEXT: v_pk_fma_f64 v[2:5], v[0:3], 0, v[4:7]
+; GFX1251-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1251-GISEL-NEXT: global_store_b128 v0, v[2:5], s[0:1]
+; GFX1251-GISEL-NEXT: s_endpgm
+entry:
+ %tid = tail call i32 @llvm.amdgcn.workitem.id.x()
+ %cmp = icmp eq i32 %tid, 0
+ %v = select i1 %cmp, <2 x double> zeroinitializer, <2 x double> splat (double 1.000000e+00)
+ %nv = fneg <2 x double> %v
+ %r = tail call <2 x double> @llvm.fma.v2f64(<2 x double> %v, <2 x double> zeroinitializer, <2 x double> %nv)
+ store <2 x double> %r, ptr addrspace(1) %out, align 16
+ ret void
+}
More information about the llvm-commits
mailing list