[llvm] [AMDGPU] Duplicate packed fp32 instructions (PR #212857)
Stanislav Mekhanoshin via llvm-commits
llvm-commits at lists.llvm.org
Thu Jul 30 11:48:11 PDT 2026
https://github.com/rampitec updated https://github.com/llvm/llvm-project/pull/212857
>From c46282cb3f787330112ebdf477a08e78817da1a4 Mon Sep 17 00:00:00 2001
From: Stanislav Mekhanoshin <Stanislav.Mekhanoshin at amd.com>
Date: Wed, 29 Jul 2026 13:01:05 -0700
Subject: [PATCH] [AMDGPU] Duplicate packed fp32 instructions
These have different semantics on gfx9 and gfx12 with respect to
scalar operands.
---
llvm/lib/Target/AMDGPU/AMDGPU.td | 9 +++++-
.../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 8 ++---
.../AMDGPU/AMDGPUTargetTransformInfo.cpp | 14 ++++-----
.../AMDGPU/AsmParser/AMDGPUAsmParser.cpp | 2 +-
llvm/lib/Target/AMDGPU/GCNSubtarget.h | 4 +++
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 2 +-
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 8 ++---
llvm/lib/Target/AMDGPU/SIInstructions.td | 30 +++++++++++++------
llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp | 3 ++
.../Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp | 18 +++++------
llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h | 8 +++--
llvm/lib/Target/AMDGPU/VOP3PInstructions.td | 13 +++++---
.../CodeGen/AMDGPU/bug-pk-f32-imm-fold.mir | 12 ++++----
.../AMDGPU/coexec-sched-effective-stall.mir | 10 +++----
.../coexec-sched-flavor-classification.mir | 2 +-
15 files changed, 89 insertions(+), 54 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.td b/llvm/lib/Target/AMDGPU/AMDGPU.td
index 8eba93af714dc..07b5fdcd649b8 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.td
@@ -745,6 +745,10 @@ defm PackedFP32Ops : AMDGPUSubtargetFeature<"packed-fp32-ops",
"Support packed fp32 instructions"
>;
+defm PackedFP32SingleSGPROps : AMDGPUSubtargetFeature<"packed-fp32-single-sgpr-ops",
+ "Support packed fp32 instructions with SGPR operands propagating low to high 32 bit"
+>;
+
defm PackedFP64Ops : AMDGPUSubtargetFeature<"packed-fp64-ops",
"Support packed fp64 instructions"
>;
@@ -2224,7 +2228,7 @@ def FeatureISAVersion12_50_Common : FeatureSet<
FeatureLDSBankCount32,
FeatureDLInsts,
FeatureFmacF64Inst,
- FeaturePackedFP32Ops,
+ FeaturePackedFP32SingleSGPROps,
FeatureDot7Insts,
FeatureDot8Insts,
FeatureWavefrontSize32,
@@ -2943,6 +2947,9 @@ def HasSignedDotInsts : Predicate<"Subtarget->hasDot1Insts() || Subtarget->hasDo
def NotHasIEEEMinimumMaximumInsts : Predicate<"!Subtarget->hasIEEEMinimumMaximumInsts()">;
+def HasAnyPackedFP32Ops : Predicate<"Subtarget->hasAnyPackedFP32Ops()">,
+ AssemblerPredicate<(any_of FeaturePackedFP32Ops, FeaturePackedFP32SingleSGPROps)>;
+
def NotHasCvtFP8VOP1Bug : Predicate<"!Subtarget->hasCvtFP8VOP1Bug()">;
def NeedsAlignedVGPRs : Predicate<"Subtarget->needsAlignedVGPRs()">,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 9c1e49b13aaab..48dbc3325ba03 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -1004,7 +1004,7 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
FDIVActions.customFor({S16});
}
- if (ST.hasPackedFP32Ops()) {
+ if (ST.hasAnyPackedFP32Ops()) {
FPOpActions.legalFor({V2F32});
FPOpActions.clampMaxNumElementsStrict(0, F32, 2);
}
@@ -1079,9 +1079,9 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
auto &FNegAbs = getActionDefinitionsBuilder({G_FNEG, G_FABS});
FNegAbs.legalFor(FPTypesPK16)
- .legalFor(ST.hasPackedFP32Ops(), {V2S32})
+ .legalFor(ST.hasAnyPackedFP32Ops(), {V2S32})
.clampMaxNumElementsStrict(0, S16, 2);
- if (ST.hasPackedFP32Ops())
+ if (ST.hasAnyPackedFP32Ops())
FNegAbs.clampMaxNumElementsStrict(0, S32, 2);
FNegAbs.scalarize(0).clampScalar(0, S16, S64);
@@ -1182,7 +1182,7 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
.lowerFor({F64, F16, V2F16});
}
- if (ST.hasPackedFP32Ops())
+ if (ST.hasAnyPackedFP32Ops())
FSubActions.lowerFor({V2F32}).clampMaxNumElements(0, F32, 2);
FSubActions.clampMaxNumElements(0, F16, 2).scalarize(0).clampScalar(0, F32,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index 2d6ab7ec65f4b..5fc7ef1b8bac2 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -317,8 +317,8 @@ GCNTTIImpl::getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const {
case TargetTransformInfo::RGK_FixedWidthVector:
return TypeSize::getFixed((ST->hasPackedFP64Ops() || ST->hasPackedU64Ops())
? 128
- : ST->hasPackedFP32Ops() ? 64
- : 32);
+ : ST->hasAnyPackedFP32Ops() ? 64
+ : 32);
case TargetTransformInfo::RGK_ScalableVector:
return TypeSize::getScalable(0);
}
@@ -334,9 +334,9 @@ unsigned GCNTTIImpl::getMaximumVF(unsigned ElemWidth, unsigned Opcode) const {
return 32 * 4 / ElemWidth;
// For a given width return the max 0number of elements that can be combined
// into a wider bit value:
- return (ElemWidth == 8 && ST->has16BitInsts()) ? 4
- : (ElemWidth == 16 && ST->has16BitInsts()) ? 2
- : (ElemWidth == 32 && ST->hasPackedFP32Ops()) ? 2
+ return (ElemWidth == 8 && ST->has16BitInsts()) ? 4
+ : (ElemWidth == 16 && ST->has16BitInsts()) ? 2
+ : (ElemWidth == 32 && ST->hasAnyPackedFP32Ops()) ? 2
: (ElemWidth == 64 &&
(ST->hasPackedFP64Ops() || ST->hasPackedU64Ops()))
? 2
@@ -605,7 +605,7 @@ InstructionCost GCNTTIImpl::getArithmeticInstrCost(
[[fallthrough]];
case ISD::FADD:
case ISD::FSUB:
- if (ST->hasPackedFP32Ops() && SLT == MVT::f32)
+ if (ST->hasAnyPackedFP32Ops() && SLT == MVT::f32)
NElts = (NElts + 1) / 2;
if (ST->hasBF16PackedInsts() && SLT == MVT::bf16)
NElts = (NElts + 1) / 2;
@@ -880,7 +880,7 @@ GCNTTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
(ST->hasPackedU64Ops() && SLT == MVT::i64)) {
NElts = (NElts + 1) / 2;
} else if (SLT == MVT::f32) {
- bool HasPk2FP32Op = ST->hasPackedFP32Ops() &&
+ bool HasPk2FP32Op = ST->hasAnyPackedFP32Ops() &&
IID != Intrinsic::minimumnum &&
IID != Intrinsic::maximumnum;
NElts = HasPk2FP32Op ? (NElts + 1) / 2 : NElts;
diff --git a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
index 27139491c2d26..365152f461ad6 100644
--- a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
+++ b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
@@ -4919,7 +4919,7 @@ bool AMDGPUAsmParser::validateOpSel(const MCInst &Inst) {
// Packed math FP32 instructions typically accept SGPRs or VGPRs as source
// operands. On gfx12+, if a source operand uses SGPRs, the HW can only read
// the first SGPR and use it for both the low and high operations.
- if (isPackedFP32Inst(Opc) && isGFX12Plus()) {
+ if (isPackedSingleSGPRFP32Inst(Opc)) {
int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1);
int OpSelIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::op_sel);
diff --git a/llvm/lib/Target/AMDGPU/GCNSubtarget.h b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
index 78ffad6875057..03638802ca6d0 100644
--- a/llvm/lib/Target/AMDGPU/GCNSubtarget.h
+++ b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
@@ -749,6 +749,10 @@ class GCNSubtarget final : public AMDGPUGenSubtargetInfo,
bool hasSubClampInsts() const { return hasGFX10_3Insts(); }
+ bool hasAnyPackedFP32Ops() const {
+ return hasPackedFP32Ops() || hasPackedFP32SingleSGPROps();
+ };
+
/// \returns SGPR allocation granularity supported by the subtarget.
unsigned getSGPRAllocGranule() const {
return AMDGPU::getSGPRAllocGranule(getTargetID().getGPUKind());
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 5091340e2c894..1d01419549168 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -908,7 +908,7 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
VT, Custom);
}
- if (Subtarget->hasPackedFP32Ops()) {
+ if (Subtarget->hasAnyPackedFP32Ops()) {
setOperationAction({ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FNEG},
MVT::v2f32, Legal);
setOperationAction({ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FNEG},
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 6eb927ae5cf07..2a4ab971b245b 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -5927,7 +5927,7 @@ bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
}
// See SIInstrInfo::isLegalSingleSGPRReadInstOperand for more information.
- if (AMDGPU::isGFX12Plus(ST) && AMDGPU::isSingleSGPRReadInst(Opcode)) {
+ if (AMDGPU::isSingleSGPRReadInst(Opcode)) {
for (unsigned I = 0; I < 3; ++I) {
if (!isLegalSingleSGPRReadInstOperand(MRI, MI, I))
return false;
@@ -6286,7 +6286,7 @@ void SIInstrInfo::legalizeOpWithMove(MachineInstr &MI, unsigned OpIdx) const {
Register Reg = MRI.createVirtualRegister(VRC);
DebugLoc DL = MBB->findDebugLoc(I);
- if (Size == 128 && AMDGPU::isPacked64BitInst(MI.getOpcode()) &&
+ if (Size == 128 && AMDGPU::isPackedSingleSGPR64BitInst(MI.getOpcode()) &&
isLegalSingleSGPRReadInstOperand(MRI, MI, VOP3OpIdxToSrcN(MI, OpIdx))) {
// Special case for V_PK_*64 instructions: these do not have OPSEL but SGPR
// sources behave like OPSEL is set replicating low 64-bits into high. VGPR
@@ -6390,7 +6390,7 @@ bool SIInstrInfo::isLegalRegOperand(const MachineInstr &MI, unsigned OpIdx,
unsigned Opc = MI.getOpcode();
// See SIInstrInfo::isLegalSingleSGPRReadInstOperand for more information.
- if (AMDGPU::isGFX12Plus(ST) && MO.isReg() && RI.isSGPRReg(MRI, MO.getReg()) &&
+ if (MO.isReg() && RI.isSGPRReg(MRI, MO.getReg()) &&
AMDGPU::isSingleSGPRReadInst(MI.getOpcode()) &&
!isLegalSingleSGPRReadInstOperand(MRI, MI, VOP3OpIdxToSrcN(MI, OpIdx),
&MO))
@@ -6882,7 +6882,7 @@ void SIInstrInfo::legalizeOperandsVOP3(MachineRegisterInfo &MRI,
// Fix the register class of single-sgpr-read instructions on gfx12+. See
// SIInstrInfo::isLegalSingleSGPRReadInstOperand for more information.
- if (AMDGPU::isGFX12Plus(ST) && AMDGPU::isSingleSGPRReadInst(Opc)) {
+ if (AMDGPU::isSingleSGPRReadInst(Opc)) {
for (unsigned I = 0; I < 3; ++I) {
if (!isLegalSingleSGPRReadInstOperand(MRI, MI, /*SrcN=*/I))
legalizeOpWithMove(MI, VOP3Idx[I]);
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index d42e123e61eba..0153b24cda286 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -2397,7 +2397,7 @@ def : GCNPat <
(V_XOR_B32_e64 (i32 (EXTRACT_SUBREG VReg_64:$src, sub1)),
(i32 (S_MOV_B32 (i32 0x80000000)))), sub1)
> {
- let SubtargetPredicate = HasPackedFP32Ops;
+ let SubtargetPredicate = HasAnyPackedFP32Ops;
}
def : GCNPat <
@@ -3819,17 +3819,29 @@ def : GCNPat<
>;
let SubtargetPredicate = HasPackedFP32Ops in {
-def : GCNPat<
- (fcanonicalize (v2f32 (VOP3PMods v2f32:$src, i32:$src_mods))),
- (V_PK_MUL_F32 0, (i64 CONST.FP32_ONE), $src_mods, $src)
->;
+ def : GCNPat<
+ (fcanonicalize (v2f32 (VOP3PMods v2f32:$src, i32:$src_mods))),
+ (V_PK_MUL_F32 0, (i64 CONST.FP32_ONE), $src_mods, $src)
+ >;
-def : GCNPat<
- (fcanonicalize (v2f32 (fneg (VOP3PMods v2f32:$src, i32:$src_mods)))),
- (V_PK_MUL_F32 0, (i64 CONST.FP32_NEG_ONE), $src_mods, $src)
->;
+ def : GCNPat<
+ (fcanonicalize (v2f32 (fneg (VOP3PMods v2f32:$src, i32:$src_mods)))),
+ (V_PK_MUL_F32 0, (i64 CONST.FP32_NEG_ONE), $src_mods, $src)
+ >;
} // End SubtargetPredicate = HasPackedFP32Ops
+let SubtargetPredicate = HasPackedFP32SingleSGPROps in {
+ def : GCNPat<
+ (fcanonicalize (v2f32 (VOP3PMods v2f32:$src, i32:$src_mods))),
+ (V_PK_MUL_F32_gfx1250 0, (i64 CONST.FP32_ONE), $src_mods, $src)
+ >;
+
+ def : GCNPat<
+ (fcanonicalize (v2f32 (fneg (VOP3PMods v2f32:$src, i32:$src_mods)))),
+ (V_PK_MUL_F32_gfx1250 0, (i64 CONST.FP32_NEG_ONE), $src_mods, $src)
+ >;
+} // End SubtargetPredicate = HasPackedFP32SingleSGPROps
+
// TODO: Handle fneg like other types.
let SubtargetPredicate = isNotGFX12Plus in {
def : GCNPat<
diff --git a/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp b/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
index a496c9a4daa71..e43e753e1314d 100644
--- a/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
+++ b/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
@@ -576,10 +576,13 @@ uint32_t SIPreEmitPeephole::mapToUnpackedOpcode(MachineInstr &I) {
// e32 instructions don't allow source modifiers.
switch (Opcode) {
case AMDGPU::V_PK_ADD_F32:
+ case AMDGPU::V_PK_ADD_F32_gfx1250:
return AMDGPU::V_ADD_F32_e64;
case AMDGPU::V_PK_MUL_F32:
+ case AMDGPU::V_PK_MUL_F32_gfx1250:
return AMDGPU::V_MUL_F32_e64;
case AMDGPU::V_PK_FMA_F32:
+ case AMDGPU::V_PK_FMA_F32_gfx1250:
return AMDGPU::V_FMA_F32_e64;
default:
return std::numeric_limits<uint32_t>::max();
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
index cf1fa87d64f8e..38329d6295538 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
@@ -3732,21 +3732,21 @@ unsigned getLdsDwGranularity(const MCSubtargetInfo &ST) {
return 64; // In sync with getAddressableLocalMemorySize
}
-bool isPackedFP32Inst(unsigned Opc) {
+bool isPackedSingleSGPRFP32Inst(unsigned Opc) {
switch (Opc) {
- case AMDGPU::V_PK_ADD_F32:
- case AMDGPU::V_PK_ADD_F32_gfx12:
- case AMDGPU::V_PK_MUL_F32:
- case AMDGPU::V_PK_MUL_F32_gfx12:
- case AMDGPU::V_PK_FMA_F32:
- case AMDGPU::V_PK_FMA_F32_gfx12:
+ case AMDGPU::V_PK_ADD_F32_gfx1250:
+ case AMDGPU::V_PK_ADD_F32_gfx1250_gfx12:
+ case AMDGPU::V_PK_MUL_F32_gfx1250:
+ case AMDGPU::V_PK_MUL_F32_gfx1250_gfx12:
+ case AMDGPU::V_PK_FMA_F32_gfx1250:
+ case AMDGPU::V_PK_FMA_F32_gfx1250_gfx12:
return true;
default:
return false;
}
}
-bool isPacked64BitInst(unsigned Opc) {
+bool isPackedSingleSGPR64BitInst(unsigned Opc) {
switch (Opc) {
case AMDGPU::V_PK_ADD_F64:
case AMDGPU::V_PK_ADD_F64_gfx1250:
@@ -3771,7 +3771,7 @@ bool isPacked64BitInst(unsigned Opc) {
}
bool isSingleSGPRReadInst(unsigned Opc) {
- return isPackedFP32Inst(Opc) || isPacked64BitInst(Opc);
+ return isPackedSingleSGPRFP32Inst(Opc) || isPackedSingleSGPR64BitInst(Opc);
}
const std::array<unsigned, 3> &ClusterDimsAttr::getDims() const {
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
index 4f435948e1b8a..44476d4d555c0 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
@@ -1688,9 +1688,13 @@ bool isArgPassedInSGPR(const Argument *Arg);
bool isArgPassedInSGPR(const CallBase *CB, unsigned ArgNo);
-LLVM_READONLY bool isPackedFP32Inst(unsigned Opc);
+/// The opcode is a packed fp32 instruction which only reads low 32 bits of
+/// a scalar operand and propagates it to high channel.
+LLVM_READONLY bool isPackedSingleSGPRFP32Inst(unsigned Opc);
-LLVM_READONLY bool isPacked64BitInst(unsigned Opc);
+/// The opcode is a packed 64-bit instruction which only reads low 64 bits of
+/// a scalar operand and propagates it to high channel.
+LLVM_READONLY bool isPackedSingleSGPR64BitInst(unsigned Opc);
/// Packed instructions that read a single SGPR for SGPR operands, except for
/// 64-bit elements which read two SGPRs.
diff --git a/llvm/lib/Target/AMDGPU/VOP3PInstructions.td b/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
index e9043985bee12..cb0e1369f94a1 100644
--- a/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
@@ -1449,7 +1449,12 @@ let isCommutable = 1, isReMaterializable = 1 in {
defm V_PK_FMA_F32 : VOP3PInst<"v_pk_fma_f32", VOP3P_Profile<VOP_V2F32_V2F32_V2F32_V2F32, VOP3_PACKED>, any_fma>;
defm V_PK_MUL_F32 : VOP3PInst<"v_pk_mul_f32", VOP3P_Profile<VOP_V2F32_V2F32_V2F32, VOP3_PACKED>, any_fmul>;
defm V_PK_ADD_F32 : VOP3PInst<"v_pk_add_f32", VOP3P_Profile<VOP_V2F32_V2F32_V2F32, VOP3_PACKED>, any_fadd>;
- } // End SubtargetPredicate = HasPackedFP32Ops
+ }
+ let SubtargetPredicate = HasPackedFP32SingleSGPROps in {
+ defm V_PK_FMA_F32_gfx1250 : VOP3PInst<"v_pk_fma_f32", VOP3P_Profile<VOP_V2F32_V2F32_V2F32_V2F32, VOP3_PACKED>, any_fma>;
+ defm V_PK_MUL_F32_gfx1250 : VOP3PInst<"v_pk_mul_f32", VOP3P_Profile<VOP_V2F32_V2F32_V2F32, VOP3_PACKED>, any_fmul>;
+ defm V_PK_ADD_F32_gfx1250 : VOP3PInst<"v_pk_add_f32", VOP3P_Profile<VOP_V2F32_V2F32_V2F32, VOP3_PACKED>, any_fadd>;
+ }
let SubtargetPredicate = HasPkMovB32, isAsCheapAsAMove = 1 in
defm V_PK_MOV_B32 : VOP3PInst<"v_pk_mov_b32", VOP3P_Profile<VOP_V2I32_V2I32_V2I32, VOP3_PACKED>>;
@@ -2740,9 +2745,9 @@ multiclass VOP3P_Realtriple_gfx1250_gfx13<bits<8> op> :
defm V_PK_MIN_NUM_F16 : VOP3P_Real_with_name_gfx12_gfx13<0x1b, "V_PK_MIN_F16", "v_pk_min_num_f16">;
defm V_PK_MAX_NUM_F16 : VOP3P_Real_with_name_gfx12_gfx13<0x1c, "V_PK_MAX_F16", "v_pk_max_num_f16">;
-defm V_PK_FMA_F32 : VOP3P_Real_gfx12<0x1f>;
-defm V_PK_MUL_F32 : VOP3P_Real_gfx12<0x28>;
-defm V_PK_ADD_F32 : VOP3P_Real_gfx12<0x29>;
+defm V_PK_FMA_F32_gfx1250 : VOP3P_Real_gfx12<0x1f>;
+defm V_PK_MUL_F32_gfx1250 : VOP3P_Real_gfx12<0x28>;
+defm V_PK_ADD_F32_gfx1250 : VOP3P_Real_gfx12<0x29>;
defm V_PK_ADD_MAX_I16 : VOP3P_Real_gfx1250<0x14>;
defm V_PK_ADD_MAX_U16 : VOP3P_Real_gfx1250<0x15>;
diff --git a/llvm/test/CodeGen/AMDGPU/bug-pk-f32-imm-fold.mir b/llvm/test/CodeGen/AMDGPU/bug-pk-f32-imm-fold.mir
index 10861bee99beb..14811bf17080f 100644
--- a/llvm/test/CodeGen/AMDGPU/bug-pk-f32-imm-fold.mir
+++ b/llvm/test/CodeGen/AMDGPU/bug-pk-f32-imm-fold.mir
@@ -12,12 +12,12 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[DEF:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF
; CHECK-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64_align2 = V_MOV_B64_PSEUDO 1065353216, implicit $exec
- ; CHECK-NEXT: [[V_PK_ADD_F32_:%[0-9]+]]:vreg_64_align2 = V_PK_ADD_F32 11, [[DEF]], 8, [[V_MOV_B]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_PK_ADD_F32_gfx1250_:%[0-9]+]]:vreg_64_align2 = V_PK_ADD_F32_gfx1250 11, [[DEF]], 8, [[V_MOV_B]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: S_ENDPGM 0
%0:vreg_64_align2 = IMPLICIT_DEF
%1:sreg_64 = S_MOV_B64 1065353216
%2:vreg_64_align2 = COPY killed %1
- %3:vreg_64_align2 = V_PK_ADD_F32 11, %0, 8, %2, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ %3:vreg_64_align2 = V_PK_ADD_F32_gfx1250 11, %0, 8, %2, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
S_ENDPGM 0
...
@@ -32,12 +32,12 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[DEF:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF
; CHECK-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64_align2 = V_MOV_B64_PSEUDO 1065353216, implicit $exec
- ; CHECK-NEXT: [[V_PK_MUL_F32_:%[0-9]+]]:vreg_64_align2 = V_PK_MUL_F32 11, [[DEF]], 8, [[V_MOV_B]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_PK_MUL_F32_gfx1250_:%[0-9]+]]:vreg_64_align2 = V_PK_MUL_F32_gfx1250 11, [[DEF]], 8, [[V_MOV_B]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: S_ENDPGM 0
%0:vreg_64_align2 = IMPLICIT_DEF
%1:sreg_64 = S_MOV_B64 1065353216
%2:vreg_64_align2 = COPY killed %1
- %3:vreg_64_align2 = V_PK_MUL_F32 11, %0, 8, %2, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ %3:vreg_64_align2 = V_PK_MUL_F32_gfx1250 11, %0, 8, %2, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
S_ENDPGM 0
...
@@ -53,12 +53,12 @@ body: |
; CHECK-NEXT: [[DEF:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF1:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF
; CHECK-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64_align2 = V_MOV_B64_PSEUDO 1065353216, implicit $exec
- ; CHECK-NEXT: [[V_PK_FMA_F32_:%[0-9]+]]:vreg_64_align2 = V_PK_FMA_F32 0, [[DEF]], 8, [[DEF1]], 11, [[V_MOV_B]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_PK_FMA_F32_gfx1250_:%[0-9]+]]:vreg_64_align2 = V_PK_FMA_F32_gfx1250 0, [[DEF]], 8, [[DEF1]], 11, [[V_MOV_B]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: S_ENDPGM 0
%0:vreg_64_align2 = IMPLICIT_DEF
%1:vreg_64_align2 = IMPLICIT_DEF
%2:sreg_64 = S_MOV_B64 1065353216
%3:vreg_64_align2 = COPY killed %2
- %4:vreg_64_align2 = V_PK_FMA_F32 0, %0, 8, %1, 11, %3, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ %4:vreg_64_align2 = V_PK_FMA_F32_gfx1250 0, %0, 8, %1, 11, %3, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
S_ENDPGM 0
...
diff --git a/llvm/test/CodeGen/AMDGPU/coexec-sched-effective-stall.mir b/llvm/test/CodeGen/AMDGPU/coexec-sched-effective-stall.mir
index d19c27e83b4d1..60ccfb53490e9 100644
--- a/llvm/test/CodeGen/AMDGPU/coexec-sched-effective-stall.mir
+++ b/llvm/test/CodeGen/AMDGPU/coexec-sched-effective-stall.mir
@@ -28,9 +28,9 @@ body: |
; DEFAULT-NEXT: [[DEF8:%[0-9]+]]:vreg_256_align2 = IMPLICIT_DEF
; DEFAULT-NEXT: [[DEF9:%[0-9]+]]:vgpr_32_lo256 = IMPLICIT_DEF
; DEFAULT-NEXT: [[DEF10:%[0-9]+]]:vgpr_32_lo256 = IMPLICIT_DEF
- ; DEFAULT-NEXT: [[V_PK_ADD_F32_:%[0-9]+]]:vreg_64_align2 = V_PK_ADD_F32 8, [[GLOBAL_LOAD_DWORDX2_]], 8, [[GLOBAL_LOAD_DWORDX2_]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; DEFAULT-NEXT: [[V_PK_ADD_F32_gfx1250_:%[0-9]+]]:vreg_64_align2 = V_PK_ADD_F32_gfx1250 8, [[GLOBAL_LOAD_DWORDX2_]], 8, [[GLOBAL_LOAD_DWORDX2_]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; DEFAULT-NEXT: early-clobber %14:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr [[DEF6]], [[DEF7]], 0, [[DEF8]], [[DEF9]], [[DEF10]], 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
- ; DEFAULT-NEXT: S_ENDPGM 0, implicit [[V_PK_ADD_F32_]], implicit %13, implicit %14
+ ; DEFAULT-NEXT: S_ENDPGM 0, implicit [[V_PK_ADD_F32_gfx1250_]], implicit %13, implicit %14
;
; COEXEC-LABEL: name: test-sched-effective-stall
; COEXEC: [[DEF:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
@@ -47,8 +47,8 @@ body: |
; COEXEC-NEXT: [[DEF10:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF
; COEXEC-NEXT: [[GLOBAL_LOAD_DWORDX2_:%[0-9]+]]:vreg_64_align2 = GLOBAL_LOAD_DWORDX2 [[DEF10]], 0, 0, implicit $exec
; COEXEC-NEXT: early-clobber %14:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr [[DEF5]], [[DEF6]], 0, [[DEF7]], [[DEF8]], [[DEF9]], 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
- ; COEXEC-NEXT: [[V_PK_ADD_F32_:%[0-9]+]]:vreg_64_align2 = V_PK_ADD_F32 8, [[GLOBAL_LOAD_DWORDX2_]], 8, [[GLOBAL_LOAD_DWORDX2_]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; COEXEC-NEXT: S_ENDPGM 0, implicit [[V_PK_ADD_F32_]], implicit %13, implicit %14
+ ; COEXEC-NEXT: [[V_PK_ADD_F32_gfx1250_:%[0-9]+]]:vreg_64_align2 = V_PK_ADD_F32_gfx1250 8, [[GLOBAL_LOAD_DWORDX2_]], 8, [[GLOBAL_LOAD_DWORDX2_]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; COEXEC-NEXT: S_ENDPGM 0, implicit [[V_PK_ADD_F32_gfx1250_]], implicit %13, implicit %14
%0:vreg_512_align2 = IMPLICIT_DEF
%1:vreg_512_align2 = IMPLICIT_DEF
%2:vreg_256_align2 = IMPLICIT_DEF
@@ -61,7 +61,7 @@ body: |
%9:vgpr_32_lo256 = IMPLICIT_DEF
%10:vreg_64_align2 = IMPLICIT_DEF
%11:vreg_64_align2 = GLOBAL_LOAD_DWORDX2 %10, 0, 0, implicit $exec
- %12:vreg_64_align2 = V_PK_ADD_F32 8, %11, 8, %11, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ %12:vreg_64_align2 = V_PK_ADD_F32_gfx1250 8, %11, 8, %11, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
%13:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr %0, %1, 0, %2, %3, %4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
%14:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr %5, %6, 0, %7, %8, %9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
S_ENDPGM 0, implicit %12, implicit %13, implicit %14
diff --git a/llvm/test/CodeGen/AMDGPU/coexec-sched-flavor-classification.mir b/llvm/test/CodeGen/AMDGPU/coexec-sched-flavor-classification.mir
index 9a0d16c142801..b776030b41dd8 100644
--- a/llvm/test/CodeGen/AMDGPU/coexec-sched-flavor-classification.mir
+++ b/llvm/test/CodeGen/AMDGPU/coexec-sched-flavor-classification.mir
@@ -49,7 +49,7 @@ body: |
TENSOR_LOAD_TO_LDS_d2 %1, %3, 0, 0, implicit-def dead $tensorcnt, implicit $exec, implicit $tensorcnt
GLOBAL_LOAD_ASYNC_TO_LDS_B32 %2, %0, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt
- %30:vreg_64_align2 = V_PK_ADD_F32 8, %11, 8, %11, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ %30:vreg_64_align2 = V_PK_ADD_F32_gfx1250 8, %11, 8, %11, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
%31:vgpr_32 = V_PERM_B32_e64 %10, %12, 84148480, implicit $exec
%32:vreg_64_align2 = V_MIN_I64_e64 %11, %13, implicit $exec
%33:vreg_64_align2 = V_CVT_SCALEF32_PK8_FP8_F32_e64 %4, %10, implicit $mode, implicit $exec
More information about the llvm-commits
mailing list