[llvm] [AMDGPU] Add pre-RA unpacking of V_PK_*_F32 ops as a scheduling stage (PR #225338)
Romanov Vlad via llvm-commits
llvm-commits at lists.llvm.org
Tue Sep 22 02:22:00 PDT 2026
https://github.com/romanovvlad updated https://github.com/llvm/llvm-project/pull/225338
>From 1d8717b7e8b19bf0dcba364bf734bbd1c28a46a4 Mon Sep 17 00:00:00 2001
From: Vlad <Vladislav.Romanov at amd.com>
Date: Tue, 22 Sep 2026 02:31:13 -0500
Subject: [PATCH] [AMDGPU] Add pre-RA unpacking of V_PK_*_F32 ops as a
scheduling stage
Add UnpackPKOps scheduling stage that splits V_PK_ADD/MUL/FMA_F32
into two independent scalar V_ADD/MUL/FMA_F32_e64 ops in regions
where VGPR pressure exceeds the limit. Each packed op producing
a vreg_64 is replaced by two scalar ops producing independent
vgpr_32 values, shortening live ranges and reducing peak register
pressure.
---
.../AMDGPU/AMDGPUCoExecSchedStrategy.cpp | 1 +
llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 251 +++++++++++++++++-
llvm/lib/Target/AMDGPU/GCNSchedStrategy.h | 17 +-
llvm/test/CodeGen/AMDGPU/pre-ra-pk-unpack.mir | 139 ++++++++++
4 files changed, 406 insertions(+), 2 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/pre-ra-pk-unpack.mir
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
index 50d444068e01d..c6566f1374493 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
@@ -1065,6 +1065,7 @@ AMDGPUCoExecSchedStrategy::AMDGPUCoExecSchedStrategy(
const MachineSchedContext *C)
: GCNSchedStrategy(C) {
SchedStages.push_back(GCNSchedStageID::ILPInitialSchedule);
+ SchedStages.push_back(GCNSchedStageID::UnpackPKOps);
SchedStages.push_back(GCNSchedStageID::RewriteMFMAForm);
SchedStages.push_back(GCNSchedStageID::PreRARematerialize);
// Use more accurate GCN pressure trackers.
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index 6a1250f672b50..510fd524ab056 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -1055,6 +1055,8 @@ GCNScheduleDAGMILive::createSchedStage(GCNSchedStageID SchedStageID) {
case GCNSchedStageID::MemoryClauseInitialSchedule:
return std::make_unique<MemoryClauseInitialScheduleStage>(SchedStageID,
*this);
+ case GCNSchedStageID::UnpackPKOps:
+ return std::make_unique<UnpackPKOpsStage>(SchedStageID, *this);
}
llvm_unreachable("Unknown SchedStageID.");
@@ -1297,6 +1299,9 @@ raw_ostream &llvm::operator<<(raw_ostream &OS, const GCNSchedStageID &StageID) {
case GCNSchedStageID::MemoryClauseInitialSchedule:
OS << "Max memory clause Initial Schedule";
break;
+ case GCNSchedStageID::UnpackPKOps:
+ OS << "Unpack PK Ops";
+ break;
}
return OS;
@@ -1377,11 +1382,255 @@ void RewriteMFMAFormStage::findReachingUses(
}
}
+bool UnpackPKOpsStage::initGCNSchedStage() {
+ unpackPKOpsForPressure();
+ return false;
+}
+
+// Split V_PK_ADD/MUL/FMA_F32 into pairs of scalar V_ADD/MUL/FMA_F32_e64 in
+// regions where VGPR pressure exceeds the architectural limit.
+//
+// A packed op like V_PK_MUL_F32 computes two f32 multiplies and writes a
+// vreg_64 result. When many such results are live simultaneously, the wide
+// live ranges inflate VGPR pressure. Splitting into two independent vgpr_32
+// results lets the register allocator schedule their lifetimes independently,
+// reducing peak pressure at the cost of more instructions.
+//
+// The transformation rewrites all users of the original vreg_64:
+// - Subreg users (.sub0/.sub1): rewritten directly to use LoReg/HiReg.
+// - Full vreg_64 users: a pair of COPYs reconstructs the vreg_64 right
+// before the user instruction.
+// - Other defs of the same vreg_64 (partial subreg redefs like
+// %dst.sub0 = COPY ...): rewritten to define LoReg/HiReg instead.
+//
+// If any other def of the destination is a full-register def (not a subreg
+// def), the split is skipped for that instruction since we cannot redirect
+// a full-reg def to two independent registers.
+bool UnpackPKOpsStage::unpackPKOpsForPressure() {
+ const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
+ const SIInstrInfo *TII = ST.getInstrInfo();
+ MachineRegisterInfo &MRI = DAG.MRI;
+ unsigned ArchVGPRLimit = ST.getAddressableNumArchVGPRs();
+ bool Changed = false;
+
+ for (unsigned Region = 0; Region < DAG.Regions.size(); Region++) {
+ // Only unpack in regions where pressure exceeds the VGPR budget.
+ if (DAG.Pressure[Region].getArchVGPRNum() <= ArchVGPRLimit)
+ continue;
+
+ auto [Begin, End] = DAG.Regions[Region];
+
+ // Collect all packed f32 ops in this region.
+ SmallVector<MachineInstr *, 16> ToUnpack;
+ for (auto I = Begin; I != End; ++I) {
+ unsigned Opc = I->getOpcode();
+ if (Opc != AMDGPU::V_PK_ADD_F32 && Opc != AMDGPU::V_PK_MUL_F32 &&
+ Opc != AMDGPU::V_PK_FMA_F32)
+ continue;
+ MachineOperand &Dst = I->getOperand(0);
+ if (!Dst.getReg().isVirtual())
+ continue;
+ ToUnpack.push_back(&*I);
+ }
+
+ for (MachineInstr *MI : ToUnpack) {
+ unsigned Opc = MI->getOpcode();
+ unsigned ScalarOpc;
+ if (Opc == AMDGPU::V_PK_ADD_F32)
+ ScalarOpc = AMDGPU::V_ADD_F32_e64;
+ else if (Opc == AMDGPU::V_PK_MUL_F32)
+ ScalarOpc = AMDGPU::V_MUL_F32_e64;
+ else
+ ScalarOpc = AMDGPU::V_FMA_F32_e64;
+
+ Register DstReg = MI->getOperand(0).getReg();
+ const TargetRegisterClass *DstRC = MRI.getRegClass(DstReg);
+ // Only split vreg_64 results — wider register classes would need
+ // more complex handling.
+ if (TII->getRegisterInfo().getRegSizeInBits(*DstRC) != 64)
+ continue;
+
+ MachineOperand *Src0 = TII->getNamedOperand(*MI, AMDGPU::OpName::src0);
+ MachineOperand *Src1 = TII->getNamedOperand(*MI, AMDGPU::OpName::src1);
+ MachineOperand *Src0Mods =
+ TII->getNamedOperand(*MI, AMDGPU::OpName::src0_modifiers);
+ MachineOperand *Src1Mods =
+ TII->getNamedOperand(*MI, AMDGPU::OpName::src1_modifiers);
+ MachineOperand *Clamp = TII->getNamedOperand(*MI, AMDGPU::OpName::clamp);
+ if (!Src0 || !Src1 || !Src0Mods || !Src1Mods || !Clamp)
+ continue;
+
+ // Check that all other defs of DstReg are subreg defs (sub0/sub1).
+ // A full-register def elsewhere would need to write both LoReg and
+ // HiReg simultaneously, which we can't express after splitting.
+ bool CanSplit = true;
+ SmallVector<MachineOperand *, 8> OtherDefs;
+ for (MachineOperand &DefMO : MRI.def_operands(DstReg)) {
+ if (DefMO.getParent() == MI)
+ continue;
+ unsigned SubIdx = DefMO.getSubReg();
+ if (SubIdx != AMDGPU::sub0 && SubIdx != AMDGPU::sub1) {
+ CanSplit = false;
+ break;
+ }
+ OtherDefs.push_back(&DefMO);
+ }
+ if (!CanSplit) {
+ LLVM_DEBUG(dbgs() << "UnpackPK: skip (non-subreg other def) " << *MI);
+ continue;
+ }
+
+ MachineBasicBlock &MBB = *MI->getParent();
+ DebugLoc DL = MI->getDebugLoc();
+
+ // Create two independent vgpr_32 destinations — one for each lane
+ // of the original packed result.
+ Register LoReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
+ Register HiReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
+
+ // Build one scalar op for either the lo (sub0) or hi (sub1) lane.
+ // Translates packed source modifiers (NEG/NEG_HI, OP_SEL) into the
+ // corresponding scalar modifier for the selected lane. For virtual
+ // register sources, adds a subreg index to select the right 32-bit
+ // half of the 64-bit source.
+ auto BuildScalar = [&](Register OutReg, bool IsHi) {
+ unsigned S0Mods = Src0Mods->getImm();
+ unsigned S1Mods = Src1Mods->getImm();
+ // In packed encoding, NEG applies to lo elements, NEG_HI to hi.
+ unsigned S0Neg = IsHi ? SISrcMods::NEG_HI : SISrcMods::NEG;
+ unsigned S1Neg = IsHi ? SISrcMods::NEG_HI : SISrcMods::NEG;
+ // OP_SEL selects which 32-bit half of the source to use.
+ unsigned OpSel = IsHi ? SISrcMods::OP_SEL_1 : SISrcMods::OP_SEL_0;
+
+ unsigned NewS0Mods = (S0Mods & S0Neg) ? (unsigned)SISrcMods::NEG : 0u;
+ unsigned NewS1Mods = (S1Mods & S1Neg) ? (unsigned)SISrcMods::NEG : 0u;
+
+ // For virtual register sources, extract the right 32-bit half via
+ // subreg. If the source already has a subreg (e.g., %r.sub6_sub7 from
+ // a vreg_512), compose it with the lane subreg to get the correct
+ // element (e.g., sub6_sub7 + sub0 = sub6).
+ // For immediates and physical regs, use the operand as-is.
+ const SIRegisterInfo &TRI = TII->getRegisterInfo();
+ auto GetSrcOp = [&](const MachineOperand &MO,
+ unsigned Mods) -> MachineOperand {
+ if (MO.isImm())
+ return MachineOperand::CreateImm(MO.getImm());
+ Register Reg = MO.getReg();
+ if (Reg.isVirtual()) {
+ unsigned LaneSubIdx = (Mods & OpSel) ? AMDGPU::sub1 : AMDGPU::sub0;
+ unsigned OrigSubIdx = MO.getSubReg();
+ unsigned FinalSubIdx =
+ OrigSubIdx ? TRI.composeSubRegIndices(OrigSubIdx, LaneSubIdx)
+ : LaneSubIdx;
+ return MachineOperand::CreateReg(Reg, false, false, false, false,
+ false, false, FinalSubIdx);
+ }
+ return MO;
+ };
+
+ MachineInstrBuilder NewMI =
+ BuildMI(MBB, MI, DL, TII->get(ScalarOpc), OutReg);
+ NewMI.addImm(NewS0Mods);
+ NewMI.add(GetSrcOp(*Src0, S0Mods));
+ NewMI.addImm(NewS1Mods);
+ NewMI.add(GetSrcOp(*Src1, S1Mods));
+
+ // V_PK_FMA_F32 has a third source operand.
+ if (Opc == AMDGPU::V_PK_FMA_F32) {
+ MachineOperand *Src2 =
+ TII->getNamedOperand(*MI, AMDGPU::OpName::src2);
+ MachineOperand *Src2Mods =
+ TII->getNamedOperand(*MI, AMDGPU::OpName::src2_modifiers);
+ unsigned S2Neg = IsHi ? SISrcMods::NEG_HI : SISrcMods::NEG;
+ unsigned NewS2Mods =
+ (Src2Mods->getImm() & S2Neg) ? (unsigned)SISrcMods::NEG : 0u;
+ NewMI.addImm(NewS2Mods);
+ NewMI.add(GetSrcOp(*Src2, Src2Mods->getImm()));
+ }
+
+ NewMI.addImm(Clamp->getImm());
+ NewMI.addImm(0);
+ NewMI->setFlags(MI->getFlags());
+ };
+
+ BuildScalar(LoReg, /*IsHi=*/false);
+ BuildScalar(HiReg, /*IsHi=*/true);
+
+ // Redirect partial subreg redefs of the original vreg_64 to the
+ // corresponding split register (sub0 → LoReg, sub1 → HiReg).
+ for (MachineOperand *DefMO : OtherDefs) {
+ if (DefMO->getSubReg() == AMDGPU::sub0) {
+ DefMO->setReg(LoReg);
+ DefMO->setSubReg(0);
+ DefMO->setIsUndef(false);
+ } else {
+ DefMO->setReg(HiReg);
+ DefMO->setSubReg(0);
+ DefMO->setIsUndef(false);
+ }
+ }
+
+ // Collect all uses before rewriting — iterating while modifying the
+ // use list is not safe.
+ SmallVector<MachineOperand *, 16> AllUses;
+ for (MachineOperand &UseMO : MRI.use_operands(DstReg))
+ if (UseMO.getParent() != MI)
+ AllUses.push_back(&UseMO);
+
+ for (MachineOperand *UseMO : AllUses) {
+ unsigned SubIdx = UseMO->getSubReg();
+ if (SubIdx == AMDGPU::sub0) {
+ // Subreg use of the lo half — directly use LoReg.
+ UseMO->setReg(LoReg);
+ UseMO->setSubReg(0);
+ } else if (SubIdx == AMDGPU::sub1) {
+ // Subreg use of the hi half — directly use HiReg.
+ UseMO->setReg(HiReg);
+ UseMO->setSubReg(0);
+ } else {
+ // Full vreg_64 use — reconstruct the pair right before this user
+ // via two subreg COPYs into a fresh vreg_64.
+ MachineInstr *UserMI = UseMO->getParent();
+ MachineBasicBlock &UserMBB = *UserMI->getParent();
+ Register NewFull = MRI.createVirtualRegister(MRI.getRegClass(DstReg));
+ auto &LoCopy = *BuildMI(UserMBB, UserMI, UserMI->getDebugLoc(),
+ TII->get(TargetOpcode::COPY))
+ .addReg(NewFull, RegState::Define, AMDGPU::sub0)
+ .addReg(LoReg);
+ LoCopy.getOperand(0).setIsUndef(true);
+ BuildMI(UserMBB, UserMI, UserMI->getDebugLoc(),
+ TII->get(TargetOpcode::COPY))
+ .addReg(NewFull, RegState::Define, AMDGPU::sub1)
+ .addReg(HiReg);
+ UseMO->setReg(NewFull);
+ }
+ }
+
+ LLVM_DEBUG(dbgs() << "UnpackPK: split " << *MI);
+ MI->eraseFromParent();
+ Changed = true;
+ }
+ }
+
+ if (Changed) {
+ // Rebuild slot indexes and live intervals since we inserted/removed
+ // instructions. Then recompute region pressures with the new schedule.
+ DAG.getLIS()->getSlotIndexes()->reanalyze(DAG.MF);
+ DAG.getLIS()->reanalyze(DAG.MF);
+ for (unsigned I = 0; I < DAG.Regions.size(); ++I)
+ DAG.Pressure[I] = DAG.getRealRegPressure(I);
+ LLVM_DEBUG(dbgs() << "UnpackPK: done, reanalyzed\n");
+ }
+
+ return Changed;
+}
+
bool RewriteMFMAFormStage::initGCNSchedStage() {
+ const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
+
// We only need to run this pass if the architecture supports AGPRs.
// Additionally, we don't use AGPRs at occupancy levels above 1 so there
// is no need for this pass in that case, either.
- const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
if (!ST.hasGFX90AInsts() || MFI.getMinWavesPerEU() > 1)
return false;
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
index 2b70d03103909..efe4b0a507164 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
@@ -35,7 +35,8 @@ enum class GCNSchedStageID : unsigned {
ClusteredLowOccupancyReschedule = 3,
PreRARematerialize = 4,
ILPInitialSchedule = 5,
- MemoryClauseInitialSchedule = 6
+ MemoryClauseInitialSchedule = 6,
+ UnpackPKOps = 7
};
#ifndef NDEBUG
@@ -266,6 +267,7 @@ class GCNScheduleDAGMILive final : public ScheduleDAGMILive {
friend class ClusteredLowOccStage;
friend class PreRARematStage;
friend class ILPInitialScheduleStage;
+ friend class UnpackPKOpsStage;
friend class RegionPressureMap;
const GCNSubtarget &ST;
@@ -794,6 +796,19 @@ class MemoryClauseInitialScheduleStage : public GCNSchedStage {
: GCNSchedStage(StageID, DAG) {}
};
+/// Unpack V_PK_ADD/MUL/FMA_F32 into scalar ops in regions where VGPR
+/// pressure exceeds the architectural limit, reducing peak register usage.
+class UnpackPKOpsStage : public GCNSchedStage {
+public:
+ bool initGCNSchedStage() override;
+
+ UnpackPKOpsStage(GCNSchedStageID StageID, GCNScheduleDAGMILive &DAG)
+ : GCNSchedStage(StageID, DAG) {}
+
+private:
+ bool unpackPKOpsForPressure();
+};
+
class GCNPostScheduleDAGMILive final : public ScheduleDAGMI {
private:
std::vector<std::unique_ptr<ScheduleDAGMutation>> SavedMutations;
diff --git a/llvm/test/CodeGen/AMDGPU/pre-ra-pk-unpack.mir b/llvm/test/CodeGen/AMDGPU/pre-ra-pk-unpack.mir
new file mode 100644
index 0000000000000..37b25580388e8
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/pre-ra-pk-unpack.mir
@@ -0,0 +1,139 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=amdgpu9.50-amd-amdhsa -run-pass=machine-scheduler \
+# RUN: -amdgpu-sched-strategy=coexec %s -o - | FileCheck %s
+
+--- |
+ define void @pk_unpack_high_pressure() #0 {
+ entry:
+ unreachable
+ }
+
+ define void @pk_unpack_low_pressure() #0 {
+ entry:
+ unreachable
+ }
+
+ attributes #0 = { "amdgpu-waves-per-eu"="1,1" "amdgpu-flat-work-group-size"="64,64" }
+...
+---
+# High VGPR pressure region (>256): pk ops should be unpacked into scalar ops.
+name: pk_unpack_high_pressure
+tracksRegLiveness: true
+machineFunctionInfo:
+ isEntryFunction: true
+body: |
+ bb.0:
+ ; Create high VGPR pressure with large IMPLICIT_DEFs kept alive.
+ ; CHECK-LABEL: name: pk_unpack_high_pressure
+ ; CHECK: [[DEF:%[0-9]+]]:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF1:%[0-9]+]]:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF2:%[0-9]+]]:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF3:%[0-9]+]]:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF4:%[0-9]+]]:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF5:%[0-9]+]]:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF6:%[0-9]+]]:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF7:%[0-9]+]]:vreg_1024 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF8:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF9:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF10:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF11:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[V_MUL_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_MUL_F32_e64 0, [[DEF8]].sub0, 0, [[DEF11]].sub6, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MUL_F32_e64_1:%[0-9]+]]:vgpr_32 = nofpexcept V_MUL_F32_e64 0, [[DEF8]].sub1, 0, [[DEF11]].sub7, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY [[V_MUL_F32_e64_]]
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[V_MUL_F32_e64_1]]
+ ; CHECK-NEXT: [[V_MUL_F32_e64_2:%[0-9]+]]:vgpr_32 = nofpexcept V_MUL_F32_e64 0, [[DEF8]].sub0, 0, [[DEF9]].sub0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MUL_F32_e64_3:%[0-9]+]]:vgpr_32 = nofpexcept V_MUL_F32_e64 0, [[DEF8]].sub1, 0, [[DEF9]].sub1, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_MUL_F32_e64_2]]
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[V_MUL_F32_e64_3]]
+ ; CHECK-NEXT: [[V_ADD_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[DEF8]].sub0, 0, [[DEF10]].sub0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_ADD_F32_e64_1:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[DEF8]].sub1, 0, [[DEF10]].sub1, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[V_ADD_F32_e64_]]
+ ; CHECK-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[V_ADD_F32_e64_1]]
+ ; CHECK-NEXT: [[V_FMA_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_FMA_F32_e64 0, [[DEF8]].sub0, 0, [[DEF9]].sub0, 0, [[DEF10]].sub0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_FMA_F32_e64_1:%[0-9]+]]:vgpr_32 = nofpexcept V_FMA_F32_e64 0, [[DEF8]].sub1, 0, [[DEF9]].sub1, 0, [[DEF10]].sub1, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MUL_F32_e64_4:%[0-9]+]]:vgpr_32 = nofpexcept V_MUL_F32_e64 0, [[DEF9]].sub0, 0, [[DEF10]].sub0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MUL_F32_e64_5:%[0-9]+]]:vgpr_32 = nofpexcept V_MUL_F32_e64 0, [[DEF9]].sub1, 0, [[DEF10]].sub1, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[V_FMA_F32_e64_]]
+ ; CHECK-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[V_FMA_F32_e64_1]]
+ ; CHECK-NEXT: undef [[COPY8:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MUL_F32_e64_4]]
+ ; CHECK-NEXT: [[COPY8:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MUL_F32_e64_5]]
+ ; CHECK-NEXT: KILL [[DEF]], [[DEF1]], [[DEF2]], [[DEF3]], [[DEF4]], [[DEF5]], [[DEF6]], [[DEF7]], [[COPY2]], [[COPY3]], [[COPY4]], [[COPY5]], [[COPY6]], [[COPY7]], [[COPY8]], [[COPY]], [[COPY1]]
+ ; CHECK-NEXT: S_ENDPGM 0
+ %0:vreg_1024 = IMPLICIT_DEF
+ %1:vreg_1024 = IMPLICIT_DEF
+ %2:vreg_1024 = IMPLICIT_DEF
+ %3:vreg_1024 = IMPLICIT_DEF
+ %4:vreg_1024 = IMPLICIT_DEF
+ %5:vreg_1024 = IMPLICIT_DEF
+ %6:vreg_1024 = IMPLICIT_DEF
+ %7:vreg_1024 = IMPLICIT_DEF
+
+ ; Sources for pk ops.
+ %10:vreg_64_align2 = IMPLICIT_DEF
+ %11:vreg_64_align2 = IMPLICIT_DEF
+ %12:vreg_64_align2 = IMPLICIT_DEF
+
+ ; V_PK_MUL_F32 — should be split into two V_MUL_F32_e64.
+ %20:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, %10, 8, %11, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+
+ ; V_PK_ADD_F32 — should be split into two V_ADD_F32_e64.
+ %21:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, %10, 8, %12, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+
+ ; V_PK_FMA_F32 — should be split into two V_FMA_F32_e64.
+ %22:vreg_64_align2 = nofpexcept V_PK_FMA_F32 8, %10, 8, %11, 8, %12, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+
+ ; V_PK_MUL_F32 with a full vreg_64 user — reconstruction COPYs expected.
+ %23:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, %11, 8, %12, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+
+ ; V_PK_MUL_F32 with subreg source — tests subreg composition.
+ ; %13 is vreg_512; .sub6_sub7 selects a 64-bit pair. After unpacking,
+ ; the lo scalar op should read %13.sub6, the hi should read %13.sub7.
+ %13:vreg_512_align2 = IMPLICIT_DEF
+ %24:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, %10, 8, %13.sub6_sub7, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+
+ ; Subreg uses of the unpacked results.
+ %30:vgpr_32 = COPY %20.sub0
+ %31:vgpr_32 = COPY %20.sub1
+ %32:vgpr_32 = COPY %21.sub0
+ %33:vgpr_32 = COPY %21.sub1
+ %34:vgpr_32 = COPY %22.sub0
+ %35:vgpr_32 = COPY %22.sub1
+ %36:vgpr_32 = COPY %24.sub0
+ %37:vgpr_32 = COPY %24.sub1
+
+ KILL %0, %1, %2, %3, %4, %5, %6, %7, %30, %31, %32, %33, %34, %35, %23, %36, %37
+
+ S_ENDPGM 0
+...
+---
+# Low VGPR pressure region (<256): pk ops should be preserved.
+name: pk_unpack_low_pressure
+tracksRegLiveness: true
+machineFunctionInfo:
+ isEntryFunction: true
+body: |
+ bb.0:
+ ; Low pressure: only a few small regs.
+ ; CHECK-LABEL: name: pk_unpack_low_pressure
+ ; CHECK: [[DEF:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF1:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF2:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[V_PK_MUL_F32_:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, [[DEF]], 8, [[DEF1]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY [[V_PK_MUL_F32_]].sub0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[V_PK_MUL_F32_]].sub1
+ ; CHECK-NEXT: KILL [[DEF2]], [[COPY]], [[COPY1]]
+ ; CHECK-NEXT: S_ENDPGM 0
+ %0:vreg_64_align2 = IMPLICIT_DEF
+ %1:vreg_64_align2 = IMPLICIT_DEF
+ %2:vreg_64_align2 = IMPLICIT_DEF
+
+ ; V_PK_MUL_F32 — should NOT be unpacked (pressure below limit).
+ %10:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, %0, 8, %1, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+
+ %20:vgpr_32 = COPY %10.sub0
+ %21:vgpr_32 = COPY %10.sub1
+
+ KILL %2, %20, %21
+
+ S_ENDPGM 0
+...
More information about the llvm-commits
mailing list