[llvm] [AMDGPU] Unpack V_PK_MOV_B32 for MFMA co-issue (PR #211937)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Sep 25 07:55:47 PDT 2026
https://github.com/robertvirany updated https://github.com/llvm/llvm-project/pull/211937
>From 44c60d25d8278e71a0fc722d530589ba33140d5a Mon Sep 17 00:00:00 2001
From: Robert Virany <robertvirany at gmail.com>
Date: Tue, 21 Jul 2026 15:13:54 -0600
Subject: [PATCH 1/3] [AMDGPU] Precommit V_PK_MOV_B32 unpacking test
---
...ck-non-coissue-insts-post-ra-scheduler.mir | 52 +++++++++++++++++++
1 file changed, 52 insertions(+)
diff --git a/llvm/test/CodeGen/AMDGPU/unpack-non-coissue-insts-post-ra-scheduler.mir b/llvm/test/CodeGen/AMDGPU/unpack-non-coissue-insts-post-ra-scheduler.mir
index 5685df559b082..87fa8f0eda151 100644
--- a/llvm/test/CodeGen/AMDGPU/unpack-non-coissue-insts-post-ra-scheduler.mir
+++ b/llvm/test/CodeGen/AMDGPU/unpack-non-coissue-insts-post-ra-scheduler.mir
@@ -438,6 +438,58 @@ body: |
renamable $vgpr10_vgpr11 = nofpexcept V_PK_ADD_F32 8, killed $sgpr10_sgpr11, 8, $vgpr0_vgpr1, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
S_ENDPGM 0
+...
+---
+name: test_pk_mov_unpacking_b32
+tracksRegLiveness: true
+
+liveins:
+ - { reg: '$sgpr4_sgpr5' }
+
+body: |
+ bb.0.entry:
+ liveins: $sgpr4_sgpr5
+ ; GFX950-LABEL: name: test_pk_mov_unpacking_b32
+ ; GFX950: liveins: $sgpr4_sgpr5
+ ; GFX950-NEXT: {{ $}}
+ ; GFX950-NEXT: early-clobber renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43 = S_LOAD_DWORDX8_IMM_ec killed renamable $sgpr4_sgpr5, 0, 0
+ ; GFX950-NEXT: renamable $vgpr18 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX950-NEXT: S_WAITCNT .Lgkmcnt_0
+ ; GFX950-NEXT: renamable $sgpr44_sgpr45_sgpr46_sgpr47 = S_LOAD_DWORDX4_IMM renamable $sgpr40_sgpr41, 0, 0
+ ; GFX950-NEXT: renamable $sgpr48_sgpr49_sgpr50_sgpr51 = S_LOAD_DWORDX4_IMM renamable $sgpr42_sgpr43, 0, 0
+ ; GFX950-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15 = S_LOAD_DWORDX16_IMM_ec killed renamable $sgpr42_sgpr43, 0, 0
+ ; GFX950-NEXT: early-clobber renamable $sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31 = S_LOAD_DWORDX16_IMM_ec killed renamable $sgpr40_sgpr41, 0, 0
+ ; GFX950-NEXT: S_WAITCNT .Lgkmcnt_0
+ ; GFX950-NEXT: $vgpr0_vgpr1 = V_MOV_B64_e32 $sgpr44_sgpr45, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3, implicit $sgpr44_sgpr45_sgpr46_sgpr47
+ ; GFX950-NEXT: $vgpr2_vgpr3 = V_MOV_B64_e32 killed $sgpr46_sgpr47, implicit $exec, implicit $sgpr44_sgpr45_sgpr46_sgpr47, implicit $exec
+ ; GFX950-NEXT: $vgpr4_vgpr5 = V_MOV_B64_e32 $sgpr48_sgpr49, implicit $exec, implicit-def $vgpr4_vgpr5_vgpr6_vgpr7, implicit $sgpr48_sgpr49_sgpr50_sgpr51
+ ; GFX950-NEXT: $vgpr6_vgpr7 = V_MOV_B64_e32 killed $sgpr50_sgpr51, implicit $exec, implicit $sgpr48_sgpr49_sgpr50_sgpr51, implicit $exec
+ ; GFX950-NEXT: early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X16_F16_e64 killed $vgpr0_vgpr1_vgpr2_vgpr3, killed $vgpr4_vgpr5_vgpr6_vgpr7, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NEXT: $vgpr4 = V_MOV_B32_e32 killed $sgpr14, implicit $exec, implicit $exec
+ ; GFX950-NEXT: $vgpr5 = V_MOV_B32_e32 killed $sgpr15, implicit $exec, implicit $exec
+ ; GFX950-NEXT: $vgpr16 = V_MOV_B32_e32 killed $sgpr31, implicit $exec
+ ; GFX950-NEXT: $vgpr17 = V_MOV_B32_e32 killed $vgpr4, implicit $exec
+ ; GFX950-NEXT: S_ENDPGM 0
+ ;
+ early-clobber renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43 = S_LOAD_DWORDX8_IMM_ec killed renamable $sgpr4_sgpr5, 0, 0
+ renamable $vgpr18 = V_MOV_B32_e32 0, implicit $exec
+ S_WAITCNT 49279
+ renamable $sgpr44_sgpr45_sgpr46_sgpr47 = S_LOAD_DWORDX4_IMM renamable $sgpr40_sgpr41, 0, 0
+ renamable $sgpr48_sgpr49_sgpr50_sgpr51 = S_LOAD_DWORDX4_IMM renamable $sgpr42_sgpr43, 0, 0
+ early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15 = S_LOAD_DWORDX16_IMM_ec killed renamable $sgpr42_sgpr43, 0, 0
+ early-clobber renamable $sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31 = S_LOAD_DWORDX16_IMM_ec killed renamable $sgpr40_sgpr41, 0, 0
+ S_WAITCNT 49279
+ $vgpr0_vgpr1 = V_MOV_B64_e32 $sgpr44_sgpr45, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3, implicit $sgpr44_sgpr45_sgpr46_sgpr47
+ $vgpr2_vgpr3 = V_MOV_B64_e32 killed $sgpr46_sgpr47, implicit $exec, implicit $sgpr44_sgpr45_sgpr46_sgpr47, implicit $exec
+ $vgpr4_vgpr5 = V_MOV_B64_e32 $sgpr48_sgpr49, implicit $exec, implicit-def $vgpr4_vgpr5_vgpr6_vgpr7, implicit $sgpr48_sgpr49_sgpr50_sgpr51
+ $vgpr6_vgpr7 = V_MOV_B64_e32 killed $sgpr50_sgpr51, implicit $exec, implicit $sgpr48_sgpr49_sgpr50_sgpr51, implicit $exec
+ renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X16_F16_e64 killed $vgpr0_vgpr1_vgpr2_vgpr3, killed $vgpr4_vgpr5_vgpr6_vgpr7, 0, 0, 0, 0, implicit $mode, implicit $exec
+ $vgpr4 = V_MOV_B32_e32 killed $sgpr14, implicit $exec, implicit $exec
+ $vgpr5 = V_MOV_B32_e32 killed $sgpr15, implicit $exec, implicit $exec
+ renamable $vgpr16_vgpr17 = V_PK_MOV_B32 12, killed $sgpr30_sgpr31, 8, killed $vgpr4_vgpr5, 0, 0, 0, 0, 0, implicit $exec
+ S_ENDPGM 0
+
+
...
---
name: test_pk_fma_unpacking_f32
>From f7bdf9703a28629fe515b4a7b7865dc09e5ec381 Mon Sep 17 00:00:00 2001
From: Robert Virany <robertvirany at gmail.com>
Date: Fri, 24 Jul 2026 15:51:10 -0600
Subject: [PATCH 2/3] [AMDGPU] Unpack V_PK_MOV_B32 for MFMA co-issue
---
llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp | 56 ++++++++++++++++---
...ck-non-coissue-insts-post-ra-scheduler.mir | 2 +
2 files changed, 50 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp b/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
index 9b67cdd6be6f4..9f7d132576498 100644
--- a/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
+++ b/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
@@ -21,8 +21,12 @@
#include "AMDGPU.h"
#include "GCNSubtarget.h"
#include "llvm/ADT/Statistic.h"
+#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
+#include "SIDefines.h"
+#include "llvm/ADT/SetVector.h"
#include "llvm/CodeGen/MachineDominators.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineLoopInfo.h"
#include "llvm/CodeGen/TargetSchedule.h"
#include "llvm/Support/BranchProbability.h"
@@ -77,10 +81,10 @@ class SIPreEmitPeephole {
// Here, we have overwritten v0 before we use it. This function checks if
// unpacking can lead to such a situation.
bool canUnpackingClobberRegister(const MachineInstr &MI);
- // Unpack and insert F32 packed instructions, such as V_PK_MUL, V_PK_ADD, and
- // V_PK_FMA. Currently, only V_PK_MUL, V_PK_ADD, V_PK_FMA are supported for
- // this transformation.
- void performF32Unpacking(MachineInstr &I);
+ // Unpack and insert packed instructions, such as V_PK_MUL, V_PK_ADD,
+ // V_PK_FMA, and V_PK_MOV. Currently, V_PK_MUL, V_PK_ADD, V_PK_FMA, and
+ // V_PK_MOV are supported for this transformation.
+ void performUnpacking(MachineInstr &I);
// Select corresponding unpacked instruction
uint32_t mapToUnpackedOpcode(MachineInstr &I);
// Creates the unpacked instruction to be inserted. Adds source modifiers to
@@ -608,6 +612,20 @@ bool SIPreEmitPeephole::canUnpackingClobberRegister(const MachineInstr &MI) {
// op_sel_hi modifiers.
Register UnpackedDstReg = TRI->getSubReg(DstReg, AMDGPU::sub0);
+ if (OpCode == AMDGPU::V_PK_MOV_B32) {
+ const MachineOperand *Src1MO =
+ TII->getNamedOperand(MI, AMDGPU::OpName::src1);
+ if (!Src1MO || !Src1MO->isReg())
+ return false;
+
+ unsigned Src1Mods =
+ TII->getNamedOperand(MI, AMDGPU::OpName::src1_modifiers)->getImm();
+ Register HiSrcReg = Src1Mods & SISrcMods::OP_SEL_0
+ ? TRI->getSubReg(Src1MO->getReg(), AMDGPU::sub1)
+ : TRI->getSubReg(Src1MO->getReg(), AMDGPU::sub0);
+ return TRI->regsOverlap(UnpackedDstReg, HiSrcReg);
+ }
+
const MachineOperand *Src0MO = TII->getNamedOperand(MI, AMDGPU::OpName::src0);
if (Src0MO && Src0MO->isReg()) {
Register SrcReg0 = Src0MO->getReg();
@@ -655,10 +673,10 @@ bool SIPreEmitPeephole::canUnpackingClobberRegister(const MachineInstr &MI) {
uint32_t SIPreEmitPeephole::mapToUnpackedOpcode(MachineInstr &I) {
unsigned Opcode = I.getOpcode();
- // Use 64 bit encoding to allow use of VOP3 instructions.
+ switch (Opcode) {
+ // Use 64 bit encoding to allow use of arithmetic VOP3 instructions.
// VOP3 e64 instructions allow source modifiers
// e32 instructions don't allow source modifiers.
- switch (Opcode) {
case AMDGPU::V_PK_ADD_F32:
case AMDGPU::V_PK_ADD_F32_gfx1250:
return AMDGPU::V_ADD_F32_e64;
@@ -668,6 +686,8 @@ uint32_t SIPreEmitPeephole::mapToUnpackedOpcode(MachineInstr &I) {
case AMDGPU::V_PK_FMA_F32:
case AMDGPU::V_PK_FMA_F32_gfx1250:
return AMDGPU::V_FMA_F32_e64;
+ case AMDGPU::V_PK_MOV_B32:
+ return AMDGPU::V_MOV_B32_e32;
default:
return std::numeric_limits<uint32_t>::max();
}
@@ -787,7 +807,7 @@ void SIPreEmitPeephole::collectUnpackingCandidates(
}
}
-void SIPreEmitPeephole::performF32Unpacking(MachineInstr &I) {
+void SIPreEmitPeephole::performUnpacking(MachineInstr &I) {
const MachineOperand &DstOp = I.getOperand(0);
uint32_t UnpackedOpcode = mapToUnpackedOpcode(I);
@@ -825,6 +845,26 @@ MachineInstrBuilder SIPreEmitPeephole::createUnpackedMI(MachineInstr &I,
Register UnpackedDstReg = IsHiBits ? TRI->getSubReg(DstReg, AMDGPU::sub1)
: TRI->getSubReg(DstReg, AMDGPU::sub0);
+ if (UnpackedOpcode == AMDGPU::V_MOV_B32_e32) {
+ MachineInstrBuilder NewMI = BuildMI(MBB, I, DL, TII->get(UnpackedOpcode));
+ NewMI.addDef(UnpackedDstReg);
+ const MachineOperand &SrcMO = IsHiBits ? *SrcMO1 : *SrcMO0;
+ unsigned SrcMods =
+ TII->getNamedOperand(I, IsHiBits ? AMDGPU::OpName::src1_modifiers
+ : AMDGPU::OpName::src0_modifiers)
+ ->getImm();
+
+ if (SrcMO.isImm()) {
+ NewMI.addImm(SrcMO.getImm());
+ } else {
+ unsigned SrcSubReg =
+ SrcMods & SISrcMods::OP_SEL_0 ? AMDGPU::sub1 : AMDGPU::sub0;
+ Register UnpackedSrcReg = TRI->getSubReg(SrcMO.getReg(), SrcSubReg);
+ NewMI.addReg(UnpackedSrcReg, getRegState(SrcMO));
+ }
+ return NewMI;
+ }
+
int64_t ClampVal = TII->getNamedOperand(I, AMDGPU::OpName::clamp)->getImm();
unsigned Src0Mods =
TII->getNamedOperand(I, AMDGPU::OpName::src0_modifiers)->getImm();
@@ -946,7 +986,7 @@ bool SIPreEmitPeephole::run(MachineFunction &MF, MachineLoopInfo *LoopInfo) {
collectUnpackingCandidates(MI, InstrsToUnpack, NumMFMACycles);
}
for (MachineInstr *MI : InstrsToUnpack) {
- performF32Unpacking(*MI);
+ performUnpacking(*MI);
}
}
diff --git a/llvm/test/CodeGen/AMDGPU/unpack-non-coissue-insts-post-ra-scheduler.mir b/llvm/test/CodeGen/AMDGPU/unpack-non-coissue-insts-post-ra-scheduler.mir
index 87fa8f0eda151..02897a0125430 100644
--- a/llvm/test/CodeGen/AMDGPU/unpack-non-coissue-insts-post-ra-scheduler.mir
+++ b/llvm/test/CodeGen/AMDGPU/unpack-non-coissue-insts-post-ra-scheduler.mir
@@ -469,6 +469,7 @@ body: |
; GFX950-NEXT: $vgpr5 = V_MOV_B32_e32 killed $sgpr15, implicit $exec, implicit $exec
; GFX950-NEXT: $vgpr16 = V_MOV_B32_e32 killed $sgpr31, implicit $exec
; GFX950-NEXT: $vgpr17 = V_MOV_B32_e32 killed $vgpr4, implicit $exec
+ ; GFX950-NEXT: renamable $vgpr16_vgpr17 = V_PK_MOV_B32 12, killed $sgpr28_sgpr29, 8, killed $vgpr16_vgpr17, 0, 0, 0, 0, 0, implicit $exec
; GFX950-NEXT: S_ENDPGM 0
;
early-clobber renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43 = S_LOAD_DWORDX8_IMM_ec killed renamable $sgpr4_sgpr5, 0, 0
@@ -487,6 +488,7 @@ body: |
$vgpr4 = V_MOV_B32_e32 killed $sgpr14, implicit $exec, implicit $exec
$vgpr5 = V_MOV_B32_e32 killed $sgpr15, implicit $exec, implicit $exec
renamable $vgpr16_vgpr17 = V_PK_MOV_B32 12, killed $sgpr30_sgpr31, 8, killed $vgpr4_vgpr5, 0, 0, 0, 0, 0, implicit $exec
+ renamable $vgpr16_vgpr17 = V_PK_MOV_B32 12, killed $sgpr28_sgpr29, 8, killed $vgpr16_vgpr17, 0, 0, 0, 0, 0, implicit $exec
S_ENDPGM 0
>From 52c3894a9526c22f3290c882793a2caeee2a9d57 Mon Sep 17 00:00:00 2001
From: Robert Virany <robertvirany at gmail.com>
Date: Fri, 25 Sep 2026 08:44:05 -0600
Subject: [PATCH 3/3] [AMDGPU] Add High-First unpack for V_PK_MOV_B32 and tests
---
llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp | 67 ++++++++++++-----
...ck-non-coissue-insts-post-ra-scheduler.mir | 72 ++++++++++++++++++-
2 files changed, 122 insertions(+), 17 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp b/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
index 9f7d132576498..6386d712b939b 100644
--- a/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
+++ b/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
@@ -73,6 +73,7 @@ class SIPreEmitPeephole {
void collectUnpackingCandidates(MachineInstr &BeginMI,
SetVector<MachineInstr *> &InstrsToUnpack,
uint16_t NumMFMACycles);
+ bool canUnpackPkMov(const MachineInstr &MI, bool &HighFirst) const;
// v_pk_fma_f32 v[0:1], v[0:1], v[2:3], v[2:3] op_sel:[1,1,1]
// op_sel_hi:[0,0,0]
// ==>
@@ -600,6 +601,38 @@ bool SIPreEmitPeephole::removeRedundantModeWrites(
return Changed;
}
+bool SIPreEmitPeephole::canUnpackPkMov(const MachineInstr &MI,
+ bool &HighFirst) const {
+ Register Dst = MI.getOperand(0).getReg();
+ Register LoDst = TRI->getSubReg(Dst, AMDGPU::sub0);
+ Register HiDst = TRI->getSubReg(Dst, AMDGPU::sub1);
+
+ auto SelectedSrc = [&](AMDGPU::OpName SrcName,
+ AMDGPU::OpName ModsName) -> Register {
+ const MachineOperand *Src = TII->getNamedOperand(MI, SrcName);
+ if (!Src || !Src->isReg())
+ return Register();
+
+ unsigned Mods = TII->getNamedOperand(MI, ModsName)->getImm();
+ unsigned SubReg = Mods & SISrcMods::OP_SEL_0 ? AMDGPU::sub1 : AMDGPU::sub0;
+ return TRI->getSubReg(Src->getReg(), SubReg);
+ };
+
+ Register HiSrc =
+ SelectedSrc(AMDGPU::OpName::src1, AMDGPU::OpName::src1_modifiers);
+ bool LowFirstClobbers = HiSrc && TRI->regsOverlap(LoDst, HiSrc);
+
+ Register LoSrc =
+ SelectedSrc(AMDGPU::OpName::src0, AMDGPU::OpName::src0_modifiers);
+ bool HighFirstClobbers = LoSrc && TRI->regsOverlap(HiDst, LoSrc);
+
+ if (LowFirstClobbers && HighFirstClobbers)
+ return false;
+
+ HighFirst = LowFirstClobbers;
+ return true;
+}
+
bool SIPreEmitPeephole::canUnpackingClobberRegister(const MachineInstr &MI) {
unsigned OpCode = MI.getOpcode();
Register DstReg = MI.getOperand(0).getReg();
@@ -613,17 +646,8 @@ bool SIPreEmitPeephole::canUnpackingClobberRegister(const MachineInstr &MI) {
Register UnpackedDstReg = TRI->getSubReg(DstReg, AMDGPU::sub0);
if (OpCode == AMDGPU::V_PK_MOV_B32) {
- const MachineOperand *Src1MO =
- TII->getNamedOperand(MI, AMDGPU::OpName::src1);
- if (!Src1MO || !Src1MO->isReg())
- return false;
-
- unsigned Src1Mods =
- TII->getNamedOperand(MI, AMDGPU::OpName::src1_modifiers)->getImm();
- Register HiSrcReg = Src1Mods & SISrcMods::OP_SEL_0
- ? TRI->getSubReg(Src1MO->getReg(), AMDGPU::sub1)
- : TRI->getSubReg(Src1MO->getReg(), AMDGPU::sub0);
- return TRI->regsOverlap(UnpackedDstReg, HiSrcReg);
+ bool HighFirst{};
+ return !canUnpackPkMov(MI, HighFirst);
}
const MachineOperand *Src0MO = TII->getNamedOperand(MI, AMDGPU::OpName::src0);
@@ -814,14 +838,25 @@ void SIPreEmitPeephole::performUnpacking(MachineInstr &I) {
assert(UnpackedOpcode != std::numeric_limits<uint32_t>::max() &&
"Unsupported Opcode");
- MachineInstrBuilder Op0LOp1L =
- createUnpackedMI(I, UnpackedOpcode, /*IsHiBits=*/false);
- MachineOperand LoDstOp = Op0LOp1L->getOperand(0);
+ bool HighFirst{};
+ if (I.getOpcode() == AMDGPU::V_PK_MOV_B32) {
+ bool CanUnpack = canUnpackPkMov(I, HighFirst);
+ assert(CanUnpack && "Unsafe V_PK_MOV_B32 reached unpacking");
+ (void)CanUnpack;
+ }
+ MachineInstrBuilder Op0LOp1L, Op0HOp1H;
+ if (HighFirst) {
+ Op0HOp1H = createUnpackedMI(I, UnpackedOpcode, /*IsHiBits=*/true);
+ Op0LOp1L = createUnpackedMI(I, UnpackedOpcode, /*IsHiBits=*/false);
+ } else {
+ Op0LOp1L = createUnpackedMI(I, UnpackedOpcode, /*IsHiBits=*/false);
+ Op0HOp1H = createUnpackedMI(I, UnpackedOpcode, /*IsHiBits=*/true);
+ }
+
+ MachineOperand LoDstOp = Op0LOp1L->getOperand(0);
LoDstOp.setIsUndef(DstOp.isUndef());
- MachineInstrBuilder Op0HOp1H =
- createUnpackedMI(I, UnpackedOpcode, /*IsHiBits=*/true);
MachineOperand HiDstOp = Op0HOp1H->getOperand(0);
uint32_t IFlags = I.getFlags();
diff --git a/llvm/test/CodeGen/AMDGPU/unpack-non-coissue-insts-post-ra-scheduler.mir b/llvm/test/CodeGen/AMDGPU/unpack-non-coissue-insts-post-ra-scheduler.mir
index 02897a0125430..12e417099bb50 100644
--- a/llvm/test/CodeGen/AMDGPU/unpack-non-coissue-insts-post-ra-scheduler.mir
+++ b/llvm/test/CodeGen/AMDGPU/unpack-non-coissue-insts-post-ra-scheduler.mir
@@ -469,9 +469,74 @@ body: |
; GFX950-NEXT: $vgpr5 = V_MOV_B32_e32 killed $sgpr15, implicit $exec, implicit $exec
; GFX950-NEXT: $vgpr16 = V_MOV_B32_e32 killed $sgpr31, implicit $exec
; GFX950-NEXT: $vgpr17 = V_MOV_B32_e32 killed $vgpr4, implicit $exec
- ; GFX950-NEXT: renamable $vgpr16_vgpr17 = V_PK_MOV_B32 12, killed $sgpr28_sgpr29, 8, killed $vgpr16_vgpr17, 0, 0, 0, 0, 0, implicit $exec
+ ; GFX950-NEXT: $vgpr17 = V_MOV_B32_e32 killed $vgpr16, implicit $exec
+ ; GFX950-NEXT: $vgpr16 = V_MOV_B32_e32 killed $sgpr29, implicit $exec
+ ; GFX950-NEXT: renamable $vgpr16_vgpr17 = V_PK_MOV_B32 12, killed $vgpr16_vgpr17, 8, killed $vgpr16_vgpr17, 0, 0, 0, 0, 0, implicit $exec
+ ; GFX950-NEXT: renamable $vgpr19 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX950-NEXT: early-clobber renamable $agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 = V_MFMA_F32_32X32X16_F16_e64 $vgpr16_vgpr17_vgpr18_vgpr19, $vgpr16_vgpr17_vgpr18_vgpr19, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX950-NEXT: $vgpr16 = V_MOV_B32_e32 killed $vgpr17, implicit $exec
+ ; GFX950-NEXT: $vgpr17 = V_MOV_B32_e32 killed $sgpr24, implicit $exec
+ ; GFX950-NEXT: $vgpr22 = V_MOV_B32_e32 7, implicit $exec
+ ; GFX950-NEXT: $vgpr23 = V_MOV_B32_e32 killed $sgpr26, implicit $exec
; GFX950-NEXT: S_ENDPGM 0
;
+ ; GFX942-LABEL: name: test_pk_mov_unpacking_b32
+ ; GFX942: liveins: $sgpr4_sgpr5
+ ; GFX942-NEXT: {{ $}}
+ ; GFX942-NEXT: early-clobber renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43 = S_LOAD_DWORDX8_IMM_ec killed renamable $sgpr4_sgpr5, 0, 0
+ ; GFX942-NEXT: renamable $vgpr18 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX942-NEXT: S_WAITCNT .Lgkmcnt_0
+ ; GFX942-NEXT: renamable $sgpr44_sgpr45_sgpr46_sgpr47 = S_LOAD_DWORDX4_IMM renamable $sgpr40_sgpr41, 0, 0
+ ; GFX942-NEXT: renamable $sgpr48_sgpr49_sgpr50_sgpr51 = S_LOAD_DWORDX4_IMM renamable $sgpr42_sgpr43, 0, 0
+ ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15 = S_LOAD_DWORDX16_IMM_ec killed renamable $sgpr42_sgpr43, 0, 0
+ ; GFX942-NEXT: early-clobber renamable $sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31 = S_LOAD_DWORDX16_IMM_ec killed renamable $sgpr40_sgpr41, 0, 0
+ ; GFX942-NEXT: S_WAITCNT .Lgkmcnt_0
+ ; GFX942-NEXT: $vgpr0_vgpr1 = V_MOV_B64_e32 $sgpr44_sgpr45, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3, implicit $sgpr44_sgpr45_sgpr46_sgpr47
+ ; GFX942-NEXT: $vgpr2_vgpr3 = V_MOV_B64_e32 killed $sgpr46_sgpr47, implicit $exec, implicit $sgpr44_sgpr45_sgpr46_sgpr47, implicit $exec
+ ; GFX942-NEXT: $vgpr4_vgpr5 = V_MOV_B64_e32 $sgpr48_sgpr49, implicit $exec, implicit-def $vgpr4_vgpr5_vgpr6_vgpr7, implicit $sgpr48_sgpr49_sgpr50_sgpr51
+ ; GFX942-NEXT: $vgpr6_vgpr7 = V_MOV_B64_e32 killed $sgpr50_sgpr51, implicit $exec, implicit $sgpr48_sgpr49_sgpr50_sgpr51, implicit $exec
+ ; GFX942-NEXT: early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X16_F16_e64 killed $vgpr0_vgpr1_vgpr2_vgpr3, killed $vgpr4_vgpr5_vgpr6_vgpr7, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NEXT: $vgpr4 = V_MOV_B32_e32 killed $sgpr14, implicit $exec, implicit $exec
+ ; GFX942-NEXT: $vgpr5 = V_MOV_B32_e32 killed $sgpr15, implicit $exec, implicit $exec
+ ; GFX942-NEXT: $vgpr16 = V_MOV_B32_e32 killed $sgpr31, implicit $exec
+ ; GFX942-NEXT: $vgpr17 = V_MOV_B32_e32 killed $vgpr4, implicit $exec
+ ; GFX942-NEXT: $vgpr17 = V_MOV_B32_e32 killed $vgpr16, implicit $exec
+ ; GFX942-NEXT: $vgpr16 = V_MOV_B32_e32 killed $sgpr29, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr16_vgpr17 = V_PK_MOV_B32 12, killed $vgpr16_vgpr17, 8, killed $vgpr16_vgpr17, 0, 0, 0, 0, 0, implicit $exec
+ ; GFX942-NEXT: renamable $vgpr19 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX942-NEXT: early-clobber renamable $agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 = V_MFMA_F32_32X32X16_F16_e64 $vgpr16_vgpr17_vgpr18_vgpr19, $vgpr16_vgpr17_vgpr18_vgpr19, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX942-NEXT: $vgpr16 = V_MOV_B32_e32 killed $vgpr17, implicit $exec
+ ; GFX942-NEXT: $vgpr17 = V_MOV_B32_e32 killed $sgpr24, implicit $exec
+ ; GFX942-NEXT: $vgpr22 = V_MOV_B32_e32 7, implicit $exec
+ ; GFX942-NEXT: $vgpr23 = V_MOV_B32_e32 killed $sgpr26, implicit $exec
+ ; GFX942-NEXT: S_ENDPGM 0
+ ;
+ ; GFX90A-LABEL: name: test_pk_mov_unpacking_b32
+ ; GFX90A: liveins: $sgpr4_sgpr5
+ ; GFX90A-NEXT: {{ $}}
+ ; GFX90A-NEXT: early-clobber renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43 = S_LOAD_DWORDX8_IMM_ec killed renamable $sgpr4_sgpr5, 0, 0
+ ; GFX90A-NEXT: renamable $vgpr18 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX90A-NEXT: S_WAITCNT .Lgkmcnt_0
+ ; GFX90A-NEXT: renamable $sgpr44_sgpr45_sgpr46_sgpr47 = S_LOAD_DWORDX4_IMM renamable $sgpr40_sgpr41, 0, 0
+ ; GFX90A-NEXT: renamable $sgpr48_sgpr49_sgpr50_sgpr51 = S_LOAD_DWORDX4_IMM renamable $sgpr42_sgpr43, 0, 0
+ ; GFX90A-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15 = S_LOAD_DWORDX16_IMM_ec killed renamable $sgpr42_sgpr43, 0, 0
+ ; GFX90A-NEXT: early-clobber renamable $sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31 = S_LOAD_DWORDX16_IMM_ec killed renamable $sgpr40_sgpr41, 0, 0
+ ; GFX90A-NEXT: S_WAITCNT .Lgkmcnt_0
+ ; GFX90A-NEXT: $vgpr0_vgpr1 = V_MOV_B64_e32 $sgpr44_sgpr45, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3, implicit $sgpr44_sgpr45_sgpr46_sgpr47
+ ; GFX90A-NEXT: $vgpr2_vgpr3 = V_MOV_B64_e32 killed $sgpr46_sgpr47, implicit $exec, implicit $sgpr44_sgpr45_sgpr46_sgpr47, implicit $exec
+ ; GFX90A-NEXT: $vgpr4_vgpr5 = V_MOV_B64_e32 $sgpr48_sgpr49, implicit $exec, implicit-def $vgpr4_vgpr5_vgpr6_vgpr7, implicit $sgpr48_sgpr49_sgpr50_sgpr51
+ ; GFX90A-NEXT: $vgpr6_vgpr7 = V_MOV_B64_e32 killed $sgpr50_sgpr51, implicit $exec, implicit $sgpr48_sgpr49_sgpr50_sgpr51, implicit $exec
+ ; GFX90A-NEXT: early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X16_F16_e64 killed $vgpr0_vgpr1_vgpr2_vgpr3, killed $vgpr4_vgpr5_vgpr6_vgpr7, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX90A-NEXT: $vgpr4 = V_MOV_B32_e32 killed $sgpr14, implicit $exec, implicit $exec
+ ; GFX90A-NEXT: $vgpr5 = V_MOV_B32_e32 killed $sgpr15, implicit $exec, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr16_vgpr17 = V_PK_MOV_B32 12, killed $sgpr30_sgpr31, 8, killed $vgpr4_vgpr5, 0, 0, 0, 0, 0, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr16_vgpr17 = V_PK_MOV_B32 12, killed $sgpr28_sgpr29, 8, killed $vgpr16_vgpr17, 0, 0, 0, 0, 0, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr16_vgpr17 = V_PK_MOV_B32 12, killed $vgpr16_vgpr17, 8, killed $vgpr16_vgpr17, 0, 0, 0, 0, 0, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr19 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX90A-NEXT: early-clobber renamable $agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 = V_MFMA_F32_32X32X16_F16_e64 $vgpr16_vgpr17_vgpr18_vgpr19, $vgpr16_vgpr17_vgpr18_vgpr19, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr16_vgpr17 = V_PK_MOV_B32 12, killed $vgpr16_vgpr17, 8, killed $sgpr24_sgpr25, 0, 0, 0, 0, 0, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr22_vgpr23 = V_PK_MOV_B32 8, 7, 8, killed $sgpr26_sgpr27, 0, 0, 0, 0, 0, implicit $exec
+ ; GFX90A-NEXT: S_ENDPGM 0
early-clobber renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43 = S_LOAD_DWORDX8_IMM_ec killed renamable $sgpr4_sgpr5, 0, 0
renamable $vgpr18 = V_MOV_B32_e32 0, implicit $exec
S_WAITCNT 49279
@@ -489,6 +554,11 @@ body: |
$vgpr5 = V_MOV_B32_e32 killed $sgpr15, implicit $exec, implicit $exec
renamable $vgpr16_vgpr17 = V_PK_MOV_B32 12, killed $sgpr30_sgpr31, 8, killed $vgpr4_vgpr5, 0, 0, 0, 0, 0, implicit $exec
renamable $vgpr16_vgpr17 = V_PK_MOV_B32 12, killed $sgpr28_sgpr29, 8, killed $vgpr16_vgpr17, 0, 0, 0, 0, 0, implicit $exec
+ renamable $vgpr16_vgpr17 = V_PK_MOV_B32 12, killed $vgpr16_vgpr17, 8, killed $vgpr16_vgpr17, 0, 0, 0, 0, 0, implicit $exec
+ renamable $vgpr19 = V_MOV_B32_e32 0, implicit $exec
+ early-clobber renamable $agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 = V_MFMA_F32_32X32X16_F16_e64 $vgpr16_vgpr17_vgpr18_vgpr19, $vgpr16_vgpr17_vgpr18_vgpr19, 0, 0, 0, 0, implicit $mode, implicit $exec
+ renamable $vgpr16_vgpr17 = V_PK_MOV_B32 12, killed $vgpr16_vgpr17, 8, killed $sgpr24_sgpr25, 0, 0, 0, 0, 0, implicit $exec
+ renamable $vgpr22_vgpr23 = V_PK_MOV_B32 8, 7, 8, killed $sgpr26_sgpr27, 0, 0, 0, 0, 0, implicit $exec
S_ENDPGM 0
More information about the llvm-commits
mailing list