[llvm] [AMDGPU] Fold v_perm pair into v_swap (PR #181966)
Frederick Vu via llvm-commits
llvm-commits at lists.llvm.org
Mon Apr 13 22:54:57 PDT 2026
https://github.com/FrederickVu updated https://github.com/llvm/llvm-project/pull/181966
>From 90dd20ba8a3c529c173ef47109b782b73b7cd158 Mon Sep 17 00:00:00 2001
From: Frederick Vu <100011202+FrederickVu at users.noreply.github.com>
Date: Fri, 13 Feb 2026 12:35:33 -0800
Subject: [PATCH 1/7] Fold v_perm pair into v_swap
---
.../Target/AMDGPU/SIShrinkInstructions.cpp | 242 ++++++++++++++++++
.../CodeGen/AMDGPU/shrink-perm-to-swap.mir | 140 ++++++++++
2 files changed, 382 insertions(+)
create mode 100644 llvm/test/CodeGen/AMDGPU/shrink-perm-to-swap.mir
diff --git a/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp b/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp
index 14ed778f44f3a..48c87be448960 100644
--- a/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp
+++ b/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp
@@ -13,6 +13,8 @@
#include "GCNSubtarget.h"
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
#include "Utils/AMDGPUBaseInfo.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/DenseSet.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
@@ -37,6 +39,10 @@ class SIShrinkInstructions {
const SIRegisterInfo *TRI;
bool IsPostRA;
+ using PendingSwapMap =
+ SmallDenseMap<std::pair<uint64_t, uint64_t>,
+ std::pair<MachineInstr *, uint32_t>, 4>;
+
bool foldImmediates(MachineInstr &MI, bool TryToCommute = true) const;
bool shouldShrinkTrue16(MachineInstr &MI) const;
bool isKImmOperand(const MachineOperand &Src) const;
@@ -58,6 +64,8 @@ class SIShrinkInstructions {
unsigned I) const;
void dropInstructionKeepingImpDefs(MachineInstr &MI) const;
MachineInstr *matchSwap(MachineInstr &MovT) const;
+ MachineInstr *matchSwapB16(MachineInstr &Perm,
+ PendingSwapMap &SwapCandidates) const;
public:
SIShrinkInstructions() = default;
@@ -843,6 +851,227 @@ MachineInstr *SIShrinkInstructions::matchSwap(MachineInstr &MovT) const {
return nullptr;
}
+MachineInstr *SIShrinkInstructions::matchSwapB16(
+ MachineInstr &Perm, PendingSwapMap &SwapCandidates) const {
+ assert(Perm.getOpcode() == AMDGPU::V_PERM_B32_e64);
+ if (IsPostRA)
+ return nullptr;
+
+ if (ST->getGeneration() < AMDGPUSubtarget::GFX11 ||
+ !ST->useRealTrue16Insts() ||
+ TII->pseudoToMCOpcode(AMDGPU::V_SWAP_B16) == -1)
+ return nullptr;
+
+ const int Src0Idx =
+ AMDGPU::getNamedOperandIdx(Perm.getOpcode(), AMDGPU::OpName::src0);
+ const int Src1Idx =
+ AMDGPU::getNamedOperandIdx(Perm.getOpcode(), AMDGPU::OpName::src1);
+ const int Src2Idx =
+ AMDGPU::getNamedOperandIdx(Perm.getOpcode(), AMDGPU::OpName::src2);
+
+ // Trace mask immediate back through COPYs and moves if necessary/possible.
+ auto getPermMaskImm = [&](MachineInstr &MI, uint32_t &Mask) {
+ const MachineOperand &Op = MI.getOperand(Src2Idx);
+ if (Op.isImm()) {
+ Mask = static_cast<uint32_t>(Op.getImm());
+ return true;
+ }
+ if (!Op.isReg() || !Op.getReg().isVirtual())
+ return false;
+
+ Register Reg = Op.getReg();
+ SmallDenseSet<Register, 4> Seen;
+ while (Reg.isVirtual() && Seen.insert(Reg).second) {
+ MachineInstr *Def = MRI->getUniqueVRegDef(Reg);
+ if (!Def)
+ return false;
+ if (Def->isMoveImmediate()) {
+ const MachineOperand &ImmOp = Def->getOperand(1);
+ if (!ImmOp.isImm())
+ return false;
+ Mask = static_cast<uint32_t>(ImmOp.getImm());
+ return true;
+ }
+ if (!Def->isCopy() || !Def->getOperand(1).isReg())
+ return false;
+
+ if (Def->getOperand(0).getSubReg() != AMDGPU::NoSubRegister ||
+ Def->getOperand(1).getSubReg() != AMDGPU::NoSubRegister)
+ return false;
+ Reg = Def->getOperand(1).getReg();
+ }
+ return false;
+ };
+
+ const MachineOperand &S0 = Perm.getOperand(Src0Idx);
+ const MachineOperand &S1 = Perm.getOperand(Src1Idx);
+ if (!S0.isReg() || !S1.isReg())
+ return nullptr;
+ Register Src0Reg = S0.getReg();
+ Register Src1Reg = S1.getReg();
+ unsigned Src0Sub = S0.getSubReg();
+ unsigned Src1Sub = S1.getSubReg();
+ if (!Src0Reg.isVirtual() || !Src1Reg.isVirtual())
+ return nullptr;
+ if (!TRI->isVGPR(*MRI, Src0Reg) || !TRI->isVGPR(*MRI, Src1Reg))
+ return nullptr;
+ if (Src0Reg == Src1Reg && Src0Sub == Src1Sub)
+ return nullptr;
+
+ uint32_t Mask = 0;
+ if (!getPermMaskImm(Perm, Mask))
+ return nullptr;
+
+ // For two v_perms with common operands {src0, src1} and complementary,
+ // eligible masks {S0Mask, S1Mask}, we emit a v_swap_b16 which swaps
+ // src0.S0Sub and src1.S1Sub. S0 and S1 indicate the Perm whose destination
+ // register will be replaced by an INSERT_SUBREG which has src0 or src1 as
+ // its base.
+ struct SwapCase {
+ uint32_t S1Mask;
+ uint32_t S0Mask;
+ unsigned S0Sub;
+ unsigned S1Sub;
+ };
+ static constexpr SwapCase Cases[] = {
+ {0x05040100u, 0x07060302u, AMDGPU::hi16, AMDGPU::lo16},
+ {0x07060100u, 0x03020504u, AMDGPU::hi16, AMDGPU::hi16},
+ {0x03020706u, 0x01000504u, AMDGPU::lo16, AMDGPU::hi16},
+ };
+
+ // If current v_perm's mask is not in above list, bail.
+ bool IsSwapEligible = false;
+ for (const SwapCase &C : Cases) {
+ if (Mask == C.S1Mask || Mask == C.S0Mask) {
+ IsSwapEligible = true;
+ break;
+ }
+ }
+ if (!IsSwapEligible)
+ return nullptr;
+
+ auto PackRegKey = [](Register Reg, unsigned Sub) {
+ return (uint64_t(Reg.id()) << 32) | Sub;
+ };
+ // Allow for paired v_perms to have swapped src0 and src1 operands.
+ std::pair<uint64_t, uint64_t> Key = {PackRegKey(Src0Reg, Src0Sub),
+ PackRegKey(Src1Reg, Src1Sub)};
+ std::pair<uint64_t, uint64_t> RevKey = {PackRegKey(Src1Reg, Src1Sub),
+ PackRegKey(Src0Reg, Src0Sub)};
+
+ // Current v_perm is now candidate. Search for another v_perm with same src0
+ // and src1 operands or record current v_perm and continue.
+ auto It = SwapCandidates.find(Key);
+ bool Reversed = false;
+ if (It == SwapCandidates.end()) {
+ It = SwapCandidates.find(RevKey);
+ if (It != SwapCandidates.end())
+ Reversed = true;
+ }
+ if (It == SwapCandidates.end()) {
+ SwapCandidates[Key] = {&Perm, Mask};
+ return nullptr;
+ }
+
+ // Define P0 as previously found v_perm and P1 as current v_perm.
+ auto [P0, M0] = It->second;
+ if (Reversed)
+ M0 ^= 0x04040404;
+ MachineInstr *P1 = &Perm;
+ uint32_t M1 = Mask;
+
+ // Check if P0 and P1 masks are complementary.
+ unsigned S1Sub = AMDGPU::NoSubRegister;
+ unsigned S0Sub = AMDGPU::NoSubRegister;
+ Register S1Dst, S0Dst;
+ for (const SwapCase &C : Cases) {
+ if (M0 == C.S0Mask && M1 == C.S1Mask) {
+ S1Sub = C.S1Sub;
+ S0Sub = C.S0Sub;
+ S1Dst = P1->getOperand(0).getReg();
+ S0Dst = P0->getOperand(0).getReg();
+ break;
+ } else if (M0 == C.S1Mask && M1 == C.S0Mask) {
+ S1Sub = C.S1Sub;
+ S0Sub = C.S0Sub;
+ S1Dst = P0->getOperand(0).getReg();
+ S0Dst = P1->getOperand(0).getReg();
+ break;
+ }
+ }
+ // If non-complementary, replace P0 with P1 in candidates map.
+ if (S1Sub == AMDGPU::NoSubRegister) {
+ It->second = {&Perm, Mask};
+ return nullptr;
+ }
+
+ SwapCandidates.erase(It);
+
+ // Ensure that we can use Lo128 registers for operands of the v_swap.
+ if (!MRI->constrainRegClass(S1Dst, &AMDGPU::VGPR_32_Lo128RegClass) ||
+ !MRI->constrainRegClass(S0Dst, &AMDGPU::VGPR_32_Lo128RegClass))
+ return nullptr;
+ if (!Src1Sub &&
+ !MRI->constrainRegClass(Src1Reg, &AMDGPU::VGPR_32_Lo128RegClass))
+ return nullptr;
+ if (!Src0Sub &&
+ !MRI->constrainRegClass(Src0Reg, &AMDGPU::VGPR_32_Lo128RegClass))
+ return nullptr;
+
+ MachineBasicBlock &MBB = *P0->getParent();
+ MachineBasicBlock::iterator I = P0->getIterator();
+ const DebugLoc &DL = P0->getDebugLoc();
+
+ // Extract the two 16-bit halves to be swapped, S1In and S0In.
+ Register S1In = MRI->createVirtualRegister(&AMDGPU::VGPR_16_Lo128RegClass);
+ Register S0In = MRI->createVirtualRegister(&AMDGPU::VGPR_16_Lo128RegClass);
+ unsigned S1InSub = TRI->composeSubRegIndices(Src1Sub, S1Sub);
+ unsigned S0InSub = TRI->composeSubRegIndices(Src0Sub, S0Sub);
+ BuildMI(MBB, I, DL, TII->get(TargetOpcode::COPY), S1In)
+ .addReg(Src1Reg, {}, S1InSub);
+ BuildMI(MBB, I, DL, TII->get(TargetOpcode::COPY), S0In)
+ .addReg(Src0Reg, {}, S0InSub);
+
+ // Swap. S1Out = S0In; S0Out = S1In;
+ Register S1Out = MRI->createVirtualRegister(&AMDGPU::VGPR_16_Lo128RegClass);
+ Register S0Out = MRI->createVirtualRegister(&AMDGPU::VGPR_16_Lo128RegClass);
+ auto *SwapMI = BuildMI(MBB, I, DL, TII->get(AMDGPU::V_SWAP_B16))
+ .addDef(S1Out)
+ .addDef(S0Out)
+ .addReg(S0In)
+ .addReg(S1In)
+ .getInstr();
+
+ // If P0/P1's operands were subregisters, COPY into new 32-bit registers.
+ Register S1Base = Src1Reg;
+ Register S0Base = Src0Reg;
+ if (Src1Sub) {
+ S1Base = MRI->createVirtualRegister(&AMDGPU::VGPR_32_Lo128RegClass);
+ BuildMI(MBB, I, DL, TII->get(TargetOpcode::COPY), S1Base)
+ .addReg(Src1Reg, {}, Src1Sub);
+ }
+ if (Src0Sub) {
+ S0Base = MRI->createVirtualRegister(&AMDGPU::VGPR_32_Lo128RegClass);
+ BuildMI(MBB, I, DL, TII->get(TargetOpcode::COPY), S0Base)
+ .addReg(Src0Reg, {}, Src0Sub);
+ }
+
+ BuildMI(MBB, I, DL, TII->get(TargetOpcode::INSERT_SUBREG), S1Dst)
+ .addReg(S1Base)
+ .addReg(S1Out)
+ .addImm(S1Sub);
+
+ BuildMI(MBB, I, DL, TII->get(TargetOpcode::INSERT_SUBREG), S0Dst)
+ .addReg(S0Base)
+ .addReg(S0Out)
+ .addImm(S0Sub);
+
+ dropInstructionKeepingImpDefs(*P1);
+ dropInstructionKeepingImpDefs(*P0);
+
+ return SwapMI->getNextNode() ? SwapMI->getNextNode() : SwapMI;
+}
+
// If an instruction has dead sdst replace it with NULL register on gfx1030+
bool SIShrinkInstructions::tryReplaceDeadSDST(MachineInstr &MI) const {
if (!ST->hasGFX10_3Insts())
@@ -872,11 +1101,16 @@ bool SIShrinkInstructions::run(MachineFunction &MF) {
bool Changed = false;
for (MachineBasicBlock &MBB : MF) {
+ PendingSwapMap SwapCandidates;
MachineBasicBlock::iterator I, Next;
for (I = MBB.begin(); I != MBB.end(); I = Next) {
Next = std::next(I);
MachineInstr &MI = *I;
+ if (!SwapCandidates.empty() &&
+ instModifiesReg(&MI, AMDGPU::EXEC, AMDGPU::NoSubRegister))
+ SwapCandidates.clear();
+
if (MI.getOpcode() == AMDGPU::V_MOV_B32_e32) {
// If this has a literal constant source that is the same as the
// reversed bits of an inline immediate, replace with a bitreverse of
@@ -919,6 +1153,14 @@ bool SIShrinkInstructions::run(MachineFunction &MF) {
Changed |= (CK == ChangeKind::UpdateInst);
}
+ if (MI.getOpcode() == AMDGPU::V_PERM_B32_e64) {
+ if (auto *NextMI = matchSwapB16(MI, SwapCandidates)) {
+ Next = NextMI->getIterator();
+ Changed = true;
+ continue;
+ }
+ }
+
// Try to use S_ADDK_I32 and S_MULK_I32.
if (MI.getOpcode() == AMDGPU::S_ADD_I32 ||
MI.getOpcode() == AMDGPU::S_MUL_I32 ||
diff --git a/llvm/test/CodeGen/AMDGPU/shrink-perm-to-swap.mir b/llvm/test/CodeGen/AMDGPU/shrink-perm-to-swap.mir
new file mode 100644
index 0000000000000..3a0f482384fd8
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/shrink-perm-to-swap.mir
@@ -0,0 +1,140 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -mattr=+real-true16 -run-pass=si-shrink-instructions -verify-machineinstrs %s -o - | FileCheck %s
+
+# Test folding complementary V_PERM_B32_e64 pairs into V_SWAP_B16.
+
+---
+name: perm_to_swap
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr1
+ ; CHECK-LABEL: name: perm_to_swap
+ ; CHECK: liveins: $vgpr0, $vgpr1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr_32_lo128 = COPY $vgpr0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32_lo128 = COPY $vgpr1
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr_16_lo128 = COPY [[COPY1]].lo16
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr_16_lo128 = COPY [[COPY]].hi16
+ ; CHECK-NEXT: [[V_SWAP_B16_:%[0-9]+]]:vgpr_16_lo128, [[V_SWAP_B16_1:%[0-9]+]]:vgpr_16_lo128 = V_SWAP_B16 [[COPY3]], [[COPY2]], implicit $exec
+ ; CHECK-NEXT: [[INSERT_SUBREG:%[0-9]+]]:vgpr_32_lo128 = INSERT_SUBREG [[COPY1]], [[V_SWAP_B16_]], %subreg.lo16
+ ; CHECK-NEXT: [[INSERT_SUBREG1:%[0-9]+]]:vgpr_32_lo128 = INSERT_SUBREG [[COPY]], [[V_SWAP_B16_1]], %subreg.hi16
+ ; CHECK-NEXT: S_NOP 0, implicit [[INSERT_SUBREG]], implicit [[INSERT_SUBREG1]]
+ %0:vgpr_32 = COPY $vgpr0
+ %1:vgpr_32 = COPY $vgpr1
+ %2:vgpr_32 = V_PERM_B32_e64 %0, %1, 84148480, implicit $exec ; 0x05040100
+ %3:vgpr_32 = V_PERM_B32_e64 %0, %1, 117834498, implicit $exec ; 0x07060302
+ S_NOP 0, implicit %2, implicit %3
+...
+
+---
+name: perm_to_swap_subreg
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0_vgpr1_vgpr2_vgpr3
+ ; CHECK-LABEL: name: perm_to_swap_subreg
+ ; CHECK: liveins: $vgpr0_vgpr1_vgpr2_vgpr3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_16_lo128 = COPY [[COPY]].sub3_hi16
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr_16_lo128 = COPY [[COPY]].sub1_lo16
+ ; CHECK-NEXT: [[V_SWAP_B16_:%[0-9]+]]:vgpr_16_lo128, [[V_SWAP_B16_1:%[0-9]+]]:vgpr_16_lo128 = V_SWAP_B16 [[COPY2]], [[COPY1]], implicit $exec
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr_32_lo128 = COPY [[COPY]].sub3
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr_32_lo128 = COPY [[COPY]].sub1
+ ; CHECK-NEXT: [[INSERT_SUBREG:%[0-9]+]]:vgpr_32_lo128 = INSERT_SUBREG [[COPY3]], [[V_SWAP_B16_]], %subreg.hi16
+ ; CHECK-NEXT: [[INSERT_SUBREG1:%[0-9]+]]:vgpr_32_lo128 = INSERT_SUBREG [[COPY4]], [[V_SWAP_B16_1]], %subreg.lo16
+ ; CHECK-NEXT: [[COPY5:%[0-9]+]]:vgpr_16_lo128 = COPY [[COPY]].sub2_hi16
+ ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vgpr_16_lo128 = COPY [[COPY]].lo16
+ ; CHECK-NEXT: [[V_SWAP_B16_2:%[0-9]+]]:vgpr_16_lo128, [[V_SWAP_B16_3:%[0-9]+]]:vgpr_16_lo128 = V_SWAP_B16 [[COPY6]], [[COPY5]], implicit $exec
+ ; CHECK-NEXT: [[COPY7:%[0-9]+]]:vgpr_32_lo128 = COPY [[COPY]].sub2
+ ; CHECK-NEXT: [[COPY8:%[0-9]+]]:vgpr_32_lo128 = COPY [[COPY]].sub0
+ ; CHECK-NEXT: [[INSERT_SUBREG2:%[0-9]+]]:vgpr_32_lo128 = INSERT_SUBREG [[COPY7]], [[V_SWAP_B16_2]], %subreg.hi16
+ ; CHECK-NEXT: [[INSERT_SUBREG3:%[0-9]+]]:vgpr_32_lo128 = INSERT_SUBREG [[COPY8]], [[V_SWAP_B16_3]], %subreg.lo16
+ ; CHECK-NEXT: S_NOP 0, implicit [[INSERT_SUBREG]], implicit [[INSERT_SUBREG1]], implicit [[INSERT_SUBREG2]], implicit [[INSERT_SUBREG3]]
+ %0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ %1:vgpr_32 = V_PERM_B32_e64 %0.sub1, %0.sub3, 50464518, implicit $exec ; 0x03020706
+ %2:vgpr_32 = V_PERM_B32_e64 %0.sub1, %0.sub3, 16778500, implicit $exec ; 0x01000504
+ %3:vgpr_32 = V_PERM_B32_e64 %0.sub0, %0.sub2, 50464518, implicit $exec ; 0x03020706
+ %4:vgpr_32 = V_PERM_B32_e64 %0.sub0, %0.sub2, 16778500, implicit $exec ; 0x01000504
+ S_NOP 0, implicit %1, implicit %2, implicit %3, implicit %4
+...
+
+---
+name: perm_to_swap_separated
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; CHECK-LABEL: name: perm_to_swap_separated
+ ; CHECK: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr_32_lo128 = COPY $vgpr0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32_lo128 = COPY $vgpr1
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr_16_lo128 = COPY [[COPY1]].lo16
+ ; CHECK-NEXT: [[COPY5:%[0-9]+]]:vgpr_16_lo128 = COPY [[COPY]].hi16
+ ; CHECK-NEXT: [[V_SWAP_B16_:%[0-9]+]]:vgpr_16_lo128, [[V_SWAP_B16_1:%[0-9]+]]:vgpr_16_lo128 = V_SWAP_B16 [[COPY5]], [[COPY4]], implicit $exec
+ ; CHECK-NEXT: [[INSERT_SUBREG:%[0-9]+]]:vgpr_32_lo128 = INSERT_SUBREG [[COPY1]], [[V_SWAP_B16_]], %subreg.lo16
+ ; CHECK-NEXT: [[INSERT_SUBREG1:%[0-9]+]]:vgpr_32_lo128 = INSERT_SUBREG [[COPY]], [[V_SWAP_B16_1]], %subreg.hi16
+ ; CHECK-NEXT: [[V_ADD_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[COPY2]], [[COPY3]], implicit $exec
+ ; CHECK-NEXT: S_NOP 0, implicit [[INSERT_SUBREG]], implicit [[V_ADD_U32_e32_]], implicit [[INSERT_SUBREG1]]
+ %0:vgpr_32 = COPY $vgpr0
+ %1:vgpr_32 = COPY $vgpr1
+ %2:vgpr_32 = COPY $vgpr2
+ %3:vgpr_32 = COPY $vgpr3
+ %4:vgpr_32 = V_PERM_B32_e64 %0, %1, 84148480, implicit $exec ; 0x05040100
+ %5:vgpr_32 = V_ADD_U32_e64 %2, %3, 0, implicit $exec
+ %6:vgpr_32 = V_PERM_B32_e64 %0, %1, 117834498, implicit $exec ; 0x07060302
+ S_NOP 0, implicit %4, implicit %5, implicit %6
+...
+
+---
+name: perm_to_swap_reversed_operands
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr1
+ ; CHECK-LABEL: name: perm_to_swap_reversed_operands
+ ; CHECK: liveins: $vgpr0, $vgpr1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr_32_lo128 = COPY $vgpr0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32_lo128 = COPY $vgpr1
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr_16_lo128 = COPY [[COPY]].hi16
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr_16_lo128 = COPY [[COPY1]].lo16
+ ; CHECK-NEXT: [[V_SWAP_B16_:%[0-9]+]]:vgpr_16_lo128, [[V_SWAP_B16_1:%[0-9]+]]:vgpr_16_lo128 = V_SWAP_B16 [[COPY3]], [[COPY2]], implicit $exec
+ ; CHECK-NEXT: [[INSERT_SUBREG:%[0-9]+]]:vgpr_32_lo128 = INSERT_SUBREG [[COPY]], [[V_SWAP_B16_]], %subreg.hi16
+ ; CHECK-NEXT: [[INSERT_SUBREG1:%[0-9]+]]:vgpr_32_lo128 = INSERT_SUBREG [[COPY1]], [[V_SWAP_B16_1]], %subreg.lo16
+ ; CHECK-NEXT: S_NOP 0, implicit [[INSERT_SUBREG1]], implicit [[INSERT_SUBREG]]
+ %0:vgpr_32 = COPY $vgpr0
+ %1:vgpr_32 = COPY $vgpr1
+ %2:vgpr_32 = V_PERM_B32_e64 %0, %1, 84148480, implicit $exec ; 0x05040100
+ %3:vgpr_32 = V_PERM_B32_e64 %1, %0, 50464518, implicit $exec ; 0x03020706 (reversed operands)
+ S_NOP 0, implicit %2, implicit %3
+...
+
+---
+name: perm_no_swap_exec_mod
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr1, $sgpr0
+ ; CHECK-LABEL: name: perm_no_swap_exec_mod
+ ; CHECK: liveins: $vgpr0, $vgpr1, $sgpr0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+ ; CHECK-NEXT: [[V_PERM_B32_e64_:%[0-9]+]]:vgpr_32 = V_PERM_B32_e64 [[COPY]], [[COPY1]], 84148480, implicit $exec
+ ; CHECK-NEXT: $exec_lo = S_MOV_B32 [[COPY2]]
+ ; CHECK-NEXT: [[V_PERM_B32_e64_1:%[0-9]+]]:vgpr_32 = V_PERM_B32_e64 [[COPY]], [[COPY1]], 117834498, implicit $exec
+ ; CHECK-NEXT: S_NOP 0, implicit [[V_PERM_B32_e64_]], implicit [[V_PERM_B32_e64_1]]
+ %0:vgpr_32 = COPY $vgpr0
+ %1:vgpr_32 = COPY $vgpr1
+ %2:sgpr_32 = COPY $sgpr0
+ %3:vgpr_32 = V_PERM_B32_e64 %0, %1, 84148480, implicit $exec ; 0x05040100
+ $exec_lo = S_MOV_B32 %2
+ %4:vgpr_32 = V_PERM_B32_e64 %0, %1, 117834498, implicit $exec ; 0x07060302
+ S_NOP 0, implicit %3, implicit %4
+...
>From 75c1a329ae638dde22998edb90879851e039670d Mon Sep 17 00:00:00 2001
From: Frederick Vu <100011202+FrederickVu at users.noreply.github.com>
Date: Wed, 18 Feb 2026 10:20:54 -0800
Subject: [PATCH 2/7] Add v_swap_b16 v4i8 shuffle test
---
llvm/test/CodeGen/AMDGPU/v_swap_b16.ll | 67 ++++++++++++++++++++++++++
1 file changed, 67 insertions(+)
diff --git a/llvm/test/CodeGen/AMDGPU/v_swap_b16.ll b/llvm/test/CodeGen/AMDGPU/v_swap_b16.ll
index 55986328491ec..934b55e0cd24f 100644
--- a/llvm/test/CodeGen/AMDGPU/v_swap_b16.ll
+++ b/llvm/test/CodeGen/AMDGPU/v_swap_b16.ll
@@ -110,3 +110,70 @@ loop:
ret:
ret half %x
}
+
+define void @swap_shuffle_v4i8(ptr %out0, ptr %out1, ptr %in0, ptr %in1) {
+; GFX11-TRUE16-LABEL: swap_shuffle_v4i8:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[4:5]
+; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[6:7]
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_swap_b16 v5.h, v4.l
+; GFX11-TRUE16-NEXT: flat_store_b32 v[0:1], v4
+; GFX11-TRUE16-NEXT: flat_store_b32 v[2:3], v5
+; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: swap_shuffle_v4i8:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: flat_load_b32 v4, v[4:5]
+; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[6:7]
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_perm_b32 v6, v4, v5, 0x1000504
+; GFX11-FAKE16-NEXT: v_perm_b32 v4, v4, v5, 0x3020706
+; GFX11-FAKE16-NEXT: flat_store_b32 v[0:1], v6
+; GFX11-FAKE16-NEXT: flat_store_b32 v[2:3], v4
+; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-TRUE16-LABEL: swap_shuffle_v4i8:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[4:5]
+; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[6:7]
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: v_swap_b16 v5.h, v4.l
+; GFX12-TRUE16-NEXT: flat_store_b32 v[0:1], v4
+; GFX12-TRUE16-NEXT: flat_store_b32 v[2:3], v5
+; GFX12-TRUE16-NEXT: s_wait_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: swap_shuffle_v4i8:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: flat_load_b32 v4, v[4:5]
+; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[6:7]
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: v_perm_b32 v6, v4, v5, 0x1000504
+; GFX12-FAKE16-NEXT: v_perm_b32 v4, v4, v5, 0x3020706
+; GFX12-FAKE16-NEXT: flat_store_b32 v[0:1], v6
+; GFX12-FAKE16-NEXT: flat_store_b32 v[2:3], v4
+; GFX12-FAKE16-NEXT: s_wait_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+ %a = load <4 x i8>, ptr %in0, align 4
+ %b = load <4 x i8>, ptr %in1, align 4
+ %lo = shufflevector <4 x i8> %a, <4 x i8> %b, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+ %hi = shufflevector <4 x i8> %a, <4 x i8> %b, <4 x i32> <i32 2, i32 3, i32 6, i32 7>
+ store <4 x i8> %lo, ptr %out0, align 4
+ store <4 x i8> %hi, ptr %out1, align 4
+ ret void
+}
>From ed95a4be4cbb419fb5528631fd8867f49e640443 Mon Sep 17 00:00:00 2001
From: Frederick Vu <100011202+FrederickVu at users.noreply.github.com>
Date: Fri, 20 Feb 2026 10:34:53 -0800
Subject: [PATCH 3/7] Fix formatting
---
llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp b/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp
index 48c87be448960..7624964696c33 100644
--- a/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp
+++ b/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp
@@ -39,9 +39,8 @@ class SIShrinkInstructions {
const SIRegisterInfo *TRI;
bool IsPostRA;
- using PendingSwapMap =
- SmallDenseMap<std::pair<uint64_t, uint64_t>,
- std::pair<MachineInstr *, uint32_t>, 4>;
+ using PendingSwapMap = SmallDenseMap<std::pair<uint64_t, uint64_t>,
+ std::pair<MachineInstr *, uint32_t>, 4>;
bool foldImmediates(MachineInstr &MI, bool TryToCommute = true) const;
bool shouldShrinkTrue16(MachineInstr &MI) const;
@@ -851,8 +850,9 @@ MachineInstr *SIShrinkInstructions::matchSwap(MachineInstr &MovT) const {
return nullptr;
}
-MachineInstr *SIShrinkInstructions::matchSwapB16(
- MachineInstr &Perm, PendingSwapMap &SwapCandidates) const {
+MachineInstr *
+SIShrinkInstructions::matchSwapB16(MachineInstr &Perm,
+ PendingSwapMap &SwapCandidates) const {
assert(Perm.getOpcode() == AMDGPU::V_PERM_B32_e64);
if (IsPostRA)
return nullptr;
@@ -955,9 +955,9 @@ MachineInstr *SIShrinkInstructions::matchSwapB16(
};
// Allow for paired v_perms to have swapped src0 and src1 operands.
std::pair<uint64_t, uint64_t> Key = {PackRegKey(Src0Reg, Src0Sub),
- PackRegKey(Src1Reg, Src1Sub)};
+ PackRegKey(Src1Reg, Src1Sub)};
std::pair<uint64_t, uint64_t> RevKey = {PackRegKey(Src1Reg, Src1Sub),
- PackRegKey(Src0Reg, Src0Sub)};
+ PackRegKey(Src0Reg, Src0Sub)};
// Current v_perm is now candidate. Search for another v_perm with same src0
// and src1 operands or record current v_perm and continue.
>From 24a0b1d8b74925960c90d9763e4388018d3aab23 Mon Sep 17 00:00:00 2001
From: Frederick Vu <100011202+FrederickVu at users.noreply.github.com>
Date: Wed, 8 Apr 2026 01:48:54 +0000
Subject: [PATCH 4/7] Address comments
---
.../Target/AMDGPU/SIShrinkInstructions.cpp | 126 ++++++-------
llvm/test/CodeGen/AMDGPU/v_swap_b16.ll | 170 +++++++++++++-----
2 files changed, 196 insertions(+), 100 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp b/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp
index 7624964696c33..befd07c334435 100644
--- a/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp
+++ b/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp
@@ -850,6 +850,51 @@ MachineInstr *SIShrinkInstructions::matchSwap(MachineInstr &MovT) const {
return nullptr;
}
+// Trace the selector operand of a V_PERM_B32 back through COPYs to find the
+// immediate mask.
+static bool getPermMaskImm(MachineInstr &Perm, MachineRegisterInfo *MRI,
+ uint32_t &Mask) {
+ const int Src2Idx =
+ AMDGPU::getNamedOperandIdx(Perm.getOpcode(), AMDGPU::OpName::src2);
+ const MachineOperand &Op = Perm.getOperand(Src2Idx);
+ if (Op.isImm()) {
+ Mask = static_cast<uint32_t>(Op.getImm());
+ return true;
+ }
+ if (!Op.isReg() || !Op.getReg().isVirtual())
+ return false;
+
+ Register Reg = Op.getReg();
+ SmallDenseSet<Register, 4> Seen;
+ while (Reg.isVirtual() && Seen.insert(Reg).second) {
+ MachineInstr *Def = MRI->getUniqueVRegDef(Reg);
+ if (!Def)
+ return false;
+ if (Def->isMoveImmediate()) {
+ const MachineOperand &ImmOp = Def->getOperand(1);
+ if (!ImmOp.isImm())
+ return false;
+ Mask = static_cast<uint32_t>(ImmOp.getImm());
+ return true;
+ }
+ if (!Def->isCopy() || !Def->getOperand(1).isReg())
+ return false;
+
+ if (Def->getOperand(0).getSubReg() != AMDGPU::NoSubRegister ||
+ Def->getOperand(1).getSubReg() != AMDGPU::NoSubRegister)
+ return false;
+ Reg = Def->getOperand(1).getReg();
+ }
+ return false;
+}
+
+// Matches two v_perms that together swap 16-bit halves between two inputs. For
+// example:
+//
+// v_perm v2, v0, v1, 0x5040100
+// v_perm v3, v0, v1, 0x7060302
+// =>
+// v_swap_b16 v0.h, v1.l
MachineInstr *
SIShrinkInstructions::matchSwapB16(MachineInstr &Perm,
PendingSwapMap &SwapCandidates) const {
@@ -857,8 +902,7 @@ SIShrinkInstructions::matchSwapB16(MachineInstr &Perm,
if (IsPostRA)
return nullptr;
- if (ST->getGeneration() < AMDGPUSubtarget::GFX11 ||
- !ST->useRealTrue16Insts() ||
+ if (!ST->useRealTrue16Insts() ||
TII->pseudoToMCOpcode(AMDGPU::V_SWAP_B16) == -1)
return nullptr;
@@ -866,42 +910,6 @@ SIShrinkInstructions::matchSwapB16(MachineInstr &Perm,
AMDGPU::getNamedOperandIdx(Perm.getOpcode(), AMDGPU::OpName::src0);
const int Src1Idx =
AMDGPU::getNamedOperandIdx(Perm.getOpcode(), AMDGPU::OpName::src1);
- const int Src2Idx =
- AMDGPU::getNamedOperandIdx(Perm.getOpcode(), AMDGPU::OpName::src2);
-
- // Trace mask immediate back through COPYs and moves if necessary/possible.
- auto getPermMaskImm = [&](MachineInstr &MI, uint32_t &Mask) {
- const MachineOperand &Op = MI.getOperand(Src2Idx);
- if (Op.isImm()) {
- Mask = static_cast<uint32_t>(Op.getImm());
- return true;
- }
- if (!Op.isReg() || !Op.getReg().isVirtual())
- return false;
-
- Register Reg = Op.getReg();
- SmallDenseSet<Register, 4> Seen;
- while (Reg.isVirtual() && Seen.insert(Reg).second) {
- MachineInstr *Def = MRI->getUniqueVRegDef(Reg);
- if (!Def)
- return false;
- if (Def->isMoveImmediate()) {
- const MachineOperand &ImmOp = Def->getOperand(1);
- if (!ImmOp.isImm())
- return false;
- Mask = static_cast<uint32_t>(ImmOp.getImm());
- return true;
- }
- if (!Def->isCopy() || !Def->getOperand(1).isReg())
- return false;
-
- if (Def->getOperand(0).getSubReg() != AMDGPU::NoSubRegister ||
- Def->getOperand(1).getSubReg() != AMDGPU::NoSubRegister)
- return false;
- Reg = Def->getOperand(1).getReg();
- }
- return false;
- };
const MachineOperand &S0 = Perm.getOperand(Src0Idx);
const MachineOperand &S1 = Perm.getOperand(Src1Idx);
@@ -919,7 +927,7 @@ SIShrinkInstructions::matchSwapB16(MachineInstr &Perm,
return nullptr;
uint32_t Mask = 0;
- if (!getPermMaskImm(Perm, Mask))
+ if (!getPermMaskImm(Perm, MRI, Mask))
return nullptr;
// For two v_perms with common operands {src0, src1} and complementary,
@@ -939,15 +947,9 @@ SIShrinkInstructions::matchSwapB16(MachineInstr &Perm,
{0x03020706u, 0x01000504u, AMDGPU::lo16, AMDGPU::hi16},
};
- // If current v_perm's mask is not in above list, bail.
- bool IsSwapEligible = false;
- for (const SwapCase &C : Cases) {
- if (Mask == C.S1Mask || Mask == C.S0Mask) {
- IsSwapEligible = true;
- break;
- }
- }
- if (!IsSwapEligible)
+ if (!llvm::any_of(Cases, [Mask](const SwapCase &C) {
+ return Mask == C.S1Mask || Mask == C.S0Mask;
+ }))
return nullptr;
auto PackRegKey = [](Register Reg, unsigned Sub) {
@@ -1008,14 +1010,18 @@ SIShrinkInstructions::matchSwapB16(MachineInstr &Perm,
SwapCandidates.erase(It);
// Ensure that we can use Lo128 registers for operands of the v_swap.
- if (!MRI->constrainRegClass(S1Dst, &AMDGPU::VGPR_32_Lo128RegClass) ||
- !MRI->constrainRegClass(S0Dst, &AMDGPU::VGPR_32_Lo128RegClass))
+ const MCInstrDesc &SwapDesc = TII->get(AMDGPU::V_SWAP_B16);
+ const TargetRegisterClass *Swap16RC = TII->getRegClass(SwapDesc, 0);
+ const TargetRegisterClass *Swap32RC =
+ TRI->getMatchingSuperRegClass(MRI->getRegClass(S0Dst), Swap16RC, S0Sub);
+ if (!Swap32RC)
+ return nullptr;
+ if (!MRI->constrainRegClass(S1Dst, Swap32RC) ||
+ !MRI->constrainRegClass(S0Dst, Swap32RC))
return nullptr;
- if (!Src1Sub &&
- !MRI->constrainRegClass(Src1Reg, &AMDGPU::VGPR_32_Lo128RegClass))
+ if (!Src1Sub && !MRI->constrainRegClass(Src1Reg, Swap32RC))
return nullptr;
- if (!Src0Sub &&
- !MRI->constrainRegClass(Src0Reg, &AMDGPU::VGPR_32_Lo128RegClass))
+ if (!Src0Sub && !MRI->constrainRegClass(Src0Reg, Swap32RC))
return nullptr;
MachineBasicBlock &MBB = *P0->getParent();
@@ -1023,8 +1029,8 @@ SIShrinkInstructions::matchSwapB16(MachineInstr &Perm,
const DebugLoc &DL = P0->getDebugLoc();
// Extract the two 16-bit halves to be swapped, S1In and S0In.
- Register S1In = MRI->createVirtualRegister(&AMDGPU::VGPR_16_Lo128RegClass);
- Register S0In = MRI->createVirtualRegister(&AMDGPU::VGPR_16_Lo128RegClass);
+ Register S1In = MRI->createVirtualRegister(Swap16RC);
+ Register S0In = MRI->createVirtualRegister(Swap16RC);
unsigned S1InSub = TRI->composeSubRegIndices(Src1Sub, S1Sub);
unsigned S0InSub = TRI->composeSubRegIndices(Src0Sub, S0Sub);
BuildMI(MBB, I, DL, TII->get(TargetOpcode::COPY), S1In)
@@ -1033,8 +1039,8 @@ SIShrinkInstructions::matchSwapB16(MachineInstr &Perm,
.addReg(Src0Reg, {}, S0InSub);
// Swap. S1Out = S0In; S0Out = S1In;
- Register S1Out = MRI->createVirtualRegister(&AMDGPU::VGPR_16_Lo128RegClass);
- Register S0Out = MRI->createVirtualRegister(&AMDGPU::VGPR_16_Lo128RegClass);
+ Register S1Out = MRI->createVirtualRegister(Swap16RC);
+ Register S0Out = MRI->createVirtualRegister(Swap16RC);
auto *SwapMI = BuildMI(MBB, I, DL, TII->get(AMDGPU::V_SWAP_B16))
.addDef(S1Out)
.addDef(S0Out)
@@ -1046,12 +1052,12 @@ SIShrinkInstructions::matchSwapB16(MachineInstr &Perm,
Register S1Base = Src1Reg;
Register S0Base = Src0Reg;
if (Src1Sub) {
- S1Base = MRI->createVirtualRegister(&AMDGPU::VGPR_32_Lo128RegClass);
+ S1Base = MRI->createVirtualRegister(Swap32RC);
BuildMI(MBB, I, DL, TII->get(TargetOpcode::COPY), S1Base)
.addReg(Src1Reg, {}, Src1Sub);
}
if (Src0Sub) {
- S0Base = MRI->createVirtualRegister(&AMDGPU::VGPR_32_Lo128RegClass);
+ S0Base = MRI->createVirtualRegister(Swap32RC);
BuildMI(MBB, I, DL, TII->get(TargetOpcode::COPY), S0Base)
.addReg(Src0Reg, {}, Src0Sub);
}
diff --git a/llvm/test/CodeGen/AMDGPU/v_swap_b16.ll b/llvm/test/CodeGen/AMDGPU/v_swap_b16.ll
index 934b55e0cd24f..b39f7e2145ac1 100644
--- a/llvm/test/CodeGen/AMDGPU/v_swap_b16.ll
+++ b/llvm/test/CodeGen/AMDGPU/v_swap_b16.ll
@@ -111,69 +111,159 @@ ret:
ret half %x
}
-define void @swap_shuffle_v4i8(ptr %out0, ptr %out1, ptr %in0, ptr %in1) {
-; GFX11-TRUE16-LABEL: swap_shuffle_v4i8:
+define { i32, i32 } @swap16_hi_lo(i32 %a, i32 %b) {
+; GFX11-TRUE16-LABEL: swap16_hi_lo:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[4:5]
-; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[6:7]
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_swap_b16 v5.h, v4.l
-; GFX11-TRUE16-NEXT: flat_store_b32 v[0:1], v4
-; GFX11-TRUE16-NEXT: flat_store_b32 v[2:3], v5
-; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_swap_b16 v1.h, v0.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-FAKE16-LABEL: swap_shuffle_v4i8:
+; GFX11-FAKE16-LABEL: swap16_hi_lo:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: flat_load_b32 v4, v[4:5]
-; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[6:7]
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_perm_b32 v6, v4, v5, 0x1000504
-; GFX11-FAKE16-NEXT: v_perm_b32 v4, v4, v5, 0x3020706
-; GFX11-FAKE16-NEXT: flat_store_b32 v[0:1], v6
-; GFX11-FAKE16-NEXT: flat_store_b32 v[2:3], v4
-; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, v0, v1, 0x1000504
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, v0, v1, 0x3020706
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, v2
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-TRUE16-LABEL: swap_shuffle_v4i8:
+; GFX12-TRUE16-LABEL: swap16_hi_lo:
; GFX12-TRUE16: ; %bb.0:
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[4:5]
-; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[6:7]
+; GFX12-TRUE16-NEXT: v_swap_b16 v1.h, v0.l
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: swap16_hi_lo:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_perm_b32 v2, v0, v1, 0x1000504
+; GFX12-FAKE16-NEXT: v_perm_b32 v1, v0, v1, 0x3020706
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, v2
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+ %aV = bitcast i32 %a to <4 x i8>
+ %bV = bitcast i32 %b to <4 x i8>
+ %loV = shufflevector <4 x i8> %aV, <4 x i8> %bV, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+ %hiV = shufflevector <4 x i8> %aV, <4 x i8> %bV, <4 x i32> <i32 2, i32 3, i32 6, i32 7>
+ %lo = bitcast <4 x i8> %loV to i32
+ %hi = bitcast <4 x i8> %hiV to i32
+ %r0 = insertvalue { i32, i32 } poison, i32 %lo, 0
+ %r1 = insertvalue { i32, i32 } %r0, i32 %hi, 1
+ ret { i32, i32 } %r1
+}
+
+define { i32, i32 } @swap16_hi_hi(i32 %a, i32 %b) {
+; GFX11-TRUE16-LABEL: swap16_hi_hi:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_swap_b16 v2.h, v1.h
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v1 :: v_dual_mov_b32 v1, v2
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: swap16_hi_hi:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, v0, v1, 0x7060100
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, v1, v0, 0x7060100
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, v2
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-TRUE16-LABEL: swap16_hi_hi:
+; GFX12-TRUE16: ; %bb.0:
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_swap_b16 v5.h, v4.l
-; GFX12-TRUE16-NEXT: flat_store_b32 v[0:1], v4
-; GFX12-TRUE16-NEXT: flat_store_b32 v[2:3], v5
-; GFX12-TRUE16-NEXT: s_wait_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_swap_b16 v2.h, v1.h
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v0, v1 :: v_dual_mov_b32 v1, v2
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-FAKE16-LABEL: swap_shuffle_v4i8:
+; GFX12-FAKE16-LABEL: swap16_hi_hi:
; GFX12-FAKE16: ; %bb.0:
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: flat_load_b32 v4, v[4:5]
-; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[6:7]
+; GFX12-FAKE16-NEXT: v_perm_b32 v2, v0, v1, 0x7060100
+; GFX12-FAKE16-NEXT: v_perm_b32 v1, v1, v0, 0x7060100
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, v2
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+ %c = call i32 @llvm.amdgcn.perm(i32 %a, i32 %b, i32 u0x7060100)
+ %d = call i32 @llvm.amdgcn.perm(i32 %b, i32 %a, i32 u0x7060100)
+ %r0 = insertvalue { i32, i32 } poison, i32 %c, 0
+ %r1 = insertvalue { i32, i32 } %r0, i32 %d, 1
+ ret { i32, i32 } %r1
+}
+
+define { i32, i32, i32 } @swap16_reuse(i32 %a, i32 %b) {
+; GFX11-TRUE16-LABEL: swap16_reuse:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v3, v1, 0x7060302
+; GFX11-TRUE16-NEXT: v_swap_b16 v3.l, v1.h
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v1 :: v_dual_mov_b32 v1, v3
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: swap16_reuse:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_perm_b32 v4, v0, v1, 0x3020706
+; GFX11-FAKE16-NEXT: v_perm_b32 v3, v1, v0, 0x5040100
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, v0, v1, 0x7060302
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v3
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-TRUE16-LABEL: swap16_reuse:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_perm_b32 v2, v3, v1, 0x7060302
+; GFX12-TRUE16-NEXT: v_swap_b16 v3.l, v1.h
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v0, v1 :: v_dual_mov_b32 v1, v3
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: swap16_reuse:
+; GFX12-FAKE16: ; %bb.0:
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_perm_b32 v6, v4, v5, 0x1000504
-; GFX12-FAKE16-NEXT: v_perm_b32 v4, v4, v5, 0x3020706
-; GFX12-FAKE16-NEXT: flat_store_b32 v[0:1], v6
-; GFX12-FAKE16-NEXT: flat_store_b32 v[2:3], v4
-; GFX12-FAKE16-NEXT: s_wait_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_perm_b32 v4, v0, v1, 0x3020706
+; GFX12-FAKE16-NEXT: v_perm_b32 v3, v1, v0, 0x5040100
+; GFX12-FAKE16-NEXT: v_perm_b32 v2, v0, v1, 0x7060302
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v3
; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
- %a = load <4 x i8>, ptr %in0, align 4
- %b = load <4 x i8>, ptr %in1, align 4
- %lo = shufflevector <4 x i8> %a, <4 x i8> %b, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
- %hi = shufflevector <4 x i8> %a, <4 x i8> %b, <4 x i32> <i32 2, i32 3, i32 6, i32 7>
- store <4 x i8> %lo, ptr %out0, align 4
- store <4 x i8> %hi, ptr %out1, align 4
- ret void
+ %x = call i32 @llvm.amdgcn.perm(i32 %a, i32 %b, i32 u0x7060302)
+ %c = call i32 @llvm.amdgcn.perm(i32 %a, i32 %b, i32 u0x3020706)
+ %d = call i32 @llvm.amdgcn.perm(i32 %b, i32 %a, i32 u0x5040100)
+ %r0 = insertvalue { i32, i32, i32 } poison, i32 %c, 0
+ %r1 = insertvalue { i32, i32, i32 } %r0, i32 %d, 1
+ %r2 = insertvalue { i32, i32, i32 } %r1, i32 %x, 2
+ ret { i32, i32, i32 } %r2
}
>From 9c2bac0c8fe0e8bf95848b00e00fc6e3da55d70b Mon Sep 17 00:00:00 2001
From: Frederick Vu <100011202+FrederickVu at users.noreply.github.com>
Date: Wed, 8 Apr 2026 20:31:53 +0000
Subject: [PATCH 5/7] Use subregs directly in swap and simplify mask retrieval
---
.../Target/AMDGPU/SIShrinkInstructions.cpp | 119 ++++++------------
.../CodeGen/AMDGPU/shrink-perm-to-swap.mir | 48 +++----
2 files changed, 59 insertions(+), 108 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp b/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp
index befd07c334435..1022c8f06bbb8 100644
--- a/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp
+++ b/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp
@@ -850,44 +850,6 @@ MachineInstr *SIShrinkInstructions::matchSwap(MachineInstr &MovT) const {
return nullptr;
}
-// Trace the selector operand of a V_PERM_B32 back through COPYs to find the
-// immediate mask.
-static bool getPermMaskImm(MachineInstr &Perm, MachineRegisterInfo *MRI,
- uint32_t &Mask) {
- const int Src2Idx =
- AMDGPU::getNamedOperandIdx(Perm.getOpcode(), AMDGPU::OpName::src2);
- const MachineOperand &Op = Perm.getOperand(Src2Idx);
- if (Op.isImm()) {
- Mask = static_cast<uint32_t>(Op.getImm());
- return true;
- }
- if (!Op.isReg() || !Op.getReg().isVirtual())
- return false;
-
- Register Reg = Op.getReg();
- SmallDenseSet<Register, 4> Seen;
- while (Reg.isVirtual() && Seen.insert(Reg).second) {
- MachineInstr *Def = MRI->getUniqueVRegDef(Reg);
- if (!Def)
- return false;
- if (Def->isMoveImmediate()) {
- const MachineOperand &ImmOp = Def->getOperand(1);
- if (!ImmOp.isImm())
- return false;
- Mask = static_cast<uint32_t>(ImmOp.getImm());
- return true;
- }
- if (!Def->isCopy() || !Def->getOperand(1).isReg())
- return false;
-
- if (Def->getOperand(0).getSubReg() != AMDGPU::NoSubRegister ||
- Def->getOperand(1).getSubReg() != AMDGPU::NoSubRegister)
- return false;
- Reg = Def->getOperand(1).getReg();
- }
- return false;
-}
-
// Matches two v_perms that together swap 16-bit halves between two inputs. For
// example:
//
@@ -910,6 +872,8 @@ SIShrinkInstructions::matchSwapB16(MachineInstr &Perm,
AMDGPU::getNamedOperandIdx(Perm.getOpcode(), AMDGPU::OpName::src0);
const int Src1Idx =
AMDGPU::getNamedOperandIdx(Perm.getOpcode(), AMDGPU::OpName::src1);
+ const int Src2Idx =
+ AMDGPU::getNamedOperandIdx(Perm.getOpcode(), AMDGPU::OpName::src2);
const MachineOperand &S0 = Perm.getOperand(Src0Idx);
const MachineOperand &S1 = Perm.getOperand(Src1Idx);
@@ -921,14 +885,14 @@ SIShrinkInstructions::matchSwapB16(MachineInstr &Perm,
unsigned Src1Sub = S1.getSubReg();
if (!Src0Reg.isVirtual() || !Src1Reg.isVirtual())
return nullptr;
- if (!TRI->isVGPR(*MRI, Src0Reg) || !TRI->isVGPR(*MRI, Src1Reg))
- return nullptr;
if (Src0Reg == Src1Reg && Src0Sub == Src1Sub)
return nullptr;
- uint32_t Mask = 0;
- if (!getPermMaskImm(Perm, MRI, Mask))
+ std::optional<int64_t> MaybeMask =
+ TII->getImmOrMaterializedImm(Perm.getOperand(Src2Idx));
+ if (!MaybeMask)
return nullptr;
+ uint32_t Mask = static_cast<uint32_t>(*MaybeMask);
// For two v_perms with common operands {src0, src1} and complementary,
// eligible masks {S0Mask, S1Mask}, we emit a v_swap_b16 which swaps
@@ -1012,56 +976,53 @@ SIShrinkInstructions::matchSwapB16(MachineInstr &Perm,
// Ensure that we can use Lo128 registers for operands of the v_swap.
const MCInstrDesc &SwapDesc = TII->get(AMDGPU::V_SWAP_B16);
const TargetRegisterClass *Swap16RC = TII->getRegClass(SwapDesc, 0);
- const TargetRegisterClass *Swap32RC =
- TRI->getMatchingSuperRegClass(MRI->getRegClass(S0Dst), Swap16RC, S0Sub);
- if (!Swap32RC)
- return nullptr;
- if (!MRI->constrainRegClass(S1Dst, Swap32RC) ||
- !MRI->constrainRegClass(S0Dst, Swap32RC))
- return nullptr;
- if (!Src1Sub && !MRI->constrainRegClass(Src1Reg, Swap32RC))
- return nullptr;
- if (!Src0Sub && !MRI->constrainRegClass(Src0Reg, Swap32RC))
+
+ const TargetRegisterClass *Src032RC =
+ Src0Sub ? TRI->getSubRegisterClass(MRI->getRegClass(Src0Reg), Src0Sub)
+ : MRI->getRegClass(Src0Reg);
+ const TargetRegisterClass *Src132RC =
+ Src1Sub ? TRI->getSubRegisterClass(MRI->getRegClass(Src1Reg), Src1Sub)
+ : MRI->getRegClass(Src1Reg);
+ if (!Src032RC || !Src132RC)
return nullptr;
+ const TargetRegisterClass *Base0RC =
+ TRI->getMatchingSuperRegClass(Src032RC, Swap16RC, S0Sub);
+ const TargetRegisterClass *Base1RC =
+ TRI->getMatchingSuperRegClass(Src132RC, Swap16RC, S1Sub);
+
MachineBasicBlock &MBB = *P0->getParent();
MachineBasicBlock::iterator I = P0->getIterator();
const DebugLoc &DL = P0->getDebugLoc();
- // Extract the two 16-bit halves to be swapped, S1In and S0In.
- Register S1In = MRI->createVirtualRegister(Swap16RC);
- Register S0In = MRI->createVirtualRegister(Swap16RC);
- unsigned S1InSub = TRI->composeSubRegIndices(Src1Sub, S1Sub);
- unsigned S0InSub = TRI->composeSubRegIndices(Src0Sub, S0Sub);
- BuildMI(MBB, I, DL, TII->get(TargetOpcode::COPY), S1In)
- .addReg(Src1Reg, {}, S1InSub);
- BuildMI(MBB, I, DL, TII->get(TargetOpcode::COPY), S0In)
- .addReg(Src0Reg, {}, S0InSub);
-
- // Swap. S1Out = S0In; S0Out = S1In;
+ // If P0/P1's operands were subregisters, COPY into Low128 32-bit registers.
+ Register S0Base = Src0Reg;
+ Register S1Base = Src1Reg;
+ if (Src0Sub) {
+ S0Base = MRI->createVirtualRegister(Base0RC);
+ BuildMI(MBB, I, DL, TII->get(TargetOpcode::COPY), S0Base)
+ .addReg(Src0Reg, {}, Src0Sub);
+ } else if (!MRI->constrainRegClass(Src0Reg, Base0RC)) {
+ return nullptr;
+ }
+ if (Src1Sub) {
+ S1Base = MRI->createVirtualRegister(Base1RC);
+ BuildMI(MBB, I, DL, TII->get(TargetOpcode::COPY), S1Base)
+ .addReg(Src1Reg, {}, Src1Sub);
+ } else if (!MRI->constrainRegClass(Src1Reg, Base1RC)) {
+ return nullptr;
+ }
+
+ // Swap. S1Out = Src0.S0Sub; S0Out = Src1.S1Sub;
Register S1Out = MRI->createVirtualRegister(Swap16RC);
Register S0Out = MRI->createVirtualRegister(Swap16RC);
auto *SwapMI = BuildMI(MBB, I, DL, TII->get(AMDGPU::V_SWAP_B16))
.addDef(S1Out)
.addDef(S0Out)
- .addReg(S0In)
- .addReg(S1In)
+ .addReg(S0Base, {}, S0Sub)
+ .addReg(S1Base, {}, S1Sub)
.getInstr();
- // If P0/P1's operands were subregisters, COPY into new 32-bit registers.
- Register S1Base = Src1Reg;
- Register S0Base = Src0Reg;
- if (Src1Sub) {
- S1Base = MRI->createVirtualRegister(Swap32RC);
- BuildMI(MBB, I, DL, TII->get(TargetOpcode::COPY), S1Base)
- .addReg(Src1Reg, {}, Src1Sub);
- }
- if (Src0Sub) {
- S0Base = MRI->createVirtualRegister(Swap32RC);
- BuildMI(MBB, I, DL, TII->get(TargetOpcode::COPY), S0Base)
- .addReg(Src0Reg, {}, Src0Sub);
- }
-
BuildMI(MBB, I, DL, TII->get(TargetOpcode::INSERT_SUBREG), S1Dst)
.addReg(S1Base)
.addReg(S1Out)
diff --git a/llvm/test/CodeGen/AMDGPU/shrink-perm-to-swap.mir b/llvm/test/CodeGen/AMDGPU/shrink-perm-to-swap.mir
index 3a0f482384fd8..da5ec5f9184c0 100644
--- a/llvm/test/CodeGen/AMDGPU/shrink-perm-to-swap.mir
+++ b/llvm/test/CodeGen/AMDGPU/shrink-perm-to-swap.mir
@@ -14,11 +14,9 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr_32_lo128 = COPY $vgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32_lo128 = COPY $vgpr1
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr_16_lo128 = COPY [[COPY1]].lo16
- ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr_16_lo128 = COPY [[COPY]].hi16
- ; CHECK-NEXT: [[V_SWAP_B16_:%[0-9]+]]:vgpr_16_lo128, [[V_SWAP_B16_1:%[0-9]+]]:vgpr_16_lo128 = V_SWAP_B16 [[COPY3]], [[COPY2]], implicit $exec
- ; CHECK-NEXT: [[INSERT_SUBREG:%[0-9]+]]:vgpr_32_lo128 = INSERT_SUBREG [[COPY1]], [[V_SWAP_B16_]], %subreg.lo16
- ; CHECK-NEXT: [[INSERT_SUBREG1:%[0-9]+]]:vgpr_32_lo128 = INSERT_SUBREG [[COPY]], [[V_SWAP_B16_1]], %subreg.hi16
+ ; CHECK-NEXT: [[V_SWAP_B16_:%[0-9]+]]:vgpr_16_lo128, [[V_SWAP_B16_1:%[0-9]+]]:vgpr_16_lo128 = V_SWAP_B16 [[COPY]].hi16, [[COPY1]].lo16, implicit $exec
+ ; CHECK-NEXT: [[INSERT_SUBREG:%[0-9]+]]:vgpr_32 = INSERT_SUBREG [[COPY1]], [[V_SWAP_B16_]], %subreg.lo16
+ ; CHECK-NEXT: [[INSERT_SUBREG1:%[0-9]+]]:vgpr_32 = INSERT_SUBREG [[COPY]], [[V_SWAP_B16_1]], %subreg.hi16
; CHECK-NEXT: S_NOP 0, implicit [[INSERT_SUBREG]], implicit [[INSERT_SUBREG1]]
%0:vgpr_32 = COPY $vgpr0
%1:vgpr_32 = COPY $vgpr1
@@ -37,20 +35,16 @@ body: |
; CHECK: liveins: $vgpr0_vgpr1_vgpr2_vgpr3
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_16_lo128 = COPY [[COPY]].sub3_hi16
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr_16_lo128 = COPY [[COPY]].sub1_lo16
- ; CHECK-NEXT: [[V_SWAP_B16_:%[0-9]+]]:vgpr_16_lo128, [[V_SWAP_B16_1:%[0-9]+]]:vgpr_16_lo128 = V_SWAP_B16 [[COPY2]], [[COPY1]], implicit $exec
- ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr_32_lo128 = COPY [[COPY]].sub3
- ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr_32_lo128 = COPY [[COPY]].sub1
- ; CHECK-NEXT: [[INSERT_SUBREG:%[0-9]+]]:vgpr_32_lo128 = INSERT_SUBREG [[COPY3]], [[V_SWAP_B16_]], %subreg.hi16
- ; CHECK-NEXT: [[INSERT_SUBREG1:%[0-9]+]]:vgpr_32_lo128 = INSERT_SUBREG [[COPY4]], [[V_SWAP_B16_1]], %subreg.lo16
- ; CHECK-NEXT: [[COPY5:%[0-9]+]]:vgpr_16_lo128 = COPY [[COPY]].sub2_hi16
- ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vgpr_16_lo128 = COPY [[COPY]].lo16
- ; CHECK-NEXT: [[V_SWAP_B16_2:%[0-9]+]]:vgpr_16_lo128, [[V_SWAP_B16_3:%[0-9]+]]:vgpr_16_lo128 = V_SWAP_B16 [[COPY6]], [[COPY5]], implicit $exec
- ; CHECK-NEXT: [[COPY7:%[0-9]+]]:vgpr_32_lo128 = COPY [[COPY]].sub2
- ; CHECK-NEXT: [[COPY8:%[0-9]+]]:vgpr_32_lo128 = COPY [[COPY]].sub0
- ; CHECK-NEXT: [[INSERT_SUBREG2:%[0-9]+]]:vgpr_32_lo128 = INSERT_SUBREG [[COPY7]], [[V_SWAP_B16_2]], %subreg.hi16
- ; CHECK-NEXT: [[INSERT_SUBREG3:%[0-9]+]]:vgpr_32_lo128 = INSERT_SUBREG [[COPY8]], [[V_SWAP_B16_3]], %subreg.lo16
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32_lo128 = COPY [[COPY]].sub1
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr_32_lo128 = COPY [[COPY]].sub3
+ ; CHECK-NEXT: [[V_SWAP_B16_:%[0-9]+]]:vgpr_16_lo128, [[V_SWAP_B16_1:%[0-9]+]]:vgpr_16_lo128 = V_SWAP_B16 [[COPY1]].lo16, [[COPY2]].hi16, implicit $exec
+ ; CHECK-NEXT: [[INSERT_SUBREG:%[0-9]+]]:vgpr_32 = INSERT_SUBREG [[COPY2]], [[V_SWAP_B16_]], %subreg.hi16
+ ; CHECK-NEXT: [[INSERT_SUBREG1:%[0-9]+]]:vgpr_32 = INSERT_SUBREG [[COPY1]], [[V_SWAP_B16_1]], %subreg.lo16
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr_32_lo128 = COPY [[COPY]].sub0
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr_32_lo128 = COPY [[COPY]].sub2
+ ; CHECK-NEXT: [[V_SWAP_B16_2:%[0-9]+]]:vgpr_16_lo128, [[V_SWAP_B16_3:%[0-9]+]]:vgpr_16_lo128 = V_SWAP_B16 [[COPY3]].lo16, [[COPY4]].hi16, implicit $exec
+ ; CHECK-NEXT: [[INSERT_SUBREG2:%[0-9]+]]:vgpr_32 = INSERT_SUBREG [[COPY4]], [[V_SWAP_B16_2]], %subreg.hi16
+ ; CHECK-NEXT: [[INSERT_SUBREG3:%[0-9]+]]:vgpr_32 = INSERT_SUBREG [[COPY3]], [[V_SWAP_B16_3]], %subreg.lo16
; CHECK-NEXT: S_NOP 0, implicit [[INSERT_SUBREG]], implicit [[INSERT_SUBREG1]], implicit [[INSERT_SUBREG2]], implicit [[INSERT_SUBREG3]]
%0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
%1:vgpr_32 = V_PERM_B32_e64 %0.sub1, %0.sub3, 50464518, implicit $exec ; 0x03020706
@@ -73,11 +67,9 @@ body: |
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32_lo128 = COPY $vgpr1
; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
- ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr_16_lo128 = COPY [[COPY1]].lo16
- ; CHECK-NEXT: [[COPY5:%[0-9]+]]:vgpr_16_lo128 = COPY [[COPY]].hi16
- ; CHECK-NEXT: [[V_SWAP_B16_:%[0-9]+]]:vgpr_16_lo128, [[V_SWAP_B16_1:%[0-9]+]]:vgpr_16_lo128 = V_SWAP_B16 [[COPY5]], [[COPY4]], implicit $exec
- ; CHECK-NEXT: [[INSERT_SUBREG:%[0-9]+]]:vgpr_32_lo128 = INSERT_SUBREG [[COPY1]], [[V_SWAP_B16_]], %subreg.lo16
- ; CHECK-NEXT: [[INSERT_SUBREG1:%[0-9]+]]:vgpr_32_lo128 = INSERT_SUBREG [[COPY]], [[V_SWAP_B16_1]], %subreg.hi16
+ ; CHECK-NEXT: [[V_SWAP_B16_:%[0-9]+]]:vgpr_16_lo128, [[V_SWAP_B16_1:%[0-9]+]]:vgpr_16_lo128 = V_SWAP_B16 [[COPY]].hi16, [[COPY1]].lo16, implicit $exec
+ ; CHECK-NEXT: [[INSERT_SUBREG:%[0-9]+]]:vgpr_32 = INSERT_SUBREG [[COPY1]], [[V_SWAP_B16_]], %subreg.lo16
+ ; CHECK-NEXT: [[INSERT_SUBREG1:%[0-9]+]]:vgpr_32 = INSERT_SUBREG [[COPY]], [[V_SWAP_B16_1]], %subreg.hi16
; CHECK-NEXT: [[V_ADD_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[COPY2]], [[COPY3]], implicit $exec
; CHECK-NEXT: S_NOP 0, implicit [[INSERT_SUBREG]], implicit [[V_ADD_U32_e32_]], implicit [[INSERT_SUBREG1]]
%0:vgpr_32 = COPY $vgpr0
@@ -101,11 +93,9 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr_32_lo128 = COPY $vgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32_lo128 = COPY $vgpr1
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr_16_lo128 = COPY [[COPY]].hi16
- ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr_16_lo128 = COPY [[COPY1]].lo16
- ; CHECK-NEXT: [[V_SWAP_B16_:%[0-9]+]]:vgpr_16_lo128, [[V_SWAP_B16_1:%[0-9]+]]:vgpr_16_lo128 = V_SWAP_B16 [[COPY3]], [[COPY2]], implicit $exec
- ; CHECK-NEXT: [[INSERT_SUBREG:%[0-9]+]]:vgpr_32_lo128 = INSERT_SUBREG [[COPY]], [[V_SWAP_B16_]], %subreg.hi16
- ; CHECK-NEXT: [[INSERT_SUBREG1:%[0-9]+]]:vgpr_32_lo128 = INSERT_SUBREG [[COPY1]], [[V_SWAP_B16_1]], %subreg.lo16
+ ; CHECK-NEXT: [[V_SWAP_B16_:%[0-9]+]]:vgpr_16_lo128, [[V_SWAP_B16_1:%[0-9]+]]:vgpr_16_lo128 = V_SWAP_B16 [[COPY1]].lo16, [[COPY]].hi16, implicit $exec
+ ; CHECK-NEXT: [[INSERT_SUBREG:%[0-9]+]]:vgpr_32 = INSERT_SUBREG [[COPY]], [[V_SWAP_B16_]], %subreg.hi16
+ ; CHECK-NEXT: [[INSERT_SUBREG1:%[0-9]+]]:vgpr_32 = INSERT_SUBREG [[COPY1]], [[V_SWAP_B16_1]], %subreg.lo16
; CHECK-NEXT: S_NOP 0, implicit [[INSERT_SUBREG1]], implicit [[INSERT_SUBREG]]
%0:vgpr_32 = COPY $vgpr0
%1:vgpr_32 = COPY $vgpr1
>From 889637d39f057deccf26e0bfec259882079b7fb4 Mon Sep 17 00:00:00 2001
From: Frederick Vu <100011202+FrederickVu at users.noreply.github.com>
Date: Mon, 13 Apr 2026 20:33:30 +0000
Subject: [PATCH 6/7] Address comments and fix swap cases
---
.../Target/AMDGPU/SIShrinkInstructions.cpp | 78 ++++++++-----------
.../CodeGen/AMDGPU/shrink-perm-to-swap.mir | 70 +++++++++++------
llvm/test/CodeGen/AMDGPU/v_swap_b16.ll | 8 +-
3 files changed, 82 insertions(+), 74 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp b/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp
index 1022c8f06bbb8..55816050b647b 100644
--- a/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp
+++ b/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp
@@ -895,10 +895,8 @@ SIShrinkInstructions::matchSwapB16(MachineInstr &Perm,
uint32_t Mask = static_cast<uint32_t>(*MaybeMask);
// For two v_perms with common operands {src0, src1} and complementary,
- // eligible masks {S0Mask, S1Mask}, we emit a v_swap_b16 which swaps
- // src0.S0Sub and src1.S1Sub. S0 and S1 indicate the Perm whose destination
- // register will be replaced by an INSERT_SUBREG which has src0 or src1 as
- // its base.
+ // eligible masks, denote by S0 the 32-bit output of the v_perm with mask
+ // S0Mask and S0Sub the half of S0 which is swapped, and similarly for S1.
struct SwapCase {
uint32_t S1Mask;
uint32_t S0Mask;
@@ -906,12 +904,12 @@ SIShrinkInstructions::matchSwapB16(MachineInstr &Perm,
unsigned S1Sub;
};
static constexpr SwapCase Cases[] = {
- {0x05040100u, 0x07060302u, AMDGPU::hi16, AMDGPU::lo16},
+ {0x05040100u, 0x07060302u, AMDGPU::lo16, AMDGPU::hi16},
{0x07060100u, 0x03020504u, AMDGPU::hi16, AMDGPU::hi16},
- {0x03020706u, 0x01000504u, AMDGPU::lo16, AMDGPU::hi16},
+ {0x03020706u, 0x01000504u, AMDGPU::hi16, AMDGPU::lo16},
};
- if (!llvm::any_of(Cases, [Mask](const SwapCase &C) {
+ if (llvm::none_of(Cases, [Mask](const SwapCase &C) {
return Mask == C.S1Mask || Mask == C.S0Mask;
}))
return nullptr;
@@ -977,61 +975,49 @@ SIShrinkInstructions::matchSwapB16(MachineInstr &Perm,
const MCInstrDesc &SwapDesc = TII->get(AMDGPU::V_SWAP_B16);
const TargetRegisterClass *Swap16RC = TII->getRegClass(SwapDesc, 0);
- const TargetRegisterClass *Src032RC =
- Src0Sub ? TRI->getSubRegisterClass(MRI->getRegClass(Src0Reg), Src0Sub)
- : MRI->getRegClass(Src0Reg);
- const TargetRegisterClass *Src132RC =
- Src1Sub ? TRI->getSubRegisterClass(MRI->getRegClass(Src1Reg), Src1Sub)
- : MRI->getRegClass(Src1Reg);
- if (!Src032RC || !Src132RC)
- return nullptr;
+ unsigned S0InSub = TRI->composeSubRegIndices(Src0Sub, S0Sub);
+ unsigned S1InSub = TRI->composeSubRegIndices(Src1Sub, S1Sub);
+
+ const TargetRegisterClass *Src0LowRC = TRI->getMatchingSuperRegClass(
+ MRI->getRegClass(Src0Reg), Swap16RC, S0InSub);
+ const TargetRegisterClass *Src1LowRC = TRI->getMatchingSuperRegClass(
+ MRI->getRegClass(Src1Reg), Swap16RC, S1InSub);
- const TargetRegisterClass *Base0RC =
- TRI->getMatchingSuperRegClass(Src032RC, Swap16RC, S0Sub);
- const TargetRegisterClass *Base1RC =
- TRI->getMatchingSuperRegClass(Src132RC, Swap16RC, S1Sub);
+ if (!Src0LowRC || !MRI->constrainRegClass(Src0Reg, Src0LowRC) || !Src1LowRC ||
+ !MRI->constrainRegClass(Src1Reg, Src1LowRC))
+ return nullptr;
MachineBasicBlock &MBB = *P0->getParent();
MachineBasicBlock::iterator I = P0->getIterator();
const DebugLoc &DL = P0->getDebugLoc();
- // If P0/P1's operands were subregisters, COPY into Low128 32-bit registers.
- Register S0Base = Src0Reg;
- Register S1Base = Src1Reg;
- if (Src0Sub) {
- S0Base = MRI->createVirtualRegister(Base0RC);
- BuildMI(MBB, I, DL, TII->get(TargetOpcode::COPY), S0Base)
- .addReg(Src0Reg, {}, Src0Sub);
- } else if (!MRI->constrainRegClass(Src0Reg, Base0RC)) {
- return nullptr;
- }
- if (Src1Sub) {
- S1Base = MRI->createVirtualRegister(Base1RC);
- BuildMI(MBB, I, DL, TII->get(TargetOpcode::COPY), S1Base)
- .addReg(Src1Reg, {}, Src1Sub);
- } else if (!MRI->constrainRegClass(Src1Reg, Base1RC)) {
- return nullptr;
- }
-
- // Swap. S1Out = Src0.S0Sub; S0Out = Src1.S1Sub;
+ // Swap. S1Out = Src0.S0InSub; S0Out = Src1.S1InSub;
Register S1Out = MRI->createVirtualRegister(Swap16RC);
Register S0Out = MRI->createVirtualRegister(Swap16RC);
auto *SwapMI = BuildMI(MBB, I, DL, TII->get(AMDGPU::V_SWAP_B16))
.addDef(S1Out)
.addDef(S0Out)
- .addReg(S0Base, {}, S0Sub)
- .addReg(S1Base, {}, S1Sub)
+ .addReg(Src0Reg, {}, S0InSub)
+ .addReg(Src1Reg, {}, S1InSub)
.getInstr();
- BuildMI(MBB, I, DL, TII->get(TargetOpcode::INSERT_SUBREG), S1Dst)
- .addReg(S1Base)
+ auto otherSub16 = [](unsigned sub) {
+ return sub == AMDGPU::lo16 ? AMDGPU::hi16 : AMDGPU::lo16;
+ };
+ unsigned S1KeepSub = TRI->composeSubRegIndices(Src1Sub, otherSub16(S1Sub));
+ unsigned S0KeepSub = TRI->composeSubRegIndices(Src0Sub, otherSub16(S0Sub));
+
+ BuildMI(MBB, I, DL, TII->get(TargetOpcode::REG_SEQUENCE), S1Dst)
.addReg(S1Out)
- .addImm(S1Sub);
+ .addImm(S1Sub)
+ .addReg(Src1Reg, {}, S1KeepSub)
+ .addImm(otherSub16(S1Sub));
- BuildMI(MBB, I, DL, TII->get(TargetOpcode::INSERT_SUBREG), S0Dst)
- .addReg(S0Base)
+ BuildMI(MBB, I, DL, TII->get(TargetOpcode::REG_SEQUENCE), S0Dst)
.addReg(S0Out)
- .addImm(S0Sub);
+ .addImm(S0Sub)
+ .addReg(Src0Reg, {}, S0KeepSub)
+ .addImm(otherSub16(S0Sub));
dropInstructionKeepingImpDefs(*P1);
dropInstructionKeepingImpDefs(*P0);
diff --git a/llvm/test/CodeGen/AMDGPU/shrink-perm-to-swap.mir b/llvm/test/CodeGen/AMDGPU/shrink-perm-to-swap.mir
index da5ec5f9184c0..fb75c209ec7d2 100644
--- a/llvm/test/CodeGen/AMDGPU/shrink-perm-to-swap.mir
+++ b/llvm/test/CodeGen/AMDGPU/shrink-perm-to-swap.mir
@@ -14,10 +14,10 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr_32_lo128 = COPY $vgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32_lo128 = COPY $vgpr1
- ; CHECK-NEXT: [[V_SWAP_B16_:%[0-9]+]]:vgpr_16_lo128, [[V_SWAP_B16_1:%[0-9]+]]:vgpr_16_lo128 = V_SWAP_B16 [[COPY]].hi16, [[COPY1]].lo16, implicit $exec
- ; CHECK-NEXT: [[INSERT_SUBREG:%[0-9]+]]:vgpr_32 = INSERT_SUBREG [[COPY1]], [[V_SWAP_B16_]], %subreg.lo16
- ; CHECK-NEXT: [[INSERT_SUBREG1:%[0-9]+]]:vgpr_32 = INSERT_SUBREG [[COPY]], [[V_SWAP_B16_1]], %subreg.hi16
- ; CHECK-NEXT: S_NOP 0, implicit [[INSERT_SUBREG]], implicit [[INSERT_SUBREG1]]
+ ; CHECK-NEXT: [[V_SWAP_B16_:%[0-9]+]]:vgpr_16_lo128, [[V_SWAP_B16_1:%[0-9]+]]:vgpr_16_lo128 = V_SWAP_B16 [[COPY]].lo16, [[COPY1]].hi16, implicit $exec
+ ; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[V_SWAP_B16_]], %subreg.hi16, [[COPY1]].lo16, %subreg.lo16
+ ; CHECK-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[V_SWAP_B16_1]], %subreg.lo16, [[COPY]].hi16, %subreg.hi16
+ ; CHECK-NEXT: S_NOP 0, implicit [[REG_SEQUENCE]], implicit [[REG_SEQUENCE1]]
%0:vgpr_32 = COPY $vgpr0
%1:vgpr_32 = COPY $vgpr1
%2:vgpr_32 = V_PERM_B32_e64 %0, %1, 84148480, implicit $exec ; 0x05040100
@@ -34,18 +34,14 @@ body: |
; CHECK-LABEL: name: perm_to_swap_subreg
; CHECK: liveins: $vgpr0_vgpr1_vgpr2_vgpr3
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32_lo128 = COPY [[COPY]].sub1
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr_32_lo128 = COPY [[COPY]].sub3
- ; CHECK-NEXT: [[V_SWAP_B16_:%[0-9]+]]:vgpr_16_lo128, [[V_SWAP_B16_1:%[0-9]+]]:vgpr_16_lo128 = V_SWAP_B16 [[COPY1]].lo16, [[COPY2]].hi16, implicit $exec
- ; CHECK-NEXT: [[INSERT_SUBREG:%[0-9]+]]:vgpr_32 = INSERT_SUBREG [[COPY2]], [[V_SWAP_B16_]], %subreg.hi16
- ; CHECK-NEXT: [[INSERT_SUBREG1:%[0-9]+]]:vgpr_32 = INSERT_SUBREG [[COPY1]], [[V_SWAP_B16_1]], %subreg.lo16
- ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr_32_lo128 = COPY [[COPY]].sub0
- ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr_32_lo128 = COPY [[COPY]].sub2
- ; CHECK-NEXT: [[V_SWAP_B16_2:%[0-9]+]]:vgpr_16_lo128, [[V_SWAP_B16_3:%[0-9]+]]:vgpr_16_lo128 = V_SWAP_B16 [[COPY3]].lo16, [[COPY4]].hi16, implicit $exec
- ; CHECK-NEXT: [[INSERT_SUBREG2:%[0-9]+]]:vgpr_32 = INSERT_SUBREG [[COPY4]], [[V_SWAP_B16_2]], %subreg.hi16
- ; CHECK-NEXT: [[INSERT_SUBREG3:%[0-9]+]]:vgpr_32 = INSERT_SUBREG [[COPY3]], [[V_SWAP_B16_3]], %subreg.lo16
- ; CHECK-NEXT: S_NOP 0, implicit [[INSERT_SUBREG]], implicit [[INSERT_SUBREG1]], implicit [[INSERT_SUBREG2]], implicit [[INSERT_SUBREG3]]
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:av_128_with_sub0_sub1_sub2_in_av_96_align2_and_av_96_with_sub2_in_vgpr_32_lo128 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+ ; CHECK-NEXT: [[V_SWAP_B16_:%[0-9]+]]:vgpr_16_lo128, [[V_SWAP_B16_1:%[0-9]+]]:vgpr_16_lo128 = V_SWAP_B16 [[COPY]].sub1_hi16, [[COPY]].sub3_lo16, implicit $exec
+ ; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[V_SWAP_B16_]], %subreg.lo16, [[COPY]].sub3_hi16, %subreg.hi16
+ ; CHECK-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[V_SWAP_B16_1]], %subreg.hi16, [[COPY]].sub1_lo16, %subreg.lo16
+ ; CHECK-NEXT: [[V_SWAP_B16_2:%[0-9]+]]:vgpr_16_lo128, [[V_SWAP_B16_3:%[0-9]+]]:vgpr_16_lo128 = V_SWAP_B16 [[COPY]].hi16, [[COPY]].sub2_lo16, implicit $exec
+ ; CHECK-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[V_SWAP_B16_2]], %subreg.lo16, [[COPY]].sub2_hi16, %subreg.hi16
+ ; CHECK-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[V_SWAP_B16_3]], %subreg.hi16, [[COPY]].lo16, %subreg.lo16
+ ; CHECK-NEXT: S_NOP 0, implicit [[REG_SEQUENCE]], implicit [[REG_SEQUENCE1]], implicit [[REG_SEQUENCE2]], implicit [[REG_SEQUENCE3]]
%0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
%1:vgpr_32 = V_PERM_B32_e64 %0.sub1, %0.sub3, 50464518, implicit $exec ; 0x03020706
%2:vgpr_32 = V_PERM_B32_e64 %0.sub1, %0.sub3, 16778500, implicit $exec ; 0x01000504
@@ -67,11 +63,11 @@ body: |
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32_lo128 = COPY $vgpr1
; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
- ; CHECK-NEXT: [[V_SWAP_B16_:%[0-9]+]]:vgpr_16_lo128, [[V_SWAP_B16_1:%[0-9]+]]:vgpr_16_lo128 = V_SWAP_B16 [[COPY]].hi16, [[COPY1]].lo16, implicit $exec
- ; CHECK-NEXT: [[INSERT_SUBREG:%[0-9]+]]:vgpr_32 = INSERT_SUBREG [[COPY1]], [[V_SWAP_B16_]], %subreg.lo16
- ; CHECK-NEXT: [[INSERT_SUBREG1:%[0-9]+]]:vgpr_32 = INSERT_SUBREG [[COPY]], [[V_SWAP_B16_1]], %subreg.hi16
+ ; CHECK-NEXT: [[V_SWAP_B16_:%[0-9]+]]:vgpr_16_lo128, [[V_SWAP_B16_1:%[0-9]+]]:vgpr_16_lo128 = V_SWAP_B16 [[COPY]].lo16, [[COPY1]].hi16, implicit $exec
+ ; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[V_SWAP_B16_]], %subreg.hi16, [[COPY1]].lo16, %subreg.lo16
+ ; CHECK-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[V_SWAP_B16_1]], %subreg.lo16, [[COPY]].hi16, %subreg.hi16
; CHECK-NEXT: [[V_ADD_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[COPY2]], [[COPY3]], implicit $exec
- ; CHECK-NEXT: S_NOP 0, implicit [[INSERT_SUBREG]], implicit [[V_ADD_U32_e32_]], implicit [[INSERT_SUBREG1]]
+ ; CHECK-NEXT: S_NOP 0, implicit [[REG_SEQUENCE]], implicit [[V_ADD_U32_e32_]], implicit [[REG_SEQUENCE1]]
%0:vgpr_32 = COPY $vgpr0
%1:vgpr_32 = COPY $vgpr1
%2:vgpr_32 = COPY $vgpr2
@@ -93,10 +89,10 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr_32_lo128 = COPY $vgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32_lo128 = COPY $vgpr1
- ; CHECK-NEXT: [[V_SWAP_B16_:%[0-9]+]]:vgpr_16_lo128, [[V_SWAP_B16_1:%[0-9]+]]:vgpr_16_lo128 = V_SWAP_B16 [[COPY1]].lo16, [[COPY]].hi16, implicit $exec
- ; CHECK-NEXT: [[INSERT_SUBREG:%[0-9]+]]:vgpr_32 = INSERT_SUBREG [[COPY]], [[V_SWAP_B16_]], %subreg.hi16
- ; CHECK-NEXT: [[INSERT_SUBREG1:%[0-9]+]]:vgpr_32 = INSERT_SUBREG [[COPY1]], [[V_SWAP_B16_1]], %subreg.lo16
- ; CHECK-NEXT: S_NOP 0, implicit [[INSERT_SUBREG1]], implicit [[INSERT_SUBREG]]
+ ; CHECK-NEXT: [[V_SWAP_B16_:%[0-9]+]]:vgpr_16_lo128, [[V_SWAP_B16_1:%[0-9]+]]:vgpr_16_lo128 = V_SWAP_B16 [[COPY1]].hi16, [[COPY]].lo16, implicit $exec
+ ; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[V_SWAP_B16_]], %subreg.lo16, [[COPY]].hi16, %subreg.hi16
+ ; CHECK-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[V_SWAP_B16_1]], %subreg.hi16, [[COPY1]].lo16, %subreg.lo16
+ ; CHECK-NEXT: S_NOP 0, implicit [[REG_SEQUENCE1]], implicit [[REG_SEQUENCE]]
%0:vgpr_32 = COPY $vgpr0
%1:vgpr_32 = COPY $vgpr1
%2:vgpr_32 = V_PERM_B32_e64 %0, %1, 84148480, implicit $exec ; 0x05040100
@@ -128,3 +124,29 @@ body: |
%4:vgpr_32 = V_PERM_B32_e64 %0, %1, 117834498, implicit $exec ; 0x07060302
S_NOP 0, implicit %3, implicit %4
...
+
+---
+name: perm_to_swap_materialized_mask
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr1
+ ; CHECK-LABEL: name: perm_to_swap_materialized_mask
+ ; CHECK: liveins: $vgpr0, $vgpr1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr_32_lo128 = COPY $vgpr0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32_lo128 = COPY $vgpr1
+ ; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 84148480, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 117834498, implicit $exec
+ ; CHECK-NEXT: [[V_SWAP_B16_:%[0-9]+]]:vgpr_16_lo128, [[V_SWAP_B16_1:%[0-9]+]]:vgpr_16_lo128 = V_SWAP_B16 [[COPY]].lo16, [[COPY1]].hi16, implicit $exec
+ ; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[V_SWAP_B16_]], %subreg.hi16, [[COPY1]].lo16, %subreg.lo16
+ ; CHECK-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[V_SWAP_B16_1]], %subreg.lo16, [[COPY]].hi16, %subreg.hi16
+ ; CHECK-NEXT: S_NOP 0, implicit [[REG_SEQUENCE]], implicit [[REG_SEQUENCE1]]
+ %0:vgpr_32 = COPY $vgpr0
+ %1:vgpr_32 = COPY $vgpr1
+ %2:vgpr_32 = V_MOV_B32_e32 84148480, implicit $exec ; 0x05040100
+ %3:vgpr_32 = V_MOV_B32_e32 117834498, implicit $exec ; 0x07060302
+ %4:vgpr_32 = V_PERM_B32_e64 %0, %1, %2, implicit $exec
+ %5:vgpr_32 = V_PERM_B32_e64 %0, %1, %3, implicit $exec
+ S_NOP 0, implicit %4, implicit %5
+...
diff --git a/llvm/test/CodeGen/AMDGPU/v_swap_b16.ll b/llvm/test/CodeGen/AMDGPU/v_swap_b16.ll
index b39f7e2145ac1..93e43ab7b168f 100644
--- a/llvm/test/CodeGen/AMDGPU/v_swap_b16.ll
+++ b/llvm/test/CodeGen/AMDGPU/v_swap_b16.ll
@@ -115,7 +115,7 @@ define { i32, i32 } @swap16_hi_lo(i32 %a, i32 %b) {
; GFX11-TRUE16-LABEL: swap16_hi_lo:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_swap_b16 v1.h, v0.l
+; GFX11-TRUE16-NEXT: v_swap_b16 v1.l, v0.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: swap16_hi_lo:
@@ -134,7 +134,7 @@ define { i32, i32 } @swap16_hi_lo(i32 %a, i32 %b) {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_swap_b16 v1.h, v0.l
+; GFX12-TRUE16-NEXT: v_swap_b16 v1.l, v0.h
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: swap16_hi_lo:
@@ -218,7 +218,7 @@ define { i32, i32, i32 } @swap16_reuse(i32 %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v3, v1, 0x7060302
-; GFX11-TRUE16-NEXT: v_swap_b16 v3.l, v1.h
+; GFX11-TRUE16-NEXT: v_swap_b16 v3.h, v1.l
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v1 :: v_dual_mov_b32 v1, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -242,7 +242,7 @@ define { i32, i32, i32 } @swap16_reuse(i32 %a, i32 %b) {
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_perm_b32 v2, v3, v1, 0x7060302
-; GFX12-TRUE16-NEXT: v_swap_b16 v3.l, v1.h
+; GFX12-TRUE16-NEXT: v_swap_b16 v3.h, v1.l
; GFX12-TRUE16-NEXT: v_dual_mov_b32 v0, v1 :: v_dual_mov_b32 v1, v3
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
>From fb7961128367c0dd5e0c3b467ac937789ffb493d Mon Sep 17 00:00:00 2001
From: Frederick Vu <100011202+FrederickVu at users.noreply.github.com>
Date: Tue, 14 Apr 2026 05:51:14 +0000
Subject: [PATCH 7/7] Address commentsy
---
llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp | 16 ++++++++--------
1 file changed, 8 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp b/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp
index 55816050b647b..ef59dd7938108 100644
--- a/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp
+++ b/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp
@@ -853,8 +853,8 @@ MachineInstr *SIShrinkInstructions::matchSwap(MachineInstr &MovT) const {
// Matches two v_perms that together swap 16-bit halves between two inputs. For
// example:
//
-// v_perm v2, v0, v1, 0x5040100
-// v_perm v3, v0, v1, 0x7060302
+// v_perm_b32 v2, v0, v1, 0x5040100
+// v_perm_b32 v3, v0, v1, 0x7060302
// =>
// v_swap_b16 v0.h, v1.l
MachineInstr *
@@ -994,12 +994,12 @@ SIShrinkInstructions::matchSwapB16(MachineInstr &Perm,
// Swap. S1Out = Src0.S0InSub; S0Out = Src1.S1InSub;
Register S1Out = MRI->createVirtualRegister(Swap16RC);
Register S0Out = MRI->createVirtualRegister(Swap16RC);
- auto *SwapMI = BuildMI(MBB, I, DL, TII->get(AMDGPU::V_SWAP_B16))
- .addDef(S1Out)
- .addDef(S0Out)
- .addReg(Src0Reg, {}, S0InSub)
- .addReg(Src1Reg, {}, S1InSub)
- .getInstr();
+ MachineInstr *SwapMI = BuildMI(MBB, I, DL, TII->get(AMDGPU::V_SWAP_B16))
+ .addDef(S1Out)
+ .addDef(S0Out)
+ .addReg(Src0Reg, {}, S0InSub)
+ .addReg(Src1Reg, {}, S1InSub)
+ .getInstr();
auto otherSub16 = [](unsigned sub) {
return sub == AMDGPU::lo16 ? AMDGPU::hi16 : AMDGPU::lo16;
More information about the llvm-commits
mailing list