[llvm-branch-commits] [llvm] [AMDGPU] Insert exec-forced V_NOP after V_PERM_PK16 (gfx1250 hazard) (PR #214406)
via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Wed Aug 5 23:34:07 PDT 2026
https://github.com/hidekisaito created https://github.com/llvm/llvm-project/pull/214406
On gfx1250 the V_PERM_PK16 family (V_PERM_PK16_B4/B6/B8_U4) has a hazard: the instruction must be immediately followed by a "safe" instruction that issues on the pipe which clears the hazard. Insert V_NOP with forced non-zero EXEC as needed.
This PR depends on #207862, to help identify whether MI is a "safe" instruction or not.
Assisted-by: Opus 4.8 Medium
>From 6250bdf0f139e9d7c7250bd3f54384e0b472906e Mon Sep 17 00:00:00 2001
From: Hideki Saito <hidekido at amd.com>
Date: Wed, 5 Aug 2026 22:11:04 -0500
Subject: [PATCH] [AMDGPU] Insert exec-forced V_NOP after V_PERM_PK16 (gfx1250
hazard)
On gfx1250 the V_PERM_PK16 family (V_PERM_PK16_B4/B6/B8_U4) has a hazard:
the instruction must be immediately followed by a "safe" instruction that
issues on the pipe which clears the hazard. Insert V_NOP with forced
non-zero EXEC as needed.
Assisted-by: Opus 4.8 Medium
---
.../lib/Target/AMDGPU/GCNHazardRecognizer.cpp | 59 +++
llvm/lib/Target/AMDGPU/GCNHazardRecognizer.h | 4 +
llvm/lib/Target/AMDGPU/GCNSubtarget.h | 8 +
llvm/lib/Target/AMDGPU/SIFrameLowering.cpp | 6 +-
llvm/lib/Target/AMDGPU/SIInstrInfo.h | 49 +++
llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp | 25 +-
.../CodeGen/AMDGPU/llvm.amdgcn.perm.pk.ll | 16 +
.../CodeGen/AMDGPU/vperm-pk16-exec-hazard.ll | 80 ++++
.../CodeGen/AMDGPU/vperm-pk16-exec-hazard.mir | 348 ++++++++++++++++++
9 files changed, 591 insertions(+), 4 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/vperm-pk16-exec-hazard.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/vperm-pk16-exec-hazard.mir
diff --git a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
index a3842f43a62fa..aeaa7a9101daf 100644
--- a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
@@ -1781,6 +1781,8 @@ void GCNHazardRecognizer::fixHazards(MachineInstr *MI) {
fixShift64HighRegBug(MI);
fixVALUMaskWriteHazard(MI);
fixRequiredExportPriority(MI);
+ if (ST.hasVPermPk16Hazard())
+ fixVPermPk16Hazard(MI);
if (ST.requiresWaitIdleBeforeGetReg())
fixGetRegWaitIdle(MI);
if (ST.hasDsAtomicAsyncBarrierArriveB64PipeBug())
@@ -4185,6 +4187,63 @@ bool GCNHazardRecognizer::fixRequiredExportPriority(MachineInstr *MI) {
return true;
}
+// Advance past meta instructions (debug values, labels, CFI, KILL, etc.) to the
+// next instruction that actually issues. Unlike skipDebugInstructionsForward /
+// next_nodbg, this skips the full isMetaInstruction() set.
+static MachineBasicBlock::iterator
+skipMetaInstructionsForward(MachineBasicBlock::iterator I,
+ MachineBasicBlock::iterator End) {
+ while (I != End && I->isMetaInstruction())
+ ++I;
+ return I;
+}
+
+void GCNHazardRecognizer::emitVPermPk16ExecNonZeroVNop(
+ MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertPt,
+ const DebugLoc &DL) {
+ MachineFunction *MFn = MBB.getParent();
+
+ const SIMachineFunctionInfo *MFI = MFn->getInfo<SIMachineFunctionInfo>();
+ Register ExecCopy = MFI->getSGPRForEXECCopy();
+ assert(ExecCopy &&
+ "Function with V_PERM_PK16 must reserve an SGPR for the EXEC copy");
+
+ // Insert V_NOP with forced non-zero EXEC.
+ TII.insertScratchExecCopy(*MFn, MBB, InsertPt, DL, ExecCopy,
+ /*IsSCCLive=*/true, /*Indexes=*/nullptr);
+ BuildMI(MBB, InsertPt, DL, TII.get(AMDGPU::V_NOP_e32));
+ TII.restoreExec(*MFn, MBB, InsertPt, DL, ExecCopy, /*Indexes=*/nullptr);
+}
+
+bool GCNHazardRecognizer::fixVPermPk16Hazard(MachineInstr *MI) {
+ // Requirement #1 of 2:
+ // The cross-wave entry-block mitigation is delegated to the mandatory
+ // unclaused-VMEM entry prologue (GLOBAL_PREFETCH_B8 + V_NOP).
+ assert(ST.hasRequiresInitialUnclausedVmem() &&
+ "V_PERM_PK16-hazard subtarget must provide the unclaused-VMEM entry "
+ "prologue to satisfy the cross-wave entry mitigation");
+
+ if (!SIInstrInfo::isVPermPk16(MI->getOpcode()))
+ return false;
+
+ MachineBasicBlock *MBB = MI->getParent();
+
+ // Requirement #2 of 2:
+ // V_PERM_PK16 must be immediately followed by a safe instruction.
+ // Try verifying this is the case.
+ MachineBasicBlock::iterator NextI =
+ skipMetaInstructionsForward(std::next(MI->getIterator()), MBB->end());
+ if (NextI != MBB->end() && TII.isVPermPk16SafeInstr(*NextI))
+ return false;
+
+ // If the next instruction is not safe, insert a V_NOP immediately after the
+ // V_PERM_PK16. Because a V_NOP degenerates to an S_NOP under a zero EXEC
+ // mask (and would then fail to clear the hazard), force EXEC non-zero around
+ // it.
+ emitVPermPk16ExecNonZeroVNop(*MBB, NextI, MI->getDebugLoc());
+ return true;
+}
+
bool GCNHazardRecognizer::fixGetRegWaitIdle(MachineInstr *MI) {
if (!isSGetReg(MI->getOpcode()))
return false;
diff --git a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.h b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.h
index dc8b6b3e23aec..fdc6700e7508b 100644
--- a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.h
+++ b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.h
@@ -231,6 +231,10 @@ class GCNHazardRecognizer final : public ScheduleHazardRecognizer {
bool isCoexecutionHazardFor(const MachineInstr &I,
const MachineInstr &MI) const;
bool fixShift64HighRegBug(MachineInstr *MI);
+ bool fixVPermPk16Hazard(MachineInstr *MI);
+ void emitVPermPk16ExecNonZeroVNop(MachineBasicBlock &MBB,
+ MachineBasicBlock::iterator InsertPt,
+ const DebugLoc &DL);
bool fixVALUMaskWriteHazard(MachineInstr *MI);
bool fixRequiredExportPriority(MachineInstr *MI);
bool fixGetRegWaitIdle(MachineInstr *MI);
diff --git a/llvm/lib/Target/AMDGPU/GCNSubtarget.h b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
index c42ca8e19ef9c..6ae1bdc602a10 100644
--- a/llvm/lib/Target/AMDGPU/GCNSubtarget.h
+++ b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
@@ -998,6 +998,14 @@ class GCNSubtarget final : public AMDGPUGenSubtargetInfo,
return HasGFX1250Insts && getGeneration() == GFX12;
}
+ // V_PERM_PK16 leaves a hazard that must be cleared by an immediately following
+ // "safe" instruction (or an inserted V_NOP). The V_PERM_PK16 instructions are available
+ // whenever the tensor convert LUT instructions are available, but the hazard only
+ // affects the GFX12 (gfx1250) part; gfx13 issues them safely.
+ bool hasVPermPk16Hazard() const {
+ return hasTensorCvtLutInsts() && getGeneration() == GFX12;
+ }
+
/// \returns true if the subtarget requires a wait for xcnt before VMEM
/// accesses that must never be repeated in the event of a page fault/re-try.
/// Atomic stores/rmw and all volatile accesses fall under this criteria.
diff --git a/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp b/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp
index 158a83fd61cf4..1a120ef2b9cdb 100644
--- a/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp
@@ -1953,12 +1953,16 @@ void SIFrameLowering::determineCalleeSaves(MachineFunction &MF,
const SIInstrInfo *TII = ST.getInstrInfo();
bool NeedExecCopyReservedReg = false;
+ // Functions affected by the V_PERM_PK16 hazard need SGPRForEXECCopy.
+ const bool CheckVPermPk16 = ST.hasVPermPk16Hazard();
+
MachineInstr *ReturnMI = nullptr;
for (MachineBasicBlock &MBB : MF) {
for (MachineInstr &MI : MBB) {
// TODO: Walking through all MBBs here would be a bad heuristic. Better
// handle them elsewhere.
- if (TII->isWWMRegSpillOpcode(MI.getOpcode()))
+ if (TII->isWWMRegSpillOpcode(MI.getOpcode()) ||
+ (CheckVPermPk16 && SIInstrInfo::isVPermPk16(MI.getOpcode())))
NeedExecCopyReservedReg = true;
else if (MI.getOpcode() == AMDGPU::SI_RETURN ||
MI.getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG ||
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
index 71fefdf06dea3..b10ef2c3da6e1 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
@@ -1179,6 +1179,55 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
Opcode == AMDGPU::V_S_SQRT_F16_e64;
}
+ static bool isPseudoScalarTrans(unsigned Opcode) {
+ return isF16PseudoScalarTrans(Opcode) ||
+ Opcode == AMDGPU::V_S_EXP_F32_e64 ||
+ Opcode == AMDGPU::V_S_LOG_F32_e64 ||
+ Opcode == AMDGPU::V_S_RCP_F32_e64 ||
+ Opcode == AMDGPU::V_S_RSQ_F32_e64 ||
+ Opcode == AMDGPU::V_S_SQRT_F32_e64;
+ }
+
+ static bool isF64Trans(unsigned Opcode) {
+ return Opcode == AMDGPU::V_RCP_F64_e32 ||
+ Opcode == AMDGPU::V_RCP_F64_e64 ||
+ Opcode == AMDGPU::V_RSQ_F64_e32 ||
+ Opcode == AMDGPU::V_RSQ_F64_e64 ||
+ Opcode == AMDGPU::V_SQRT_F64_e32 ||
+ Opcode == AMDGPU::V_SQRT_F64_e64;
+ }
+
+ static bool isVPermPk16(unsigned Opcode) {
+ return Opcode == AMDGPU::V_PERM_PK16_B4_U4_e64 ||
+ Opcode == AMDGPU::V_PERM_PK16_B6_U4_e64 ||
+ Opcode == AMDGPU::V_PERM_PK16_B8_U4_e64;
+ }
+
+ // \returns true if \p MI clears the V_PERM_PK16 hazard when it immediately
+ // follows a V_PERM_PK16 (i.e. \p MI is a "safe" instruction).
+ bool isVPermPk16SafeInstr(const MachineInstr &MI) const {
+ unsigned Opc = MI.getOpcode();
+
+ // Only VALU ops issue on the pipe that clears the V_PERM_PK16 hazard.
+ if (!isVALU(MI, /*AllowLDSDMA=*/false))
+ return false;
+ // OP_XDL: matrix (WMMA/SWMMAC/DOT) ops clear the hazard.
+ if (isXDL(MI))
+ return true;
+ // Pseudo-scalar transcendentals (OP32_SCL_T) do NOT clear the hazard.
+ if (isPseudoScalarTrans(Opc))
+ return false;
+ // OP_32_T: genuine transcendentals clear the hazard, except the F64
+ // transcendentals (which belong to the multi-pass FP64 class).
+ if (isTRANS(MI))
+ return !isF64Trans(Opc);
+
+ // Everything else that is a single-pass VALU op (OP16_1, OP32_1, OP_CMACC,
+ // OP_DUAL_1) is safe. Multi-pass ops block the VALU pipe for more than one
+ // cycle (getBlockingCycles > 1) and do not clear the hazard.
+ return getBlockingCycles(MI) < 2;
+ }
+
static bool doesNotReadTiedSource(const MachineInstr &MI) {
return SIInstrFlags::isTiedSourceNotRead(MI);
}
diff --git a/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp b/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp
index 2fa7feb9d3327..0f699e6b7902f 100644
--- a/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp
+++ b/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp
@@ -446,6 +446,14 @@ bool SILowerSGPRSpillsLegacy::runOnMachineFunction(MachineFunction &MF) {
return SILowerSGPRSpills(LIS, Indexes, MDT, MCI).run(MF);
}
+static bool hasVPermPk16(const MachineFunction &MF) {
+ for (const MachineBasicBlock &MBB : MF)
+ for (const MachineInstr &MI : MBB)
+ if (SIInstrInfo::isVPermPk16(MI.getOpcode()))
+ return true;
+ return false;
+}
+
bool SILowerSGPRSpills::run(MachineFunction &MF) {
const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
TII = ST.getInstrInfo();
@@ -472,6 +480,9 @@ bool SILowerSGPRSpills::run(MachineFunction &MF) {
bool MadeChange = false;
bool SpilledToVirtVGPRLanes = false;
+ const bool CheckVPermPk16 = ST.hasVPermPk16Hazard();
+ bool HasVPermPk16 = false;
+
// TODO: CSR VGPRs will never be spilled to AGPRs. These can probably be
// handled as SpilledToReg in regular PrologEpilogInserter.
const bool HasSGPRSpillToVGPR = TRI->spillSGPRToVGPR() &&
@@ -496,6 +507,9 @@ bool SILowerSGPRSpills::run(MachineFunction &MF) {
for (MachineBasicBlock &MBB : MF) {
for (MachineInstr &MI : llvm::make_early_inc_range(MBB)) {
+ if (CheckVPermPk16 && SIInstrInfo::isVPermPk16(MI.getOpcode()))
+ HasVPermPk16 = true;
+
if (MI.getOpcode() == AMDGPU::ENTER_STRICT_WWM ||
MI.getOpcode() == AMDGPU::ENTER_STRICT_WQM) {
HasStrictWWMRegion = true;
@@ -621,9 +635,14 @@ bool SILowerSGPRSpills::run(MachineFunction &MF) {
TRI->getHWRegIndex(FuncInfo->getSGPRForEXECCopy()))
FuncInfo->setSGPRForEXECCopy(UnusedLowSGPR);
} else {
- // No SGPR spills to virtual VGPR lanes and hence there won't be any WWM
- // spills/copies. Reset the SGPR reserved for EXEC copy.
- FuncInfo->setSGPRForEXECCopy(AMDGPU::NoRegister);
+ const bool NeedsVPermPk16Fix =
+ CheckVPermPk16 &&
+ (HasVPermPk16 || (!HasSGPRSpillToVGPR && hasVPermPk16(MF)));
+ if (!NeedsVPermPk16Fix)
+ // No SGPR spills to virtual VGPR lanes and hence there won't be any WWM
+ // spills/copies. Also, function is not affected by the V_PERM_PK16
+ // hazard. Reset the SGPR reserved for EXEC copy.
+ FuncInfo->setSGPRForEXECCopy(AMDGPU::NoRegister);
}
SaveBlocks.clear();
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.perm.pk.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.perm.pk.ll
index 863830a8cfaa8..1a5d9ae8b2e30 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.perm.pk.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.perm.pk.ll
@@ -12,6 +12,10 @@ define void @test_perm_pk16_b4_u4(i32 %a, i32 %b, <2 x i32> %c, ptr %out) {
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_perm_pk16_b4_u4 v[0:1], v0, v1, v[2:3]
+; GFX1250-NEXT: s_mov_b32 s0, exec_lo
+; GFX1250-NEXT: s_mov_b32 exec_lo, -1
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: s_mov_b32 exec_lo, s0
; GFX1250-NEXT: flat_store_b64 v[4:5], v[0:1]
; GFX1250-NEXT: s_wait_dscnt 0x0
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
@@ -30,6 +34,10 @@ define void @test_perm_pk16_b6_u4(i32 %a, i64 %b, <2 x i32> %c, ptr %out) {
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v6, v5
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-SDAG-NEXT: v_perm_pk16_b6_u4 v[0:2], v0, v[2:3], v[8:9]
+; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo
+; GFX1250-SDAG-NEXT: s_mov_b32 exec_lo, -1
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: s_mov_b32 exec_lo, s0
; GFX1250-SDAG-NEXT: flat_store_b96 v[6:7], v[0:2]
; GFX1250-SDAG-NEXT: s_wait_dscnt 0x0
; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
@@ -43,6 +51,10 @@ define void @test_perm_pk16_b6_u4(i32 %a, i64 %b, <2 x i32> %c, ptr %out) {
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v5 :: v_dual_mov_b32 v5, v6
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1250-GISEL-NEXT: v_perm_pk16_b6_u4 v[0:2], v0, v[8:9], v[2:3]
+; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
+; GFX1250-GISEL-NEXT: s_mov_b32 exec_lo, -1
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: s_mov_b32 exec_lo, s0
; GFX1250-GISEL-NEXT: flat_store_b96 v[4:5], v[0:2]
; GFX1250-GISEL-NEXT: s_wait_dscnt 0x0
; GFX1250-GISEL-NEXT: s_set_pc_i64 s[30:31]
@@ -57,6 +69,10 @@ define void @test_perm_pk16_b8_u4(i64 %a, i64 %b, <2 x i32> %c, ptr %out) {
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_perm_pk16_b8_u4 v[0:3], v[0:1], v[2:3], v[4:5]
+; GFX1250-NEXT: s_mov_b32 s0, exec_lo
+; GFX1250-NEXT: s_mov_b32 exec_lo, -1
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: s_mov_b32 exec_lo, s0
; GFX1250-NEXT: flat_store_b128 v[6:7], v[0:3]
; GFX1250-NEXT: s_wait_dscnt 0x0
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/vperm-pk16-exec-hazard.ll b/llvm/test/CodeGen/AMDGPU/vperm-pk16-exec-hazard.ll
new file mode 100644
index 0000000000000..7fab27c9f41dd
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/vperm-pk16-exec-hazard.ll
@@ -0,0 +1,80 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=amdgpu12.50 < %s | FileCheck -check-prefix=GFX1250 %s
+; RUN: llc -mtriple=amdgpu13.10 < %s | FileCheck -check-prefix=GFX1310 %s
+
+; End-to-end coverage for the V_PERM_PK16 hazard. A store
+; (non-safe) follows the perm, so an exec-forced V_NOP must be inserted. Entry
+; functions additionally get a defensive mitigation at the start.
+;
+; The hazard only affects the GFX12 (gfx1250) part. gfx13 (gfx1310) has the same
+; instructions but issues them safely, so it is negative coverage: no mitigation.
+
+declare <2 x i32> @llvm.amdgcn.perm.pk16.b4.u4(i32, i32, <2 x i32>)
+
+define void @perm_func(i32 %a, i32 %b, <2 x i32> %c, ptr %out) {
+; GFX1250-LABEL: perm_func:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_perm_pk16_b4_u4 v[0:1], v0, v1, v[2:3]
+; GFX1250-NEXT: s_mov_b32 s0, exec_lo
+; GFX1250-NEXT: s_mov_b32 exec_lo, -1
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: s_mov_b32 exec_lo, s0
+; GFX1250-NEXT: flat_store_b64 v[4:5], v[0:1]
+; GFX1250-NEXT: s_wait_dscnt 0x0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1310-LABEL: perm_func:
+; GFX1310: ; %bb.0:
+; GFX1310-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1310-NEXT: s_wait_expcnt 0x0
+; GFX1310-NEXT: s_wait_samplecnt 0x0
+; GFX1310-NEXT: s_wait_bvhcnt 0x0
+; GFX1310-NEXT: s_wait_kmcnt 0x0
+; GFX1310-NEXT: v_perm_pk16_b4_u4 v[0:1], v0, v1, v[2:3]
+; GFX1310-NEXT: flat_store_b64 v[4:5], v[0:1]
+; GFX1310-NEXT: s_wait_dscnt 0x0
+; GFX1310-NEXT: s_set_pc_i64 s[30:31]
+ %r = tail call <2 x i32> @llvm.amdgcn.perm.pk16.b4.u4(i32 %a, i32 %b, <2 x i32> %c)
+ store <2 x i32> %r, ptr %out, align 8
+ ret void
+}
+
+define amdgpu_kernel void @perm_kernel(i32 %a, i32 %b, <2 x i32> %c, ptr %out) {
+; GFX1250-LABEL: perm_kernel:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_clause 0x1
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x34 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v0, s2
+; GFX1250-NEXT: v_mov_b32_e32 v1, s3
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_perm_pk16_b4_u4 v[0:1], s0, s1, v[0:1]
+; GFX1250-NEXT: s_mov_b32 s105, exec_lo
+; GFX1250-NEXT: s_mov_b32 exec_lo, -1
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: s_mov_b32 exec_lo, s105
+; GFX1250-NEXT: flat_store_b64 v2, v[0:1], s[6:7]
+; GFX1250-NEXT: s_endpgm
+;
+; GFX1310-LABEL: perm_kernel:
+; GFX1310: ; %bb.0:
+; GFX1310-NEXT: s_clause 0x1
+; GFX1310-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX1310-NEXT: s_load_b64 s[4:5], s[4:5], 0x34 nv
+; GFX1310-NEXT: s_wait_kmcnt 0x0
+; GFX1310-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v0, s2
+; GFX1310-NEXT: v_mov_b32_e32 v1, s3
+; GFX1310-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1310-NEXT: v_perm_pk16_b4_u4 v[0:1], s0, s1, v[0:1]
+; GFX1310-NEXT: flat_store_b64 v2, v[0:1], s[4:5]
+; GFX1310-NEXT: s_endpgm
+ %r = tail call <2 x i32> @llvm.amdgcn.perm.pk16.b4.u4(i32 %a, i32 %b, <2 x i32> %c)
+ store <2 x i32> %r, ptr %out, align 8
+ ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/vperm-pk16-exec-hazard.mir b/llvm/test/CodeGen/AMDGPU/vperm-pk16-exec-hazard.mir
new file mode 100644
index 0000000000000..a15d4e9a0586a
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/vperm-pk16-exec-hazard.mir
@@ -0,0 +1,348 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgpu12.50 -run-pass=si-insert-waitcnts,post-RA-hazard-rec %s -o - | FileCheck -check-prefix=GFX1250 %s
+# RUN: llc -mtriple=amdgpu13.10 -run-pass=si-insert-waitcnts,post-RA-hazard-rec %s -o - | FileCheck -check-prefix=GFX1310 %s
+
+# V_PERM_PK16 must be immediately followed by a "safe" instruction, which clears
+# the hazard regardless of EXEC. A follower is safe iff it issues on
+# the VALU pipe and belongs to one of the safe HW SP op-type buckets: OP_XDL,
+# OP_32_T (genuine transcendentals), OP16_1, OP32_1, OP_CMACC, or OP_DUAL_1.
+# The unsafe buckets are the multi-pass ops (GFX1250 BlockingCycles >= 2): FP64,
+# 64-bit integer ops, I32_MUL (V_MUL_LO/HI_U32), QSAD/MQSAD, CVT_SCALE_PK*, and
+# V_PERM_PK16 itself, plus the pseudo-scalar transcendentals (OP32_SCL_T). When
+# no safe instruction follows, a V_NOP is inserted and EXEC is forced non-zero
+# around it (a V_NOP degenerates to an S_NOP under a zero EXEC mask).
+#
+# The cross-wave entry-block mitigation is delegated to the mandatory
+# unclaused-VMEM entry prologue (GLOBAL_PREFETCH_B8 + V_NOP) emitted by
+# SIInsertWaitcnts on every entry function, so this test runs that pass before
+# the hazard recognizer to exercise the full sequence.
+#
+# The hazard only affects the GFX12 (gfx1250) part. gfx13 (gfx1310) has the same
+# instructions but issues them safely, so it serves as negative coverage: no
+# mitigation is inserted in any of the cases below.
+
+# OP32_1 / OP_CMACC (V_MOV_B32) is safe: no mitigation.
+---
+name: perm_then_mov_safe
+machineFunctionInfo:
+ isEntryFunction: false
+ sgprForEXECCopy: '$sgpr0'
+body: |
+ bb.0:
+ ; GFX1250-LABEL: name: perm_then_mov_safe
+ ; GFX1250: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; GFX1250-NEXT: S_WAIT_KMCNT 0
+ ; GFX1250-NEXT: $vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
+ ; GFX1250-NEXT: $vgpr4 = V_MOV_B32_e32 0, implicit $exec
+ ;
+ ; GFX1310-LABEL: name: perm_then_mov_safe
+ ; GFX1310: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; GFX1310-NEXT: S_WAIT_EXPCNT 0
+ ; GFX1310-NEXT: S_WAIT_SAMPLECNT 0
+ ; GFX1310-NEXT: S_WAIT_BVHCNT 0
+ ; GFX1310-NEXT: S_WAIT_KMCNT 0
+ ; GFX1310-NEXT: $vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
+ ; GFX1310-NEXT: $vgpr4 = V_MOV_B32_e32 0, implicit $exec
+ $vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
+ $vgpr4 = V_MOV_B32_e32 0, implicit $exec
+...
+
+# A packed VOP3P op (OP32_1) is safe: no mitigation.
+---
+name: perm_then_packed_safe
+machineFunctionInfo:
+ isEntryFunction: false
+ sgprForEXECCopy: '$sgpr0'
+body: |
+ bb.0:
+ ; GFX1250-LABEL: name: perm_then_packed_safe
+ ; GFX1250: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; GFX1250-NEXT: S_WAIT_KMCNT 0
+ ; GFX1250-NEXT: $vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
+ ; GFX1250-NEXT: $vgpr4 = V_PK_ADD_U16 8, $vgpr10, 8, $vgpr11, 0, 0, 0, 0, 0, implicit $exec
+ ;
+ ; GFX1310-LABEL: name: perm_then_packed_safe
+ ; GFX1310: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; GFX1310-NEXT: S_WAIT_EXPCNT 0
+ ; GFX1310-NEXT: S_WAIT_SAMPLECNT 0
+ ; GFX1310-NEXT: S_WAIT_BVHCNT 0
+ ; GFX1310-NEXT: S_WAIT_KMCNT 0
+ ; GFX1310-NEXT: $vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
+ ; GFX1310-NEXT: $vgpr4 = V_PK_ADD_U16 8, $vgpr10, 8, $vgpr11, 0, 0, 0, 0, 0, implicit $exec
+ $vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
+ $vgpr4 = V_PK_ADD_U16 8, $vgpr10, 8, $vgpr11, 0, 0, 0, 0, 0, implicit $exec
+...
+
+# A 16-bit op (OP_CMACC/OP16_1, V_ADD_F16) is safe: no mitigation.
+---
+name: perm_then_op16_safe
+machineFunctionInfo:
+ isEntryFunction: false
+ sgprForEXECCopy: '$sgpr0'
+body: |
+ bb.0:
+ ; GFX1250-LABEL: name: perm_then_op16_safe
+ ; GFX1250: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; GFX1250-NEXT: S_WAIT_KMCNT 0
+ ; GFX1250-NEXT: $vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
+ ; GFX1250-NEXT: $vgpr4 = V_ADD_F16_fake16_e64 0, $vgpr10, 0, $vgpr11, 0, 0, implicit $mode, implicit $exec
+ ;
+ ; GFX1310-LABEL: name: perm_then_op16_safe
+ ; GFX1310: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; GFX1310-NEXT: S_WAIT_EXPCNT 0
+ ; GFX1310-NEXT: S_WAIT_SAMPLECNT 0
+ ; GFX1310-NEXT: S_WAIT_BVHCNT 0
+ ; GFX1310-NEXT: S_WAIT_KMCNT 0
+ ; GFX1310-NEXT: $vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
+ ; GFX1310-NEXT: $vgpr4 = V_ADD_F16_fake16_e64 0, $vgpr10, 0, $vgpr11, 0, 0, implicit $mode, implicit $exec
+ $vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
+ $vgpr4 = V_ADD_F16_fake16_e64 0, $vgpr10, 0, $vgpr11, 0, 0, implicit $mode, implicit $exec
+...
+
+# A scalar FMA-accumulate op (OP_CMACC, V_FMAC_F32) is safe: no mitigation.
+---
+name: perm_then_cmacc_safe
+machineFunctionInfo:
+ isEntryFunction: false
+ sgprForEXECCopy: '$sgpr0'
+body: |
+ bb.0:
+ ; GFX1250-LABEL: name: perm_then_cmacc_safe
+ ; GFX1250: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; GFX1250-NEXT: S_WAIT_KMCNT 0
+ ; GFX1250-NEXT: $vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
+ ; GFX1250-NEXT: $vgpr4 = V_FMAC_F32_e64 0, $vgpr10, 0, $vgpr11, 0, $vgpr4, 0, 0, implicit $mode, implicit $exec
+ ;
+ ; GFX1310-LABEL: name: perm_then_cmacc_safe
+ ; GFX1310: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; GFX1310-NEXT: S_WAIT_EXPCNT 0
+ ; GFX1310-NEXT: S_WAIT_SAMPLECNT 0
+ ; GFX1310-NEXT: S_WAIT_BVHCNT 0
+ ; GFX1310-NEXT: S_WAIT_KMCNT 0
+ ; GFX1310-NEXT: $vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
+ ; GFX1310-NEXT: $vgpr4 = V_FMAC_F32_e64 0, $vgpr10, 0, $vgpr11, 0, $vgpr4, 0, 0, implicit $mode, implicit $exec
+ $vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
+ $vgpr4 = V_FMAC_F32_e64 0, $vgpr10, 0, $vgpr11, 0, $vgpr4, 0, 0, implicit $mode, implicit $exec
+...
+
+# A lane/control op (OP32_1, V_READFIRSTLANE_B32) is safe: no mitigation.
+---
+name: perm_then_lane_safe
+machineFunctionInfo:
+ isEntryFunction: false
+ sgprForEXECCopy: '$sgpr0'
+body: |
+ bb.0:
+ ; GFX1250-LABEL: name: perm_then_lane_safe
+ ; GFX1250: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; GFX1250-NEXT: S_WAIT_KMCNT 0
+ ; GFX1250-NEXT: $vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
+ ; GFX1250-NEXT: $sgpr4 = V_READFIRSTLANE_B32 $vgpr10, implicit $exec
+ ;
+ ; GFX1310-LABEL: name: perm_then_lane_safe
+ ; GFX1310: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; GFX1310-NEXT: S_WAIT_EXPCNT 0
+ ; GFX1310-NEXT: S_WAIT_SAMPLECNT 0
+ ; GFX1310-NEXT: S_WAIT_BVHCNT 0
+ ; GFX1310-NEXT: S_WAIT_KMCNT 0
+ ; GFX1310-NEXT: $vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
+ ; GFX1310-NEXT: $sgpr4 = V_READFIRSTLANE_B32 $vgpr10, implicit $exec
+ $vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
+ $sgpr4 = V_READFIRSTLANE_B32 $vgpr10, implicit $exec
+...
+
+# A genuine TRANS instruction (OP_32_T, V_RCP_F32) is safe: no mitigation.
+---
+name: perm_then_trans_safe
+machineFunctionInfo:
+ isEntryFunction: false
+ sgprForEXECCopy: '$sgpr0'
+body: |
+ bb.0:
+ ; GFX1250-LABEL: name: perm_then_trans_safe
+ ; GFX1250: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; GFX1250-NEXT: S_WAIT_KMCNT 0
+ ; GFX1250-NEXT: $vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
+ ; GFX1250-NEXT: $vgpr4 = V_RCP_F32_e32 $vgpr10, implicit $mode, implicit $exec
+ ;
+ ; GFX1310-LABEL: name: perm_then_trans_safe
+ ; GFX1310: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; GFX1310-NEXT: S_WAIT_EXPCNT 0
+ ; GFX1310-NEXT: S_WAIT_SAMPLECNT 0
+ ; GFX1310-NEXT: S_WAIT_BVHCNT 0
+ ; GFX1310-NEXT: S_WAIT_KMCNT 0
+ ; GFX1310-NEXT: $vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
+ ; GFX1310-NEXT: $vgpr4 = V_RCP_F32_e32 $vgpr10, implicit $mode, implicit $exec
+ $vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
+ $vgpr4 = V_RCP_F32_e32 $vgpr10, implicit $mode, implicit $exec
+...
+
+# A multi-pass op (I32_MUL, V_MUL_LO_U32) is NOT safe: insert an exec-forced
+# V_NOP.
+---
+name: perm_then_i32mul_nonsafe
+machineFunctionInfo:
+ isEntryFunction: false
+ sgprForEXECCopy: '$sgpr0'
+body: |
+ bb.0:
+ ; GFX1250-LABEL: name: perm_then_i32mul_nonsafe
+ ; GFX1250: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; GFX1250-NEXT: S_WAIT_KMCNT 0
+ ; GFX1250-NEXT: $vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
+ ; GFX1250-NEXT: $sgpr0 = S_MOV_B32 killed $exec_lo
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32 -1
+ ; GFX1250-NEXT: V_NOP_e32 implicit $exec
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32 killed $sgpr0
+ ; GFX1250-NEXT: $vgpr4 = V_MUL_LO_U32_e64 $vgpr10, $vgpr11, implicit $exec
+ ;
+ ; GFX1310-LABEL: name: perm_then_i32mul_nonsafe
+ ; GFX1310: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; GFX1310-NEXT: S_WAIT_EXPCNT 0
+ ; GFX1310-NEXT: S_WAIT_SAMPLECNT 0
+ ; GFX1310-NEXT: S_WAIT_BVHCNT 0
+ ; GFX1310-NEXT: S_WAIT_KMCNT 0
+ ; GFX1310-NEXT: $vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
+ ; GFX1310-NEXT: $vgpr4 = V_MUL_LO_U32_e64 $vgpr10, $vgpr11, implicit $exec
+ $vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
+ $vgpr4 = V_MUL_LO_U32_e64 $vgpr10, $vgpr11, implicit $exec
+...
+
+# A multi-pass op (QSAD, V_MQSAD_PK_U16_U8 - the post-silicon bug case) is NOT
+# safe: insert an exec-forced V_NOP.
+---
+name: perm_then_qsad_nonsafe
+machineFunctionInfo:
+ isEntryFunction: false
+ sgprForEXECCopy: '$sgpr0'
+body: |
+ bb.0:
+ ; GFX1250-LABEL: name: perm_then_qsad_nonsafe
+ ; GFX1250: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; GFX1250-NEXT: S_WAIT_KMCNT 0
+ ; GFX1250-NEXT: $vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
+ ; GFX1250-NEXT: $sgpr0 = S_MOV_B32 killed $exec_lo
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32 -1
+ ; GFX1250-NEXT: V_NOP_e32 implicit $exec
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32 killed $sgpr0
+ ; GFX1250-NEXT: early-clobber $vgpr4_vgpr5 = V_MQSAD_PK_U16_U8_e64 $vgpr10_vgpr11, $vgpr12, $vgpr14_vgpr15, 0, implicit $exec
+ ;
+ ; GFX1310-LABEL: name: perm_then_qsad_nonsafe
+ ; GFX1310: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; GFX1310-NEXT: S_WAIT_EXPCNT 0
+ ; GFX1310-NEXT: S_WAIT_SAMPLECNT 0
+ ; GFX1310-NEXT: S_WAIT_BVHCNT 0
+ ; GFX1310-NEXT: S_WAIT_KMCNT 0
+ ; GFX1310-NEXT: $vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
+ ; GFX1310-NEXT: early-clobber $vgpr4_vgpr5 = V_MQSAD_PK_U16_U8_e64 $vgpr10_vgpr11, $vgpr12, $vgpr14_vgpr15, 0, implicit $exec
+ $vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
+ $vgpr4_vgpr5 = V_MQSAD_PK_U16_U8_e64 $vgpr10_vgpr11, $vgpr12, $vgpr14_vgpr15, 0, implicit $exec
+...
+
+# An F64 transcendental is NOT safe: insert an exec-forced V_NOP.
+---
+name: perm_then_f64_trans
+machineFunctionInfo:
+ isEntryFunction: false
+ sgprForEXECCopy: '$sgpr0'
+body: |
+ bb.0:
+ ; GFX1250-LABEL: name: perm_then_f64_trans
+ ; GFX1250: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; GFX1250-NEXT: S_WAIT_KMCNT 0
+ ; GFX1250-NEXT: $vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
+ ; GFX1250-NEXT: $sgpr0 = S_MOV_B32 killed $exec_lo
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32 -1
+ ; GFX1250-NEXT: V_NOP_e32 implicit $exec
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32 killed $sgpr0
+ ; GFX1250-NEXT: $vgpr4_vgpr5 = V_RCP_F64_e32 $vgpr10_vgpr11, implicit $mode, implicit $exec
+ ;
+ ; GFX1310-LABEL: name: perm_then_f64_trans
+ ; GFX1310: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; GFX1310-NEXT: S_WAIT_EXPCNT 0
+ ; GFX1310-NEXT: S_WAIT_SAMPLECNT 0
+ ; GFX1310-NEXT: S_WAIT_BVHCNT 0
+ ; GFX1310-NEXT: S_WAIT_KMCNT 0
+ ; GFX1310-NEXT: $vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
+ ; GFX1310-NEXT: $vgpr4_vgpr5 = V_RCP_F64_e32 $vgpr10_vgpr11, implicit $mode, implicit $exec
+ $vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
+ $vgpr4_vgpr5 = V_RCP_F64_e32 $vgpr10_vgpr11, implicit $mode, implicit $exec
+...
+
+# A pseudo-scalar transcendental (OP32_SCL_T, V_S_EXP_F32) is NOT safe: insert
+# an exec-forced V_NOP.
+---
+name: perm_then_pseudo_scalar_trans_nonsafe
+machineFunctionInfo:
+ isEntryFunction: false
+ sgprForEXECCopy: '$sgpr0'
+body: |
+ bb.0:
+ ; GFX1250-LABEL: name: perm_then_pseudo_scalar_trans_nonsafe
+ ; GFX1250: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; GFX1250-NEXT: S_WAIT_KMCNT 0
+ ; GFX1250-NEXT: $vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
+ ; GFX1250-NEXT: $sgpr0 = S_MOV_B32 killed $exec_lo
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32 -1
+ ; GFX1250-NEXT: V_NOP_e32 implicit $exec
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32 killed $sgpr0
+ ; GFX1250-NEXT: $sgpr4 = V_S_EXP_F32_e64 0, $sgpr10, 0, 0, implicit $mode, implicit $exec
+ ;
+ ; GFX1310-LABEL: name: perm_then_pseudo_scalar_trans_nonsafe
+ ; GFX1310: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
+ ; GFX1310-NEXT: S_WAIT_EXPCNT 0
+ ; GFX1310-NEXT: S_WAIT_SAMPLECNT 0
+ ; GFX1310-NEXT: S_WAIT_BVHCNT 0
+ ; GFX1310-NEXT: S_WAIT_KMCNT 0
+ ; GFX1310-NEXT: $vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
+ ; GFX1310-NEXT: $sgpr4 = V_S_EXP_F32_e64 0, $sgpr10, 0, 0, implicit $mode, implicit $exec
+ $vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
+ $sgpr4 = V_S_EXP_F32_e64 0, $sgpr10, 0, 0, implicit $mode, implicit $exec
+...
+
+# Entry function: defensive mitigation at the very start plus the per-perm one.
+---
+name: perm_kernel_entry
+machineFunctionInfo:
+ isEntryFunction: true
+ sgprForEXECCopy: '$sgpr0'
+body: |
+ bb.0:
+ ; GFX1250-LABEL: name: perm_kernel_entry
+ ; GFX1250: GLOBAL_PREFETCH_B8_SADDR $sgpr_null64, undef $vgpr0, 0, 8, implicit $exec
+ ; GFX1250-NEXT: V_NOP_e32 implicit $exec
+ ; GFX1250-NEXT: $vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
+ ; GFX1250-NEXT: $sgpr0 = S_MOV_B32 killed $exec_lo
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32 -1
+ ; GFX1250-NEXT: V_NOP_e32 implicit $exec
+ ; GFX1250-NEXT: $exec_lo = S_MOV_B32 killed $sgpr0
+ ; GFX1250-NEXT: S_ENDPGM 0
+ ;
+ ; GFX1310-LABEL: name: perm_kernel_entry
+ ; GFX1310: $vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
+ ; GFX1310-NEXT: S_ENDPGM 0
+ $vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
+ S_ENDPGM 0
+...
+
+# Entry function with no V_PERM_PK16 at all: the defensive entry-block mitigation
+# is still inserted. Minimal mitigation for the potentially cross-wave hazard.
+---
+name: no_perm_kernel_entry
+machineFunctionInfo:
+ isEntryFunction: true
+ sgprForEXECCopy: '$sgpr0'
+body: |
+ bb.0:
+ ; GFX1250-LABEL: name: no_perm_kernel_entry
+ ; GFX1250: GLOBAL_PREFETCH_B8_SADDR $sgpr_null64, undef $vgpr0, 0, 8, implicit $exec
+ ; GFX1250-NEXT: V_NOP_e32 implicit $exec
+ ; GFX1250-NEXT: $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX1250-NEXT: S_ENDPGM 0
+ ;
+ ; GFX1310-LABEL: name: no_perm_kernel_entry
+ ; GFX1310: $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX1310-NEXT: S_ENDPGM 0
+ $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ S_ENDPGM 0
+...
More information about the llvm-branch-commits
mailing list