[llvm] [AMDGPU] merge 16bit mov pairs in post-RA peephole (PR #208625)
via llvm-commits
llvm-commits at lists.llvm.org
Sat Jul 11 10:52:15 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-amdgpu
Author: Guo Chen (broxigarchen)
<details>
<summary>Changes</summary>
Address https://github.com/llvm/llvm-project/issues/207011. Add 8 patterns to merge 16bit mov pairs
Co-Authored-By: Claude
---
Patch is 342.32 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/208625.diff
49 Files Affected:
- (modified) llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp (+293)
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.intersect_ray.ll (+8-11)
- (modified) llvm/test/CodeGen/AMDGPU/add.v2i16.ll (+7-11)
- (modified) llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll (+33-49)
- (modified) llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll (+18-34)
- (modified) llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.32bit.ll (+2-4)
- (modified) llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.48bit.ll (+4-7)
- (modified) llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll (+19-25)
- (modified) llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll (+5-8)
- (modified) llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll (+6-8)
- (modified) llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll (+8-12)
- (modified) llvm/test/CodeGen/AMDGPU/bf16.ll (+9-15)
- (modified) llvm/test/CodeGen/AMDGPU/bswap.ll (+2-3)
- (modified) llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll (+12-18)
- (modified) llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll (+12-18)
- (modified) llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll (+12-18)
- (modified) llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll (+5-8)
- (modified) llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll (+40-60)
- (modified) llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll (+40-60)
- (modified) llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll (+40-60)
- (modified) llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll (+40-60)
- (modified) llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll (+2-4)
- (modified) llvm/test/CodeGen/AMDGPU/function-args.ll (+2-3)
- (modified) llvm/test/CodeGen/AMDGPU/function-returns.ll (+1-2)
- (modified) llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll (+40-60)
- (modified) llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll (+40-60)
- (modified) llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll (+40-60)
- (modified) llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll (+40-60)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.gather4.a16.dim.ll (+5-12)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.msaa.load.ll (+3-6)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.a16.dim.ll (+21-42)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.encode.ll (+8-12)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.ll (+2-4)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.v3f16.ll (+2-4)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.v3f16.ll (+1-2)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.waitcnt.out.order.ll (+4-8)
- (modified) llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll (+24-36)
- (modified) llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmax.ll (+24-36)
- (modified) llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmin.ll (+24-36)
- (modified) llvm/test/CodeGen/AMDGPU/local-atomicrmw-fsub.ll (+24-36)
- (modified) llvm/test/CodeGen/AMDGPU/repeated-divisor.ll (+2-3)
- (modified) llvm/test/CodeGen/AMDGPU/scalar_to_vector.ll (+1-2)
- (added) llvm/test/CodeGen/AMDGPU/si-pre-emit-peephole-merge-v-mov-b16.mir (+199)
- (modified) llvm/test/CodeGen/AMDGPU/sub.v2i16.ll (+7-11)
- (modified) llvm/test/CodeGen/AMDGPU/v_swap_b16.ll (+6-7)
- (modified) llvm/test/CodeGen/AMDGPU/vector-reduce-and.ll (+14-22)
- (modified) llvm/test/CodeGen/AMDGPU/vector-reduce-or.ll (+14-22)
- (modified) llvm/test/CodeGen/AMDGPU/vector-reduce-xor.ll (+14-22)
- (modified) llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll (+128-280)
``````````diff
diff --git a/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp b/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
index a496c9a4daa71..0a9ca8bfbd11c 100644
--- a/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
+++ b/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
@@ -44,6 +44,9 @@ class SIPreEmitPeephole {
void updateMLIBeforeRemovingEdge(MachineBasicBlock *From,
MachineBasicBlock *To) const;
bool optimizeSetGPR(MachineInstr &First, MachineInstr &MI) const;
+ bool mergeSingleMovB16Pair(MachineInstr &Lo, MachineInstr &Hi,
+ bool IsHiFirst) const;
+ bool mergeMovB16Pair(MachineFunction &MF) const;
bool getBlockDestinations(MachineBasicBlock &SrcMBB,
MachineBasicBlock *&TrueMBB,
MachineBasicBlock *&FalseMBB,
@@ -763,6 +766,292 @@ MachineInstrBuilder SIPreEmitPeephole::createUnpackedMI(MachineInstr &I,
return NewMI;
}
+// Helper: extract the src operand and whether it is from the hi16 half.
+// Post-RA, both V_MOV_B16_t16_e32 and V_MOV_B16_t16_e64 use VGPR_16 physical
+// registers whose encoding already encodes hi/lo (IS_HI16 bit).
+// Returns false if the source is not a physical VGPR_16 or immediate zero.
+//
+// Operand layouts (post-RA, physical registers):
+// V_MOV_B16_t16_e32: dst(0), src0(1)
+// V_MOV_B16_t16_e64: dst(0), src0_mods(1), src0(2), op_sel(3)
+static bool getMovB16Info(const MachineInstr &MI, const SIRegisterInfo *TRI,
+ MCRegister &SrcReg32, bool &SrcIsHi, bool &SrcIsImm,
+ int64_t &ImmVal) {
+ SrcIsImm = false;
+ SrcIsHi = false;
+ SrcReg32 = MCRegister();
+
+ unsigned Opc = MI.getOpcode();
+ const MachineOperand *SrcOp = nullptr;
+
+ if (Opc == AMDGPU::V_MOV_B16_t16_e64)
+ SrcOp = &MI.getOperand(2);
+ else if (Opc == AMDGPU::V_MOV_B16_t16_e32)
+ SrcOp = &MI.getOperand(1);
+ else
+ return false;
+
+ if (SrcOp->isImm()) {
+ SrcIsImm = true;
+ ImmVal = SrcOp->getImm();
+ return true;
+ }
+
+ if (!SrcOp->isReg() || !SrcOp->getReg().isPhysical())
+ return false;
+
+ MCRegister SrcReg = SrcOp->getReg().asMCReg();
+
+ // We require the source to be a 16-bit VGPR so we can determine hi/lo.
+ if (!AMDGPU::VGPR_16RegClass.contains(SrcReg))
+ return false;
+
+ SrcIsHi = AMDGPU::isHi16Reg(SrcReg, *TRI);
+ SrcReg32 = TRI->get32BitRegister(SrcReg);
+ return SrcReg32.isValid();
+}
+
+// clang-format off
+// Try to merge a pair of v_mov_b16 instructions targeting the lo16 and hi16
+// halves of the same VGPR into a single 32-bit instruction.
+//
+// Patterns:
+// v_mov_b16 v0.h, 0 v_mov_b16 v0.l, v2.l => v_and_b32 v0,0xffff,v2
+// v_mov_b16 v0.h, 0 v_mov_b16 v0.l, v2.h => v_lshrrev_b32 v0,16,v2
+// v_mov_b16 v0.l, 0 v_mov_b16 v0.h, v2.l => v_lshlrev_b32 v0,16,v2
+// v_mov_b16 v0.l, 0 v_mov_b16 v0.h, v2.h => v_and_b32 v0,0xffff0000,v2
+// v_mov_b16 v0.l, v2.l v_mov_b16 v0.h, v3.l => v_perm_b32 v0,v2,v3,0x05040100
+// v_mov_b16 v0.l, v2.l v_mov_b16 v0.h, v3.h => v_bfi_b32 v0,0x0000ffff,v2,v3
+// v_mov_b16 v0.l, v2.h v_mov_b16 v0.h, v3.l => v_alignbit_b32 v0,v3,v2,16
+// v_mov_b16 v0.l, v2.h v_mov_b16 v0.h, v3.h => v_perm_b32 v0,v2,v3,0x07060302
+// clang-format on
+bool SIPreEmitPeephole::mergeSingleMovB16Pair(MachineInstr &Lo,
+ MachineInstr &Hi,
+ bool IsHiFirst) const {
+ // Both must be v_mov_b16 true16 variants.
+ unsigned LoOpc = Lo.getOpcode();
+ unsigned HiOpc = Hi.getOpcode();
+ if ((LoOpc != AMDGPU::V_MOV_B16_t16_e32 &&
+ LoOpc != AMDGPU::V_MOV_B16_t16_e64) ||
+ (HiOpc != AMDGPU::V_MOV_B16_t16_e32 &&
+ HiOpc != AMDGPU::V_MOV_B16_t16_e64))
+ return false;
+
+ MCRegister LoDst = Lo.getOperand(0).getReg().asMCReg();
+ MCRegister HiDst = Hi.getOperand(0).getReg().asMCReg();
+
+ if (!LoDst.isValid() || !HiDst.isValid())
+ return false;
+
+ if (!AMDGPU::VGPR_16RegClass.contains(LoDst) ||
+ !AMDGPU::VGPR_16RegClass.contains(HiDst))
+ return false;
+ if (AMDGPU::isHi16Reg(LoDst, *TRI))
+ return false; // Lo is actually writing hi half
+ if (!AMDGPU::isHi16Reg(HiDst, *TRI))
+ return false; // Hi is actually writing lo half
+
+ // Both must target the same 32-bit VGPR.
+ MCRegister LoDst32 = TRI->get32BitRegister(LoDst);
+ MCRegister HiDst32 = TRI->get32BitRegister(HiDst);
+ if (!LoDst32.isValid() || LoDst32 != HiDst32)
+ return false;
+
+ MCRegister Dst32 = LoDst32;
+
+ // Extract source info for Lo and Hi.
+ MCRegister LoSrc32, HiSrc32;
+ bool LoSrcIsHi, HiSrcIsHi, LoSrcIsImm, HiSrcIsImm;
+ int64_t LoImm = 0, HiImm = 0;
+
+ if (!getMovB16Info(Lo, TRI, LoSrc32, LoSrcIsHi, LoSrcIsImm, LoImm))
+ return false;
+ if (!getMovB16Info(Hi, TRI, HiSrc32, HiSrcIsHi, HiSrcIsImm, HiImm))
+ return false;
+
+ MachineInstr &FirstMI = IsHiFirst ? Hi : Lo;
+ MachineInstr &SecondMI = IsHiFirst ? Lo : Hi;
+ bool IsSecondImm = IsHiFirst ? LoSrcIsImm : HiSrcIsImm;
+
+ MachineBasicBlock &MBB = *FirstMI.getParent();
+ const DebugLoc &DL = FirstMI.getDebugLoc();
+
+ // Check that between Lo and Hi, there are no instructions that:
+ // - modify Dst32 (except through Lo/Hi themselves)
+ // - modify LoSrc32 or HiSrc32 dependinig on order (data dependency)
+ // We scan from the instruction after the first mov up to (but not including)
+ // the second mov.
+ MCRegister SecondSrc32 = IsHiFirst ? LoSrc32 : HiSrc32;
+ for (auto It = std::next(FirstMI.getIterator()); &*It != &SecondMI; ++It) {
+ const MachineInstr &Scan = *It;
+ if (Scan.modifiesRegister(Dst32, TRI))
+ return false;
+ if (!IsSecondImm && Scan.modifiesRegister(SecondSrc32, TRI))
+ return false;
+ }
+
+ // Now match patterns and emit the replacement instruction.
+ // Insert before the first (Lo) instruction, then remove both.
+
+ // Pattern: v_mov_b16 v0.h, 0 + v_mov_b16 v0.l, v2.l
+ // => v_and_b32 v0, 0x0000ffff, v2
+ if (HiSrcIsImm && HiImm == 0 && !LoSrcIsImm && !LoSrcIsHi) {
+ BuildMI(MBB, Lo, DL, TII->get(AMDGPU::V_AND_B32_e64), Dst32)
+ .addImm(0x0000ffff)
+ .addReg(LoSrc32);
+ Lo.eraseFromParent();
+ Hi.eraseFromParent();
+ return true;
+ }
+
+ // Pattern: v_mov_b16 v0.h, 0 + v_mov_b16 v0.l, v2.h
+ // => v_lshrrev_b32 v0, 16, v2
+ if (HiSrcIsImm && HiImm == 0 && !LoSrcIsImm && LoSrcIsHi) {
+ BuildMI(MBB, Lo, DL, TII->get(AMDGPU::V_LSHRREV_B32_e64), Dst32)
+ .addImm(16)
+ .addReg(LoSrc32);
+ Lo.eraseFromParent();
+ Hi.eraseFromParent();
+ return true;
+ }
+
+ // Pattern: v_mov_b16 v0.l, 0 + v_mov_b16 v0.h, v2.l
+ // => v_lshlrev_b32 v0, 16, v2
+ if (LoSrcIsImm && LoImm == 0 && !HiSrcIsImm && !HiSrcIsHi) {
+ BuildMI(MBB, Lo, DL, TII->get(AMDGPU::V_LSHLREV_B32_e64), Dst32)
+ .addImm(16)
+ .addReg(HiSrc32);
+ Lo.eraseFromParent();
+ Hi.eraseFromParent();
+ return true;
+ }
+
+ // Pattern: v_mov_b16 v0.l, 0 + v_mov_b16 v0.h, v2.h
+ // => v_and_b32 v0, 0xffff0000, v2
+ if (LoSrcIsImm && LoImm == 0 && !HiSrcIsImm && HiSrcIsHi) {
+ BuildMI(MBB, Lo, DL, TII->get(AMDGPU::V_AND_B32_e64), Dst32)
+ .addImm(0xffff0000)
+ .addReg(HiSrc32);
+ Lo.eraseFromParent();
+ Hi.eraseFromParent();
+ return true;
+ }
+
+ // Pattern: v_mov_b16 v0.l, v2.l + v_mov_b16 v0.h, v3.l
+ // => v_perm_b32 v0, v2, v3, 0x05040100
+ if (!LoSrcIsImm && !LoSrcIsHi && !HiSrcIsImm && !HiSrcIsHi) {
+ BuildMI(MBB, Lo, DL, TII->get(AMDGPU::V_PERM_B32_e64), Dst32)
+ .addReg(LoSrc32)
+ .addReg(HiSrc32)
+ .addImm(0x05040100);
+ Lo.eraseFromParent();
+ Hi.eraseFromParent();
+ return true;
+ }
+
+ // Pattern: v_mov_b16 v0.l, v2.l + v_mov_b16 v0.h, v3.h
+ // => v_bfi_b32 v0, 0x0000ffff, v2, v3
+ if (!LoSrcIsImm && !LoSrcIsHi && !HiSrcIsImm && HiSrcIsHi) {
+ BuildMI(MBB, Lo, DL, TII->get(AMDGPU::V_BFI_B32_e64), Dst32)
+ .addImm(0x0000ffff)
+ .addReg(LoSrc32)
+ .addReg(HiSrc32);
+ Lo.eraseFromParent();
+ Hi.eraseFromParent();
+ return true;
+ }
+
+ // Pattern: v_mov_b16 v0.l, v2.h + v_mov_b16 v0.h, v3.l
+ // => v_alignbit_b32 v0,v3,v2,16
+ if (!LoSrcIsImm && LoSrcIsHi && !HiSrcIsImm && !HiSrcIsHi) {
+ BuildMI(MBB, Lo, DL, TII->get(AMDGPU::V_ALIGNBIT_B32_t16_e64), Dst32)
+ .addImm(0) // SrcMod0
+ .addReg(HiSrc32)
+ .addImm(0) // SrcMod1
+ .addReg(LoSrc32)
+ .addImm(0) // SrcMod2
+ .addImm(16)
+ .addImm(0) // Clamp
+ .addImm(0); // Opsel
+ Lo.eraseFromParent();
+ Hi.eraseFromParent();
+ return true;
+ }
+
+ // Pattern: v_mov_b16 v0.l, v2.h + v_mov_b16 v0.h, v3.h
+ // => v_perm_b32 v0, v2, v3, 0x07060302
+ if (!LoSrcIsImm && LoSrcIsHi && !HiSrcIsImm && HiSrcIsHi) {
+ BuildMI(MBB, Lo, DL, TII->get(AMDGPU::V_PERM_B32_e64), Dst32)
+ .addReg(LoSrc32)
+ .addReg(HiSrc32)
+ .addImm(0x07060302);
+ Lo.eraseFromParent();
+ Hi.eraseFromParent();
+ return true;
+ }
+
+ return false;
+}
+
+// Merge pairs of v_mov_b16 targeting the lo16 and hi16 halves of the same
+// VGPR into a single 32-bit instruction (true16 mode only).
+bool SIPreEmitPeephole::mergeMovB16Pair(MachineFunction &MF) const {
+ bool Changed = false;
+ for (MachineBasicBlock &MBB : MF) {
+ // Map from 32-bit VGPR to the pending v_mov_b16 and its age.
+ // Age tracks how many non-mov-b16 instructions have passed since the
+ // lo16 write, used to bound the search window.
+ struct Pending {
+ MachineInstr *MI;
+ unsigned Age; // instructions since was seen
+ unsigned IsHi;
+ };
+ // Search window size
+ const unsigned ScanLimit = 16;
+ SmallDenseMap<MCRegister, Pending> PendingWrites;
+
+ for (auto &MI : make_early_inc_range(MBB)) {
+ unsigned Opc = MI.getOpcode();
+ bool IsMovB16 = (Opc == AMDGPU::V_MOV_B16_t16_e32 ||
+ Opc == AMDGPU::V_MOV_B16_t16_e64);
+
+ if (!IsMovB16) {
+ // Age all pending lo writes and invalidate stale or clobbered ones.
+ for (auto &[key, value] : PendingWrites)
+ value.Age++;
+
+ PendingWrites.remove_if([&](const auto &KV) {
+ return (KV.second.Age >= ScanLimit ||
+ MI.modifiesRegister(KV.first, TRI));
+ });
+ continue;
+ }
+
+ MCRegister DstReg = MI.getOperand(0).getReg().asMCReg();
+
+ bool DstIsHi = AMDGPU::isHi16Reg(DstReg, *TRI);
+ MCRegister Dst32 = TRI->get32BitRegister(DstReg);
+ if (!Dst32.isValid())
+ continue;
+
+ auto It = PendingWrites.find(Dst32);
+ if (It != PendingWrites.end() && It->second.IsHi != DstIsHi) {
+ // Look for a matching pending write.
+ MachineInstr &LoMI = !DstIsHi ? MI : *It->second.MI;
+ MachineInstr &HiMI = DstIsHi ? MI : *It->second.MI;
+ bool IsHiFirst = It->second.IsHi;
+ if (mergeSingleMovB16Pair(LoMI, HiMI, IsHiFirst))
+ Changed = true;
+ PendingWrites.erase(It);
+ } else {
+ PendingWrites[Dst32] = {&MI, 0, DstIsHi};
+ }
+ }
+ }
+
+ return Changed;
+}
+
PreservedAnalyses
llvm::SIPreEmitPeepholePass::run(MachineFunction &MF,
MachineFunctionAnalysisManager &MFAM) {
@@ -836,6 +1125,10 @@ bool SIPreEmitPeephole::run(MachineFunction &MF, MachineLoopInfo *LoopInfo) {
}
}
+ // Try merge B16 Pair in true16 mode
+ if (ST.useRealTrue16Insts())
+ Changed |= mergeMovB16Pair(MF);
+
// TODO: Fold this into previous block, if possible. Evaluate and handle any
// side effects.
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.intersect_ray.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.intersect_ray.ll
index 2b9094d1664a3..34a0ee06dd84a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.intersect_ray.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.intersect_ray.ll
@@ -66,10 +66,9 @@ define amdgpu_ps <4 x float> @image_bvh_intersect_ray_a16(i32 %node_ptr, float %
; GFX11-TRUE16-LABEL: image_bvh_intersect_ray_a16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v9, v5 :: v_dual_mov_b32 v10, v8
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v7.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v7, v9, 0x5040100
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.h, v6.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v9.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v7.h
; GFX11-TRUE16-NEXT: image_bvh_intersect_ray v[0:3], [v0, v1, v[2:4], v[8:10]], s[0:3] a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -141,10 +140,9 @@ define amdgpu_ps <4 x float> @image_bvh64_intersect_ray_a16(i64 %node_ptr, float
; GFX11-TRUE16-LABEL: image_bvh64_intersect_ray_a16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, v6 :: v_dual_mov_b32 v11, v9
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v8.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, v8, v10, 0x5040100
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.h, v7.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v10.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v8.h
; GFX11-TRUE16-NEXT: image_bvh64_intersect_ray v[0:3], [v[0:1], v2, v[3:5], v[9:11]], s[0:3] a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -361,8 +359,8 @@ define amdgpu_ps <4 x float> @image_bvh_intersect_ray_a16_vgpr_descr(i32 %node_p
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v16, v0 :: v_dual_mov_b32 v17, v1
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v13, v2 :: v_dual_mov_b32 v14, v3
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v15, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.h, v19.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_perm_b32 v18, v7, v19, 0x5040100
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v7.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.h, v6.l
; GFX11-TRUE16-NEXT: s_mov_b32 s1, exec_lo
@@ -630,9 +628,8 @@ define amdgpu_ps <4 x float> @image_bvh64_intersect_ray_a16_vgpr_descr(i64 %node
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v6 :: v_dual_mov_b32 v6, v9
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v17, v0 :: v_dual_mov_b32 v18, v1
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v19, v2 :: v_dual_mov_b32 v14, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v8.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v5.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v8, v5, 0x5040100
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v8.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v7.l
; GFX11-TRUE16-NEXT: s_mov_b32 s1, exec_lo
diff --git a/llvm/test/CodeGen/AMDGPU/add.v2i16.ll b/llvm/test/CodeGen/AMDGPU/add.v2i16.ll
index 96906ca0a5bbf..5c83e26c1803c 100644
--- a/llvm/test/CodeGen/AMDGPU/add.v2i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/add.v2i16.ll
@@ -671,10 +671,9 @@ define amdgpu_kernel void @v_test_add_v2i16_zext_to_v2i32(ptr addrspace(1) %out,
; GFX11-TRUE16-NEXT: global_load_b32 v0, v0, s[4:5] glc dlc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_pk_add_u16 v2, v1, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v1, 16, v2
; GFX11-TRUE16-NEXT: global_store_b64 v3, v[0:1], s[0:1]
; GFX11-TRUE16-NEXT: s_endpgm
;
@@ -788,9 +787,8 @@ define amdgpu_kernel void @v_test_add_v2i16_zext_to_v2i64(ptr addrspace(1) %out,
; GFX11-TRUE16-NEXT: global_load_b32 v0, v0, s[4:5] glc dlc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_pk_add_u16 v0, v2, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v2, v1, v0, 16
; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX11-TRUE16-NEXT: global_store_b128 v1, v[0:3], s[0:1]
; GFX11-TRUE16-NEXT: s_endpgm
@@ -1004,14 +1002,12 @@ define amdgpu_kernel void @v_test_add_v2i16_sext_to_v2i64(ptr addrspace(1) %out,
; GFX11-TRUE16-NEXT: global_load_b32 v0, v0, s[4:5]
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_pk_add_u16 v0, v1, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v1, 16, v0
; GFX11-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_i32 v2, v1, 0, 16
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_ashrrev_i32_e32 v3, 31, v2
; GFX11-TRUE16-NEXT: global_store_b128 v4, v[0:3], s[0:1]
; GFX11-TRUE16-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
index a733d427cd01c..8c108b82af271 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
@@ -163158,9 +163158,8 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc_lo
; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v7, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v50.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v51.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v50, v51, 0x5040100
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v3, v9 :: v_dual_add_nc_u32 v4, v5, v7
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, v6, v8
; GFX11-TRUE16-NEXT: v_add_f32_e64 v6, 0x40c00000, s28
@@ -163192,8 +163191,8 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v9
; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v10, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v8
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v54.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v55.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v54, v55, 0x5040100
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v5, v7, vcc_lo
; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v8, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
@@ -163413,7 +163412,7 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX11-TRUE16-NEXT: v_bfe_u32 v22, v23, 16, 1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v103, 16, v13
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v100, 16, v11
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v99.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v21, v99, v112, 0x5040100
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v16, v16, v17, vcc_lo
; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v18, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
@@ -163426,15 +163425,15 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX11-TRUE16-NEXT: v_add_f32_e64 v22, 0x40c00000, s0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
; GFX11-TRUE16-NEXT: s_and_b32 s0, s3, 0xffff0000
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.h, v112.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v113, 16, v15
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 ...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/208625
More information about the llvm-commits
mailing list