[llvm] [AMDGPU] merge 16bit mov pairs in post-RA peephole (PR #208625)
Guo Chen via llvm-commits
llvm-commits at lists.llvm.org
Thu Jul 16 08:55:11 PDT 2026
https://github.com/broxigarchen updated https://github.com/llvm/llvm-project/pull/208625
>From 5887fea854e4645a3dda7cbbb1d2daa069cce422 Mon Sep 17 00:00:00 2001
From: guochen2 <guochen2 at amd.com>
Date: Thu, 9 Jul 2026 19:09:46 -0400
Subject: [PATCH 1/5] preephole hole for merge v_mov_b16
---
llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp | 293 +++++++++++++
llvm/test/CodeGen/AMDGPU/GlobalISel/ashr.ll | 3 +-
.../test/CodeGen/AMDGPU/GlobalISel/fptrunc.ll | 14 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll | 3 +-
.../AMDGPU/GlobalISel/insertelement.i16.ll | 32 +-
.../GlobalISel/llvm.amdgcn.intersect_ray.ll | 19 +-
llvm/test/CodeGen/AMDGPU/add.v2i16.ll | 18 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll | 82 ++--
.../CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll | 52 +--
.../CodeGen/AMDGPU/amdgcn.bitcast.32bit.ll | 6 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.48bit.ll | 11 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll | 44 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll | 13 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll | 14 +-
.../atomic_optimizations_global_pointer.ll | 20 +-
llvm/test/CodeGen/AMDGPU/bf16.ll | 24 +-
llvm/test/CodeGen/AMDGPU/bswap.ll | 5 +-
.../buffer-fat-pointer-atomicrmw-fadd.ll | 30 +-
.../buffer-fat-pointer-atomicrmw-fmax.ll | 30 +-
.../buffer-fat-pointer-atomicrmw-fmin.ll | 30 +-
llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll | 13 +-
.../CodeGen/AMDGPU/flat-atomicrmw-fadd.ll | 100 ++---
.../CodeGen/AMDGPU/flat-atomicrmw-fmax.ll | 100 ++---
.../CodeGen/AMDGPU/flat-atomicrmw-fmin.ll | 100 ++---
.../CodeGen/AMDGPU/flat-atomicrmw-fsub.ll | 100 ++---
llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll | 6 +-
llvm/test/CodeGen/AMDGPU/function-args.ll | 5 +-
llvm/test/CodeGen/AMDGPU/function-returns.ll | 3 +-
.../CodeGen/AMDGPU/global-atomicrmw-fadd.ll | 100 ++---
.../CodeGen/AMDGPU/global-atomicrmw-fmax.ll | 100 ++---
.../CodeGen/AMDGPU/global-atomicrmw-fmin.ll | 100 ++---
.../CodeGen/AMDGPU/global-atomicrmw-fsub.ll | 100 ++---
.../llvm.amdgcn.image.gather4.a16.dim.ll | 17 +-
.../AMDGPU/llvm.amdgcn.image.msaa.load.ll | 9 +-
.../llvm.amdgcn.image.sample.a16.dim.ll | 63 +--
.../llvm.amdgcn.image.sample.g16.encode.ll | 20 +-
.../AMDGPU/llvm.amdgcn.image.sample.g16.ll | 6 +-
....amdgcn.struct.buffer.load.format.v3f16.ll | 6 +-
...gcn.struct.ptr.buffer.load.format.v3f16.ll | 3 +-
.../AMDGPU/llvm.amdgcn.waitcnt.out.order.ll | 12 +-
.../CodeGen/AMDGPU/local-atomicrmw-fadd.ll | 60 ++-
.../CodeGen/AMDGPU/local-atomicrmw-fmax.ll | 60 ++-
.../CodeGen/AMDGPU/local-atomicrmw-fmin.ll | 60 ++-
.../CodeGen/AMDGPU/local-atomicrmw-fsub.ll | 60 ++-
llvm/test/CodeGen/AMDGPU/repeated-divisor.ll | 5 +-
llvm/test/CodeGen/AMDGPU/scalar_to_vector.ll | 3 +-
.../si-pre-emit-peephole-merge-v-mov-b16.mir | 199 +++++++++
llvm/test/CodeGen/AMDGPU/sub.v2i16.ll | 18 +-
llvm/test/CodeGen/AMDGPU/v_swap_b16.ll | 13 +-
llvm/test/CodeGen/AMDGPU/vector-reduce-and.ll | 36 +-
llvm/test/CodeGen/AMDGPU/vector-reduce-or.ll | 36 +-
llvm/test/CodeGen/AMDGPU/vector-reduce-xor.ll | 36 +-
.../CodeGen/AMDGPU/vector_shuffle.packed.ll | 408 ++++++------------
53 files changed, 1327 insertions(+), 1373 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/si-pre-emit-peephole-merge-v-mov-b16.mir
diff --git a/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp b/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
index a496c9a4daa71..0a9ca8bfbd11c 100644
--- a/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
+++ b/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
@@ -44,6 +44,9 @@ class SIPreEmitPeephole {
void updateMLIBeforeRemovingEdge(MachineBasicBlock *From,
MachineBasicBlock *To) const;
bool optimizeSetGPR(MachineInstr &First, MachineInstr &MI) const;
+ bool mergeSingleMovB16Pair(MachineInstr &Lo, MachineInstr &Hi,
+ bool IsHiFirst) const;
+ bool mergeMovB16Pair(MachineFunction &MF) const;
bool getBlockDestinations(MachineBasicBlock &SrcMBB,
MachineBasicBlock *&TrueMBB,
MachineBasicBlock *&FalseMBB,
@@ -763,6 +766,292 @@ MachineInstrBuilder SIPreEmitPeephole::createUnpackedMI(MachineInstr &I,
return NewMI;
}
+// Helper: extract the src operand and whether it is from the hi16 half.
+// Post-RA, both V_MOV_B16_t16_e32 and V_MOV_B16_t16_e64 use VGPR_16 physical
+// registers whose encoding already encodes hi/lo (IS_HI16 bit).
+// Returns false if the source is not a physical VGPR_16 or immediate zero.
+//
+// Operand layouts (post-RA, physical registers):
+// V_MOV_B16_t16_e32: dst(0), src0(1)
+// V_MOV_B16_t16_e64: dst(0), src0_mods(1), src0(2), op_sel(3)
+static bool getMovB16Info(const MachineInstr &MI, const SIRegisterInfo *TRI,
+ MCRegister &SrcReg32, bool &SrcIsHi, bool &SrcIsImm,
+ int64_t &ImmVal) {
+ SrcIsImm = false;
+ SrcIsHi = false;
+ SrcReg32 = MCRegister();
+
+ unsigned Opc = MI.getOpcode();
+ const MachineOperand *SrcOp = nullptr;
+
+ if (Opc == AMDGPU::V_MOV_B16_t16_e64)
+ SrcOp = &MI.getOperand(2);
+ else if (Opc == AMDGPU::V_MOV_B16_t16_e32)
+ SrcOp = &MI.getOperand(1);
+ else
+ return false;
+
+ if (SrcOp->isImm()) {
+ SrcIsImm = true;
+ ImmVal = SrcOp->getImm();
+ return true;
+ }
+
+ if (!SrcOp->isReg() || !SrcOp->getReg().isPhysical())
+ return false;
+
+ MCRegister SrcReg = SrcOp->getReg().asMCReg();
+
+ // We require the source to be a 16-bit VGPR so we can determine hi/lo.
+ if (!AMDGPU::VGPR_16RegClass.contains(SrcReg))
+ return false;
+
+ SrcIsHi = AMDGPU::isHi16Reg(SrcReg, *TRI);
+ SrcReg32 = TRI->get32BitRegister(SrcReg);
+ return SrcReg32.isValid();
+}
+
+// clang-format off
+// Try to merge a pair of v_mov_b16 instructions targeting the lo16 and hi16
+// halves of the same VGPR into a single 32-bit instruction.
+//
+// Patterns:
+// v_mov_b16 v0.h, 0 v_mov_b16 v0.l, v2.l => v_and_b32 v0,0xffff,v2
+// v_mov_b16 v0.h, 0 v_mov_b16 v0.l, v2.h => v_lshrrev_b32 v0,16,v2
+// v_mov_b16 v0.l, 0 v_mov_b16 v0.h, v2.l => v_lshlrev_b32 v0,16,v2
+// v_mov_b16 v0.l, 0 v_mov_b16 v0.h, v2.h => v_and_b32 v0,0xffff0000,v2
+// v_mov_b16 v0.l, v2.l v_mov_b16 v0.h, v3.l => v_perm_b32 v0,v2,v3,0x05040100
+// v_mov_b16 v0.l, v2.l v_mov_b16 v0.h, v3.h => v_bfi_b32 v0,0x0000ffff,v2,v3
+// v_mov_b16 v0.l, v2.h v_mov_b16 v0.h, v3.l => v_alignbit_b32 v0,v3,v2,16
+// v_mov_b16 v0.l, v2.h v_mov_b16 v0.h, v3.h => v_perm_b32 v0,v2,v3,0x07060302
+// clang-format on
+bool SIPreEmitPeephole::mergeSingleMovB16Pair(MachineInstr &Lo,
+ MachineInstr &Hi,
+ bool IsHiFirst) const {
+ // Both must be v_mov_b16 true16 variants.
+ unsigned LoOpc = Lo.getOpcode();
+ unsigned HiOpc = Hi.getOpcode();
+ if ((LoOpc != AMDGPU::V_MOV_B16_t16_e32 &&
+ LoOpc != AMDGPU::V_MOV_B16_t16_e64) ||
+ (HiOpc != AMDGPU::V_MOV_B16_t16_e32 &&
+ HiOpc != AMDGPU::V_MOV_B16_t16_e64))
+ return false;
+
+ MCRegister LoDst = Lo.getOperand(0).getReg().asMCReg();
+ MCRegister HiDst = Hi.getOperand(0).getReg().asMCReg();
+
+ if (!LoDst.isValid() || !HiDst.isValid())
+ return false;
+
+ if (!AMDGPU::VGPR_16RegClass.contains(LoDst) ||
+ !AMDGPU::VGPR_16RegClass.contains(HiDst))
+ return false;
+ if (AMDGPU::isHi16Reg(LoDst, *TRI))
+ return false; // Lo is actually writing hi half
+ if (!AMDGPU::isHi16Reg(HiDst, *TRI))
+ return false; // Hi is actually writing lo half
+
+ // Both must target the same 32-bit VGPR.
+ MCRegister LoDst32 = TRI->get32BitRegister(LoDst);
+ MCRegister HiDst32 = TRI->get32BitRegister(HiDst);
+ if (!LoDst32.isValid() || LoDst32 != HiDst32)
+ return false;
+
+ MCRegister Dst32 = LoDst32;
+
+ // Extract source info for Lo and Hi.
+ MCRegister LoSrc32, HiSrc32;
+ bool LoSrcIsHi, HiSrcIsHi, LoSrcIsImm, HiSrcIsImm;
+ int64_t LoImm = 0, HiImm = 0;
+
+ if (!getMovB16Info(Lo, TRI, LoSrc32, LoSrcIsHi, LoSrcIsImm, LoImm))
+ return false;
+ if (!getMovB16Info(Hi, TRI, HiSrc32, HiSrcIsHi, HiSrcIsImm, HiImm))
+ return false;
+
+ MachineInstr &FirstMI = IsHiFirst ? Hi : Lo;
+ MachineInstr &SecondMI = IsHiFirst ? Lo : Hi;
+ bool IsSecondImm = IsHiFirst ? LoSrcIsImm : HiSrcIsImm;
+
+ MachineBasicBlock &MBB = *FirstMI.getParent();
+ const DebugLoc &DL = FirstMI.getDebugLoc();
+
+ // Check that between Lo and Hi, there are no instructions that:
+ // - modify Dst32 (except through Lo/Hi themselves)
+ // - modify LoSrc32 or HiSrc32 dependinig on order (data dependency)
+ // We scan from the instruction after the first mov up to (but not including)
+ // the second mov.
+ MCRegister SecondSrc32 = IsHiFirst ? LoSrc32 : HiSrc32;
+ for (auto It = std::next(FirstMI.getIterator()); &*It != &SecondMI; ++It) {
+ const MachineInstr &Scan = *It;
+ if (Scan.modifiesRegister(Dst32, TRI))
+ return false;
+ if (!IsSecondImm && Scan.modifiesRegister(SecondSrc32, TRI))
+ return false;
+ }
+
+ // Now match patterns and emit the replacement instruction.
+ // Insert before the first (Lo) instruction, then remove both.
+
+ // Pattern: v_mov_b16 v0.h, 0 + v_mov_b16 v0.l, v2.l
+ // => v_and_b32 v0, 0x0000ffff, v2
+ if (HiSrcIsImm && HiImm == 0 && !LoSrcIsImm && !LoSrcIsHi) {
+ BuildMI(MBB, Lo, DL, TII->get(AMDGPU::V_AND_B32_e64), Dst32)
+ .addImm(0x0000ffff)
+ .addReg(LoSrc32);
+ Lo.eraseFromParent();
+ Hi.eraseFromParent();
+ return true;
+ }
+
+ // Pattern: v_mov_b16 v0.h, 0 + v_mov_b16 v0.l, v2.h
+ // => v_lshrrev_b32 v0, 16, v2
+ if (HiSrcIsImm && HiImm == 0 && !LoSrcIsImm && LoSrcIsHi) {
+ BuildMI(MBB, Lo, DL, TII->get(AMDGPU::V_LSHRREV_B32_e64), Dst32)
+ .addImm(16)
+ .addReg(LoSrc32);
+ Lo.eraseFromParent();
+ Hi.eraseFromParent();
+ return true;
+ }
+
+ // Pattern: v_mov_b16 v0.l, 0 + v_mov_b16 v0.h, v2.l
+ // => v_lshlrev_b32 v0, 16, v2
+ if (LoSrcIsImm && LoImm == 0 && !HiSrcIsImm && !HiSrcIsHi) {
+ BuildMI(MBB, Lo, DL, TII->get(AMDGPU::V_LSHLREV_B32_e64), Dst32)
+ .addImm(16)
+ .addReg(HiSrc32);
+ Lo.eraseFromParent();
+ Hi.eraseFromParent();
+ return true;
+ }
+
+ // Pattern: v_mov_b16 v0.l, 0 + v_mov_b16 v0.h, v2.h
+ // => v_and_b32 v0, 0xffff0000, v2
+ if (LoSrcIsImm && LoImm == 0 && !HiSrcIsImm && HiSrcIsHi) {
+ BuildMI(MBB, Lo, DL, TII->get(AMDGPU::V_AND_B32_e64), Dst32)
+ .addImm(0xffff0000)
+ .addReg(HiSrc32);
+ Lo.eraseFromParent();
+ Hi.eraseFromParent();
+ return true;
+ }
+
+ // Pattern: v_mov_b16 v0.l, v2.l + v_mov_b16 v0.h, v3.l
+ // => v_perm_b32 v0, v2, v3, 0x05040100
+ if (!LoSrcIsImm && !LoSrcIsHi && !HiSrcIsImm && !HiSrcIsHi) {
+ BuildMI(MBB, Lo, DL, TII->get(AMDGPU::V_PERM_B32_e64), Dst32)
+ .addReg(LoSrc32)
+ .addReg(HiSrc32)
+ .addImm(0x05040100);
+ Lo.eraseFromParent();
+ Hi.eraseFromParent();
+ return true;
+ }
+
+ // Pattern: v_mov_b16 v0.l, v2.l + v_mov_b16 v0.h, v3.h
+ // => v_bfi_b32 v0, 0x0000ffff, v2, v3
+ if (!LoSrcIsImm && !LoSrcIsHi && !HiSrcIsImm && HiSrcIsHi) {
+ BuildMI(MBB, Lo, DL, TII->get(AMDGPU::V_BFI_B32_e64), Dst32)
+ .addImm(0x0000ffff)
+ .addReg(LoSrc32)
+ .addReg(HiSrc32);
+ Lo.eraseFromParent();
+ Hi.eraseFromParent();
+ return true;
+ }
+
+ // Pattern: v_mov_b16 v0.l, v2.h + v_mov_b16 v0.h, v3.l
+ // => v_alignbit_b32 v0,v3,v2,16
+ if (!LoSrcIsImm && LoSrcIsHi && !HiSrcIsImm && !HiSrcIsHi) {
+ BuildMI(MBB, Lo, DL, TII->get(AMDGPU::V_ALIGNBIT_B32_t16_e64), Dst32)
+ .addImm(0) // SrcMod0
+ .addReg(HiSrc32)
+ .addImm(0) // SrcMod1
+ .addReg(LoSrc32)
+ .addImm(0) // SrcMod2
+ .addImm(16)
+ .addImm(0) // Clamp
+ .addImm(0); // Opsel
+ Lo.eraseFromParent();
+ Hi.eraseFromParent();
+ return true;
+ }
+
+ // Pattern: v_mov_b16 v0.l, v2.h + v_mov_b16 v0.h, v3.h
+ // => v_perm_b32 v0, v2, v3, 0x07060302
+ if (!LoSrcIsImm && LoSrcIsHi && !HiSrcIsImm && HiSrcIsHi) {
+ BuildMI(MBB, Lo, DL, TII->get(AMDGPU::V_PERM_B32_e64), Dst32)
+ .addReg(LoSrc32)
+ .addReg(HiSrc32)
+ .addImm(0x07060302);
+ Lo.eraseFromParent();
+ Hi.eraseFromParent();
+ return true;
+ }
+
+ return false;
+}
+
+// Merge pairs of v_mov_b16 targeting the lo16 and hi16 halves of the same
+// VGPR into a single 32-bit instruction (true16 mode only).
+bool SIPreEmitPeephole::mergeMovB16Pair(MachineFunction &MF) const {
+ bool Changed = false;
+ for (MachineBasicBlock &MBB : MF) {
+ // Map from 32-bit VGPR to the pending v_mov_b16 and its age.
+ // Age tracks how many non-mov-b16 instructions have passed since the
+ // lo16 write, used to bound the search window.
+ struct Pending {
+ MachineInstr *MI;
+ unsigned Age; // instructions since was seen
+ unsigned IsHi;
+ };
+ // Search window size
+ const unsigned ScanLimit = 16;
+ SmallDenseMap<MCRegister, Pending> PendingWrites;
+
+ for (auto &MI : make_early_inc_range(MBB)) {
+ unsigned Opc = MI.getOpcode();
+ bool IsMovB16 = (Opc == AMDGPU::V_MOV_B16_t16_e32 ||
+ Opc == AMDGPU::V_MOV_B16_t16_e64);
+
+ if (!IsMovB16) {
+ // Age all pending lo writes and invalidate stale or clobbered ones.
+ for (auto &[key, value] : PendingWrites)
+ value.Age++;
+
+ PendingWrites.remove_if([&](const auto &KV) {
+ return (KV.second.Age >= ScanLimit ||
+ MI.modifiesRegister(KV.first, TRI));
+ });
+ continue;
+ }
+
+ MCRegister DstReg = MI.getOperand(0).getReg().asMCReg();
+
+ bool DstIsHi = AMDGPU::isHi16Reg(DstReg, *TRI);
+ MCRegister Dst32 = TRI->get32BitRegister(DstReg);
+ if (!Dst32.isValid())
+ continue;
+
+ auto It = PendingWrites.find(Dst32);
+ if (It != PendingWrites.end() && It->second.IsHi != DstIsHi) {
+ // Look for a matching pending write.
+ MachineInstr &LoMI = !DstIsHi ? MI : *It->second.MI;
+ MachineInstr &HiMI = DstIsHi ? MI : *It->second.MI;
+ bool IsHiFirst = It->second.IsHi;
+ if (mergeSingleMovB16Pair(LoMI, HiMI, IsHiFirst))
+ Changed = true;
+ PendingWrites.erase(It);
+ } else {
+ PendingWrites[Dst32] = {&MI, 0, DstIsHi};
+ }
+ }
+ }
+
+ return Changed;
+}
+
PreservedAnalyses
llvm::SIPreEmitPeepholePass::run(MachineFunction &MF,
MachineFunctionAnalysisManager &MFAM) {
@@ -836,6 +1125,10 @@ bool SIPreEmitPeephole::run(MachineFunction &MF, MachineLoopInfo *LoopInfo) {
}
}
+ // Try merge B16 Pair in true16 mode
+ if (ST.useRealTrue16Insts())
+ Changed |= mergeMovB16Pair(MF);
+
// TODO: Fold this into previous block, if possible. Evaluate and handle any
// side effects.
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/ashr.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/ashr.ll
index a7c6d8b158c70..6aa882a15b136 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/ashr.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/ashr.ll
@@ -2008,8 +2008,7 @@ define <4 x i2> @v_ashr_v4i2(<4 x i2> %value, <4 x i2> %amount) {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v3, 0x5040100
; GFX11-TRUE16-NEXT: v_and_b16 v2.h, v5.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, v4.l, 3
; GFX11-TRUE16-NEXT: v_pk_lshlrev_b16 v0, 14, v0 op_sel_hi:[0,1]
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fptrunc.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fptrunc.ll
index ff53b3dfdeb5e..6e60d2aa8b287 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fptrunc.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fptrunc.ll
@@ -386,12 +386,12 @@ define amdgpu_ps half @fptrunc_f64_to_f16_div(double %a) {
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, 0x7e00 :: v_dual_add_nc_u32 v2, v2, v4
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, 0x7c00, v5, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_gt_i32_e32 vcc_lo, 31, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v4, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, 0x7c00, v2, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0x40f, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v0, 0x8000, v4, v0
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -495,8 +495,7 @@ define amdgpu_ps half @fptrunc_f64_to_f16_div(double %a) {
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, 0x7c00, v5, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_gt_i32_e32 vcc_lo, 31, v3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.h, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v4, 16, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, 0x7c00, v2, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0x40f, v3
@@ -595,11 +594,10 @@ define amdgpu_ps half @fptrunc_f64_to_f16_div(double %a) {
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_or_b32_e32 v4, v4, v5
; GFX1250-TRUE16-NEXT: v_dual_mov_b32 v5, 0x7e00 :: v_dual_add_nc_u32 v2, v2, v4
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX1250-TRUE16-NEXT: v_cndmask_b32_e32 v0, 0x7c00, v5, vcc_lo
; GFX1250-TRUE16-NEXT: v_cmp_gt_i32_e32 vcc_lo, 31, v3
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v4.h, 0
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.h
+; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e64 v4, 16, v1
; GFX1250-TRUE16-NEXT: v_cndmask_b32_e32 v2, 0x7c00, v2, vcc_lo
; GFX1250-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0x40f, v3
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll
index 2d6d662777526..e413201a5d34f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll
@@ -4848,8 +4848,7 @@ define i64 @v_fshr_i64_48(i64 %lhs, i64 %rhs) {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_lshlrev_b64 v[0:1], 16, v[0:1]
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v3.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v2, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v2
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.i16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.i16.ll
index 6d6d51a66bfa4..bf0ed51126ade 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.i16.ll
@@ -243,8 +243,7 @@ define amdgpu_ps void @insertelement_s_v2i16_v_s(ptr addrspace(4) inreg %ptr, i1
; GFX11-LABEL: insertelement_s_v2i16_v_s:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b32 s0, s[2:3], 0x0
-; GFX11-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-NEXT: v_mov_b16_e32 v2.h, v0.l
+; GFX11-NEXT: v_perm_b32 v2, v0, v0, 0x5040100
; GFX11-NEXT: s_lshl_b32 s1, s4, 4
; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0
; GFX11-NEXT: s_lshl_b32 s1, 0xffff, s1
@@ -404,10 +403,9 @@ define amdgpu_ps void @insertelement_s_v2i16_v_v(ptr addrspace(4) inreg %ptr, i1
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b32 s0, s[2:3], 0x0
; GFX11-NEXT: v_lshlrev_b32_e32 v1, 4, v1
-; GFX11-NEXT: v_mov_b16_e32 v3.l, v0.l
-; GFX11-NEXT: v_mov_b16_e32 v3.h, v0.l
+; GFX11-NEXT: v_perm_b32 v3, v0, v0, 0x5040100
; GFX11-NEXT: v_mov_b32_e32 v0, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_lshlrev_b32_e64 v2, v1, 0xffff
; GFX11-NEXT: v_mov_b32_e32 v1, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
@@ -566,8 +564,7 @@ define amdgpu_ps void @insertelement_v_v2i16_v_s(ptr addrspace(1) %ptr, i16 %val
; GFX11-LABEL: insertelement_v_v2i16_v_s:
; GFX11: ; %bb.0:
; GFX11-NEXT: global_load_b32 v3, v[0:1], off
-; GFX11-NEXT: v_mov_b16_e32 v4.l, v2.l
-; GFX11-NEXT: v_mov_b16_e32 v4.h, v2.l
+; GFX11-NEXT: v_perm_b32 v4, v2, v2, 0x5040100
; GFX11-NEXT: s_lshl_b32 s0, s2, 4
; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0
; GFX11-NEXT: s_lshl_b32 s0, 0xffff, s0
@@ -649,9 +646,8 @@ define amdgpu_ps void @insertelement_v_v2i16_v_v(ptr addrspace(1) %ptr, i16 %val
; GFX11: ; %bb.0:
; GFX11-NEXT: global_load_b32 v4, v[0:1], off
; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_lshlrev_b32 v0, 4, v3
-; GFX11-NEXT: v_mov_b16_e32 v5.l, v2.l
-; GFX11-NEXT: v_mov_b16_e32 v5.h, v2.l
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_perm_b32 v5, v2, v2, 0x5040100
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_lshlrev_b32_e64 v3, v0, 0xffff
; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: s_waitcnt vmcnt(0)
@@ -882,8 +878,7 @@ define amdgpu_ps void @insertelement_s_v4i16_v_s(ptr addrspace(4) inreg %ptr, i1
; GFX11-LABEL: insertelement_s_v4i16_v_s:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
-; GFX11-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-NEXT: v_mov_b16_e32 v2.h, v0.l
+; GFX11-NEXT: v_perm_b32 v2, v0, v0, 0x5040100
; GFX11-NEXT: s_lshl_b32 s2, s4, 4
; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0
; GFX11-NEXT: s_lshl_b64 s[2:3], 0xffff, s2
@@ -1059,10 +1054,9 @@ define amdgpu_ps void @insertelement_s_v4i16_v_v(ptr addrspace(4) inreg %ptr, i1
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
; GFX11-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_lshlrev_b32 v1, 4, v1
-; GFX11-NEXT: v_mov_b16_e32 v5.l, v0.l
-; GFX11-NEXT: v_mov_b16_e32 v5.h, v0.l
+; GFX11-NEXT: v_perm_b32 v5, v0, v0, 0x5040100
; GFX11-NEXT: v_mov_b32_e32 v3, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_lshlrev_b64 v[1:2], v1, 0xffff
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_bfi_b32 v2, v2, v5, s1
@@ -1233,8 +1227,7 @@ define amdgpu_ps void @insertelement_v_v4i16_v_s(ptr addrspace(1) %ptr, i16 %val
; GFX11-LABEL: insertelement_v_v4i16_v_s:
; GFX11: ; %bb.0:
; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off
-; GFX11-NEXT: v_mov_b16_e32 v4.l, v2.l
-; GFX11-NEXT: v_mov_b16_e32 v4.h, v2.l
+; GFX11-NEXT: v_perm_b32 v4, v2, v2, 0x5040100
; GFX11-NEXT: s_lshl_b32 s0, s2, 4
; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v3, 0
; GFX11-NEXT: s_lshl_b64 s[0:1], 0xffff, s0
@@ -1322,10 +1315,9 @@ define amdgpu_ps void @insertelement_v_v4i16_v_v(ptr addrspace(1) %ptr, i16 %val
; GFX11: ; %bb.0:
; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off
; GFX11-NEXT: v_dual_mov_b32 v6, 0 :: v_dual_lshlrev_b32 v3, 4, v3
-; GFX11-NEXT: v_mov_b16_e32 v7.l, v2.l
-; GFX11-NEXT: v_mov_b16_e32 v7.h, v2.l
+; GFX11-NEXT: v_perm_b32 v7, v2, v2, 0x5040100
; GFX11-NEXT: v_mov_b32_e32 v5, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_lshlrev_b64 v[3:4], v3, 0xffff
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_bfi_b32 v1, v4, v7, v1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.intersect_ray.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.intersect_ray.ll
index 8cd03fd12ef2f..3f6fdf5ef6c81 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.intersect_ray.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.intersect_ray.ll
@@ -60,10 +60,9 @@ define amdgpu_ps <4 x float> @image_bvh_intersect_ray_a16(i32 %node_ptr, float %
; GFX11-TRUE16-LABEL: image_bvh_intersect_ray_a16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v9, v5 :: v_dual_mov_b32 v10, v8
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v7.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v7, v9, 0x5040100
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.h, v6.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v9.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v7.h
; GFX11-TRUE16-NEXT: image_bvh_intersect_ray v[0:3], [v0, v1, v[2:4], v[8:10]], s[0:3] a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -126,10 +125,9 @@ define amdgpu_ps <4 x float> @image_bvh64_intersect_ray_a16(i64 %node_ptr, float
; GFX11-TRUE16-LABEL: image_bvh64_intersect_ray_a16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, v6 :: v_dual_mov_b32 v11, v9
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v8.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, v8, v10, 0x5040100
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.h, v7.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v10.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v8.h
; GFX11-TRUE16-NEXT: image_bvh64_intersect_ray v[0:3], [v[0:1], v2, v[3:5], v[9:11]], s[0:3] a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -329,8 +327,8 @@ define amdgpu_ps <4 x float> @image_bvh_intersect_ray_a16_vgpr_descr(i32 %node_p
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v16, v0 :: v_dual_mov_b32 v17, v1
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v13, v2 :: v_dual_mov_b32 v14, v3
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v15, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.h, v19.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_perm_b32 v18, v7, v19, 0x5040100
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v7.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.h, v6.l
; GFX11-TRUE16-NEXT: s_mov_b32 s1, exec_lo
@@ -580,9 +578,8 @@ define amdgpu_ps <4 x float> @image_bvh64_intersect_ray_a16_vgpr_descr(i64 %node
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v6 :: v_dual_mov_b32 v6, v9
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v17, v0 :: v_dual_mov_b32 v18, v1
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v19, v2 :: v_dual_mov_b32 v14, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v8.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v5.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v8, v5, 0x5040100
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v8.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v7.l
; GFX11-TRUE16-NEXT: s_mov_b32 s1, exec_lo
diff --git a/llvm/test/CodeGen/AMDGPU/add.v2i16.ll b/llvm/test/CodeGen/AMDGPU/add.v2i16.ll
index 74e226cb6f120..860e14e083abf 100644
--- a/llvm/test/CodeGen/AMDGPU/add.v2i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/add.v2i16.ll
@@ -671,10 +671,9 @@ define amdgpu_kernel void @v_test_add_v2i16_zext_to_v2i32(ptr addrspace(1) %out,
; GFX11-TRUE16-NEXT: global_load_b32 v0, v0, s[4:5] glc dlc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_pk_add_u16 v2, v1, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v1, 16, v2
; GFX11-TRUE16-NEXT: global_store_b64 v3, v[0:1], s[0:1]
; GFX11-TRUE16-NEXT: s_endpgm
;
@@ -788,9 +787,8 @@ define amdgpu_kernel void @v_test_add_v2i16_zext_to_v2i64(ptr addrspace(1) %out,
; GFX11-TRUE16-NEXT: global_load_b32 v0, v0, s[4:5] glc dlc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_pk_add_u16 v0, v2, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v2, v1, v0, 16
; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX11-TRUE16-NEXT: global_store_b128 v1, v[0:3], s[0:1]
; GFX11-TRUE16-NEXT: s_endpgm
@@ -1004,14 +1002,12 @@ define amdgpu_kernel void @v_test_add_v2i16_sext_to_v2i64(ptr addrspace(1) %out,
; GFX11-TRUE16-NEXT: global_load_b32 v0, v0, s[4:5]
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_pk_add_u16 v0, v1, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v1, 16, v0
; GFX11-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_i32 v2, v1, 0, 16
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_ashrrev_i32_e32 v3, 31, v2
; GFX11-TRUE16-NEXT: global_store_b128 v4, v[0:3], s[0:1]
; GFX11-TRUE16-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
index 68699bc5679d0..bd63294186d8e 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
@@ -163158,9 +163158,8 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc_lo
; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v7, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v50.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v51.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v50, v51, 0x5040100
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v3, v9 :: v_dual_add_nc_u32 v4, v5, v7
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, v6, v8
; GFX11-TRUE16-NEXT: v_add_f32_e64 v6, 0x40c00000, s28
@@ -163192,8 +163191,8 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v9
; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v10, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v8
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v54.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v55.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v54, v55, 0x5040100
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v5, v7, vcc_lo
; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v8, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
@@ -163413,7 +163412,7 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX11-TRUE16-NEXT: v_bfe_u32 v22, v23, 16, 1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v103, 16, v13
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v100, 16, v11
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v99.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v21, v99, v112, 0x5040100
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v16, v16, v17, vcc_lo
; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v18, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
@@ -163426,15 +163425,15 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX11-TRUE16-NEXT: v_add_f32_e64 v22, 0x40c00000, s0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
; GFX11-TRUE16-NEXT: s_and_b32 s0, s3, 0xffff0000
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.h, v112.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v113, 16, v15
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v15, 0x7fff, v16
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v16, 0x7fff, v17
; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v23
; GFX11-TRUE16-NEXT: v_bfe_u32 v23, v22, 16, 1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v102.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.h, v113.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v20, v102, v113, 0x5040100
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v87.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v16, v16, v17, vcc_lo
; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v19, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
@@ -163453,14 +163452,13 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX11-TRUE16-NEXT: v_bfe_u32 v22, v23, 16, 1
; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s2, 16
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.h, v101.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v98.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v13, v98, v103, 0x5040100
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v16, v16, v17, vcc_lo
; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v18, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
; GFX11-TRUE16-NEXT: v_add_f32_e64 v19, 0x40c00000, s0
; GFX11-TRUE16-NEXT: s_and_b32 s0, s17, 0xffff0000
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v115, 16, v16
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.h, v103.l
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v15, v15, v24, vcc_lo
; GFX11-TRUE16-NEXT: v_or_b32_e32 v24, 0x400000, v18
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v16, v17, v18
@@ -163492,8 +163490,8 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v22
; GFX11-TRUE16-NEXT: v_bfe_u32 v22, v23, 16, 1
; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s16, 16
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v116.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v36.h, v129.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v36, v116, v129, 0x5040100
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v16, v16, v17, vcc_lo
; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v18, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
@@ -163511,8 +163509,8 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v23
; GFX11-TRUE16-NEXT: v_bfe_u32 v23, v22, 16, 1
; GFX11-TRUE16-NEXT: s_and_b32 s1, s19, 0xffff0000
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v118.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v35.h, v132.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v35, v118, v132, 0x5040100
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v16, v16, v17, vcc_lo
; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v19, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
@@ -163531,12 +163529,12 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v24
; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s26, 16
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v84.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v81.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v12, v81, v97, 0x5040100
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v16, v16, v17, vcc_lo
; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v18, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v24, 16, 1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.h, v97.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v130, 16, v16
; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v15, v15, v23 :: v_dual_add_nc_u32 v16, v17, v18
; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v18
@@ -163707,47 +163705,33 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v16, v16, v19, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v18, 0x7fff, v18
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v40.l, v150.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v40.h, v163.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v40, v150, v163, 0x5040100
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v162, 16, v16
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v19, v25, v26, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v47.l, v134.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v47.h, v149.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v46.l, v146.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v47, v134, v149, 0x5040100
+; GFX11-TRUE16-NEXT: v_perm_b32 v46, v146, v160, 0x5040100
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v165, 16, v19
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v18, v18, v27, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v46.h, v160.l
; GFX11-TRUE16-NEXT: v_mov_b16_e64 v45.h, v145.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v44.l, v135.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v44.h, v148.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v44, v135, v148, 0x5040100
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v177, 16, v18
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.h, v100.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v114.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.h, v117.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v115.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v28.h, v128.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v119.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v49.h, v133.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v48.l, v130.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v48.h, v144.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v179.l, v162.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v179.h, v176.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v178.l, v165.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v178.h, v177.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v29, v114, v117, 0x5040100
+; GFX11-TRUE16-NEXT: v_perm_b32 v28, v115, v128, 0x5040100
+; GFX11-TRUE16-NEXT: v_perm_b32 v49, v119, v133, 0x5040100
+; GFX11-TRUE16-NEXT: v_perm_b32 v48, v130, v144, 0x5040100
+; GFX11-TRUE16-NEXT: v_perm_b32 v179, v162, v176, 0x5040100
+; GFX11-TRUE16-NEXT: v_perm_b32 v178, v165, v177, 0x5040100
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v53.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v65.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v67.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v68.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v69.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v71.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.h, v85.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v82.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v96.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v183.l, v151.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v183.h, v164.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v182.l, v166.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v182.h, v167.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v65, v67, 0x5040100
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v68, v69, 0x5040100
+; GFX11-TRUE16-NEXT: v_perm_b32 v10, v71, v85, 0x5040100
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, v82, v96, 0x5040100
+; GFX11-TRUE16-NEXT: v_perm_b32 v183, v151, v164, 0x5040100
+; GFX11-TRUE16-NEXT: v_perm_b32 v182, v166, v167, 0x5040100
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[30:31], 24, v[40:41]
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[31:32], 24, v[46:47]
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[15:16], 24, v[20:21]
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll
index f2e9354a563d8..98f53a8163a00 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll
@@ -33961,26 +33961,21 @@ define <16 x i16> @bitcast_v32i8_to_v16i16(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_clause 0x1
; GFX11-TRUE16-NEXT: scratch_load_b32 v37, off, s32 offset:4
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.h, v27.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.h, v25.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.h, v23.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.h, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.h, v17.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v29.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v28.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v21.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v27, v21, v27, 0x5040100
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v20.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v15.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v25, v15, v25, 0x5040100
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v14.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v13.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v19, v13, v19, 0x5040100
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v12.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v11.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v17, v11, v17, 0x5040100
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v10.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v9.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v8.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v6.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v23, v6, v23, 0x5040100
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v5.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v4.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v3.l
@@ -39516,26 +39511,21 @@ define <16 x half> @bitcast_v32i8_to_v16f16(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_clause 0x1
; GFX11-TRUE16-NEXT: scratch_load_b32 v37, off, s32 offset:4
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.h, v27.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.h, v25.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.h, v23.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.h, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.h, v17.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v29.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v28.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v21.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v27, v21, v27, 0x5040100
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v20.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v15.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v25, v15, v25, 0x5040100
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v14.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v13.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v19, v13, v19, 0x5040100
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v12.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v11.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v17, v11, v17, 0x5040100
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v10.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v9.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v8.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v6.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v23, v6, v23, 0x5040100
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v5.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v4.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v3.l
@@ -42912,8 +42902,7 @@ define inreg <32 x i8> @bitcast_v16bf16_to_v32i8_scalar(<16 x bfloat> inreg %a,
; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v8, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.h, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v2.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v36, v0, v2, 0x5040100
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, v5, v8
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v34, 16, v4
@@ -42983,7 +42972,7 @@ define inreg <32 x i8> @bitcast_v16bf16_to_v32i8_scalar(<16 x bfloat> inreg %a,
; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v11, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v11
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, v3, v9
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v33.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v49, v33, v22, 0x5040100
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v7, v15, vcc_lo
; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v9
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
@@ -42991,7 +42980,7 @@ define inreg <32 x i8> @bitcast_v16bf16_to_v32i8_scalar(<16 x bfloat> inreg %a,
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, v5, v11
; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v12
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.h, v22.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x7fff, v5
@@ -43960,26 +43949,21 @@ define <16 x bfloat> @bitcast_v32i8_to_v16bf16(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_clause 0x1
; GFX11-TRUE16-NEXT: scratch_load_b32 v37, off, s32 offset:4
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.h, v27.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.h, v25.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.h, v23.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.h, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.h, v17.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v29.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v28.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v21.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v27, v21, v27, 0x5040100
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v20.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v15.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v25, v15, v25, 0x5040100
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v14.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v13.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v19, v13, v19, 0x5040100
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v12.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v11.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v17, v11, v17, 0x5040100
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v10.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v9.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v8.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v6.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v23, v6, v23, 0x5040100
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v5.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v4.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v3.l
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.32bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.32bit.ll
index 55f71864d7af3..a1cee39a3dfa3 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.32bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.32bit.ll
@@ -10265,10 +10265,8 @@ define inreg <4 x i8> @bitcast_v2bf16_to_v4i8_scalar(<2 x bfloat> inreg %a, i32
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v0, v2, 0x5040100
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.48bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.48bit.ll
index cab6fe1ea0374..3a28e939ff940 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.48bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.48bit.ll
@@ -930,17 +930,15 @@ define <3 x i16> @bitcast_v3bf16_to_v3i16(<3 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v1, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v3, v7, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, 0x7fc0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v1, v3, v1, 16
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v6, vcc_lo
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
; GFX11-TRUE16-NEXT: .LBB4_2: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -1196,11 +1194,10 @@ define inreg <3 x i16> @bitcast_v3bf16_to_v3i16_scalar(<3 x bfloat> inreg %a, i3
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v6, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, 0x7fc0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v1, v4, v1, 16
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v4.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB5_4:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll
index df420ccaa91c6..deee9db7dff82 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll
@@ -90271,9 +90271,8 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc_lo
; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v7, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v16.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v17.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v16, v17, 0x5040100
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v3, v9 :: v_dual_add_nc_u32 v4, v5, v7
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, v6, v8
; GFX11-TRUE16-NEXT: v_add_f32_e64 v6, 0x40c00000, s0
@@ -90465,26 +90464,24 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v13, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 0x7fff, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v81.l, v34.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v81, v34, v37, 0x5040100
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v48, 16, v8
; GFX11-TRUE16-NEXT: v_add_f32_e64 v8, 0x40c00000, s0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v9, v9, v13
; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s24, 16
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v81.h, v37.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v80.l, v48.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_perm_b32 v80, v48, v38, 0x5040100
; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v8, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v9, 0x7fff, v9
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v80.h, v38.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v84.l, v30.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v84.h, v33.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v84, v30, v33, 0x5040100
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v11, v11, v8
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v10, vcc_lo
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v13
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v83.l, v35.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v83.h, v36.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v86.l, v28.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v86.h, v29.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v83, v35, v36, 0x5040100
+; GFX11-TRUE16-NEXT: v_perm_b32 v86, v28, v29, 0x5040100
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v9, v9, v10 :: v_dual_add_nc_u32 v10, 0x7fff, v11
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v39, 16, v7
; GFX11-TRUE16-NEXT: v_add_f32_e64 v7, 0x40c00000, s0
@@ -90493,10 +90490,10 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; GFX11-TRUE16-NEXT: s_and_b32 s0, s27, 0xffff0000
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v69.h, v39.l
; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v7, 16, 1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v85.l, v31.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v85, v31, v32, 0x5040100
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v10, v11, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v85.h, v32.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v10, v12, v7
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v49, 16, v9
; GFX11-TRUE16-NEXT: v_add_f32_e64 v9, 0x40c00000, s0
@@ -90529,27 +90526,24 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v55, 16, v7
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v8, v11, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v68.h, v50.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v67.h, v53.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v68.l, v55.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v68, v55, v50, 0x5040100
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v25.l
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v11, v15, v51, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v51, 16, v8
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v26.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v27.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v26, v27, 0x5040100
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v20.l
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v10, v10, v52, vcc_lo
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v52, 16, v11
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v67.l, v51.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v22.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v24.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v22, v24, 0x5040100
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v54, 16, v10
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v66.l, v52.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v18.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v19.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v66, v52, v54, 0x5040100
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v18, v19, 0x5040100
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v65, 24, v67
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v66.h, v54.l
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v70, 24, v81
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v71, 8, v80
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v82, 24, v84
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll
index 0da1fde3f0549..dd365b47e4a5f 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll
@@ -18093,16 +18093,13 @@ define inreg <8 x i8> @bitcast_v4bf16_to_v8i8_scalar(<4 x bfloat> inreg %a, i32
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v8.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, v0, v2, 0x5040100
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_perm_b32 v10, v8, v6, 0x5040100
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v9
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.h, v6.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[9:10]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v10
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v10
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll
index 0e04f1ebc8e9d..5456775ec0c3a 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll
@@ -8760,8 +8760,8 @@ define inreg <12 x i8> @bitcast_v6bf16_to_v12i8_scalar(<6 x bfloat> inreg %a, i3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 0x7fff, v7
; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v13.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.h, v6.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v15, v13, v6, 0x5040100
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
@@ -8778,20 +8778,18 @@ define inreg <12 x i8> @bitcast_v6bf16_to_v12i8_scalar(<6 x bfloat> inreg %a, i3
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v7, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v15
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_perm_b32 v14, v0, v2, 0x5040100
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v3
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v8.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[14:15]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v14
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v13
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
index 25991d274a278..3de3578f15cee 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
@@ -15208,9 +15208,8 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
; GFX1164-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
-; GFX1164-TRUE16-NEXT: v_mov_b16_e32 v2.h, 0
-; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h
+; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-TRUE16-NEXT: v_lshrrev_b32_e64 v2, 16, v0
; GFX1164-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s8, v2
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-TRUE16-NEXT: v_and_or_b32 v0, v1, s9, v0
@@ -15323,9 +15322,8 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
; GFX1132-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
-; GFX1132-TRUE16-NEXT: v_mov_b16_e32 v2.h, 0
-; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h
+; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-TRUE16-NEXT: v_lshrrev_b32_e64 v2, 16, v0
; GFX1132-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s2, v2
; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-TRUE16-NEXT: v_and_or_b32 v0, v1, s8, v0
@@ -15434,9 +15432,8 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1264-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
; GFX1264-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1264-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
-; GFX1264-TRUE16-NEXT: v_mov_b16_e32 v2.h, 0
-; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1264-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h
+; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1264-TRUE16-NEXT: v_lshrrev_b32_e64 v2, 16, v0
; GFX1264-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s8, v2
; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1264-TRUE16-NEXT: v_and_or_b32 v0, v1, s9, v0
@@ -15547,9 +15544,8 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1232-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
; GFX1232-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1232-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
-; GFX1232-TRUE16-NEXT: v_mov_b16_e32 v2.h, 0
-; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1232-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h
+; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1232-TRUE16-NEXT: v_lshrrev_b32_e64 v2, 16, v0
; GFX1232-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s2, v2
; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1232-TRUE16-NEXT: v_and_or_b32 v0, v1, s8, v0
diff --git a/llvm/test/CodeGen/AMDGPU/bf16.ll b/llvm/test/CodeGen/AMDGPU/bf16.ll
index d09ec87e1a19f..bbc6340773b7d 100644
--- a/llvm/test/CodeGen/AMDGPU/bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/bf16.ll
@@ -38083,9 +38083,8 @@ define <2 x bfloat> @v_uitofp_v2i16_to_v2bf16(<2 x i16> %x) #0 {
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.h, 0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e64 v2, 16, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v0, v1
; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v1, v2
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
@@ -38132,9 +38131,8 @@ define <2 x bfloat> @v_uitofp_v2i16_to_v2bf16(<2 x i16> %x) #0 {
; GFX1250TRUE16: ; %bb.0:
; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.h, 0
; GFX1250TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX1250TRUE16-NEXT: v_lshrrev_b32_e64 v1, 16, v0
; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1250TRUE16-NEXT: v_cvt_f32_u32_e32 v0, v2
; GFX1250TRUE16-NEXT: v_cvt_f32_u32_e32 v1, v1
@@ -38283,9 +38281,8 @@ define <3 x bfloat> @v_uitofp_v3i16_to_v3bf16(<3 x i16> %x) #0 {
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.h
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e64 v3, 16, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v0, v2
; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v2, v3
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
@@ -38354,9 +38351,8 @@ define <3 x bfloat> @v_uitofp_v3i16_to_v3bf16(<3 x i16> %x) #0 {
; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1250TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX1250TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.h
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1250TRUE16-NEXT: v_lshrrev_b32_e64 v3, 16, v0
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX1250TRUE16-NEXT: v_cvt_f32_u32_e32 v0, v1
; GFX1250TRUE16-NEXT: v_cvt_f32_u32_e32 v1, v2
; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
@@ -38538,8 +38534,7 @@ define <4 x bfloat> @v_uitofp_v4i16_to_v4bf16(<4 x i16> %x) #0 {
; GFX11TRUE16-LABEL: v_uitofp_v4i16_to_v4bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.h
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e64 v3, 16, v1
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v4, v3
; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.h
@@ -38624,8 +38619,7 @@ define <4 x bfloat> @v_uitofp_v4i16_to_v4bf16(<4 x i16> %x) #0 {
; GFX1250TRUE16: ; %bb.0:
; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v2.h, 0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.h
+; GFX1250TRUE16-NEXT: v_lshrrev_b32_e64 v2, 16, v1
; GFX1250TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v0
; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX1250TRUE16-NEXT: v_cvt_f32_u32_e32 v4, v2
diff --git a/llvm/test/CodeGen/AMDGPU/bswap.ll b/llvm/test/CodeGen/AMDGPU/bswap.ll
index 9162b159dcc78..5e4bd7d240119 100644
--- a/llvm/test/CodeGen/AMDGPU/bswap.ll
+++ b/llvm/test/CodeGen/AMDGPU/bswap.ll
@@ -735,9 +735,8 @@ define i64 @v_bswap_i48(i64 %src) {
; GFX11-REAL16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-REAL16-NEXT: v_perm_b32 v2, 0, v0, 0x10203
; GFX11-REAL16-NEXT: v_perm_b32 v0, 0, v1, 0x10203
-; GFX11-REAL16-NEXT: v_mov_b16_e32 v1.h, 0
-; GFX11-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-REAL16-NEXT: v_mov_b16_e32 v1.l, v2.h
+; GFX11-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-REAL16-NEXT: v_lshrrev_b32_e64 v1, 16, v2
; GFX11-REAL16-NEXT: v_alignbit_b32 v0, v2, v0, 16
; GFX11-REAL16-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
index 45e3cee156901..d8e99eb697b5a 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
@@ -5001,10 +5001,9 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX12-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v2, 16, v0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -5155,10 +5154,9 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine
; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v2, 16, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v2
; GFX11-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
@@ -5519,10 +5517,9 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.h, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v4, 16, v0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -5670,10 +5667,9 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine
; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v4, 16, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v4
; GFX11-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
@@ -6045,9 +6041,8 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v11, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v4
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v7, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v9, v4
@@ -6291,10 +6286,9 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v11, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v7, v5
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v9, v4
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
index 1078b38f52f4c..b8c895f869963 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
@@ -4130,10 +4130,9 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX12-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v2, 16, v0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -4284,10 +4283,9 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine
; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v2, 16, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v2
; GFX11-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
@@ -4650,10 +4648,9 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.h, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v4, 16, v0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -4801,10 +4798,9 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine
; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v4, 16, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v4
; GFX11-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
@@ -5178,9 +5174,8 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v11, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v4
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v7, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v9, v4
@@ -5424,10 +5419,9 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v11, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v7, v5
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v9, v4
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
index e5a87c78d54d2..6efe5271fde53 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
@@ -4130,10 +4130,9 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX12-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v2, 16, v0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -4284,10 +4283,9 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine
; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v2, 16, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v2
; GFX11-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
@@ -4650,10 +4648,9 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.h, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v4, 16, v0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -4801,10 +4798,9 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine
; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v4, 16, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v4
; GFX11-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
@@ -5178,9 +5174,8 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v11, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v4
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v7, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v9, v4
@@ -5424,10 +5419,9 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v11, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v7, v5
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v9, v4
diff --git a/llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll b/llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll
index 19916e6d10f67..d1308410362db 100644
--- a/llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll
@@ -3242,12 +3242,11 @@ define <2 x half> @v_copysign_out_v2f16_mag_v2f64_sign_v2f16(<2 x double> %mag,
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 3, v9
; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_lt_i32_e32 vcc_lo, 5, v11
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v9, v12
; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 3, v11
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v3.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.h, 0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v12, 16, v3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, v8, v9
; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v2
@@ -4974,8 +4973,7 @@ define <3 x half> @v_copysign_out_v3f16_mag_v3f64_sign_v3f16(<3 x double> %mag,
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0x40f, v10
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_gt_i32_e32 vcc_lo, 31, v14
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v3.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, 0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v8, 16, v3
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, 0x7c00, v11, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
@@ -6116,11 +6114,10 @@ define <4 x half> @v_copysign_out_v4f16_mag_v4f64_sign_v4f16(<4 x double> %mag,
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 8, v3
; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v3, 20, 11
; GFX11-TRUE16-NEXT: v_bfe_u32 v20, v1, 20, 11
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v7.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v15, 16, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v6, 0xffe, v11, v6
; GFX11-TRUE16-NEXT: v_med3_i32 v11, v12, 0, 13
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x1000, v6
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, v11, v12
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll
index 5dda7d30e29b7..75bae5b10faf6 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll
@@ -11985,9 +11985,8 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -12131,9 +12130,8 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -12435,9 +12433,8 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -12588,9 +12585,8 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -12900,9 +12896,8 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -13054,9 +13049,8 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_gr
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -13363,10 +13357,9 @@ define void @flat_agent_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -13511,10 +13504,9 @@ define void @flat_agent_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_gr
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -13814,10 +13806,9 @@ define void @flat_agent_atomic_fadd_noret_bf16__offset12b_neg__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -13963,10 +13954,9 @@ define void @flat_agent_atomic_fadd_noret_bf16__offset12b_neg__amdgpu_no_fine_gr
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -14256,10 +14246,9 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_no
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
@@ -14375,10 +14364,9 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_no
; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
@@ -14626,9 +14614,8 @@ define void @flat_agent_atomic_fadd_noret_bf16__offset12b__align4_pos__amdgpu_no
; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v2
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -14740,9 +14727,8 @@ define void @flat_agent_atomic_fadd_noret_bf16__offset12b__align4_pos__amdgpu_no
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v2
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2046 glc
@@ -14992,10 +14978,9 @@ define void @flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
@@ -15133,10 +15118,9 @@ define void @flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
@@ -15431,9 +15415,8 @@ define bfloat @flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_g
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -15586,9 +15569,8 @@ define bfloat @flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_g
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -15898,10 +15880,9 @@ define void @flat_system_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_g
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -16048,10 +16029,9 @@ define void @flat_system_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_g
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll
index 32ecfc2b21a74..8d0c46d7cb165 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll
@@ -9843,9 +9843,8 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -9989,9 +9988,8 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -10294,9 +10292,8 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -10447,9 +10444,8 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -10760,9 +10756,8 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -10914,9 +10909,8 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_gr
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -11221,10 +11215,9 @@ define void @flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
@@ -11362,10 +11355,9 @@ define void @flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
@@ -11658,10 +11650,9 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -11806,10 +11797,9 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_gr
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -12110,10 +12100,9 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b_neg__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -12259,10 +12248,9 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b_neg__amdgpu_no_fine_gr
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -12553,10 +12541,9 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
@@ -12672,10 +12659,9 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no
; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
@@ -12924,9 +12910,8 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_no
; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v2
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -13038,9 +13023,8 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_no
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v2
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2046 glc
@@ -13297,9 +13281,8 @@ define bfloat @flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_g
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -13452,9 +13435,8 @@ define bfloat @flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_g
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -13765,10 +13747,9 @@ define void @flat_system_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_g
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -13915,10 +13896,9 @@ define void @flat_system_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_g
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll
index 8b546247186a2..5d20f7b71480e 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll
@@ -9843,9 +9843,8 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -9989,9 +9988,8 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -10294,9 +10292,8 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -10447,9 +10444,8 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -10760,9 +10756,8 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -10914,9 +10909,8 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_gr
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -11221,10 +11215,9 @@ define void @flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
@@ -11362,10 +11355,9 @@ define void @flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
@@ -11658,10 +11650,9 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -11806,10 +11797,9 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_gr
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -12110,10 +12100,9 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b_neg__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -12259,10 +12248,9 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b_neg__amdgpu_no_fine_gr
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -12553,10 +12541,9 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_no
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
@@ -12672,10 +12659,9 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_no
; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
@@ -12924,9 +12910,8 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b__align4_pos__amdgpu_no
; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v2
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -13038,9 +13023,8 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b__align4_pos__amdgpu_no
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v2
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2046 glc
@@ -13297,9 +13281,8 @@ define bfloat @flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_g
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -13452,9 +13435,8 @@ define bfloat @flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_g
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -13765,10 +13747,9 @@ define void @flat_system_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_g
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -13915,10 +13896,9 @@ define void @flat_system_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_g
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll
index 0f3ae45da7044..db4b18b40b7e8 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll
@@ -9416,9 +9416,8 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -9562,9 +9561,8 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -9866,9 +9864,8 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -10019,9 +10016,8 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -10331,9 +10327,8 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -10485,9 +10480,8 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -10791,10 +10785,9 @@ define void @flat_agent_atomic_fsub_noret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
@@ -10932,10 +10925,9 @@ define void @flat_agent_atomic_fsub_noret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
@@ -11227,10 +11219,9 @@ define void @flat_agent_atomic_fsub_noret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -11375,10 +11366,9 @@ define void @flat_agent_atomic_fsub_noret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -11678,10 +11668,9 @@ define void @flat_agent_atomic_fsub_noret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -11827,10 +11816,9 @@ define void @flat_agent_atomic_fsub_noret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -12120,10 +12108,9 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_pos__align4(ptr %ptr,
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
@@ -12239,10 +12226,9 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_pos__align4(ptr %ptr,
; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
@@ -12490,9 +12476,8 @@ define void @flat_agent_atomic_fsub_noret_bf16__offset12b__align4_pos(ptr %ptr,
; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v2
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -12604,9 +12589,8 @@ define void @flat_agent_atomic_fsub_noret_bf16__offset12b__align4_pos(ptr %ptr,
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v2
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2046 glc
@@ -12862,9 +12846,8 @@ define bfloat @flat_system_atomic_fsub_ret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -13017,9 +13000,8 @@ define bfloat @flat_system_atomic_fsub_ret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -13329,10 +13311,9 @@ define void @flat_system_atomic_fsub_noret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -13479,10 +13460,9 @@ define void @flat_system_atomic_fsub_noret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
diff --git a/llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll b/llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll
index 884cf3b9200b5..2c80ca6f6aa3f 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll
@@ -2916,9 +2916,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zero_hi(ptr inreg %sbase,
; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-TRUE16-NEXT: flat_load_u16 v1, v0, s[2:3]
; GFX1250-SDAG-TRUE16-NEXT: s_wait_xcnt 0x0
-; GFX1250-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0
+; GFX1250-SDAG-TRUE16-NEXT: v_lshlrev_b32_e64 v0, 16, v1
; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
;
; GFX1250-NOECC-SDAG-TRUE16-LABEL: flat_load_saddr_i16_d16hi_zero_hi:
@@ -2978,9 +2977,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zero_hi_immneg128(ptr inr
; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-TRUE16-NEXT: flat_load_u16 v1, v0, s[2:3] offset:-128
; GFX1250-SDAG-TRUE16-NEXT: s_wait_xcnt 0x0
-; GFX1250-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0
+; GFX1250-SDAG-TRUE16-NEXT: v_lshlrev_b32_e64 v0, 16, v1
; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
;
; GFX1250-NOECC-SDAG-TRUE16-LABEL: flat_load_saddr_i16_d16hi_zero_hi_immneg128:
diff --git a/llvm/test/CodeGen/AMDGPU/function-args.ll b/llvm/test/CodeGen/AMDGPU/function-args.ll
index 5515f798c1b36..c910a1be5d302 100644
--- a/llvm/test/CodeGen/AMDGPU/function-args.ll
+++ b/llvm/test/CodeGen/AMDGPU/function-args.ll
@@ -1800,11 +1800,10 @@ define void @void_func_v2i24(<2 x i24> %arg0) #0 {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v1, 16, v0
; GFX11-TRUE16-NEXT: s_clause 0x1
; GFX11-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0
; GFX11-TRUE16-NEXT: buffer_store_b8 v1, off, s[0:3], 0
diff --git a/llvm/test/CodeGen/AMDGPU/function-returns.ll b/llvm/test/CodeGen/AMDGPU/function-returns.ll
index fb8376ba0d6c6..a9bf830c188fb 100644
--- a/llvm/test/CodeGen/AMDGPU/function-returns.ll
+++ b/llvm/test/CodeGen/AMDGPU/function-returns.ll
@@ -2067,8 +2067,7 @@ define void @void_func_sret_max_known_zero_bits(ptr addrspace(5) sret(i8) %arg0)
; GFX11-TRUE16-LABEL: void_func_sret_max_known_zero_bits:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v1, 16, v0
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 17, v0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, 0
; GFX11-TRUE16-NEXT: ds_store_b32 v0, v1
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll
index 72e06145dc850..b0cbe66e6b855 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll
@@ -13796,9 +13796,8 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory(
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -13942,9 +13941,8 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory(
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -14386,9 +14384,8 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -14539,9 +14536,8 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -14995,9 +14991,8 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -15149,9 +15144,8 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -15595,10 +15589,9 @@ define void @global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory(
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
@@ -15736,10 +15729,9 @@ define void @global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory(
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
@@ -16170,10 +16162,9 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -16318,10 +16309,9 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -16764,10 +16754,9 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -16913,10 +16902,9 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -17326,10 +17314,9 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
@@ -17445,10 +17432,9 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_
; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
@@ -17805,9 +17791,8 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b__align4_pos__amdgpu_
; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v2
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -17919,9 +17904,8 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b__align4_pos__amdgpu_
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v2
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2046 glc
@@ -18306,9 +18290,8 @@ define bfloat @global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -18461,9 +18444,8 @@ define bfloat @global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -18915,10 +18897,9 @@ define void @global_system_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -19065,10 +19046,9 @@ define void @global_system_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll
index e34d8d2fdc5c4..9fda3e4b83e83 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll
@@ -8734,9 +8734,8 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory(
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -8880,9 +8879,8 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory(
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -9236,9 +9234,8 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -9389,9 +9386,8 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -9755,9 +9751,8 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -9909,9 +9904,8 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -10269,10 +10263,9 @@ define void @global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory(
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
@@ -10410,10 +10403,9 @@ define void @global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory(
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
@@ -10757,10 +10749,9 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -10905,10 +10896,9 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -11261,10 +11251,9 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -11410,10 +11399,9 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -11756,10 +11744,9 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
@@ -11875,10 +11862,9 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_
; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
@@ -12169,9 +12155,8 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_
; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v2
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -12283,9 +12268,8 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v2
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2046 glc
@@ -12581,9 +12565,8 @@ define bfloat @global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -12736,9 +12719,8 @@ define bfloat @global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -13102,10 +13084,9 @@ define void @global_system_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -13252,10 +13233,9 @@ define void @global_system_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll
index 31db1f8805931..7cadd1d4b98c7 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll
@@ -8734,9 +8734,8 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory(
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -8880,9 +8879,8 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory(
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -9236,9 +9234,8 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -9389,9 +9386,8 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -9755,9 +9751,8 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -9909,9 +9904,8 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -10269,10 +10263,9 @@ define void @global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory(
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
@@ -10410,10 +10403,9 @@ define void @global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory(
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
@@ -10757,10 +10749,9 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -10905,10 +10896,9 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -11261,10 +11251,9 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -11410,10 +11399,9 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -11756,10 +11744,9 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
@@ -11875,10 +11862,9 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_
; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
@@ -12169,9 +12155,8 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b__align4_pos__amdgpu_
; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v2
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -12283,9 +12268,8 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b__align4_pos__amdgpu_
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v2
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2046 glc
@@ -12581,9 +12565,8 @@ define bfloat @global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -12736,9 +12719,8 @@ define bfloat @global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -13102,10 +13084,9 @@ define void @global_system_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -13252,10 +13233,9 @@ define void @global_system_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll
index 27238cf96e18b..1a09e052dc3f5 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll
@@ -9256,9 +9256,8 @@ define bfloat @global_agent_atomic_fsub_ret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -9402,9 +9401,8 @@ define bfloat @global_agent_atomic_fsub_ret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -9756,9 +9754,8 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_pos(ptr addrspace(1)
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -9909,9 +9906,8 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_pos(ptr addrspace(1)
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -10273,9 +10269,8 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_neg(ptr addrspace(1)
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -10427,9 +10422,8 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_neg(ptr addrspace(1)
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -10785,10 +10779,9 @@ define void @global_agent_atomic_fsub_noret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
@@ -10926,10 +10919,9 @@ define void @global_agent_atomic_fsub_noret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
@@ -11271,10 +11263,9 @@ define void @global_agent_atomic_fsub_noret_bf16__offset12b_pos(ptr addrspace(1)
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -11419,10 +11410,9 @@ define void @global_agent_atomic_fsub_noret_bf16__offset12b_pos(ptr addrspace(1)
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -11773,10 +11763,9 @@ define void @global_agent_atomic_fsub_noret_bf16__offset12b_neg(ptr addrspace(1)
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -11922,10 +11911,9 @@ define void @global_agent_atomic_fsub_noret_bf16__offset12b_neg(ptr addrspace(1)
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -12266,10 +12254,9 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_pos__align4(ptr addr
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
@@ -12385,10 +12372,9 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_pos__align4(ptr addr
; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
@@ -12677,9 +12663,8 @@ define void @global_agent_atomic_fsub_noret_bf16__offset12b__align4_pos(ptr addr
; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v2
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -12791,9 +12776,8 @@ define void @global_agent_atomic_fsub_noret_bf16__offset12b__align4_pos(ptr addr
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v2
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2046 glc
@@ -13087,9 +13071,8 @@ define bfloat @global_system_atomic_fsub_ret_bf16__offset12b_pos(ptr addrspace(1
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -13242,9 +13225,8 @@ define bfloat @global_system_atomic_fsub_ret_bf16__offset12b_pos(ptr addrspace(1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -13606,10 +13588,9 @@ define void @global_system_atomic_fsub_noret_bf16__offset12b_pos(ptr addrspace(1
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -13756,10 +13737,9 @@ define void @global_system_atomic_fsub_noret_bf16__offset12b_pos(ptr addrspace(1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.gather4.a16.dim.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.gather4.a16.dim.ll
index 87a41bf3e3cc6..a758bdf23d80a 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.gather4.a16.dim.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.gather4.a16.dim.ll
@@ -101,8 +101,7 @@ define amdgpu_ps <4 x float> @gather4_cube(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX11-TRUE16-NEXT: s_mov_b32 s12, exec_lo
; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v0, v1, 0x5040100
; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s12
; GFX11-TRUE16-NEXT: image_gather4 v[0:3], v[2:3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_CUBE a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -169,8 +168,7 @@ define amdgpu_ps <4 x float> @gather4_2darray(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX11-TRUE16-NEXT: s_mov_b32 s12, exec_lo
; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v0, v1, 0x5040100
; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s12
; GFX11-TRUE16-NEXT: image_gather4 v[0:3], v[2:3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -303,8 +301,7 @@ define amdgpu_ps <4 x float> @gather4_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX11-TRUE16-NEXT: s_mov_b32 s12, exec_lo
; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v0, v1, 0x5040100
; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s12
; GFX11-TRUE16-NEXT: image_gather4_cl v[0:3], v[2:3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -573,8 +570,7 @@ define amdgpu_ps <4 x float> @gather4_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX11-TRUE16-NEXT: s_mov_b32 s12, exec_lo
; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v1, v2, 0x5040100
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s12
; GFX11-TRUE16-NEXT: image_gather4_b_cl v[0:3], v[2:4], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
@@ -703,8 +699,7 @@ define amdgpu_ps <4 x float> @gather4_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX11-TRUE16-LABEL: gather4_l_2d:
; GFX11-TRUE16: ; %bb.0: ; %main_body
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v0, v1, 0x5040100
; GFX11-TRUE16-NEXT: image_gather4_l v[0:3], v[2:3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
@@ -902,5 +897,3 @@ declare <4 x float> @llvm.amdgcn.image.gather4.c.lz.2d.v4f32.f32(i32, float, hal
attributes #0 = { nounwind }
attributes #1 = { nounwind readonly }
attributes #2 = { nounwind readnone }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GFX12: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.msaa.load.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.msaa.load.ll
index 2f288d6fb95a6..071aa783116a2 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.msaa.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.msaa.load.ll
@@ -305,8 +305,7 @@ define amdgpu_ps <4 x float> @load_2dmsaa_a16(<8 x i32> inreg %rsrc, i16 %s, i16
; GFX11-TRUE16-LABEL: load_2dmsaa_a16:
; GFX11-TRUE16: ; %bb.0: ; %main_body
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v0, v1, 0x5040100
; GFX11-TRUE16-NEXT: image_msaa_load v[0:3], v[2:3], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
@@ -339,10 +338,8 @@ main_body:
define amdgpu_ps <4 x float> @load_2darraymsaa_a16(<8 x i32> inreg %rsrc, i16 %s, i16 %t, i16 %slice, i16 %fragid) {
; GFX11-TRUE16-LABEL: load_2darraymsaa_a16:
; GFX11-TRUE16: ; %bb.0: ; %main_body
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v1.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v2, v3, 0x5040100
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v0, v1, 0x5040100
; GFX11-TRUE16-NEXT: image_msaa_load v[0:3], v[3:4], s[0:7] dmask:0x4 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.a16.dim.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.a16.dim.ll
index e3c84dcfcc027..d4ea966101893 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.a16.dim.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.a16.dim.ll
@@ -142,8 +142,7 @@ define amdgpu_ps <4 x float> @sample_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg %
; GFX11-TRUE16-NEXT: s_mov_b32 s12, exec_lo
; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v0, v1, 0x5040100
; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s12
; GFX11-TRUE16-NEXT: image_sample v[0:3], v[2:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_3D a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -210,8 +209,7 @@ define amdgpu_ps <4 x float> @sample_cube(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX11-TRUE16-NEXT: s_mov_b32 s12, exec_lo
; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v0, v1, 0x5040100
; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s12
; GFX11-TRUE16-NEXT: image_sample v[0:3], v[2:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_CUBE a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -344,8 +342,7 @@ define amdgpu_ps <4 x float> @sample_2darray(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX11-TRUE16-NEXT: s_mov_b32 s12, exec_lo
; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v0, v1, 0x5040100
; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s12
; GFX11-TRUE16-NEXT: image_sample v[0:3], v[2:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D_ARRAY a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -585,8 +582,7 @@ define amdgpu_ps <4 x float> @sample_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX11-TRUE16-NEXT: s_mov_b32 s12, exec_lo
; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v0, v1, 0x5040100
; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s12
; GFX11-TRUE16-NEXT: image_sample_cl v[0:3], v[2:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -1069,8 +1065,7 @@ define amdgpu_ps <4 x float> @sample_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX11-TRUE16-NEXT: s_mov_b32 s12, exec_lo
; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v1, v2, 0x5040100
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s12
; GFX11-TRUE16-NEXT: image_sample_b_cl v[0:3], v[2:4], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
@@ -1297,12 +1292,9 @@ define amdgpu_ps <4 x float> @sample_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX11-TRUE16-LABEL: sample_d_2d:
; GFX11-TRUE16: ; %bb.0: ; %main_body
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v1.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v4, v5, 0x5040100
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v2, v3, 0x5040100
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v0, v1, 0x5040100
; GFX11-TRUE16-NEXT: image_sample_d_g16 v[0:3], v[4:6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
@@ -1367,8 +1359,7 @@ define amdgpu_ps <4 x float> @sample_d_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX11-TRUE16-LABEL: sample_d_3d:
; GFX11-TRUE16: ; %bb.0: ; %main_body
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v7.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v6, v7, 0x5040100
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v4.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
; GFX11-TRUE16-NEXT: image_sample_d_g16 v[0:3], [v0, v2, v3, v5, v[8:9]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_3D a16
@@ -1387,8 +1378,7 @@ define amdgpu_ps <4 x float> @sample_d_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX12PLUS-TRUE16-LABEL: sample_d_3d:
; GFX12PLUS-TRUE16: ; %bb.0: ; %main_body
; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v9.l, v8.l
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v8.l, v6.l
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v8.h, v7.l
+; GFX12PLUS-TRUE16-NEXT: v_perm_b32 v8, v6, v7, 0x5040100
; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.l
; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v3.h, v4.l
; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
@@ -1573,12 +1563,9 @@ define amdgpu_ps <4 x float> @sample_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX11-TRUE16-LABEL: sample_d_cl_2d:
; GFX11-TRUE16: ; %bb.0: ; %main_body
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v1.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v4, v5, 0x5040100
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v2, v3, 0x5040100
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v0, v1, 0x5040100
; GFX11-TRUE16-NEXT: image_sample_d_cl_g16 v[0:3], v[4:7], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
@@ -1705,8 +1692,7 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX12PLUS-TRUE16-LABEL: sample_c_d_cl_2d:
; GFX12PLUS-TRUE16: ; %bb.0: ; %main_body
; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v8.l, v7.l
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.l
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v7.h, v6.l
+; GFX12PLUS-TRUE16-NEXT: v_perm_b32 v7, v5, v6, 0x5040100
; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v3.h, v4.l
; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
; GFX12PLUS-TRUE16-NEXT: image_sample_c_d_cl_g16 v[0:3], [v0, v1, v3, v[7:8]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
@@ -1793,8 +1779,7 @@ define amdgpu_ps <4 x float> @sample_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX11-TRUE16-LABEL: sample_l_2d:
; GFX11-TRUE16: ; %bb.0: ; %main_body
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v0, v1, 0x5040100
; GFX11-TRUE16-NEXT: image_sample_l v[0:3], v[2:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
@@ -2105,8 +2090,7 @@ define amdgpu_ps float @sample_c_d_o_2darray_V1(<8 x i32> inreg %rsrc, <4 x i32>
; GFX11-TRUE16-LABEL: sample_c_d_o_2darray_V1:
; GFX11-TRUE16: ; %bb.0: ; %main_body
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v7.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v6, v7, 0x5040100
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v5.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
; GFX11-TRUE16-NEXT: image_sample_c_d_o_g16 v0, [v0, v1, v2, v4, v[8:9]], s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_2D_ARRAY a16
@@ -2125,10 +2109,8 @@ define amdgpu_ps float @sample_c_d_o_2darray_V1(<8 x i32> inreg %rsrc, <4 x i32>
; GFX12PLUS-TRUE16-LABEL: sample_c_d_o_2darray_V1:
; GFX12PLUS-TRUE16: ; %bb.0: ; %main_body
; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v9.l, v8.l
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v8.l, v6.l
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v8.h, v7.l
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v7.h, v5.l
+; GFX12PLUS-TRUE16-NEXT: v_perm_b32 v8, v6, v7, 0x5040100
+; GFX12PLUS-TRUE16-NEXT: v_perm_b32 v7, v4, v5, 0x5040100
; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
; GFX12PLUS-TRUE16-NEXT: image_sample_c_d_o_g16 v0, [v0, v1, v2, v[7:9]], s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_2D_ARRAY a16
; GFX12PLUS-TRUE16-NEXT: s_wait_samplecnt 0x0
@@ -2176,8 +2158,7 @@ define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4
; GFX11-TRUE16-LABEL: sample_c_d_o_2darray_V2:
; GFX11-TRUE16: ; %bb.0: ; %main_body
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v7.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v6, v7, 0x5040100
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v5.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
; GFX11-TRUE16-NEXT: image_sample_c_d_o_g16 v[0:1], [v0, v1, v2, v4, v[8:9]], s[0:7], s[8:11] dmask:0x6 dim:SQ_RSRC_IMG_2D_ARRAY a16
@@ -2196,10 +2177,8 @@ define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4
; GFX12PLUS-TRUE16-LABEL: sample_c_d_o_2darray_V2:
; GFX12PLUS-TRUE16: ; %bb.0: ; %main_body
; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v9.l, v8.l
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v8.l, v6.l
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v8.h, v7.l
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v7.h, v5.l
+; GFX12PLUS-TRUE16-NEXT: v_perm_b32 v8, v6, v7, 0x5040100
+; GFX12PLUS-TRUE16-NEXT: v_perm_b32 v7, v4, v5, 0x5040100
; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
; GFX12PLUS-TRUE16-NEXT: image_sample_c_d_o_g16 v[0:1], [v0, v1, v2, v[7:9]], s[0:7], s[8:11] dmask:0x6 dim:SQ_RSRC_IMG_2D_ARRAY a16
; GFX12PLUS-TRUE16-NEXT: s_wait_samplecnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.encode.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.encode.ll
index 8bd42ece040d2..5da481091ffcf 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.encode.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.encode.ll
@@ -512,10 +512,9 @@ define amdgpu_ps float @sample_c_d_o_2darray_V1(<8 x i32> inreg %rsrc, <4 x i32>
; GFX12-TRUE16-LABEL: sample_c_d_o_2darray_V1:
; GFX12-TRUE16: ; %bb.0: ; %main_body
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v9, v5 ; encoding: [0x05,0x03,0x12,0x7e]
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.l ; encoding: [0x04,0x39,0x0a,0x7e]
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; encoding: [0x01,0x00,0x87,0xbf]
+; GFX12-TRUE16-NEXT: v_perm_b32 v5, v4, v9, 0x5040100 ; encoding: [0x05,0x00,0x44,0xd6,0x04,0x13,0xfe,0x03,0x00,0x01,0x04,0x05]
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l ; encoding: [0x03,0x39,0x04,0x7f]
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) ; encoding: [0x03,0x00,0x87,0xbf]
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v9.l ; encoding: [0x09,0x39,0x0a,0x7f]
; GFX12-TRUE16-NEXT: image_sample_c_d_o_g16 v0, [v0, v1, v2, v[5:8]], s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_2D_ARRAY ; encoding: [0x05,0x00,0x0f,0xe5,0x00,0x00,0x00,0x04,0x00,0x01,0x02,0x05]
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; encoding: [0x00,0x00,0xc2,0xbf]
; GFX12-TRUE16-NEXT: ; return to shader part epilog
@@ -531,10 +530,9 @@ define amdgpu_ps float @sample_c_d_o_2darray_V1(<8 x i32> inreg %rsrc, <4 x i32>
; GFX13-TRUE16-LABEL: sample_c_d_o_2darray_V1:
; GFX13-TRUE16: ; %bb.0: ; %main_body
; GFX13-TRUE16-NEXT: v_mov_b32_e32 v9, v5 ; encoding: [0x05,0x03,0x12,0x7e]
-; GFX13-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.l ; encoding: [0x04,0x39,0x0a,0x7e]
+; GFX13-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; encoding: [0x01,0x00,0xae,0xbf]
+; GFX13-TRUE16-NEXT: v_perm_b32 v5, v4, v9, 0x5040100 ; encoding: [0x05,0x00,0x44,0xd7,0x04,0x13,0xfe,0x03,0x00,0x01,0x04,0x05]
; GFX13-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l ; encoding: [0x03,0x39,0x04,0x7f]
-; GFX13-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) ; encoding: [0x03,0x00,0xae,0xbf]
-; GFX13-TRUE16-NEXT: v_mov_b16_e32 v5.h, v9.l ; encoding: [0x09,0x39,0x0a,0x7f]
; GFX13-TRUE16-NEXT: image_sample_c_d_o_g16 v0, [v0, v1, v2, v[5:8]], s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_2D_ARRAY ; encoding: [0x05,0x80,0x2e,0xe5,0x00,0x00,0x00,0x04,0x00,0x01,0x02,0x05]
; GFX13-TRUE16-NEXT: s_wait_samplecnt 0x0 ; encoding: [0x00,0x00,0xc2,0xbf]
; GFX13-TRUE16-NEXT: ; return to shader part epilog
@@ -583,10 +581,9 @@ define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4
; GFX12-TRUE16-LABEL: sample_c_d_o_2darray_V2:
; GFX12-TRUE16: ; %bb.0: ; %main_body
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v9, v5 ; encoding: [0x05,0x03,0x12,0x7e]
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.l ; encoding: [0x04,0x39,0x0a,0x7e]
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; encoding: [0x01,0x00,0x87,0xbf]
+; GFX12-TRUE16-NEXT: v_perm_b32 v5, v4, v9, 0x5040100 ; encoding: [0x05,0x00,0x44,0xd6,0x04,0x13,0xfe,0x03,0x00,0x01,0x04,0x05]
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l ; encoding: [0x03,0x39,0x04,0x7f]
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) ; encoding: [0x03,0x00,0x87,0xbf]
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v9.l ; encoding: [0x09,0x39,0x0a,0x7f]
; GFX12-TRUE16-NEXT: image_sample_c_d_o_g16 v[0:1], [v0, v1, v2, v[5:8]], s[0:7], s[8:11] dmask:0x6 dim:SQ_RSRC_IMG_2D_ARRAY ; encoding: [0x05,0x00,0x8f,0xe5,0x00,0x00,0x00,0x04,0x00,0x01,0x02,0x05]
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; encoding: [0x00,0x00,0xc2,0xbf]
; GFX12-TRUE16-NEXT: ; return to shader part epilog
@@ -602,10 +599,9 @@ define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4
; GFX13-TRUE16-LABEL: sample_c_d_o_2darray_V2:
; GFX13-TRUE16: ; %bb.0: ; %main_body
; GFX13-TRUE16-NEXT: v_mov_b32_e32 v9, v5 ; encoding: [0x05,0x03,0x12,0x7e]
-; GFX13-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.l ; encoding: [0x04,0x39,0x0a,0x7e]
+; GFX13-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; encoding: [0x01,0x00,0xae,0xbf]
+; GFX13-TRUE16-NEXT: v_perm_b32 v5, v4, v9, 0x5040100 ; encoding: [0x05,0x00,0x44,0xd7,0x04,0x13,0xfe,0x03,0x00,0x01,0x04,0x05]
; GFX13-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l ; encoding: [0x03,0x39,0x04,0x7f]
-; GFX13-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) ; encoding: [0x03,0x00,0xae,0xbf]
-; GFX13-TRUE16-NEXT: v_mov_b16_e32 v5.h, v9.l ; encoding: [0x09,0x39,0x0a,0x7f]
; GFX13-TRUE16-NEXT: image_sample_c_d_o_g16 v[0:1], [v0, v1, v2, v[5:8]], s[0:7], s[8:11] dmask:0x6 dim:SQ_RSRC_IMG_2D_ARRAY ; encoding: [0x05,0x80,0xae,0xe5,0x00,0x00,0x00,0x04,0x00,0x01,0x02,0x05]
; GFX13-TRUE16-NEXT: s_wait_samplecnt 0x0 ; encoding: [0x00,0x00,0xc2,0xbf]
; GFX13-TRUE16-NEXT: ; return to shader part epilog
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.ll
index 0f547e619f260..b73592cc3ad32 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.ll
@@ -360,9 +360,8 @@ define amdgpu_ps float @sample_c_d_o_2darray_V1(<8 x i32> inreg %rsrc, <4 x i32>
; GFX12PLUS-TRUE16-LABEL: sample_c_d_o_2darray_V1:
; GFX12PLUS-TRUE16: ; %bb.0: ; %main_body
; GFX12PLUS-TRUE16-NEXT: v_mov_b32_e32 v9, v5
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.l
+; GFX12PLUS-TRUE16-NEXT: v_perm_b32 v5, v4, v9, 0x5040100
; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v5.h, v9.l
; GFX12PLUS-TRUE16-NEXT: image_sample_c_d_o_g16 v0, [v0, v1, v2, v[5:8]], s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_2D_ARRAY
; GFX12PLUS-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12PLUS-TRUE16-NEXT: ; return to shader part epilog
@@ -411,9 +410,8 @@ define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4
; GFX12PLUS-TRUE16-LABEL: sample_c_d_o_2darray_V2:
; GFX12PLUS-TRUE16: ; %bb.0: ; %main_body
; GFX12PLUS-TRUE16-NEXT: v_mov_b32_e32 v9, v5
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.l
+; GFX12PLUS-TRUE16-NEXT: v_perm_b32 v5, v4, v9, 0x5040100
; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v5.h, v9.l
; GFX12PLUS-TRUE16-NEXT: image_sample_c_d_o_g16 v[0:1], [v0, v1, v2, v[5:8]], s[0:7], s[8:11] dmask:0x6 dim:SQ_RSRC_IMG_2D_ARRAY
; GFX12PLUS-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12PLUS-TRUE16-NEXT: ; return to shader part epilog
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.v3f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.v3f16.ll
index 6cdee67f6382c..7e8b2e12c3353 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.v3f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.v3f16.ll
@@ -110,9 +110,8 @@ define amdgpu_gs void @main(<4 x i32> %arg, i32 %arg1) {
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB0_1
; GFX11-TRUE16-NEXT: ; %bb.2:
; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v5.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v0, 16, v5
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_and_b32 v1, 0xffff, v6
; GFX11-TRUE16-NEXT: ds_store_2addr_b32 v2, v0, v1 offset0:7 offset1:8
;
@@ -163,9 +162,8 @@ define amdgpu_gs void @main(<4 x i32> %arg, i32 %arg1) {
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB0_1
; GFX12-TRUE16-NEXT: ; %bb.2:
; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v5.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v0, 16, v5
; GFX12-TRUE16-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_and_b32 v1, 0xffff, v6
; GFX12-TRUE16-NEXT: ds_store_2addr_b32 v2, v0, v1 offset0:7 offset1:8
;
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.v3f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.v3f16.ll
index 7e6ba8bde9fed..5270e67304e8b 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.v3f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.v3f16.ll
@@ -107,9 +107,8 @@ define amdgpu_gs void @main(ptr addrspace(8) %arg, i32 %arg1) {
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB0_1
; GFX11-TRUE16-NEXT: ; %bb.2:
; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v5.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v0, 16, v5
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_and_b32 v1, 0xffff, v6
; GFX11-TRUE16-NEXT: ds_store_2addr_b32 v2, v0, v1 offset0:7 offset1:8
;
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.waitcnt.out.order.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.waitcnt.out.order.ll
index e17e1c621da2f..3a3b6340b8cf5 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.waitcnt.out.order.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.waitcnt.out.order.ll
@@ -86,8 +86,7 @@ define amdgpu_ps <3 x float> @sample_load(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX11-TRUE16-LABEL: sample_load:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v0, v1, 0x5040100
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, 0
; GFX11-TRUE16-NEXT: image_msaa_load v[0:3], v[2:3], s[12:19] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm a16
; GFX11-TRUE16-NEXT: image_sample_lz v2, [v4, v4], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
@@ -106,8 +105,7 @@ define amdgpu_ps <3 x float> @sample_load(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX1150-TRUE16-LABEL: sample_load:
; GFX1150-TRUE16: ; %bb.0:
; GFX1150-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX1150-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX1150-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX1150-TRUE16-NEXT: v_perm_b32 v2, v0, v1, 0x5040100
; GFX1150-TRUE16-NEXT: v_mov_b32_e32 v4, 0
; GFX1150-TRUE16-NEXT: image_msaa_load v[0:3], v[2:3], s[12:19] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm a16
; GFX1150-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -160,8 +158,7 @@ define amdgpu_ps <3 x float> @load_sample(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX11-TRUE16-LABEL: load_sample:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v0, v1, 0x5040100
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, 0
; GFX11-TRUE16-NEXT: image_msaa_load v[0:3], v[2:3], s[12:19] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm a16
; GFX11-TRUE16-NEXT: image_sample_lz v2, [v4, v4], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
@@ -180,8 +177,7 @@ define amdgpu_ps <3 x float> @load_sample(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX1150-TRUE16-LABEL: load_sample:
; GFX1150-TRUE16: ; %bb.0:
; GFX1150-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX1150-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX1150-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX1150-TRUE16-NEXT: v_perm_b32 v2, v0, v1, 0x5040100
; GFX1150-TRUE16-NEXT: v_mov_b32_e32 v4, 0
; GFX1150-TRUE16-NEXT: image_msaa_load v[0:3], v[2:3], s[12:19] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm a16
; GFX1150-TRUE16-NEXT: s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll
index 754eadf60a309..35dfb69d7acea 100644
--- a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll
@@ -3386,9 +3386,8 @@ define bfloat @local_atomic_fadd_ret_bf16(ptr addrspace(3) %ptr) nounwind {
; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v3
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v0, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v3
@@ -3523,9 +3522,8 @@ define bfloat @local_atomic_fadd_ret_bf16(ptr addrspace(3) %ptr) nounwind {
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v3
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v0, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v3
@@ -3838,9 +3836,8 @@ define bfloat @local_atomic_fadd_ret_bf16__offset(ptr addrspace(3) %ptr) nounwin
; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v3
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v1, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v3
@@ -3980,9 +3977,8 @@ define bfloat @local_atomic_fadd_ret_bf16__offset(ptr addrspace(3) %ptr) nounwin
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v3
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v1, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v3
@@ -4301,9 +4297,8 @@ define void @local_atomic_fadd_noret_bf16(ptr addrspace(3) %ptr) nounwind {
; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v4
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v0, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v4, v2, v3, v4
@@ -4435,9 +4430,8 @@ define void @local_atomic_fadd_noret_bf16(ptr addrspace(3) %ptr) nounwind {
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v4
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v0, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v4, v2, v3, v4
@@ -4740,10 +4734,9 @@ define void @local_atomic_fadd_noret_bf16__offset(ptr addrspace(3) %ptr) nounwin
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v4
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v1, v5
; GFX12-TRUE16-NEXT: v_and_or_b32 v4, v3, v2, v4
@@ -4877,10 +4870,9 @@ define void @local_atomic_fadd_noret_bf16__offset(ptr addrspace(3) %ptr) nounwin
; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v1, v5
; GFX11-TRUE16-NEXT: v_and_or_b32 v4, v3, v2, v4
@@ -5183,10 +5175,9 @@ define bfloat @local_atomic_fadd_ret_bf16__offset__align4(ptr addrspace(3) %ptr)
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX12-TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v3, 16, v1
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v1, 0xffff0000, v2, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
@@ -5296,10 +5287,9 @@ define bfloat @local_atomic_fadd_ret_bf16__offset__align4(ptr addrspace(3) %ptr)
; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v3, 16, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v1, 0xffff0000, v2, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
@@ -5558,9 +5548,8 @@ define void @local_atomic_fadd_noret_bf16__offset__align4(ptr addrspace(3) %ptr)
; GFX12-TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v3, 16, v2
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v1, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v1 offset:65534
@@ -5666,9 +5655,8 @@ define void @local_atomic_fadd_noret_bf16__offset__align4(ptr addrspace(3) %ptr)
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v3, 16, v2
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v1, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v1 offset:65534
diff --git a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmax.ll
index 1513ddefdb66f..f0b501588da28 100644
--- a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmax.ll
@@ -2978,9 +2978,8 @@ define bfloat @local_atomic_fmax_ret_bf16(ptr addrspace(3) %ptr) nounwind {
; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v3
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v0, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v3
@@ -3115,9 +3114,8 @@ define bfloat @local_atomic_fmax_ret_bf16(ptr addrspace(3) %ptr) nounwind {
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v3
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v0, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v3
@@ -3432,9 +3430,8 @@ define bfloat @local_atomic_fmax_ret_bf16__offset(ptr addrspace(3) %ptr) nounwin
; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v3
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v1, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v3
@@ -3574,9 +3571,8 @@ define bfloat @local_atomic_fmax_ret_bf16__offset(ptr addrspace(3) %ptr) nounwin
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v3
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v1, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v3
@@ -3897,9 +3893,8 @@ define void @local_atomic_fmax_noret_bf16(ptr addrspace(3) %ptr) nounwind {
; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v4
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v0, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v4, v2, v3, v4
@@ -4031,9 +4026,8 @@ define void @local_atomic_fmax_noret_bf16(ptr addrspace(3) %ptr) nounwind {
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v4
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v0, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v4, v2, v3, v4
@@ -4338,10 +4332,9 @@ define void @local_atomic_fmax_noret_bf16__offset(ptr addrspace(3) %ptr) nounwin
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v4
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v1, v5
; GFX12-TRUE16-NEXT: v_and_or_b32 v4, v3, v2, v4
@@ -4475,10 +4468,9 @@ define void @local_atomic_fmax_noret_bf16__offset(ptr addrspace(3) %ptr) nounwin
; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v1, v5
; GFX11-TRUE16-NEXT: v_and_or_b32 v4, v3, v2, v4
@@ -4783,10 +4775,9 @@ define bfloat @local_atomic_fmax_ret_bf16__offset__align4(ptr addrspace(3) %ptr)
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX12-TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v3, 16, v1
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v1, 0xffff0000, v2, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
@@ -4896,10 +4887,9 @@ define bfloat @local_atomic_fmax_ret_bf16__offset__align4(ptr addrspace(3) %ptr)
; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v3, 16, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v1, 0xffff0000, v2, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
@@ -5160,9 +5150,8 @@ define void @local_atomic_fmax_noret_bf16__offset__align4(ptr addrspace(3) %ptr)
; GFX12-TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v3, 16, v2
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v1, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v1 offset:65534
@@ -5268,9 +5257,8 @@ define void @local_atomic_fmax_noret_bf16__offset__align4(ptr addrspace(3) %ptr)
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v3, 16, v2
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v1, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v1 offset:65534
diff --git a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmin.ll
index 1e737e245f7e0..8dc13425b2035 100644
--- a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmin.ll
@@ -2978,9 +2978,8 @@ define bfloat @local_atomic_fmin_ret_bf16(ptr addrspace(3) %ptr) nounwind {
; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v3
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v0, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v3
@@ -3115,9 +3114,8 @@ define bfloat @local_atomic_fmin_ret_bf16(ptr addrspace(3) %ptr) nounwind {
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v3
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v0, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v3
@@ -3432,9 +3430,8 @@ define bfloat @local_atomic_fmin_ret_bf16__offset(ptr addrspace(3) %ptr) nounwin
; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v3
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v1, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v3
@@ -3574,9 +3571,8 @@ define bfloat @local_atomic_fmin_ret_bf16__offset(ptr addrspace(3) %ptr) nounwin
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v3
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v1, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v3
@@ -3897,9 +3893,8 @@ define void @local_atomic_fmin_noret_bf16(ptr addrspace(3) %ptr) nounwind {
; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v4
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v0, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v4, v2, v3, v4
@@ -4031,9 +4026,8 @@ define void @local_atomic_fmin_noret_bf16(ptr addrspace(3) %ptr) nounwind {
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v4
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v0, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v4, v2, v3, v4
@@ -4338,10 +4332,9 @@ define void @local_atomic_fmin_noret_bf16__offset(ptr addrspace(3) %ptr) nounwin
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v4
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v1, v5
; GFX12-TRUE16-NEXT: v_and_or_b32 v4, v3, v2, v4
@@ -4475,10 +4468,9 @@ define void @local_atomic_fmin_noret_bf16__offset(ptr addrspace(3) %ptr) nounwin
; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v1, v5
; GFX11-TRUE16-NEXT: v_and_or_b32 v4, v3, v2, v4
@@ -4783,10 +4775,9 @@ define bfloat @local_atomic_fmin_ret_bf16__offset__align4(ptr addrspace(3) %ptr)
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX12-TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v3, 16, v1
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v1, 0xffff0000, v2, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
@@ -4896,10 +4887,9 @@ define bfloat @local_atomic_fmin_ret_bf16__offset__align4(ptr addrspace(3) %ptr)
; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v3, 16, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v1, 0xffff0000, v2, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
@@ -5160,9 +5150,8 @@ define void @local_atomic_fmin_noret_bf16__offset__align4(ptr addrspace(3) %ptr)
; GFX12-TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v3, 16, v2
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v1, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v1 offset:65534
@@ -5268,9 +5257,8 @@ define void @local_atomic_fmin_noret_bf16__offset__align4(ptr addrspace(3) %ptr)
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v3, 16, v2
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v1, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v1 offset:65534
diff --git a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fsub.ll b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fsub.ll
index 4a96506eb3c6a..503b2922e0159 100644
--- a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fsub.ll
@@ -3848,9 +3848,8 @@ define bfloat @local_atomic_fsub_ret_bf16(ptr addrspace(3) %ptr) nounwind {
; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v3
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v0, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v3
@@ -3985,9 +3984,8 @@ define bfloat @local_atomic_fsub_ret_bf16(ptr addrspace(3) %ptr) nounwind {
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v3
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v0, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v3
@@ -4300,9 +4298,8 @@ define bfloat @local_atomic_fsub_ret_bf16__offset(ptr addrspace(3) %ptr) nounwin
; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v3
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v1, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v3
@@ -4442,9 +4439,8 @@ define bfloat @local_atomic_fsub_ret_bf16__offset(ptr addrspace(3) %ptr) nounwin
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v3
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v1, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v3
@@ -4763,9 +4759,8 @@ define void @local_atomic_fsub_noret_bf16(ptr addrspace(3) %ptr) nounwind {
; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v4
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v0, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v4, v2, v3, v4
@@ -4897,9 +4892,8 @@ define void @local_atomic_fsub_noret_bf16(ptr addrspace(3) %ptr) nounwind {
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v4
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v0, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v4, v2, v3, v4
@@ -5202,10 +5196,9 @@ define void @local_atomic_fsub_noret_bf16__offset(ptr addrspace(3) %ptr) nounwin
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v4
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v1, v5
; GFX12-TRUE16-NEXT: v_and_or_b32 v4, v3, v2, v4
@@ -5339,10 +5332,9 @@ define void @local_atomic_fsub_noret_bf16__offset(ptr addrspace(3) %ptr) nounwin
; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v1, v5
; GFX11-TRUE16-NEXT: v_and_or_b32 v4, v3, v2, v4
@@ -5645,10 +5637,9 @@ define bfloat @local_atomic_fsub_ret_bf16__offset__align4(ptr addrspace(3) %ptr)
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX12-TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v3, 16, v1
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v1, 0xffff0000, v2, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
@@ -5758,10 +5749,9 @@ define bfloat @local_atomic_fsub_ret_bf16__offset__align4(ptr addrspace(3) %ptr)
; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v3, 16, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v1, 0xffff0000, v2, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
@@ -6020,9 +6010,8 @@ define void @local_atomic_fsub_noret_bf16__offset__align4(ptr addrspace(3) %ptr)
; GFX12-TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v3, 16, v2
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v1, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v1 offset:65534
@@ -6128,9 +6117,8 @@ define void @local_atomic_fsub_noret_bf16__offset__align4(ptr addrspace(3) %ptr)
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v3, 16, v2
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v1, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v1 offset:65534
diff --git a/llvm/test/CodeGen/AMDGPU/repeated-divisor.ll b/llvm/test/CodeGen/AMDGPU/repeated-divisor.ll
index 3e8c626cbf5e0..cd9e40198284c 100644
--- a/llvm/test/CodeGen/AMDGPU/repeated-divisor.ll
+++ b/llvm/test/CodeGen/AMDGPU/repeated-divisor.ll
@@ -979,10 +979,9 @@ define <6 x half> @v_repeat_divisor_v3f16_x2(<3 x half> %x, <3 x half> %y, <3 x
; GFX11-TRUE16-NEXT: v_pk_mul_f16 v3, v3, v5
; GFX11-TRUE16-NEXT: v_pk_mul_f16 v1, v1, v5
; GFX11-TRUE16-NEXT: v_pk_mul_f16 v0, v0, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v2, v3, v2, 16
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_repeat_divisor_v3f16_x2:
diff --git a/llvm/test/CodeGen/AMDGPU/scalar_to_vector.ll b/llvm/test/CodeGen/AMDGPU/scalar_to_vector.ll
index 6b1a46d0fe10c..caa9c9095d7e1 100644
--- a/llvm/test/CodeGen/AMDGPU/scalar_to_vector.ll
+++ b/llvm/test/CodeGen/AMDGPU/scalar_to_vector.ll
@@ -475,8 +475,7 @@ define i64 @bitcast_combine_scalar_to_vector_v4i16(i16 %arg) {
; GFX11-NEXT: v_lshrrev_b16 v1.l, 8, v1.l
; GFX11-NEXT: v_or_b16 v2.h, v1.l, v1.h
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_mov_b16_e32 v1.l, v2.h
-; GFX11-NEXT: v_mov_b16_e32 v1.h, v2.h
+; GFX11-NEXT: v_perm_b32 v1, v2, v2, 0x7060302
; GFX11-NEXT: v_mov_b32_e32 v0, v2
; GFX11-NEXT: s_setpc_b64 s[30:31]
%arg.cast = bitcast i16 %arg to <2 x i8>
diff --git a/llvm/test/CodeGen/AMDGPU/si-pre-emit-peephole-merge-v-mov-b16.mir b/llvm/test/CodeGen/AMDGPU/si-pre-emit-peephole-merge-v-mov-b16.mir
new file mode 100644
index 0000000000000..8a09c2c241436
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/si-pre-emit-peephole-merge-v-mov-b16.mir
@@ -0,0 +1,199 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 -run-pass si-pre-emit-peephole -verify-machineinstrs -o - %s | FileCheck %s
+
+# Tests for mergeTrue16MovB16Pair: pairs of v_mov_b16 targeting lo16 and hi16
+# of the same VGPR are merged into a single 32-bit instruction (true16 mode).
+
+---
+# Pattern: v_mov_b16 v0.h, 0 + v_mov_b16 v0.l, v2.h
+# => v_lshrrev_b32 v0, 16, v2
+name: merge_hi_zero_lo_from_hi
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr2
+
+ ; CHECK-LABEL: name: merge_hi_zero_lo_from_hi
+ ; CHECK: liveins: $vgpr2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $vgpr0 = V_LSHRREV_B32_e64 16, $vgpr2, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit $vgpr0
+ $vgpr0_lo16 = V_MOV_B16_t16_e32 $vgpr2_hi16, implicit $exec
+ $vgpr0_hi16 = V_MOV_B16_t16_e32 0, implicit $exec
+ S_ENDPGM 0, implicit $vgpr0
+...
+
+---
+# Pattern: v_mov_b16 v0.l, 0 + v_mov_b16 v0.h, v2.l
+# => v_lshlrev_b32 v0, 16, v2
+name: merge_lo_zero_hi_from_lo
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr2
+
+ ; CHECK-LABEL: name: merge_lo_zero_hi_from_lo
+ ; CHECK: liveins: $vgpr2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $vgpr0 = V_LSHLREV_B32_e64 16, $vgpr2, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit $vgpr0
+ $vgpr0_lo16 = V_MOV_B16_t16_e32 0, implicit $exec
+ $vgpr0_hi16 = V_MOV_B16_t16_e32 $vgpr2_lo16, implicit $exec
+ S_ENDPGM 0, implicit $vgpr0
+...
+
+---
+# Pattern: v_mov_b16 v0.l, v2.l + v_mov_b16 v0.h, v3.l
+# => v_perm_b32 v0, v3, v2, 0x05040100
+name: merge_lo_from_lo_hi_from_lo
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr2, $vgpr3
+
+ ; CHECK-LABEL: name: merge_lo_from_lo_hi_from_lo
+ ; CHECK: liveins: $vgpr2, $vgpr3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $vgpr0 = V_PERM_B32_e64 $vgpr2, $vgpr3, 84148480, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit $vgpr0
+ $vgpr0_lo16 = V_MOV_B16_t16_e32 $vgpr2_lo16, implicit $exec
+ $vgpr0_hi16 = V_MOV_B16_t16_e32 $vgpr3_lo16, implicit $exec
+ S_ENDPGM 0, implicit $vgpr0
+...
+
+---
+# Pattern: v_mov_b16 v0.l, v2.l + v_mov_b16 v0.h, v3.h
+# => v_bfi_b32 v0, 0x0000ffff, v2, v3
+name: merge_lo_from_lo_hi_from_hi
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr2, $vgpr3
+
+ ; CHECK-LABEL: name: merge_lo_from_lo_hi_from_hi
+ ; CHECK: liveins: $vgpr2, $vgpr3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $vgpr0 = V_BFI_B32_e64 65535, $vgpr2, $vgpr3, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit $vgpr0
+ $vgpr0_lo16 = V_MOV_B16_t16_e32 $vgpr2_lo16, implicit $exec
+ $vgpr0_hi16 = V_MOV_B16_t16_e32 $vgpr3_hi16, implicit $exec
+ S_ENDPGM 0, implicit $vgpr0
+...
+
+---
+# Pattern: v_mov_b16 v0.l, v2.h + v_mov_b16 v0.h, v3.l
+# => v_perm_b32 v0, v2, v3, 0x01000706
+name: merge_lo_from_hi_hi_from_lo
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr2, $vgpr3
+
+ ; CHECK-LABEL: name: merge_lo_from_hi_hi_from_lo
+ ; CHECK: liveins: $vgpr2, $vgpr3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $vgpr0 = V_ALIGNBIT_B32_t16_e64 0, $vgpr3, 0, $vgpr2, 0, 16, 0, 0, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit $vgpr0
+ $vgpr0_lo16 = V_MOV_B16_t16_e32 $vgpr2_hi16, implicit $exec
+ $vgpr0_hi16 = V_MOV_B16_t16_e32 $vgpr3_lo16, implicit $exec
+ S_ENDPGM 0, implicit $vgpr0
+...
+
+---
+# Pattern: v_mov_b16 v0.l, v2.h + v_mov_b16 v0.h, v3.h
+# => v_perm_b32 v0, v2, v3, 0x03020706
+name: merge_lo_from_hi_hi_from_hi
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr2, $vgpr3
+
+ ; CHECK-LABEL: name: merge_lo_from_hi_hi_from_hi
+ ; CHECK: liveins: $vgpr2, $vgpr3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $vgpr0 = V_PERM_B32_e64 $vgpr2, $vgpr3, 117834498, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit $vgpr0
+ $vgpr0_lo16 = V_MOV_B16_t16_e32 $vgpr2_hi16, implicit $exec
+ $vgpr0_hi16 = V_MOV_B16_t16_e32 $vgpr3_hi16, implicit $exec
+ S_ENDPGM 0, implicit $vgpr0
+...
+
+---
+# Pattern: v_mov_b16 v0.l, 0 + v_mov_b16 v0.h, v2.h
+# => v_and_b32 v0, 0xffff0000, v2
+name: merge_lo_zero_hi_from_hi
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr2
+
+ ; CHECK-LABEL: name: merge_lo_zero_hi_from_hi
+ ; CHECK: liveins: $vgpr2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $vgpr0 = V_AND_B32_e64 4294901760, $vgpr2, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit $vgpr0
+ $vgpr0_lo16 = V_MOV_B16_t16_e32 0, implicit $exec
+ $vgpr0_hi16 = V_MOV_B16_t16_e32 $vgpr2_hi16, implicit $exec
+ S_ENDPGM 0, implicit $vgpr0
+...
+
+---
+# Pattern: v_mov_b16 v0.h, 0 + v_mov_b16 v0.l, v2.l
+# => v_and_b32 v0, 0x0000ffff, v2
+name: merge_hi_zero_lo_from_lo
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr2
+
+ ; CHECK-LABEL: name: merge_hi_zero_lo_from_lo
+ ; CHECK: liveins: $vgpr2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $vgpr0 = V_AND_B32_e64 65535, $vgpr2, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit $vgpr0
+ $vgpr0_lo16 = V_MOV_B16_t16_e32 $vgpr2_lo16, implicit $exec
+ $vgpr0_hi16 = V_MOV_B16_t16_e32 0, implicit $exec
+ S_ENDPGM 0, implicit $vgpr0
+...
+
+---
+# Negative: intervening instruction that clobbers dst — must NOT merge.
+name: no_merge_dst_clobbered
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr2
+
+ ; CHECK-LABEL: name: no_merge_dst_clobbered
+ ; CHECK: liveins: $vgpr2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $vgpr0_lo16 = V_MOV_B16_t16_e32 $vgpr2_lo16, implicit $exec
+ ; CHECK-NEXT: $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: $vgpr0_hi16 = V_MOV_B16_t16_e32 $vgpr2_hi16, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit $vgpr0
+ $vgpr0_lo16 = V_MOV_B16_t16_e32 $vgpr2_lo16, implicit $exec
+ $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ $vgpr0_hi16 = V_MOV_B16_t16_e32 $vgpr2_hi16, implicit $exec
+ S_ENDPGM 0, implicit $vgpr0
+...
+
+---
+# Negative: intervening instruction that clobbers src — must NOT merge.
+name: no_merge_src_clobbered
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr2
+
+ ; CHECK-LABEL: name: no_merge_src_clobbered
+ ; CHECK: liveins: $vgpr2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $vgpr0_lo16 = V_MOV_B16_t16_e32 $vgpr2_lo16, implicit $exec
+ ; CHECK-NEXT: $vgpr2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: $vgpr0_hi16 = V_MOV_B16_t16_e32 $vgpr2_hi16, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit $vgpr0
+ $vgpr0_lo16 = V_MOV_B16_t16_e32 $vgpr2_lo16, implicit $exec
+ $vgpr2 = V_MOV_B32_e32 0, implicit $exec
+ $vgpr0_hi16 = V_MOV_B16_t16_e32 $vgpr2_hi16, implicit $exec
+ S_ENDPGM 0, implicit $vgpr0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/sub.v2i16.ll b/llvm/test/CodeGen/AMDGPU/sub.v2i16.ll
index fd136e4a313f3..332c31555acc8 100644
--- a/llvm/test/CodeGen/AMDGPU/sub.v2i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/sub.v2i16.ll
@@ -693,10 +693,9 @@ define amdgpu_kernel void @v_test_sub_v2i16_zext_to_v2i32(ptr addrspace(1) %out,
; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1
; GFX11-TRUE16-NEXT: v_pk_sub_i16 v2, v1, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v1, 16, v2
; GFX11-TRUE16-NEXT: buffer_store_b64 v[0:1], off, s[0:3], 0
; GFX11-TRUE16-NEXT: s_endpgm
;
@@ -818,9 +817,8 @@ define amdgpu_kernel void @v_test_sub_v2i16_zext_to_v2i64(ptr addrspace(1) %out,
; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1
; GFX11-TRUE16-NEXT: v_pk_sub_i16 v0, v2, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v2, v1, v0, 16
; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX11-TRUE16-NEXT: buffer_store_b128 v[0:3], off, s[0:3], 0
; GFX11-TRUE16-NEXT: s_endpgm
@@ -1044,14 +1042,12 @@ define amdgpu_kernel void @v_test_sub_v2i16_sext_to_v2i64(ptr addrspace(1) %out,
; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_pk_sub_i16 v0, v1, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v1, 16, v0
; GFX11-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_i32 v2, v1, 0, 16
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_ashrrev_i32_e32 v3, 31, v2
; GFX11-TRUE16-NEXT: buffer_store_b128 v[0:3], off, s[0:3], 0
; GFX11-TRUE16-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/v_swap_b16.ll b/llvm/test/CodeGen/AMDGPU/v_swap_b16.ll
index 297d43357c376..57cf677f9e567 100644
--- a/llvm/test/CodeGen/AMDGPU/v_swap_b16.ll
+++ b/llvm/test/CodeGen/AMDGPU/v_swap_b16.ll
@@ -8,18 +8,17 @@ define half @swap(half %a, half %b, i32 %i) {
; GFX11-TRUE16-LABEL: swap:
; GFX11-TRUE16: ; %bb.0: ; %entry
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: .LBB0_1: ; %loop
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, -1, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
; GFX11-TRUE16-NEXT: v_swap_b16 v0.l, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB0_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %ret
@@ -52,15 +51,15 @@ define half @swap(half %a, half %b, i32 %i) {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v0.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.l
+; GFX12-TRUE16-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB0_1: ; %loop
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: v_add_nc_u32_e32 v2, -1, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
; GFX12-TRUE16-NEXT: v_swap_b16 v0.l, v0.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-and.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-and.ll
index c505b76b9cfc4..62bfa4b52559e 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-and.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-and.ll
@@ -1105,8 +1105,7 @@ define i16 @test_vector_reduce_and_v3i16(<3 x i16> %v) {
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_and_v3i16:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.h, 0
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e64 v2, 16, v0
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, v0, v2
; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, v0, v1
@@ -1137,8 +1136,7 @@ define i16 @test_vector_reduce_and_v3i16(<3 x i16> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.h, 0
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h
+; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e64 v2, 16, v0
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, v0, v2
; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, v0, v1
@@ -1224,9 +1222,8 @@ define i16 @test_vector_reduce_and_v4i16(<4 x i16> %v) {
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e64 v1, 16, v0
; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, v0, v1
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -1256,9 +1253,8 @@ define i16 @test_vector_reduce_and_v4i16(<4 x i16> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e64 v1, 16, v0
; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, v0, v1
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -1390,10 +1386,9 @@ define i16 @test_vector_reduce_and_v8i16(<8 x i16> %v) {
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v1, v1, v3
; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e64 v1, 16, v0
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, v0, v1
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -1431,10 +1426,9 @@ define i16 @test_vector_reduce_and_v8i16(<8 x i16> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v1, v1, v3
; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e64 v1, 16, v0
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, v0, v1
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -1626,10 +1620,9 @@ define i16 @test_vector_reduce_and_v16i16(<16 x i16> %v) {
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v1, v1, v3
; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e64 v1, 16, v0
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, v0, v1
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -1682,10 +1675,9 @@ define i16 @test_vector_reduce_and_v16i16(<16 x i16> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v1, v1, v3
; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e64 v1, 16, v0
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, v0, v1
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-or.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-or.ll
index bf1e2583a7f4c..f07115d24b870 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-or.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-or.ll
@@ -934,8 +934,7 @@ define i16 @test_vector_reduce_or_v3i16(<3 x i16> %v) {
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_or_v3i16:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.h, 0
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e64 v2, 16, v0
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_or3_b32 v0, v0, v2, v1
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -963,8 +962,7 @@ define i16 @test_vector_reduce_or_v3i16(<3 x i16> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.h, 0
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h
+; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e64 v2, 16, v0
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_or3_b32 v0, v0, v2, v1
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -1048,9 +1046,8 @@ define i16 @test_vector_reduce_or_v4i16(<4 x i16> %v) {
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e64 v1, 16, v0
; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -1080,9 +1077,8 @@ define i16 @test_vector_reduce_or_v4i16(<4 x i16> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e64 v1, 16, v0
; GFX12-SDAG-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -1212,10 +1208,9 @@ define i16 @test_vector_reduce_or_v8i16(<8 x i16> %v) {
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_or3_b32 v0, v0, v2, v1
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e64 v1, 16, v0
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -1251,10 +1246,9 @@ define i16 @test_vector_reduce_or_v8i16(<8 x i16> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_or3_b32 v0, v0, v2, v1
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e64 v1, 16, v0
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -1441,9 +1435,8 @@ define i16 @test_vector_reduce_or_v16i16(<16 x i16> %v) {
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_or3_b32 v1, v1, v5, v3
; GFX11-SDAG-TRUE16-NEXT: v_or3_b32 v0, v0, v2, v1
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e64 v1, 16, v0
; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -1491,9 +1484,8 @@ define i16 @test_vector_reduce_or_v16i16(<16 x i16> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_or3_b32 v1, v1, v5, v3
; GFX12-SDAG-TRUE16-NEXT: v_or3_b32 v0, v0, v2, v1
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e64 v1, 16, v0
; GFX12-SDAG-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-xor.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-xor.ll
index 57e884d85d437..9cef96fb63ecc 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-xor.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-xor.ll
@@ -1083,8 +1083,7 @@ define i16 @test_vector_reduce_xor_v3i16(<3 x i16> %v) {
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_xor_v3i16:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.h, 0
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e64 v2, 16, v0
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_xor3_b32 v0, v0, v2, v1
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -1112,8 +1111,7 @@ define i16 @test_vector_reduce_xor_v3i16(<3 x i16> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.h, 0
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h
+; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e64 v2, 16, v0
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_xor3_b32 v0, v0, v2, v1
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -1197,9 +1195,8 @@ define i16 @test_vector_reduce_xor_v4i16(<4 x i16> %v) {
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_xor_b32_e32 v0, v0, v1
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e64 v1, 16, v0
; GFX11-SDAG-TRUE16-NEXT: v_xor_b32_e32 v0, v0, v1
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -1229,9 +1226,8 @@ define i16 @test_vector_reduce_xor_v4i16(<4 x i16> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: v_xor_b32_e32 v0, v0, v1
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e64 v1, 16, v0
; GFX12-SDAG-TRUE16-NEXT: v_xor_b32_e32 v0, v0, v1
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -1362,10 +1358,9 @@ define i16 @test_vector_reduce_xor_v8i16(<8 x i16> %v) {
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_xor_b32_e32 v1, v1, v3
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_xor3_b32 v0, v0, v2, v1
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e64 v1, 16, v0
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_xor_b32_e32 v0, v0, v1
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -1401,10 +1396,9 @@ define i16 @test_vector_reduce_xor_v8i16(<8 x i16> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: v_xor_b32_e32 v1, v1, v3
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_xor3_b32 v0, v0, v2, v1
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e64 v1, 16, v0
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_xor_b32_e32 v0, v0, v1
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -1595,9 +1589,8 @@ define i16 @test_vector_reduce_xor_v16i16(<16 x i16> %v) {
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_xor3_b32 v1, v1, v5, v3
; GFX11-SDAG-TRUE16-NEXT: v_xor3_b32 v0, v0, v2, v1
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e64 v1, 16, v0
; GFX11-SDAG-TRUE16-NEXT: v_xor_b32_e32 v0, v0, v1
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -1645,9 +1638,8 @@ define i16 @test_vector_reduce_xor_v16i16(<16 x i16> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_xor3_b32 v1, v1, v5, v3
; GFX12-SDAG-TRUE16-NEXT: v_xor3_b32 v0, v0, v2, v1
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e64 v1, 16, v0
; GFX12-SDAG-TRUE16-NEXT: v_xor_b32_e32 v0, v0, v1
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll b/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll
index ee346785aee2c..5f19ee77bc237 100644
--- a/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll
@@ -329,9 +329,8 @@ define <4 x half> @shuffle_v4f16_35u5(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
; GFX11-TRUE16-NEXT: global_load_b32 v1, v[2:3], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v1, 0x7060302
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: shuffle_v4f16_35u5:
@@ -992,26 +991,15 @@ define <4 x half> @shuffle_v4f16_2356(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX10-NEXT: v_mov_b32_e32 v0, v4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: shuffle_v4f16_2356:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b64 v[2:3], v[2:3], off
-; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: shuffle_v4f16_2356:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: global_load_b64 v[2:3], v[2:3], off
-; GFX11-FAKE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-FAKE16-NEXT: v_alignbit_b32 v1, v3, v2, 16
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: shuffle_v4f16_2356:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: global_load_b64 v[2:3], v[2:3], off
+; GFX11-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX11-NEXT: s_waitcnt vmcnt(1)
+; GFX11-NEXT: v_alignbit_b32 v1, v3, v2, 16
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, ptr addrspace(1) %arg0
%val1 = load <4 x half>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 2, i32 3, i32 5, i32 6>
@@ -1052,26 +1040,15 @@ define <4 x half> @shuffle_v4f16_5623(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX10-NEXT: v_mov_b32_e32 v1, v4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: shuffle_v4f16_5623:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b64 v[2:3], v[2:3], off
-; GFX11-TRUE16-NEXT: global_load_b32 v1, v[0:1], off offset:4
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: shuffle_v4f16_5623:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: global_load_b64 v[2:3], v[2:3], off
-; GFX11-FAKE16-NEXT: global_load_b32 v1, v[0:1], off offset:4
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, v3, v2, 16
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: shuffle_v4f16_5623:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: global_load_b64 v[2:3], v[2:3], off
+; GFX11-NEXT: global_load_b32 v1, v[0:1], off offset:4
+; GFX11-NEXT: s_waitcnt vmcnt(1)
+; GFX11-NEXT: v_alignbit_b32 v0, v3, v2, 16
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, ptr addrspace(1) %arg0
%val1 = load <4 x half>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 5, i32 6, i32 2, i32 3>
@@ -1107,11 +1084,9 @@ define <4 x half> @shuffle_v4f16_3456(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
; GFX11-TRUE16-NEXT: global_load_b64 v[1:2], v[2:3], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, v1, v0, 16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v1, v2, v1, 16
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: shuffle_v4f16_3456:
@@ -1159,11 +1134,9 @@ define <4 x half> @shuffle_v4f16_5634(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
; GFX11-TRUE16-NEXT: global_load_b64 v[2:3], v[2:3], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v1, v2, v0, 16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, v3, v2, 16
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: shuffle_v4f16_5634:
@@ -1224,11 +1197,10 @@ define <4 x half> @shuffle_v4f16_5734(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
; GFX11-TRUE16-NEXT: global_load_b64 v[2:3], v[2:3], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v1, v2, v0, 16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -1282,26 +1254,15 @@ define <4 x i16> @shuffle_v4i16_2356(ptr addrspace(1) %arg0, ptr addrspace(1) %a
; GFX10-NEXT: v_mov_b32_e32 v0, v4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: shuffle_v4i16_2356:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b64 v[2:3], v[2:3], off
-; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: shuffle_v4i16_2356:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: global_load_b64 v[2:3], v[2:3], off
-; GFX11-FAKE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-FAKE16-NEXT: v_alignbit_b32 v1, v3, v2, 16
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: shuffle_v4i16_2356:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: global_load_b64 v[2:3], v[2:3], off
+; GFX11-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX11-NEXT: s_waitcnt vmcnt(1)
+; GFX11-NEXT: v_alignbit_b32 v1, v3, v2, 16
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x i16>, ptr addrspace(1) %arg0
%val1 = load <4 x i16>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x i16> %val0, <4 x i16> %val1, <4 x i32> <i32 2, i32 3, i32 5, i32 6>
@@ -1423,8 +1384,7 @@ define <4 x half> @shuffle_v4f16_1010(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b64 v[1:2], v[0:1], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, v1, v1, 16
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -1481,8 +1441,7 @@ define <4 x half> @shuffle_v4f16_1100(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b64 v[0:1], v[0:1], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v0, v0, 0x5040100
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -1587,22 +1546,13 @@ define <4 x half> @shuffle_v4f16_2333(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX10-NEXT: v_perm_b32 v1, v0, v0, 0x7060302
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: shuffle_v4f16_2333:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.h
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: shuffle_v4f16_2333:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, v0, v0, 0x7060302
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: shuffle_v4f16_2333:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_perm_b32 v1, v0, v0, 0x7060302
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, ptr addrspace(1) %arg0
%val1 = load <4 x half>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 2, i32 3, i32 3, i32 3>
@@ -1636,22 +1586,13 @@ define <4 x half> @shuffle_v4f16_6667(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX10-NEXT: v_perm_b32 v1, v0, v0, 0x7060302
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: shuffle_v4f16_6667:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.h
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: shuffle_v4f16_6667:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, v0, v0, 0x7060302
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: shuffle_v4f16_6667:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_perm_b32 v1, v0, v0, 0x7060302
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, ptr addrspace(1) %arg0
%val1 = load <4 x half>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 2, i32 3, i32 3, i32 3>
@@ -1823,26 +1764,15 @@ define <4 x half> @shuffle_v8f16_13_14_2_3(ptr addrspace(1) %arg0, ptr addrspace
; GFX10-NEXT: v_mov_b32_e32 v1, v4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: shuffle_v8f16_13_14_2_3:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b64 v[2:3], v[2:3], off offset:8
-; GFX11-TRUE16-NEXT: global_load_b32 v1, v[0:1], off offset:4
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: shuffle_v8f16_13_14_2_3:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: global_load_b64 v[2:3], v[2:3], off offset:8
-; GFX11-FAKE16-NEXT: global_load_b32 v1, v[0:1], off offset:4
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, v3, v2, 16
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: shuffle_v8f16_13_14_2_3:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: global_load_b64 v[2:3], v[2:3], off offset:8
+; GFX11-NEXT: global_load_b32 v1, v[0:1], off offset:4
+; GFX11-NEXT: s_waitcnt vmcnt(1)
+; GFX11-NEXT: v_alignbit_b32 v0, v3, v2, 16
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%val0 = load <8 x half>, ptr addrspace(1) %arg0
%val1 = load <8 x half>, ptr addrspace(1) %arg1
%shuffle = shufflevector <8 x half> %val0, <8 x half> %val1, <4 x i32> <i32 13, i32 14, i32 2, i32 3>
@@ -1914,22 +1844,13 @@ define <4 x half> @shuffle_v2f16_0122(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX10-NEXT: v_alignbit_b32 v1, v0, v0, 16
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: shuffle_v2f16_0122:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: shuffle_v2f16_0122:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: global_load_b32 v0, v[0:1], off
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_alignbit_b32 v1, v0, v0, 16
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: shuffle_v2f16_0122:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: global_load_b32 v0, v[0:1], off
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_alignbit_b32 v1, v0, v0, 16
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%val0 = load <2 x half>, ptr addrspace(1) %arg0
%val1 = load <2 x half>, ptr addrspace(1) %arg1
%shuffle = shufflevector <2 x half> %val0, <2 x half> %val1, <4 x i32> <i32 0, i32 1, i32 1, i32 0>
@@ -2147,8 +2068,7 @@ define <4 x half> @shuffle_v4f16_0456(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-TRUE16-NEXT: global_load_b64 v[0:1], v[0:1], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v1, v3, v2, 16
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: shuffle_v4f16_0456:
@@ -2414,9 +2334,8 @@ define <2 x half> @hi16low16bits_v2bf16(ptr addrspace(1) %x0, ptr addrspace(1) %
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off
; GFX11-TRUE16-NEXT: global_load_b32 v1, v[2:3], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, v1, v0, 16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: hi16low16bits_v2bf16:
@@ -2570,9 +2489,8 @@ define <2 x i16> @i16_hi16low16bits(ptr addrspace(1) %x0, ptr addrspace(1) %x1)
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off
; GFX11-TRUE16-NEXT: global_load_b32 v1, v[2:3], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, v1, v0, 16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: i16_hi16low16bits:
@@ -3619,9 +3537,8 @@ define <4 x bfloat> @shuffle_v4bf16_35u5(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
; GFX11-TRUE16-NEXT: global_load_b32 v1, v[2:3], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v1, 0x7060302
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: shuffle_v4bf16_35u5:
@@ -4282,26 +4199,15 @@ define <4 x bfloat> @shuffle_v4bf16_2356(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX10-NEXT: v_mov_b32_e32 v0, v4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: shuffle_v4bf16_2356:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b64 v[2:3], v[2:3], off
-; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: shuffle_v4bf16_2356:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: global_load_b64 v[2:3], v[2:3], off
-; GFX11-FAKE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-FAKE16-NEXT: v_alignbit_b32 v1, v3, v2, 16
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: shuffle_v4bf16_2356:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: global_load_b64 v[2:3], v[2:3], off
+; GFX11-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX11-NEXT: s_waitcnt vmcnt(1)
+; GFX11-NEXT: v_alignbit_b32 v1, v3, v2, 16
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <4 x bfloat>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> <i32 2, i32 3, i32 5, i32 6>
@@ -4342,26 +4248,15 @@ define <4 x bfloat> @shuffle_v4bf16_5623(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX10-NEXT: v_mov_b32_e32 v1, v4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: shuffle_v4bf16_5623:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b64 v[2:3], v[2:3], off
-; GFX11-TRUE16-NEXT: global_load_b32 v1, v[0:1], off offset:4
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: shuffle_v4bf16_5623:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: global_load_b64 v[2:3], v[2:3], off
-; GFX11-FAKE16-NEXT: global_load_b32 v1, v[0:1], off offset:4
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, v3, v2, 16
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: shuffle_v4bf16_5623:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: global_load_b64 v[2:3], v[2:3], off
+; GFX11-NEXT: global_load_b32 v1, v[0:1], off offset:4
+; GFX11-NEXT: s_waitcnt vmcnt(1)
+; GFX11-NEXT: v_alignbit_b32 v0, v3, v2, 16
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <4 x bfloat>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> <i32 5, i32 6, i32 2, i32 3>
@@ -4397,11 +4292,9 @@ define <4 x bfloat> @shuffle_v4bf16_3456(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
; GFX11-TRUE16-NEXT: global_load_b64 v[1:2], v[2:3], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, v1, v0, 16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v1, v2, v1, 16
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: shuffle_v4bf16_3456:
@@ -4449,11 +4342,9 @@ define <4 x bfloat> @shuffle_v4bf16_5634(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
; GFX11-TRUE16-NEXT: global_load_b64 v[2:3], v[2:3], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v1, v2, v0, 16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, v3, v2, 16
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: shuffle_v4bf16_5634:
@@ -4514,11 +4405,10 @@ define <4 x bfloat> @shuffle_v4bf16_5734(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
; GFX11-TRUE16-NEXT: global_load_b64 v[2:3], v[2:3], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v1, v2, v0, 16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -4617,8 +4507,7 @@ define <4 x bfloat> @shuffle_v4bf16_1010(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b64 v[1:2], v[0:1], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, v1, v1, 16
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -4675,8 +4564,7 @@ define <4 x bfloat> @shuffle_v4bf16_1100(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b64 v[0:1], v[0:1], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v0, v0, 0x5040100
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -4781,22 +4669,13 @@ define <4 x bfloat> @shuffle_v4bf16_2333(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX10-NEXT: v_perm_b32 v1, v0, v0, 0x7060302
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: shuffle_v4bf16_2333:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.h
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: shuffle_v4bf16_2333:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, v0, v0, 0x7060302
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: shuffle_v4bf16_2333:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_perm_b32 v1, v0, v0, 0x7060302
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <4 x bfloat>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> <i32 2, i32 3, i32 3, i32 3>
@@ -4830,22 +4709,13 @@ define <4 x bfloat> @shuffle_v4bf16_6667(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX10-NEXT: v_perm_b32 v1, v0, v0, 0x7060302
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: shuffle_v4bf16_6667:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.h
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: shuffle_v4bf16_6667:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, v0, v0, 0x7060302
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: shuffle_v4bf16_6667:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_perm_b32 v1, v0, v0, 0x7060302
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <4 x bfloat>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> <i32 2, i32 3, i32 3, i32 3>
@@ -5017,26 +4887,15 @@ define <4 x bfloat> @shuffle_v8bf16_13_14_2_3(ptr addrspace(1) %arg0, ptr addrsp
; GFX10-NEXT: v_mov_b32_e32 v1, v4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: shuffle_v8bf16_13_14_2_3:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b64 v[2:3], v[2:3], off offset:8
-; GFX11-TRUE16-NEXT: global_load_b32 v1, v[0:1], off offset:4
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: shuffle_v8bf16_13_14_2_3:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: global_load_b64 v[2:3], v[2:3], off offset:8
-; GFX11-FAKE16-NEXT: global_load_b32 v1, v[0:1], off offset:4
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, v3, v2, 16
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: shuffle_v8bf16_13_14_2_3:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: global_load_b64 v[2:3], v[2:3], off offset:8
+; GFX11-NEXT: global_load_b32 v1, v[0:1], off offset:4
+; GFX11-NEXT: s_waitcnt vmcnt(1)
+; GFX11-NEXT: v_alignbit_b32 v0, v3, v2, 16
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%val0 = load <8 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <8 x bfloat>, ptr addrspace(1) %arg1
%shuffle = shufflevector <8 x bfloat> %val0, <8 x bfloat> %val1, <4 x i32> <i32 13, i32 14, i32 2, i32 3>
@@ -5108,22 +4967,13 @@ define <4 x bfloat> @shuffle_v2bf16_0122(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX10-NEXT: v_alignbit_b32 v1, v0, v0, 16
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: shuffle_v2bf16_0122:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: shuffle_v2bf16_0122:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: global_load_b32 v0, v[0:1], off
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_alignbit_b32 v1, v0, v0, 16
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: shuffle_v2bf16_0122:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: global_load_b32 v0, v[0:1], off
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_alignbit_b32 v1, v0, v0, 16
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%val0 = load <2 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <2 x bfloat>, ptr addrspace(1) %arg1
%shuffle = shufflevector <2 x bfloat> %val0, <2 x bfloat> %val1, <4 x i32> <i32 0, i32 1, i32 1, i32 0>
@@ -5707,8 +5557,7 @@ define <4 x bfloat> @shuffle_v4bf16_0456(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-TRUE16-NEXT: global_load_b64 v[0:1], v[0:1], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v1, v3, v2, 16
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: shuffle_v4bf16_0456:
@@ -5916,9 +5765,8 @@ define <2 x bfloat> @hi16low16bits(ptr addrspace(1) %x0, ptr addrspace(1) %x1) {
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off
; GFX11-TRUE16-NEXT: global_load_b32 v1, v[2:3], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, v1, v0, 16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: hi16low16bits:
>From 7c6ab5d4923fab22f01ad7d0d67e0c27d8271e81 Mon Sep 17 00:00:00 2001
From: guochen2 <guochen2 at amd.com>
Date: Mon, 13 Jul 2026 12:11:55 -0400
Subject: [PATCH 2/5] address comment
---
llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp | 137 ++-----
llvm/test/CodeGen/AMDGPU/GlobalISel/ashr.ll | 2 +-
.../test/CodeGen/AMDGPU/GlobalISel/fptrunc.ll | 6 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll | 25 +-
.../AMDGPU/GlobalISel/insertelement.i16.ll | 16 +-
.../GlobalISel/llvm.amdgcn.intersect_ray.ll | 8 +-
llvm/test/CodeGen/AMDGPU/add.v2i16.ll | 79 ++--
.../CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll | 48 +--
.../CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll | 34 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.32bit.ll | 2 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.48bit.ll | 4 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll | 24 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll | 4 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll | 4 +-
.../atomic_optimizations_global_pointer.ll | 8 +-
llvm/test/CodeGen/AMDGPU/bf16.ll | 12 +-
llvm/test/CodeGen/AMDGPU/bswap.ll | 28 +-
.../buffer-fat-pointer-atomicrmw-fadd.ll | 12 +-
.../buffer-fat-pointer-atomicrmw-fmax.ll | 12 +-
.../buffer-fat-pointer-atomicrmw-fmin.ll | 12 +-
llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll | 6 +-
.../CodeGen/AMDGPU/flat-atomicrmw-fadd.ll | 40 +-
.../CodeGen/AMDGPU/flat-atomicrmw-fmax.ll | 40 +-
.../CodeGen/AMDGPU/flat-atomicrmw-fmin.ll | 40 +-
.../CodeGen/AMDGPU/flat-atomicrmw-fsub.ll | 40 +-
llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll | 4 +-
llvm/test/CodeGen/AMDGPU/function-args.ll | 2 +-
llvm/test/CodeGen/AMDGPU/function-returns.ll | 34 +-
.../CodeGen/AMDGPU/global-atomicrmw-fadd.ll | 40 +-
.../CodeGen/AMDGPU/global-atomicrmw-fmax.ll | 40 +-
.../CodeGen/AMDGPU/global-atomicrmw-fmin.ll | 40 +-
.../CodeGen/AMDGPU/global-atomicrmw-fsub.ll | 40 +-
.../llvm.amdgcn.image.gather4.a16.dim.ll | 10 +-
.../AMDGPU/llvm.amdgcn.image.msaa.load.ll | 6 +-
.../llvm.amdgcn.image.sample.a16.dim.ll | 42 +-
.../llvm.amdgcn.image.sample.g16.encode.ll | 8 +-
.../AMDGPU/llvm.amdgcn.image.sample.g16.ll | 4 +-
....amdgcn.struct.buffer.load.format.v3f16.ll | 158 +++-----
...gcn.struct.ptr.buffer.load.format.v3f16.ll | 76 ++--
.../AMDGPU/llvm.amdgcn.waitcnt.out.order.ll | 8 +-
.../CodeGen/AMDGPU/local-atomicrmw-fadd.ll | 24 +-
.../CodeGen/AMDGPU/local-atomicrmw-fmax.ll | 24 +-
.../CodeGen/AMDGPU/local-atomicrmw-fmin.ll | 24 +-
.../CodeGen/AMDGPU/local-atomicrmw-fsub.ll | 24 +-
llvm/test/CodeGen/AMDGPU/repeated-divisor.ll | 2 +-
llvm/test/CodeGen/AMDGPU/scalar_to_vector.ll | 2 +-
.../si-pre-emit-peephole-merge-v-mov-b16.mir | 50 +--
llvm/test/CodeGen/AMDGPU/sub.v2i16.ll | 82 ++--
llvm/test/CodeGen/AMDGPU/v_swap_b16.ll | 4 +-
llvm/test/CodeGen/AMDGPU/vector-reduce-and.ll | 336 +++++-----------
llvm/test/CodeGen/AMDGPU/vector-reduce-or.ll | 304 +++++---------
llvm/test/CodeGen/AMDGPU/vector-reduce-xor.ll | 304 +++++---------
.../CodeGen/AMDGPU/vector_shuffle.packed.ll | 370 ++++++++++++------
53 files changed, 1116 insertions(+), 1589 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp b/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
index 0a9ca8bfbd11c..8799f6236e0be 100644
--- a/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
+++ b/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
@@ -775,10 +775,11 @@ MachineInstrBuilder SIPreEmitPeephole::createUnpackedMI(MachineInstr &I,
// V_MOV_B16_t16_e32: dst(0), src0(1)
// V_MOV_B16_t16_e64: dst(0), src0_mods(1), src0(2), op_sel(3)
static bool getMovB16Info(const MachineInstr &MI, const SIRegisterInfo *TRI,
- MCRegister &SrcReg32, bool &SrcIsHi, bool &SrcIsImm,
- int64_t &ImmVal) {
+ MCRegister &SrcReg16, MCRegister &SrcReg32,
+ bool &SrcIsHi, bool &SrcIsImm, int64_t &ImmVal) {
SrcIsImm = false;
SrcIsHi = false;
+ SrcReg16 = MCRegister();
SrcReg32 = MCRegister();
unsigned Opc = MI.getOpcode();
@@ -807,6 +808,7 @@ static bool getMovB16Info(const MachineInstr &MI, const SIRegisterInfo *TRI,
return false;
SrcIsHi = AMDGPU::isHi16Reg(SrcReg, *TRI);
+ SrcReg16 = SrcReg;
SrcReg32 = TRI->get32BitRegister(SrcReg);
return SrcReg32.isValid();
}
@@ -815,58 +817,30 @@ static bool getMovB16Info(const MachineInstr &MI, const SIRegisterInfo *TRI,
// Try to merge a pair of v_mov_b16 instructions targeting the lo16 and hi16
// halves of the same VGPR into a single 32-bit instruction.
//
+// Caller guarantee the pair to be two v_mov_b16 and targets the same dst32
+//
// Patterns:
// v_mov_b16 v0.h, 0 v_mov_b16 v0.l, v2.l => v_and_b32 v0,0xffff,v2
// v_mov_b16 v0.h, 0 v_mov_b16 v0.l, v2.h => v_lshrrev_b32 v0,16,v2
// v_mov_b16 v0.l, 0 v_mov_b16 v0.h, v2.l => v_lshlrev_b32 v0,16,v2
// v_mov_b16 v0.l, 0 v_mov_b16 v0.h, v2.h => v_and_b32 v0,0xffff0000,v2
-// v_mov_b16 v0.l, v2.l v_mov_b16 v0.h, v3.l => v_perm_b32 v0,v2,v3,0x05040100
-// v_mov_b16 v0.l, v2.l v_mov_b16 v0.h, v3.h => v_bfi_b32 v0,0x0000ffff,v2,v3
-// v_mov_b16 v0.l, v2.h v_mov_b16 v0.h, v3.l => v_alignbit_b32 v0,v3,v2,16
-// v_mov_b16 v0.l, v2.h v_mov_b16 v0.h, v3.h => v_perm_b32 v0,v2,v3,0x07060302
+// v_mov_b16 v0.l, v2.x v_mov_b16 v0.h, v3.y => v_pack_b32_f16 v0,v2.x,v3.y
// clang-format on
bool SIPreEmitPeephole::mergeSingleMovB16Pair(MachineInstr &Lo,
MachineInstr &Hi,
bool IsHiFirst) const {
- // Both must be v_mov_b16 true16 variants.
- unsigned LoOpc = Lo.getOpcode();
- unsigned HiOpc = Hi.getOpcode();
- if ((LoOpc != AMDGPU::V_MOV_B16_t16_e32 &&
- LoOpc != AMDGPU::V_MOV_B16_t16_e64) ||
- (HiOpc != AMDGPU::V_MOV_B16_t16_e32 &&
- HiOpc != AMDGPU::V_MOV_B16_t16_e64))
- return false;
-
+ // Lo and Hi share the same Dst32
MCRegister LoDst = Lo.getOperand(0).getReg().asMCReg();
- MCRegister HiDst = Hi.getOperand(0).getReg().asMCReg();
-
- if (!LoDst.isValid() || !HiDst.isValid())
- return false;
-
- if (!AMDGPU::VGPR_16RegClass.contains(LoDst) ||
- !AMDGPU::VGPR_16RegClass.contains(HiDst))
- return false;
- if (AMDGPU::isHi16Reg(LoDst, *TRI))
- return false; // Lo is actually writing hi half
- if (!AMDGPU::isHi16Reg(HiDst, *TRI))
- return false; // Hi is actually writing lo half
-
- // Both must target the same 32-bit VGPR.
- MCRegister LoDst32 = TRI->get32BitRegister(LoDst);
- MCRegister HiDst32 = TRI->get32BitRegister(HiDst);
- if (!LoDst32.isValid() || LoDst32 != HiDst32)
- return false;
-
- MCRegister Dst32 = LoDst32;
+ MCRegister Dst32 = TRI->get32BitRegister(LoDst);
// Extract source info for Lo and Hi.
- MCRegister LoSrc32, HiSrc32;
+ MCRegister LoSrc16, LoSrc32, HiSrc16, HiSrc32;
bool LoSrcIsHi, HiSrcIsHi, LoSrcIsImm, HiSrcIsImm;
int64_t LoImm = 0, HiImm = 0;
- if (!getMovB16Info(Lo, TRI, LoSrc32, LoSrcIsHi, LoSrcIsImm, LoImm))
+ if (!getMovB16Info(Lo, TRI, LoSrc16, LoSrc32, LoSrcIsHi, LoSrcIsImm, LoImm))
return false;
- if (!getMovB16Info(Hi, TRI, HiSrc32, HiSrcIsHi, HiSrcIsImm, HiImm))
+ if (!getMovB16Info(Hi, TRI, HiSrc16, HiSrc32, HiSrcIsHi, HiSrcIsImm, HiImm))
return false;
MachineInstr &FirstMI = IsHiFirst ? Hi : Lo;
@@ -878,25 +852,41 @@ bool SIPreEmitPeephole::mergeSingleMovB16Pair(MachineInstr &Lo,
// Check that between Lo and Hi, there are no instructions that:
// - modify Dst32 (except through Lo/Hi themselves)
- // - modify LoSrc32 or HiSrc32 dependinig on order (data dependency)
+ // - modify LoSrc16 or HiSrc16 dependinig on order (data dependency)
// We scan from the instruction after the first mov up to (but not including)
// the second mov.
- MCRegister SecondSrc32 = IsHiFirst ? LoSrc32 : HiSrc32;
- for (auto It = std::next(FirstMI.getIterator()); &*It != &SecondMI; ++It) {
- const MachineInstr &Scan = *It;
+ MCRegister SecondSrc16 = IsHiFirst ? LoSrc16 : HiSrc16;
+ for (auto &It :
+ drop_begin(make_range(FirstMI.getIterator(), SecondMI.getIterator()))) {
+ const MachineInstr &Scan = It;
if (Scan.modifiesRegister(Dst32, TRI))
return false;
- if (!IsSecondImm && Scan.modifiesRegister(SecondSrc32, TRI))
+ if (!IsSecondImm && Scan.modifiesRegister(SecondSrc16, TRI))
return false;
}
// Now match patterns and emit the replacement instruction.
// Insert before the first (Lo) instruction, then remove both.
+ // Pattern: v_mov_b16 v0.l, v2.x + v_mov_b16 v0.h, v3.y
+ // => v_pack_b32_f16 v0,v2.x,v3.y
+ if (!HiSrcIsImm && !LoSrcIsImm) {
+ BuildMI(MBB, Lo, DL, TII->get(AMDGPU::V_PACK_B32_F16_t16_e64), Dst32)
+ .addImm(0) // SrcMod
+ .addReg(LoSrc16)
+ .addImm(0) // SrcMod
+ .addReg(HiSrc16)
+ .addImm(0) // Clamp
+ .addImm(0); // Opsel
+ Lo.eraseFromParent();
+ Hi.eraseFromParent();
+ return true;
+ }
+
// Pattern: v_mov_b16 v0.h, 0 + v_mov_b16 v0.l, v2.l
// => v_and_b32 v0, 0x0000ffff, v2
if (HiSrcIsImm && HiImm == 0 && !LoSrcIsImm && !LoSrcIsHi) {
- BuildMI(MBB, Lo, DL, TII->get(AMDGPU::V_AND_B32_e64), Dst32)
+ BuildMI(MBB, Lo, DL, TII->get(AMDGPU::V_AND_B32_e32), Dst32)
.addImm(0x0000ffff)
.addReg(LoSrc32);
Lo.eraseFromParent();
@@ -907,7 +897,7 @@ bool SIPreEmitPeephole::mergeSingleMovB16Pair(MachineInstr &Lo,
// Pattern: v_mov_b16 v0.h, 0 + v_mov_b16 v0.l, v2.h
// => v_lshrrev_b32 v0, 16, v2
if (HiSrcIsImm && HiImm == 0 && !LoSrcIsImm && LoSrcIsHi) {
- BuildMI(MBB, Lo, DL, TII->get(AMDGPU::V_LSHRREV_B32_e64), Dst32)
+ BuildMI(MBB, Lo, DL, TII->get(AMDGPU::V_LSHRREV_B32_e32), Dst32)
.addImm(16)
.addReg(LoSrc32);
Lo.eraseFromParent();
@@ -918,7 +908,7 @@ bool SIPreEmitPeephole::mergeSingleMovB16Pair(MachineInstr &Lo,
// Pattern: v_mov_b16 v0.l, 0 + v_mov_b16 v0.h, v2.l
// => v_lshlrev_b32 v0, 16, v2
if (LoSrcIsImm && LoImm == 0 && !HiSrcIsImm && !HiSrcIsHi) {
- BuildMI(MBB, Lo, DL, TII->get(AMDGPU::V_LSHLREV_B32_e64), Dst32)
+ BuildMI(MBB, Lo, DL, TII->get(AMDGPU::V_LSHLREV_B32_e32), Dst32)
.addImm(16)
.addReg(HiSrc32);
Lo.eraseFromParent();
@@ -929,7 +919,7 @@ bool SIPreEmitPeephole::mergeSingleMovB16Pair(MachineInstr &Lo,
// Pattern: v_mov_b16 v0.l, 0 + v_mov_b16 v0.h, v2.h
// => v_and_b32 v0, 0xffff0000, v2
if (LoSrcIsImm && LoImm == 0 && !HiSrcIsImm && HiSrcIsHi) {
- BuildMI(MBB, Lo, DL, TII->get(AMDGPU::V_AND_B32_e64), Dst32)
+ BuildMI(MBB, Lo, DL, TII->get(AMDGPU::V_AND_B32_e32), Dst32)
.addImm(0xffff0000)
.addReg(HiSrc32);
Lo.eraseFromParent();
@@ -937,59 +927,6 @@ bool SIPreEmitPeephole::mergeSingleMovB16Pair(MachineInstr &Lo,
return true;
}
- // Pattern: v_mov_b16 v0.l, v2.l + v_mov_b16 v0.h, v3.l
- // => v_perm_b32 v0, v2, v3, 0x05040100
- if (!LoSrcIsImm && !LoSrcIsHi && !HiSrcIsImm && !HiSrcIsHi) {
- BuildMI(MBB, Lo, DL, TII->get(AMDGPU::V_PERM_B32_e64), Dst32)
- .addReg(LoSrc32)
- .addReg(HiSrc32)
- .addImm(0x05040100);
- Lo.eraseFromParent();
- Hi.eraseFromParent();
- return true;
- }
-
- // Pattern: v_mov_b16 v0.l, v2.l + v_mov_b16 v0.h, v3.h
- // => v_bfi_b32 v0, 0x0000ffff, v2, v3
- if (!LoSrcIsImm && !LoSrcIsHi && !HiSrcIsImm && HiSrcIsHi) {
- BuildMI(MBB, Lo, DL, TII->get(AMDGPU::V_BFI_B32_e64), Dst32)
- .addImm(0x0000ffff)
- .addReg(LoSrc32)
- .addReg(HiSrc32);
- Lo.eraseFromParent();
- Hi.eraseFromParent();
- return true;
- }
-
- // Pattern: v_mov_b16 v0.l, v2.h + v_mov_b16 v0.h, v3.l
- // => v_alignbit_b32 v0,v3,v2,16
- if (!LoSrcIsImm && LoSrcIsHi && !HiSrcIsImm && !HiSrcIsHi) {
- BuildMI(MBB, Lo, DL, TII->get(AMDGPU::V_ALIGNBIT_B32_t16_e64), Dst32)
- .addImm(0) // SrcMod0
- .addReg(HiSrc32)
- .addImm(0) // SrcMod1
- .addReg(LoSrc32)
- .addImm(0) // SrcMod2
- .addImm(16)
- .addImm(0) // Clamp
- .addImm(0); // Opsel
- Lo.eraseFromParent();
- Hi.eraseFromParent();
- return true;
- }
-
- // Pattern: v_mov_b16 v0.l, v2.h + v_mov_b16 v0.h, v3.h
- // => v_perm_b32 v0, v2, v3, 0x07060302
- if (!LoSrcIsImm && LoSrcIsHi && !HiSrcIsImm && HiSrcIsHi) {
- BuildMI(MBB, Lo, DL, TII->get(AMDGPU::V_PERM_B32_e64), Dst32)
- .addReg(LoSrc32)
- .addReg(HiSrc32)
- .addImm(0x07060302);
- Lo.eraseFromParent();
- Hi.eraseFromParent();
- return true;
- }
-
return false;
}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/ashr.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/ashr.ll
index 6aa882a15b136..a340a13c1fb98 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/ashr.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/ashr.ll
@@ -2008,7 +2008,7 @@ define <4 x i2> @v_ashr_v4i2(<4 x i2> %value, <4 x i2> %amount) {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v3, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v1, v2.l, v3.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.h, v5.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, v4.l, 3
; GFX11-TRUE16-NEXT: v_pk_lshlrev_b16 v0, 14, v0 op_sel_hi:[0,1]
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fptrunc.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fptrunc.ll
index 6e60d2aa8b287..b752ba13e7868 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fptrunc.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fptrunc.ll
@@ -386,7 +386,7 @@ define amdgpu_ps half @fptrunc_f64_to_f16_div(double %a) {
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, 0x7e00 :: v_dual_add_nc_u32 v2, v2, v4
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, 0x7c00, v5, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_gt_i32_e32 vcc_lo, 31, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v4, 16, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, 0x7c00, v2, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0x40f, v3
@@ -495,7 +495,7 @@ define amdgpu_ps half @fptrunc_f64_to_f16_div(double %a) {
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, 0x7c00, v5, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_gt_i32_e32 vcc_lo, 31, v3
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v4, 16, v1
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, 0x7c00, v2, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0x40f, v3
@@ -597,7 +597,7 @@ define amdgpu_ps half @fptrunc_f64_to_f16_div(double %a) {
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX1250-TRUE16-NEXT: v_cndmask_b32_e32 v0, 0x7c00, v5, vcc_lo
; GFX1250-TRUE16-NEXT: v_cmp_gt_i32_e32 vcc_lo, 31, v3
-; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e64 v4, 16, v1
+; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v1
; GFX1250-TRUE16-NEXT: v_cndmask_b32_e32 v2, 0x7c00, v2, vcc_lo
; GFX1250-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0x40f, v3
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll
index e413201a5d34f..d88873be73edf 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll
@@ -4844,23 +4844,14 @@ define i64 @v_fshr_i64_48(i64 %lhs, i64 %rhs) {
; GFX10-NEXT: v_or_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: v_fshr_i64_48:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b64 v[0:1], 16, v[0:1]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v2, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: v_fshr_i64_48:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshlrev_b64 v[0:1], 16, v[0:1]
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: v_fshr_i64_48:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshlrev_b64 v[0:1], 16, v[0:1]
+; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = call i64 @llvm.fshr.i64(i64 %lhs, i64 %rhs, i64 48)
ret i64 %result
}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.i16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.i16.ll
index bf0ed51126ade..cd9ab9deaa01c 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.i16.ll
@@ -243,7 +243,7 @@ define amdgpu_ps void @insertelement_s_v2i16_v_s(ptr addrspace(4) inreg %ptr, i1
; GFX11-LABEL: insertelement_s_v2i16_v_s:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b32 s0, s[2:3], 0x0
-; GFX11-NEXT: v_perm_b32 v2, v0, v0, 0x5040100
+; GFX11-NEXT: v_pack_b32_f16 v2, v0.l, v0.l
; GFX11-NEXT: s_lshl_b32 s1, s4, 4
; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0
; GFX11-NEXT: s_lshl_b32 s1, 0xffff, s1
@@ -403,7 +403,7 @@ define amdgpu_ps void @insertelement_s_v2i16_v_v(ptr addrspace(4) inreg %ptr, i1
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b32 s0, s[2:3], 0x0
; GFX11-NEXT: v_lshlrev_b32_e32 v1, 4, v1
-; GFX11-NEXT: v_perm_b32 v3, v0, v0, 0x5040100
+; GFX11-NEXT: v_pack_b32_f16 v3, v0.l, v0.l
; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_lshlrev_b32_e64 v2, v1, 0xffff
@@ -564,7 +564,7 @@ define amdgpu_ps void @insertelement_v_v2i16_v_s(ptr addrspace(1) %ptr, i16 %val
; GFX11-LABEL: insertelement_v_v2i16_v_s:
; GFX11: ; %bb.0:
; GFX11-NEXT: global_load_b32 v3, v[0:1], off
-; GFX11-NEXT: v_perm_b32 v4, v2, v2, 0x5040100
+; GFX11-NEXT: v_pack_b32_f16 v4, v2.l, v2.l
; GFX11-NEXT: s_lshl_b32 s0, s2, 4
; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0
; GFX11-NEXT: s_lshl_b32 s0, 0xffff, s0
@@ -646,7 +646,7 @@ define amdgpu_ps void @insertelement_v_v2i16_v_v(ptr addrspace(1) %ptr, i16 %val
; GFX11: ; %bb.0:
; GFX11-NEXT: global_load_b32 v4, v[0:1], off
; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_lshlrev_b32 v0, 4, v3
-; GFX11-NEXT: v_perm_b32 v5, v2, v2, 0x5040100
+; GFX11-NEXT: v_pack_b32_f16 v5, v2.l, v2.l
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_lshlrev_b32_e64 v3, v0, 0xffff
; GFX11-NEXT: v_mov_b32_e32 v0, 0
@@ -878,7 +878,7 @@ define amdgpu_ps void @insertelement_s_v4i16_v_s(ptr addrspace(4) inreg %ptr, i1
; GFX11-LABEL: insertelement_s_v4i16_v_s:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
-; GFX11-NEXT: v_perm_b32 v2, v0, v0, 0x5040100
+; GFX11-NEXT: v_pack_b32_f16 v2, v0.l, v0.l
; GFX11-NEXT: s_lshl_b32 s2, s4, 4
; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0
; GFX11-NEXT: s_lshl_b64 s[2:3], 0xffff, s2
@@ -1054,7 +1054,7 @@ define amdgpu_ps void @insertelement_s_v4i16_v_v(ptr addrspace(4) inreg %ptr, i1
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
; GFX11-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_lshlrev_b32 v1, 4, v1
-; GFX11-NEXT: v_perm_b32 v5, v0, v0, 0x5040100
+; GFX11-NEXT: v_pack_b32_f16 v5, v0.l, v0.l
; GFX11-NEXT: v_mov_b32_e32 v3, 0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_lshlrev_b64 v[1:2], v1, 0xffff
@@ -1227,7 +1227,7 @@ define amdgpu_ps void @insertelement_v_v4i16_v_s(ptr addrspace(1) %ptr, i16 %val
; GFX11-LABEL: insertelement_v_v4i16_v_s:
; GFX11: ; %bb.0:
; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off
-; GFX11-NEXT: v_perm_b32 v4, v2, v2, 0x5040100
+; GFX11-NEXT: v_pack_b32_f16 v4, v2.l, v2.l
; GFX11-NEXT: s_lshl_b32 s0, s2, 4
; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v3, 0
; GFX11-NEXT: s_lshl_b64 s[0:1], 0xffff, s0
@@ -1315,7 +1315,7 @@ define amdgpu_ps void @insertelement_v_v4i16_v_v(ptr addrspace(1) %ptr, i16 %val
; GFX11: ; %bb.0:
; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off
; GFX11-NEXT: v_dual_mov_b32 v6, 0 :: v_dual_lshlrev_b32 v3, 4, v3
-; GFX11-NEXT: v_perm_b32 v7, v2, v2, 0x5040100
+; GFX11-NEXT: v_pack_b32_f16 v7, v2.l, v2.l
; GFX11-NEXT: v_mov_b32_e32 v5, 0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_lshlrev_b64 v[3:4], v3, 0xffff
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.intersect_ray.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.intersect_ray.ll
index 3f6fdf5ef6c81..8131248d1fbf4 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.intersect_ray.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.intersect_ray.ll
@@ -61,7 +61,7 @@ define amdgpu_ps <4 x float> @image_bvh_intersect_ray_a16(i32 %node_ptr, float %
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v9, v5 :: v_dual_mov_b32 v10, v8
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, v7, v9, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v8, v7.l, v9.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.h, v6.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v7.h
; GFX11-TRUE16-NEXT: image_bvh_intersect_ray v[0:3], [v0, v1, v[2:4], v[8:10]], s[0:3] a16
@@ -126,7 +126,7 @@ define amdgpu_ps <4 x float> @image_bvh64_intersect_ray_a16(i64 %node_ptr, float
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, v6 :: v_dual_mov_b32 v11, v9
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, v8, v10, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v9, v8.l, v10.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.h, v7.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v8.h
; GFX11-TRUE16-NEXT: image_bvh64_intersect_ray v[0:3], [v[0:1], v2, v[3:5], v[9:11]], s[0:3] a16
@@ -328,7 +328,7 @@ define amdgpu_ps <4 x float> @image_bvh_intersect_ray_a16_vgpr_descr(i32 %node_p
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v13, v2 :: v_dual_mov_b32 v14, v3
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v15, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_perm_b32 v18, v7, v19, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v18, v7.l, v19.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v7.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.h, v6.l
; GFX11-TRUE16-NEXT: s_mov_b32 s1, exec_lo
@@ -579,7 +579,7 @@ define amdgpu_ps <4 x float> @image_bvh64_intersect_ray_a16_vgpr_descr(i64 %node
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v17, v0 :: v_dual_mov_b32 v18, v1
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v19, v2 :: v_dual_mov_b32 v14, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v8, v5, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v4, v8.l, v5.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v8.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v7.l
; GFX11-TRUE16-NEXT: s_mov_b32 s1, exec_lo
diff --git a/llvm/test/CodeGen/AMDGPU/add.v2i16.ll b/llvm/test/CodeGen/AMDGPU/add.v2i16.ll
index 860e14e083abf..a54223795c640 100644
--- a/llvm/test/CodeGen/AMDGPU/add.v2i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/add.v2i16.ll
@@ -673,7 +673,7 @@ define amdgpu_kernel void @v_test_add_v2i16_zext_to_v2i32(ptr addrspace(1) %out,
; GFX11-TRUE16-NEXT: v_pk_add_u16 v2, v1, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v2
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v1, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; GFX11-TRUE16-NEXT: global_store_b64 v3, v[0:1], s[0:1]
; GFX11-TRUE16-NEXT: s_endpgm
;
@@ -787,8 +787,8 @@ define amdgpu_kernel void @v_test_add_v2i16_zext_to_v2i64(ptr addrspace(1) %out,
; GFX11-TRUE16-NEXT: global_load_b32 v0, v0, s[4:5] glc dlc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_pk_add_u16 v0, v2, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v2, v1, v0, 16
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v2, v0.h, v1.l
; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX11-TRUE16-NEXT: global_store_b128 v1, v[0:3], s[0:1]
; GFX11-TRUE16-NEXT: s_endpgm
@@ -987,55 +987,30 @@ define amdgpu_kernel void @v_test_add_v2i16_sext_to_v2i64(ptr addrspace(1) %out,
; GFX10-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX10-NEXT: s_endpgm
;
-; GFX11-TRUE16-LABEL: v_test_add_v2i16_sext_to_v2i64:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-TRUE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: global_load_b32 v1, v0, s[2:3]
-; GFX11-TRUE16-NEXT: global_load_b32 v0, v0, s[4:5]
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_pk_add_u16 v0, v1, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v1, 16, v0
-; GFX11-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX11-TRUE16-NEXT: v_bfe_i32 v2, v1, 0, 16
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GFX11-TRUE16-NEXT: v_ashrrev_i32_e32 v3, 31, v2
-; GFX11-TRUE16-NEXT: global_store_b128 v4, v[0:3], s[0:1]
-; GFX11-TRUE16-NEXT: s_endpgm
-;
-; GFX11-FAKE16-LABEL: v_test_add_v2i16_sext_to_v2i64:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_clause 0x1
-; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-FAKE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, 0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_clause 0x1
-; GFX11-FAKE16-NEXT: global_load_b32 v1, v0, s[2:3]
-; GFX11-FAKE16-NEXT: global_load_b32 v0, v0, s[4:5]
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_pk_add_u16 v0, v1, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX11-FAKE16-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX11-FAKE16-NEXT: v_bfe_i32 v2, v1, 0, 16
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GFX11-FAKE16-NEXT: v_ashrrev_i32_e32 v3, 31, v2
-; GFX11-FAKE16-NEXT: global_store_b128 v4, v[0:3], s[0:1]
-; GFX11-FAKE16-NEXT: s_endpgm
+; GFX11-LABEL: v_test_add_v2i16_sext_to_v2i64:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-NEXT: v_mov_b32_e32 v4, 0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: global_load_b32 v1, v0, s[2:3]
+; GFX11-NEXT: global_load_b32 v0, v0, s[4:5]
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_pk_add_u16 v0, v1, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-NEXT: v_bfe_i32 v0, v0, 0, 16
+; GFX11-NEXT: v_bfe_i32 v2, v1, 0, 16
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; GFX11-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX11-NEXT: global_store_b128 v4, v[0:3], s[0:1]
+; GFX11-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%gep.out = getelementptr inbounds <2 x i64>, ptr addrspace(1) %out, i32 %tid
%gep.in0 = getelementptr inbounds <2 x i16>, ptr addrspace(1) %in0, i32 %tid
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
index bd63294186d8e..f1f93bc72f946 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
@@ -163158,7 +163158,7 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc_lo
; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v7, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v50, v51, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v2, v50.l, v51.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v3, v9 :: v_dual_add_nc_u32 v4, v5, v7
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, v6, v8
@@ -163191,7 +163191,7 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v9
; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v10, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v8
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v54, v55, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v4, v54.l, v55.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v5, v7, vcc_lo
; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v8, 16, 1
@@ -163412,7 +163412,7 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX11-TRUE16-NEXT: v_bfe_u32 v22, v23, 16, 1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v103, 16, v13
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v100, 16, v11
-; GFX11-TRUE16-NEXT: v_perm_b32 v21, v99, v112, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v21, v99.l, v112.l
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v16, v16, v17, vcc_lo
; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v18, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
@@ -163431,7 +163431,7 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v16, 0x7fff, v17
; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v23
; GFX11-TRUE16-NEXT: v_bfe_u32 v23, v22, 16, 1
-; GFX11-TRUE16-NEXT: v_perm_b32 v20, v102, v113, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v20, v102.l, v113.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v87.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v16, v16, v17, vcc_lo
@@ -163452,7 +163452,7 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX11-TRUE16-NEXT: v_bfe_u32 v22, v23, 16, 1
; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s2, 16
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.h, v101.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v13, v98, v103, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v13, v98.l, v103.l
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v16, v16, v17, vcc_lo
; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v18, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
@@ -163490,7 +163490,7 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v22
; GFX11-TRUE16-NEXT: v_bfe_u32 v22, v23, 16, 1
; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s16, 16
-; GFX11-TRUE16-NEXT: v_perm_b32 v36, v116, v129, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v36, v116.l, v129.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v16, v16, v17, vcc_lo
; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v18, 16, 1
@@ -163509,7 +163509,7 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v23
; GFX11-TRUE16-NEXT: v_bfe_u32 v23, v22, 16, 1
; GFX11-TRUE16-NEXT: s_and_b32 s1, s19, 0xffff0000
-; GFX11-TRUE16-NEXT: v_perm_b32 v35, v118, v132, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v35, v118.l, v132.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v16, v16, v17, vcc_lo
; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v19, 16, 1
@@ -163529,7 +163529,7 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v24
; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s26, 16
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v84.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v12, v81, v97, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v12, v81.l, v97.l
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v16, v16, v17, vcc_lo
; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v18, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
@@ -163705,33 +163705,33 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v16, v16, v19, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v18, 0x7fff, v18
-; GFX11-TRUE16-NEXT: v_perm_b32 v40, v150, v163, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v40, v150.l, v163.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v162, 16, v16
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v19, v25, v26, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
-; GFX11-TRUE16-NEXT: v_perm_b32 v47, v134, v149, 0x5040100
-; GFX11-TRUE16-NEXT: v_perm_b32 v46, v146, v160, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v47, v134.l, v149.l
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v46, v146.l, v160.l
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v165, 16, v19
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v18, v18, v27, vcc_lo
; GFX11-TRUE16-NEXT: v_mov_b16_e64 v45.h, v145.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v44, v135, v148, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v44, v135.l, v148.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v177, 16, v18
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.h, v100.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v29, v114, v117, 0x5040100
-; GFX11-TRUE16-NEXT: v_perm_b32 v28, v115, v128, 0x5040100
-; GFX11-TRUE16-NEXT: v_perm_b32 v49, v119, v133, 0x5040100
-; GFX11-TRUE16-NEXT: v_perm_b32 v48, v130, v144, 0x5040100
-; GFX11-TRUE16-NEXT: v_perm_b32 v179, v162, v176, 0x5040100
-; GFX11-TRUE16-NEXT: v_perm_b32 v178, v165, v177, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v29, v114.l, v117.l
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v28, v115.l, v128.l
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v49, v119.l, v133.l
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v48, v130.l, v144.l
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v179, v162.l, v176.l
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v178, v165.l, v177.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v53.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v65, v67, 0x5040100
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v68, v69, 0x5040100
-; GFX11-TRUE16-NEXT: v_perm_b32 v10, v71, v85, 0x5040100
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, v82, v96, 0x5040100
-; GFX11-TRUE16-NEXT: v_perm_b32 v183, v151, v164, 0x5040100
-; GFX11-TRUE16-NEXT: v_perm_b32 v182, v166, v167, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v6, v65.l, v67.l
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v5, v68.l, v69.l
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v10, v71.l, v85.l
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v9, v82.l, v96.l
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v183, v151.l, v164.l
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v182, v166.l, v167.l
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[30:31], 24, v[40:41]
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[31:32], 24, v[46:47]
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[15:16], 24, v[20:21]
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll
index 98f53a8163a00..b9719f62c2358 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll
@@ -33963,19 +33963,19 @@ define <16 x i16> @bitcast_v32i8_to_v16i16(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v29.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v28.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v27, v21, v27, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v27, v21.l, v27.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v20.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v25, v15, v25, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v25, v15.l, v25.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v14.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v19, v13, v19, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v19, v13.l, v19.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v12.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v17, v11, v17, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v17, v11.l, v17.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v10.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v9.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v8.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v7.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v23, v6, v23, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v23, v6.l, v23.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v5.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v4.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v3.l
@@ -39513,19 +39513,19 @@ define <16 x half> @bitcast_v32i8_to_v16f16(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v29.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v28.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v27, v21, v27, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v27, v21.l, v27.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v20.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v25, v15, v25, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v25, v15.l, v25.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v14.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v19, v13, v19, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v19, v13.l, v19.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v12.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v17, v11, v17, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v17, v11.l, v17.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v10.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v9.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v8.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v7.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v23, v6, v23, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v23, v6.l, v23.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v5.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v4.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v3.l
@@ -42902,7 +42902,7 @@ define inreg <32 x i8> @bitcast_v16bf16_to_v32i8_scalar(<16 x bfloat> inreg %a,
; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v8, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.h, v6.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v36, v0, v2, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v36, v0.l, v2.l
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, v5, v8
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v34, 16, v4
@@ -42972,7 +42972,7 @@ define inreg <32 x i8> @bitcast_v16bf16_to_v32i8_scalar(<16 x bfloat> inreg %a,
; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v11, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v11
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, v3, v9
-; GFX11-TRUE16-NEXT: v_perm_b32 v49, v33, v22, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v49, v33.l, v22.l
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v7, v15, vcc_lo
; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v9
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
@@ -43951,19 +43951,19 @@ define <16 x bfloat> @bitcast_v32i8_to_v16bf16(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v29.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v28.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v27, v21, v27, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v27, v21.l, v27.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v20.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v25, v15, v25, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v25, v15.l, v25.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v14.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v19, v13, v19, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v19, v13.l, v19.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v12.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v17, v11, v17, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v17, v11.l, v17.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v10.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v9.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v8.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v7.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v23, v6, v23, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v23, v6.l, v23.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v5.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v4.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v3.l
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.32bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.32bit.ll
index a1cee39a3dfa3..9fcafdb295d13 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.32bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.32bit.ll
@@ -10266,7 +10266,7 @@ define inreg <4 x i8> @bitcast_v2bf16_to_v4i8_scalar(<2 x bfloat> inreg %a, i32
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v0, v2, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v1, v0.l, v2.l
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.48bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.48bit.ll
index 3a28e939ff940..a8407e50594e1 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.48bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.48bit.ll
@@ -936,7 +936,7 @@ define <3 x i16> @bitcast_v3bf16_to_v3i16(<3 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, 0x7fc0
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v1, v3, v1, 16
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v1, v1.h, v3.l
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v6, vcc_lo
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
; GFX11-TRUE16-NEXT: .LBB4_2: ; %end
@@ -1195,7 +1195,7 @@ define inreg <3 x i16> @bitcast_v3bf16_to_v3i16_scalar(<3 x bfloat> inreg %a, i3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, 0x7fc0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v1, v4, v1, 16
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v1, v1.h, v4.l
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll
index deee9db7dff82..7b6e0a48b0ebd 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll
@@ -90271,7 +90271,7 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc_lo
; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v7, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v16, v17, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v2, v16.l, v17.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v3, v9 :: v_dual_add_nc_u32 v4, v5, v7
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, v6, v8
@@ -90464,23 +90464,23 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v13, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 0x7fff, v7
-; GFX11-TRUE16-NEXT: v_perm_b32 v81, v34, v37, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v81, v34.l, v37.l
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v48, 16, v8
; GFX11-TRUE16-NEXT: v_add_f32_e64 v8, 0x40c00000, s0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v9, v9, v13
; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s24, 16
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_perm_b32 v80, v48, v38, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v80, v48.l, v38.l
; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v8, 16, 1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v9, 0x7fff, v9
-; GFX11-TRUE16-NEXT: v_perm_b32 v84, v30, v33, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v84, v30.l, v33.l
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v11, v11, v8
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v10, vcc_lo
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v13
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX11-TRUE16-NEXT: v_perm_b32 v83, v35, v36, 0x5040100
-; GFX11-TRUE16-NEXT: v_perm_b32 v86, v28, v29, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v83, v35.l, v36.l
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v86, v28.l, v29.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v9, v9, v10 :: v_dual_add_nc_u32 v10, 0x7fff, v11
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v39, 16, v7
@@ -90490,7 +90490,7 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; GFX11-TRUE16-NEXT: s_and_b32 s0, s27, 0xffff0000
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v69.h, v39.l
; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v7, 16, 1
-; GFX11-TRUE16-NEXT: v_perm_b32 v85, v31, v32, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v85, v31.l, v32.l
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v10, v11, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
@@ -90528,21 +90528,21 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v67.h, v53.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v68, v55, v50, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v68, v55.l, v50.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v25.l
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v11, v15, v51, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v51, 16, v8
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v26, v27, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v5, v26.l, v27.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v20.l
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v10, v10, v52, vcc_lo
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v52, 16, v11
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v67.l, v51.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v22, v24, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v3, v22.l, v24.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v54, 16, v10
-; GFX11-TRUE16-NEXT: v_perm_b32 v66, v52, v54, 0x5040100
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v18, v19, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v66, v52.l, v54.l
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v1, v18.l, v19.l
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v65, 24, v67
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v70, 24, v81
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v71, 8, v80
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll
index dd365b47e4a5f..21f0d838d316a 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll
@@ -18094,10 +18094,10 @@ define inreg <8 x i8> @bitcast_v4bf16_to_v8i8_scalar(<4 x bfloat> inreg %a, i32
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, v0, v2, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v9, v0.l, v2.l
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_perm_b32 v10, v8, v6, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v10, v8.l, v6.l
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v9
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[9:10]
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll
index 5456775ec0c3a..c4b05da065ae3 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll
@@ -8760,7 +8760,7 @@ define inreg <12 x i8> @bitcast_v6bf16_to_v12i8_scalar(<6 x bfloat> inreg %a, i3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 0x7fff, v7
; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v15, v13, v6, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v15, v13.l, v6.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5
@@ -8783,7 +8783,7 @@ define inreg <12 x i8> @bitcast_v6bf16_to_v12i8_scalar(<6 x bfloat> inreg %a, i3
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v4
-; GFX11-TRUE16-NEXT: v_perm_b32 v14, v0, v2, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v14, v0.l, v2.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v3
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v10, vcc_lo
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
index 3de3578f15cee..2f45635121010 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
@@ -15209,7 +15209,7 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1164-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
; GFX1164-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-TRUE16-NEXT: v_lshrrev_b32_e64 v2, 16, v0
+; GFX1164-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX1164-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s8, v2
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-TRUE16-NEXT: v_and_or_b32 v0, v1, s9, v0
@@ -15323,7 +15323,7 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1132-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
; GFX1132-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-TRUE16-NEXT: v_lshrrev_b32_e64 v2, 16, v0
+; GFX1132-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX1132-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s2, v2
; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-TRUE16-NEXT: v_and_or_b32 v0, v1, s8, v0
@@ -15433,7 +15433,7 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1264-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1264-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1264-TRUE16-NEXT: v_lshrrev_b32_e64 v2, 16, v0
+; GFX1264-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX1264-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s8, v2
; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1264-TRUE16-NEXT: v_and_or_b32 v0, v1, s9, v0
@@ -15545,7 +15545,7 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1232-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1232-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1232-TRUE16-NEXT: v_lshrrev_b32_e64 v2, 16, v0
+; GFX1232-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX1232-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s2, v2
; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1232-TRUE16-NEXT: v_and_or_b32 v0, v1, s8, v0
diff --git a/llvm/test/CodeGen/AMDGPU/bf16.ll b/llvm/test/CodeGen/AMDGPU/bf16.ll
index bbc6340773b7d..8c24ad53dbefa 100644
--- a/llvm/test/CodeGen/AMDGPU/bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/bf16.ll
@@ -38083,7 +38083,7 @@ define <2 x bfloat> @v_uitofp_v2i16_to_v2bf16(<2 x i16> %x) #0 {
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v0
-; GFX11TRUE16-NEXT: v_lshrrev_b32_e64 v2, 16, v0
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v0, v1
; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v1, v2
@@ -38132,7 +38132,7 @@ define <2 x bfloat> @v_uitofp_v2i16_to_v2bf16(<2 x i16> %x) #0 {
; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1250TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v0
-; GFX1250TRUE16-NEXT: v_lshrrev_b32_e64 v1, 16, v0
+; GFX1250TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1250TRUE16-NEXT: v_cvt_f32_u32_e32 v0, v2
; GFX1250TRUE16-NEXT: v_cvt_f32_u32_e32 v1, v1
@@ -38281,7 +38281,7 @@ define <3 x bfloat> @v_uitofp_v3i16_to_v3bf16(<3 x i16> %x) #0 {
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v0
-; GFX11TRUE16-NEXT: v_lshrrev_b32_e64 v3, 16, v0
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v0, v2
; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v2, v3
@@ -38351,7 +38351,7 @@ define <3 x bfloat> @v_uitofp_v3i16_to_v3bf16(<3 x i16> %x) #0 {
; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1250TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX1250TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v0
-; GFX1250TRUE16-NEXT: v_lshrrev_b32_e64 v3, 16, v0
+; GFX1250TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0
; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX1250TRUE16-NEXT: v_cvt_f32_u32_e32 v0, v1
; GFX1250TRUE16-NEXT: v_cvt_f32_u32_e32 v1, v2
@@ -38534,7 +38534,7 @@ define <4 x bfloat> @v_uitofp_v4i16_to_v4bf16(<4 x i16> %x) #0 {
; GFX11TRUE16-LABEL: v_uitofp_v4i16_to_v4bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_lshrrev_b32_e64 v3, 16, v1
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v4, v3
; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.h
@@ -38619,7 +38619,7 @@ define <4 x bfloat> @v_uitofp_v4i16_to_v4bf16(<4 x i16> %x) #0 {
; GFX1250TRUE16: ; %bb.0:
; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250TRUE16-NEXT: v_lshrrev_b32_e64 v2, 16, v1
+; GFX1250TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX1250TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v0
; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX1250TRUE16-NEXT: v_cvt_f32_u32_e32 v4, v2
diff --git a/llvm/test/CodeGen/AMDGPU/bswap.ll b/llvm/test/CodeGen/AMDGPU/bswap.ll
index 5e4bd7d240119..b0bb15509e5f3 100644
--- a/llvm/test/CodeGen/AMDGPU/bswap.ll
+++ b/llvm/test/CodeGen/AMDGPU/bswap.ll
@@ -730,25 +730,15 @@ define i64 @v_bswap_i48(i64 %src) {
; VI-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-REAL16-LABEL: v_bswap_i48:
-; GFX11-REAL16: ; %bb.0:
-; GFX11-REAL16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-REAL16-NEXT: v_perm_b32 v2, 0, v0, 0x10203
-; GFX11-REAL16-NEXT: v_perm_b32 v0, 0, v1, 0x10203
-; GFX11-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-REAL16-NEXT: v_lshrrev_b32_e64 v1, 16, v2
-; GFX11-REAL16-NEXT: v_alignbit_b32 v0, v2, v0, 16
-; GFX11-REAL16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: v_bswap_i48:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_perm_b32 v2, 0, v0, 0x10203
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, 0, v1, 0x10203
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
-; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, v2, v0, 16
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: v_bswap_i48:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_perm_b32 v2, 0, v0, 0x10203
+; GFX11-NEXT: v_perm_b32 v0, 0, v1, 0x10203
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_lshrrev_b32_e32 v1, 16, v2
+; GFX11-NEXT: v_alignbit_b32 v0, v2, v0, 16
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%trunc = trunc i64 %src to i48
%bswap = call i48 @llvm.bswap.i48(i48 %trunc)
%zext = zext i48 %bswap to i64
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
index d8e99eb697b5a..898ab82b4db9f 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
@@ -5003,7 +5003,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v2, 16, v0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -5156,7 +5156,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine
; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v2, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v2
; GFX11-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
@@ -5519,7 +5519,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v4, 16, v0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -5669,7 +5669,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine
; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v4, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v4
; GFX11-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
@@ -6042,7 +6042,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v11, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v4
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v7, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v9, v4
@@ -6288,7 +6288,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v11, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v7, v5
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v9, v4
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
index b8c895f869963..2ac743c50f6b1 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
@@ -4132,7 +4132,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v2, 16, v0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -4285,7 +4285,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine
; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v2, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v2
; GFX11-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
@@ -4650,7 +4650,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v4, 16, v0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -4800,7 +4800,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine
; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v4, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v4
; GFX11-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
@@ -5175,7 +5175,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v11, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v4
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v7, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v9, v4
@@ -5421,7 +5421,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v11, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v7, v5
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v9, v4
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
index 6efe5271fde53..587297dfb25da 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
@@ -4132,7 +4132,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v2, 16, v0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -4285,7 +4285,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine
; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v2, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v2
; GFX11-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
@@ -4650,7 +4650,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v4, 16, v0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -4800,7 +4800,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine
; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v4, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v4
; GFX11-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
@@ -5175,7 +5175,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v11, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v4
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v7, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v9, v4
@@ -5421,7 +5421,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v11, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v7, v5
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v9, v4
diff --git a/llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll b/llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll
index d1308410362db..e11b0dc0cbddb 100644
--- a/llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll
@@ -3246,7 +3246,7 @@ define <2 x half> @v_copysign_out_v2f16_mag_v2f64_sign_v2f16(<2 x double> %mag,
; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v9, v12
; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 3, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v12, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v12, 16, v3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, v8, v9
; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v2
@@ -4973,7 +4973,7 @@ define <3 x half> @v_copysign_out_v3f16_mag_v3f64_sign_v3f16(<3 x double> %mag,
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0x40f, v10
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_gt_i32_e32 vcc_lo, 31, v14
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v8, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v3
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, 0x7c00, v11, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
@@ -6114,7 +6114,7 @@ define <4 x half> @v_copysign_out_v4f16_mag_v4f64_sign_v4f16(<4 x double> %mag,
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 8, v3
; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v3, 20, 11
; GFX11-TRUE16-NEXT: v_bfe_u32 v20, v1, 20, 11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v15, 16, v7
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 16, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v6, 0xffe, v11, v6
; GFX11-TRUE16-NEXT: v_med3_i32 v11, v12, 0, 13
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll
index 75bae5b10faf6..bc23dc4dde61d 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll
@@ -11986,7 +11986,7 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -12131,7 +12131,7 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -12434,7 +12434,7 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -12586,7 +12586,7 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -12897,7 +12897,7 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -13050,7 +13050,7 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_gr
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -13359,7 +13359,7 @@ define void @flat_agent_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -13506,7 +13506,7 @@ define void @flat_agent_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_gr
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -13808,7 +13808,7 @@ define void @flat_agent_atomic_fadd_noret_bf16__offset12b_neg__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -13956,7 +13956,7 @@ define void @flat_agent_atomic_fadd_noret_bf16__offset12b_neg__amdgpu_no_fine_gr
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -14248,7 +14248,7 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_no
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v3
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
@@ -14366,7 +14366,7 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_no
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
@@ -14615,7 +14615,7 @@ define void @flat_agent_atomic_fadd_noret_bf16__offset12b__align4_pos__amdgpu_no
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v2
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -14728,7 +14728,7 @@ define void @flat_agent_atomic_fadd_noret_bf16__offset12b__align4_pos__amdgpu_no
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2046 glc
@@ -14980,7 +14980,7 @@ define void @flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v3
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
@@ -15120,7 +15120,7 @@ define void @flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
@@ -15416,7 +15416,7 @@ define bfloat @flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_g
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -15570,7 +15570,7 @@ define bfloat @flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_g
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -15882,7 +15882,7 @@ define void @flat_system_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_g
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -16031,7 +16031,7 @@ define void @flat_system_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_g
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll
index 8d0c46d7cb165..915cacf0b0c6b 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll
@@ -9844,7 +9844,7 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -9989,7 +9989,7 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -10293,7 +10293,7 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -10445,7 +10445,7 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -10757,7 +10757,7 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -10910,7 +10910,7 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_gr
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -11217,7 +11217,7 @@ define void @flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v3
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
@@ -11357,7 +11357,7 @@ define void @flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
@@ -11652,7 +11652,7 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -11799,7 +11799,7 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_gr
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -12102,7 +12102,7 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b_neg__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -12250,7 +12250,7 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b_neg__amdgpu_no_fine_gr
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -12543,7 +12543,7 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v3
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
@@ -12661,7 +12661,7 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
@@ -12911,7 +12911,7 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_no
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v2
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -13024,7 +13024,7 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_no
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2046 glc
@@ -13282,7 +13282,7 @@ define bfloat @flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_g
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -13436,7 +13436,7 @@ define bfloat @flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_g
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -13749,7 +13749,7 @@ define void @flat_system_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_g
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -13898,7 +13898,7 @@ define void @flat_system_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_g
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll
index 5d20f7b71480e..a0f740f9d68c5 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll
@@ -9844,7 +9844,7 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -9989,7 +9989,7 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -10293,7 +10293,7 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -10445,7 +10445,7 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -10757,7 +10757,7 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -10910,7 +10910,7 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_gr
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -11217,7 +11217,7 @@ define void @flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v3
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
@@ -11357,7 +11357,7 @@ define void @flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
@@ -11652,7 +11652,7 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -11799,7 +11799,7 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_gr
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -12102,7 +12102,7 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b_neg__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -12250,7 +12250,7 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b_neg__amdgpu_no_fine_gr
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -12543,7 +12543,7 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_no
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v3
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
@@ -12661,7 +12661,7 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_no
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
@@ -12911,7 +12911,7 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b__align4_pos__amdgpu_no
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v2
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -13024,7 +13024,7 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b__align4_pos__amdgpu_no
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2046 glc
@@ -13282,7 +13282,7 @@ define bfloat @flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_g
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -13436,7 +13436,7 @@ define bfloat @flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_g
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -13749,7 +13749,7 @@ define void @flat_system_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_g
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -13898,7 +13898,7 @@ define void @flat_system_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_g
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll
index db4b18b40b7e8..d8671bcd26398 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll
@@ -9417,7 +9417,7 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -9562,7 +9562,7 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -9865,7 +9865,7 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -10017,7 +10017,7 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -10328,7 +10328,7 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -10481,7 +10481,7 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -10787,7 +10787,7 @@ define void @flat_agent_atomic_fsub_noret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v3
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
@@ -10927,7 +10927,7 @@ define void @flat_agent_atomic_fsub_noret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
@@ -11221,7 +11221,7 @@ define void @flat_agent_atomic_fsub_noret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -11368,7 +11368,7 @@ define void @flat_agent_atomic_fsub_noret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -11670,7 +11670,7 @@ define void @flat_agent_atomic_fsub_noret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -11818,7 +11818,7 @@ define void @flat_agent_atomic_fsub_noret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -12110,7 +12110,7 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_pos__align4(ptr %ptr,
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v3
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
@@ -12228,7 +12228,7 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_pos__align4(ptr %ptr,
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
@@ -12477,7 +12477,7 @@ define void @flat_agent_atomic_fsub_noret_bf16__offset12b__align4_pos(ptr %ptr,
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v2
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -12590,7 +12590,7 @@ define void @flat_agent_atomic_fsub_noret_bf16__offset12b__align4_pos(ptr %ptr,
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2046 glc
@@ -12847,7 +12847,7 @@ define bfloat @flat_system_atomic_fsub_ret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -13001,7 +13001,7 @@ define bfloat @flat_system_atomic_fsub_ret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -13313,7 +13313,7 @@ define void @flat_system_atomic_fsub_noret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -13462,7 +13462,7 @@ define void @flat_system_atomic_fsub_noret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
diff --git a/llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll b/llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll
index 2c80ca6f6aa3f..47a6e97b16167 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll
@@ -2916,7 +2916,7 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zero_hi(ptr inreg %sbase,
; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-TRUE16-NEXT: flat_load_u16 v1, v0, s[2:3]
; GFX1250-SDAG-TRUE16-NEXT: s_wait_xcnt 0x0
-; GFX1250-SDAG-TRUE16-NEXT: v_lshlrev_b32_e64 v0, 16, v1
+; GFX1250-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
;
@@ -2977,7 +2977,7 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zero_hi_immneg128(ptr inr
; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-TRUE16-NEXT: flat_load_u16 v1, v0, s[2:3] offset:-128
; GFX1250-SDAG-TRUE16-NEXT: s_wait_xcnt 0x0
-; GFX1250-SDAG-TRUE16-NEXT: v_lshlrev_b32_e64 v0, 16, v1
+; GFX1250-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
;
diff --git a/llvm/test/CodeGen/AMDGPU/function-args.ll b/llvm/test/CodeGen/AMDGPU/function-args.ll
index c910a1be5d302..4d90d3f716ed1 100644
--- a/llvm/test/CodeGen/AMDGPU/function-args.ll
+++ b/llvm/test/CodeGen/AMDGPU/function-args.ll
@@ -1803,7 +1803,7 @@ define void @void_func_v2i24(<2 x i24> %arg0) #0 {
; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v1, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; GFX11-TRUE16-NEXT: s_clause 0x1
; GFX11-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0
; GFX11-TRUE16-NEXT: buffer_store_b8 v1, off, s[0:3], 0
diff --git a/llvm/test/CodeGen/AMDGPU/function-returns.ll b/llvm/test/CodeGen/AMDGPU/function-returns.ll
index a9bf830c188fb..dd9548c42e148 100644
--- a/llvm/test/CodeGen/AMDGPU/function-returns.ll
+++ b/llvm/test/CodeGen/AMDGPU/function-returns.ll
@@ -2064,29 +2064,17 @@ define void @void_func_sret_max_known_zero_bits(ptr addrspace(5) sret(i8) %arg0)
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: void_func_sret_max_known_zero_bits:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v1, 16, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 17, v0
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-TRUE16-NEXT: ds_store_b32 v0, v1
-; GFX11-TRUE16-NEXT: ds_store_b32 v0, v0
-; GFX11-TRUE16-NEXT: ds_store_b32 v0, v2
-; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: void_func_sret_max_known_zero_bits:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 17, v0
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-FAKE16-NEXT: ds_store_b32 v0, v1
-; GFX11-FAKE16-NEXT: ds_store_b32 v0, v0
-; GFX11-FAKE16-NEXT: ds_store_b32 v0, v2
-; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: void_func_sret_max_known_zero_bits:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, 17, v0
+; GFX11-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-NEXT: ds_store_b32 v0, v1
+; GFX11-NEXT: ds_store_b32 v0, v0
+; GFX11-NEXT: ds_store_b32 v0, v2
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%arg0.int = ptrtoint ptr addrspace(5) %arg0 to i32
%lshr0 = lshr i32 %arg0.int, 16
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll
index b0cbe66e6b855..628c428fbaa1a 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll
@@ -13797,7 +13797,7 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory(
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -13942,7 +13942,7 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory(
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -14385,7 +14385,7 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -14537,7 +14537,7 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -14992,7 +14992,7 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -15145,7 +15145,7 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -15591,7 +15591,7 @@ define void @global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory(
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v3
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
@@ -15731,7 +15731,7 @@ define void @global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory(
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
@@ -16164,7 +16164,7 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -16311,7 +16311,7 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -16756,7 +16756,7 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -16904,7 +16904,7 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -17316,7 +17316,7 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v3
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
@@ -17434,7 +17434,7 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
@@ -17792,7 +17792,7 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b__align4_pos__amdgpu_
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v2
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -17905,7 +17905,7 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b__align4_pos__amdgpu_
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2046 glc
@@ -18291,7 +18291,7 @@ define bfloat @global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -18445,7 +18445,7 @@ define bfloat @global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -18899,7 +18899,7 @@ define void @global_system_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -19048,7 +19048,7 @@ define void @global_system_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll
index 9fda3e4b83e83..dd034347fa3ff 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll
@@ -8735,7 +8735,7 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory(
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -8880,7 +8880,7 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory(
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -9235,7 +9235,7 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -9387,7 +9387,7 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -9752,7 +9752,7 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -9905,7 +9905,7 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -10265,7 +10265,7 @@ define void @global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory(
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v3
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
@@ -10405,7 +10405,7 @@ define void @global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory(
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
@@ -10751,7 +10751,7 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -10898,7 +10898,7 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -11253,7 +11253,7 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -11401,7 +11401,7 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -11746,7 +11746,7 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v3
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
@@ -11864,7 +11864,7 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
@@ -12156,7 +12156,7 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v2
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -12269,7 +12269,7 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2046 glc
@@ -12566,7 +12566,7 @@ define bfloat @global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -12720,7 +12720,7 @@ define bfloat @global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -13086,7 +13086,7 @@ define void @global_system_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -13235,7 +13235,7 @@ define void @global_system_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll
index 7cadd1d4b98c7..7f2ae68f15c8d 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll
@@ -8735,7 +8735,7 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory(
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -8880,7 +8880,7 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory(
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -9235,7 +9235,7 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -9387,7 +9387,7 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -9752,7 +9752,7 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -9905,7 +9905,7 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -10265,7 +10265,7 @@ define void @global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory(
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v3
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
@@ -10405,7 +10405,7 @@ define void @global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory(
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
@@ -10751,7 +10751,7 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -10898,7 +10898,7 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -11253,7 +11253,7 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -11401,7 +11401,7 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -11746,7 +11746,7 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v3
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
@@ -11864,7 +11864,7 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
@@ -12156,7 +12156,7 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b__align4_pos__amdgpu_
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v2
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -12269,7 +12269,7 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b__align4_pos__amdgpu_
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2046 glc
@@ -12566,7 +12566,7 @@ define bfloat @global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -12720,7 +12720,7 @@ define bfloat @global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -13086,7 +13086,7 @@ define void @global_system_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -13235,7 +13235,7 @@ define void @global_system_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll
index 1a09e052dc3f5..9b9d75ff9075c 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll
@@ -9257,7 +9257,7 @@ define bfloat @global_agent_atomic_fsub_ret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -9402,7 +9402,7 @@ define bfloat @global_agent_atomic_fsub_ret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -9755,7 +9755,7 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_pos(ptr addrspace(1)
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -9907,7 +9907,7 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_pos(ptr addrspace(1)
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -10270,7 +10270,7 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_neg(ptr addrspace(1)
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -10423,7 +10423,7 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_neg(ptr addrspace(1)
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -10781,7 +10781,7 @@ define void @global_agent_atomic_fsub_noret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v3
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
@@ -10921,7 +10921,7 @@ define void @global_agent_atomic_fsub_noret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
@@ -11265,7 +11265,7 @@ define void @global_agent_atomic_fsub_noret_bf16__offset12b_pos(ptr addrspace(1)
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -11412,7 +11412,7 @@ define void @global_agent_atomic_fsub_noret_bf16__offset12b_pos(ptr addrspace(1)
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -11765,7 +11765,7 @@ define void @global_agent_atomic_fsub_noret_bf16__offset12b_neg(ptr addrspace(1)
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -11913,7 +11913,7 @@ define void @global_agent_atomic_fsub_noret_bf16__offset12b_neg(ptr addrspace(1)
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -12256,7 +12256,7 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_pos__align4(ptr addr
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v3
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
@@ -12374,7 +12374,7 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_pos__align4(ptr addr
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
@@ -12664,7 +12664,7 @@ define void @global_agent_atomic_fsub_noret_bf16__offset12b__align4_pos(ptr addr
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v2
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -12777,7 +12777,7 @@ define void @global_agent_atomic_fsub_noret_bf16__offset12b__align4_pos(ptr addr
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2046 glc
@@ -13072,7 +13072,7 @@ define bfloat @global_system_atomic_fsub_ret_bf16__offset12b_pos(ptr addrspace(1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -13226,7 +13226,7 @@ define bfloat @global_system_atomic_fsub_ret_bf16__offset12b_pos(ptr addrspace(1
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
@@ -13590,7 +13590,7 @@ define void @global_system_atomic_fsub_noret_bf16__offset12b_pos(ptr addrspace(1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
@@ -13739,7 +13739,7 @@ define void @global_system_atomic_fsub_noret_bf16__offset12b_pos(ptr addrspace(1
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v7, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.gather4.a16.dim.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.gather4.a16.dim.ll
index a758bdf23d80a..9b867b6b4d42c 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.gather4.a16.dim.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.gather4.a16.dim.ll
@@ -101,7 +101,7 @@ define amdgpu_ps <4 x float> @gather4_cube(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX11-TRUE16-NEXT: s_mov_b32 s12, exec_lo
; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v0, v1, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v2, v0.l, v1.l
; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s12
; GFX11-TRUE16-NEXT: image_gather4 v[0:3], v[2:3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_CUBE a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -168,7 +168,7 @@ define amdgpu_ps <4 x float> @gather4_2darray(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX11-TRUE16-NEXT: s_mov_b32 s12, exec_lo
; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v0, v1, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v2, v0.l, v1.l
; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s12
; GFX11-TRUE16-NEXT: image_gather4 v[0:3], v[2:3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -301,7 +301,7 @@ define amdgpu_ps <4 x float> @gather4_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX11-TRUE16-NEXT: s_mov_b32 s12, exec_lo
; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v0, v1, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v2, v0.l, v1.l
; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s12
; GFX11-TRUE16-NEXT: image_gather4_cl v[0:3], v[2:3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -570,7 +570,7 @@ define amdgpu_ps <4 x float> @gather4_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX11-TRUE16-NEXT: s_mov_b32 s12, exec_lo
; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v1, v2, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v3, v1.l, v2.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s12
; GFX11-TRUE16-NEXT: image_gather4_b_cl v[0:3], v[2:4], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
@@ -699,7 +699,7 @@ define amdgpu_ps <4 x float> @gather4_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX11-TRUE16-LABEL: gather4_l_2d:
; GFX11-TRUE16: ; %bb.0: ; %main_body
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v0, v1, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v2, v0.l, v1.l
; GFX11-TRUE16-NEXT: image_gather4_l v[0:3], v[2:3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.msaa.load.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.msaa.load.ll
index 071aa783116a2..79721497f435a 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.msaa.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.msaa.load.ll
@@ -305,7 +305,7 @@ define amdgpu_ps <4 x float> @load_2dmsaa_a16(<8 x i32> inreg %rsrc, i16 %s, i16
; GFX11-TRUE16-LABEL: load_2dmsaa_a16:
; GFX11-TRUE16: ; %bb.0: ; %main_body
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v0, v1, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v2, v0.l, v1.l
; GFX11-TRUE16-NEXT: image_msaa_load v[0:3], v[2:3], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
@@ -338,8 +338,8 @@ main_body:
define amdgpu_ps <4 x float> @load_2darraymsaa_a16(<8 x i32> inreg %rsrc, i16 %s, i16 %t, i16 %slice, i16 %fragid) {
; GFX11-TRUE16-LABEL: load_2darraymsaa_a16:
; GFX11-TRUE16: ; %bb.0: ; %main_body
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v2, v3, 0x5040100
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v0, v1, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v4, v2.l, v3.l
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v3, v0.l, v1.l
; GFX11-TRUE16-NEXT: image_msaa_load v[0:3], v[3:4], s[0:7] dmask:0x4 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.a16.dim.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.a16.dim.ll
index d4ea966101893..515d4fa38cca7 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.a16.dim.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.a16.dim.ll
@@ -142,7 +142,7 @@ define amdgpu_ps <4 x float> @sample_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg %
; GFX11-TRUE16-NEXT: s_mov_b32 s12, exec_lo
; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v0, v1, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v2, v0.l, v1.l
; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s12
; GFX11-TRUE16-NEXT: image_sample v[0:3], v[2:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_3D a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -209,7 +209,7 @@ define amdgpu_ps <4 x float> @sample_cube(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX11-TRUE16-NEXT: s_mov_b32 s12, exec_lo
; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v0, v1, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v2, v0.l, v1.l
; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s12
; GFX11-TRUE16-NEXT: image_sample v[0:3], v[2:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_CUBE a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -342,7 +342,7 @@ define amdgpu_ps <4 x float> @sample_2darray(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX11-TRUE16-NEXT: s_mov_b32 s12, exec_lo
; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v0, v1, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v2, v0.l, v1.l
; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s12
; GFX11-TRUE16-NEXT: image_sample v[0:3], v[2:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D_ARRAY a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -582,7 +582,7 @@ define amdgpu_ps <4 x float> @sample_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX11-TRUE16-NEXT: s_mov_b32 s12, exec_lo
; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v0, v1, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v2, v0.l, v1.l
; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s12
; GFX11-TRUE16-NEXT: image_sample_cl v[0:3], v[2:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -1065,7 +1065,7 @@ define amdgpu_ps <4 x float> @sample_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX11-TRUE16-NEXT: s_mov_b32 s12, exec_lo
; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v1, v2, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v3, v1.l, v2.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s12
; GFX11-TRUE16-NEXT: image_sample_b_cl v[0:3], v[2:4], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
@@ -1292,9 +1292,9 @@ define amdgpu_ps <4 x float> @sample_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX11-TRUE16-LABEL: sample_d_2d:
; GFX11-TRUE16: ; %bb.0: ; %main_body
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v4, v5, 0x5040100
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v2, v3, 0x5040100
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v0, v1, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v6, v4.l, v5.l
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v5, v2.l, v3.l
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v4, v0.l, v1.l
; GFX11-TRUE16-NEXT: image_sample_d_g16 v[0:3], v[4:6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
@@ -1359,7 +1359,7 @@ define amdgpu_ps <4 x float> @sample_d_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX11-TRUE16-LABEL: sample_d_3d:
; GFX11-TRUE16: ; %bb.0: ; %main_body
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v8.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, v6, v7, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v8, v6.l, v7.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v4.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
; GFX11-TRUE16-NEXT: image_sample_d_g16 v[0:3], [v0, v2, v3, v5, v[8:9]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_3D a16
@@ -1378,7 +1378,7 @@ define amdgpu_ps <4 x float> @sample_d_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX12PLUS-TRUE16-LABEL: sample_d_3d:
; GFX12PLUS-TRUE16: ; %bb.0: ; %main_body
; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v9.l, v8.l
-; GFX12PLUS-TRUE16-NEXT: v_perm_b32 v8, v6, v7, 0x5040100
+; GFX12PLUS-TRUE16-NEXT: v_pack_b32_f16 v8, v6.l, v7.l
; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.l
; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v3.h, v4.l
; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
@@ -1563,9 +1563,9 @@ define amdgpu_ps <4 x float> @sample_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX11-TRUE16-LABEL: sample_d_cl_2d:
; GFX11-TRUE16: ; %bb.0: ; %main_body
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v4, v5, 0x5040100
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v2, v3, 0x5040100
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v0, v1, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v6, v4.l, v5.l
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v5, v2.l, v3.l
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v4, v0.l, v1.l
; GFX11-TRUE16-NEXT: image_sample_d_cl_g16 v[0:3], v[4:7], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
@@ -1692,7 +1692,7 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX12PLUS-TRUE16-LABEL: sample_c_d_cl_2d:
; GFX12PLUS-TRUE16: ; %bb.0: ; %main_body
; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v8.l, v7.l
-; GFX12PLUS-TRUE16-NEXT: v_perm_b32 v7, v5, v6, 0x5040100
+; GFX12PLUS-TRUE16-NEXT: v_pack_b32_f16 v7, v5.l, v6.l
; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v3.h, v4.l
; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
; GFX12PLUS-TRUE16-NEXT: image_sample_c_d_cl_g16 v[0:3], [v0, v1, v3, v[7:8]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
@@ -1779,7 +1779,7 @@ define amdgpu_ps <4 x float> @sample_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX11-TRUE16-LABEL: sample_l_2d:
; GFX11-TRUE16: ; %bb.0: ; %main_body
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v0, v1, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v2, v0.l, v1.l
; GFX11-TRUE16-NEXT: image_sample_l v[0:3], v[2:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
@@ -2090,7 +2090,7 @@ define amdgpu_ps float @sample_c_d_o_2darray_V1(<8 x i32> inreg %rsrc, <4 x i32>
; GFX11-TRUE16-LABEL: sample_c_d_o_2darray_V1:
; GFX11-TRUE16: ; %bb.0: ; %main_body
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v8.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, v6, v7, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v8, v6.l, v7.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v5.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
; GFX11-TRUE16-NEXT: image_sample_c_d_o_g16 v0, [v0, v1, v2, v4, v[8:9]], s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_2D_ARRAY a16
@@ -2109,8 +2109,8 @@ define amdgpu_ps float @sample_c_d_o_2darray_V1(<8 x i32> inreg %rsrc, <4 x i32>
; GFX12PLUS-TRUE16-LABEL: sample_c_d_o_2darray_V1:
; GFX12PLUS-TRUE16: ; %bb.0: ; %main_body
; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v9.l, v8.l
-; GFX12PLUS-TRUE16-NEXT: v_perm_b32 v8, v6, v7, 0x5040100
-; GFX12PLUS-TRUE16-NEXT: v_perm_b32 v7, v4, v5, 0x5040100
+; GFX12PLUS-TRUE16-NEXT: v_pack_b32_f16 v8, v6.l, v7.l
+; GFX12PLUS-TRUE16-NEXT: v_pack_b32_f16 v7, v4.l, v5.l
; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
; GFX12PLUS-TRUE16-NEXT: image_sample_c_d_o_g16 v0, [v0, v1, v2, v[7:9]], s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_2D_ARRAY a16
; GFX12PLUS-TRUE16-NEXT: s_wait_samplecnt 0x0
@@ -2158,7 +2158,7 @@ define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4
; GFX11-TRUE16-LABEL: sample_c_d_o_2darray_V2:
; GFX11-TRUE16: ; %bb.0: ; %main_body
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v8.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, v6, v7, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v8, v6.l, v7.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v5.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
; GFX11-TRUE16-NEXT: image_sample_c_d_o_g16 v[0:1], [v0, v1, v2, v4, v[8:9]], s[0:7], s[8:11] dmask:0x6 dim:SQ_RSRC_IMG_2D_ARRAY a16
@@ -2177,8 +2177,8 @@ define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4
; GFX12PLUS-TRUE16-LABEL: sample_c_d_o_2darray_V2:
; GFX12PLUS-TRUE16: ; %bb.0: ; %main_body
; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v9.l, v8.l
-; GFX12PLUS-TRUE16-NEXT: v_perm_b32 v8, v6, v7, 0x5040100
-; GFX12PLUS-TRUE16-NEXT: v_perm_b32 v7, v4, v5, 0x5040100
+; GFX12PLUS-TRUE16-NEXT: v_pack_b32_f16 v8, v6.l, v7.l
+; GFX12PLUS-TRUE16-NEXT: v_pack_b32_f16 v7, v4.l, v5.l
; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
; GFX12PLUS-TRUE16-NEXT: image_sample_c_d_o_g16 v[0:1], [v0, v1, v2, v[7:9]], s[0:7], s[8:11] dmask:0x6 dim:SQ_RSRC_IMG_2D_ARRAY a16
; GFX12PLUS-TRUE16-NEXT: s_wait_samplecnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.encode.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.encode.ll
index 5da481091ffcf..a73b8f2f17e95 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.encode.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.encode.ll
@@ -513,7 +513,7 @@ define amdgpu_ps float @sample_c_d_o_2darray_V1(<8 x i32> inreg %rsrc, <4 x i32>
; GFX12-TRUE16: ; %bb.0: ; %main_body
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v9, v5 ; encoding: [0x05,0x03,0x12,0x7e]
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; encoding: [0x01,0x00,0x87,0xbf]
-; GFX12-TRUE16-NEXT: v_perm_b32 v5, v4, v9, 0x5040100 ; encoding: [0x05,0x00,0x44,0xd6,0x04,0x13,0xfe,0x03,0x00,0x01,0x04,0x05]
+; GFX12-TRUE16-NEXT: v_pack_b32_f16 v5, v4.l, v9.l ; encoding: [0x05,0x00,0x11,0xd7,0x04,0x13,0x02,0x02]
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l ; encoding: [0x03,0x39,0x04,0x7f]
; GFX12-TRUE16-NEXT: image_sample_c_d_o_g16 v0, [v0, v1, v2, v[5:8]], s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_2D_ARRAY ; encoding: [0x05,0x00,0x0f,0xe5,0x00,0x00,0x00,0x04,0x00,0x01,0x02,0x05]
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; encoding: [0x00,0x00,0xc2,0xbf]
@@ -531,7 +531,7 @@ define amdgpu_ps float @sample_c_d_o_2darray_V1(<8 x i32> inreg %rsrc, <4 x i32>
; GFX13-TRUE16: ; %bb.0: ; %main_body
; GFX13-TRUE16-NEXT: v_mov_b32_e32 v9, v5 ; encoding: [0x05,0x03,0x12,0x7e]
; GFX13-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; encoding: [0x01,0x00,0xae,0xbf]
-; GFX13-TRUE16-NEXT: v_perm_b32 v5, v4, v9, 0x5040100 ; encoding: [0x05,0x00,0x44,0xd7,0x04,0x13,0xfe,0x03,0x00,0x01,0x04,0x05]
+; GFX13-TRUE16-NEXT: v_pack_b32_f16 v5, v4.l, v9.l ; encoding: [0x05,0x00,0x11,0xd7,0x04,0x13,0x02,0x02]
; GFX13-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l ; encoding: [0x03,0x39,0x04,0x7f]
; GFX13-TRUE16-NEXT: image_sample_c_d_o_g16 v0, [v0, v1, v2, v[5:8]], s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_2D_ARRAY ; encoding: [0x05,0x80,0x2e,0xe5,0x00,0x00,0x00,0x04,0x00,0x01,0x02,0x05]
; GFX13-TRUE16-NEXT: s_wait_samplecnt 0x0 ; encoding: [0x00,0x00,0xc2,0xbf]
@@ -582,7 +582,7 @@ define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4
; GFX12-TRUE16: ; %bb.0: ; %main_body
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v9, v5 ; encoding: [0x05,0x03,0x12,0x7e]
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; encoding: [0x01,0x00,0x87,0xbf]
-; GFX12-TRUE16-NEXT: v_perm_b32 v5, v4, v9, 0x5040100 ; encoding: [0x05,0x00,0x44,0xd6,0x04,0x13,0xfe,0x03,0x00,0x01,0x04,0x05]
+; GFX12-TRUE16-NEXT: v_pack_b32_f16 v5, v4.l, v9.l ; encoding: [0x05,0x00,0x11,0xd7,0x04,0x13,0x02,0x02]
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l ; encoding: [0x03,0x39,0x04,0x7f]
; GFX12-TRUE16-NEXT: image_sample_c_d_o_g16 v[0:1], [v0, v1, v2, v[5:8]], s[0:7], s[8:11] dmask:0x6 dim:SQ_RSRC_IMG_2D_ARRAY ; encoding: [0x05,0x00,0x8f,0xe5,0x00,0x00,0x00,0x04,0x00,0x01,0x02,0x05]
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; encoding: [0x00,0x00,0xc2,0xbf]
@@ -600,7 +600,7 @@ define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4
; GFX13-TRUE16: ; %bb.0: ; %main_body
; GFX13-TRUE16-NEXT: v_mov_b32_e32 v9, v5 ; encoding: [0x05,0x03,0x12,0x7e]
; GFX13-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; encoding: [0x01,0x00,0xae,0xbf]
-; GFX13-TRUE16-NEXT: v_perm_b32 v5, v4, v9, 0x5040100 ; encoding: [0x05,0x00,0x44,0xd7,0x04,0x13,0xfe,0x03,0x00,0x01,0x04,0x05]
+; GFX13-TRUE16-NEXT: v_pack_b32_f16 v5, v4.l, v9.l ; encoding: [0x05,0x00,0x11,0xd7,0x04,0x13,0x02,0x02]
; GFX13-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l ; encoding: [0x03,0x39,0x04,0x7f]
; GFX13-TRUE16-NEXT: image_sample_c_d_o_g16 v[0:1], [v0, v1, v2, v[5:8]], s[0:7], s[8:11] dmask:0x6 dim:SQ_RSRC_IMG_2D_ARRAY ; encoding: [0x05,0x80,0xae,0xe5,0x00,0x00,0x00,0x04,0x00,0x01,0x02,0x05]
; GFX13-TRUE16-NEXT: s_wait_samplecnt 0x0 ; encoding: [0x00,0x00,0xc2,0xbf]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.ll
index b73592cc3ad32..eff5f0ff42f45 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.ll
@@ -360,7 +360,7 @@ define amdgpu_ps float @sample_c_d_o_2darray_V1(<8 x i32> inreg %rsrc, <4 x i32>
; GFX12PLUS-TRUE16-LABEL: sample_c_d_o_2darray_V1:
; GFX12PLUS-TRUE16: ; %bb.0: ; %main_body
; GFX12PLUS-TRUE16-NEXT: v_mov_b32_e32 v9, v5
-; GFX12PLUS-TRUE16-NEXT: v_perm_b32 v5, v4, v9, 0x5040100
+; GFX12PLUS-TRUE16-NEXT: v_pack_b32_f16 v5, v4.l, v9.l
; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
; GFX12PLUS-TRUE16-NEXT: image_sample_c_d_o_g16 v0, [v0, v1, v2, v[5:8]], s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_2D_ARRAY
; GFX12PLUS-TRUE16-NEXT: s_wait_samplecnt 0x0
@@ -410,7 +410,7 @@ define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4
; GFX12PLUS-TRUE16-LABEL: sample_c_d_o_2darray_V2:
; GFX12PLUS-TRUE16: ; %bb.0: ; %main_body
; GFX12PLUS-TRUE16-NEXT: v_mov_b32_e32 v9, v5
-; GFX12PLUS-TRUE16-NEXT: v_perm_b32 v5, v4, v9, 0x5040100
+; GFX12PLUS-TRUE16-NEXT: v_pack_b32_f16 v5, v4.l, v9.l
; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
; GFX12PLUS-TRUE16-NEXT: image_sample_c_d_o_g16 v[0:1], [v0, v1, v2, v[5:8]], s[0:7], s[8:11] dmask:0x6 dim:SQ_RSRC_IMG_2D_ARRAY
; GFX12PLUS-TRUE16-NEXT: s_wait_samplecnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.v3f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.v3f16.ll
index 7e8b2e12c3353..574b45472f9f1 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.v3f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.v3f16.ll
@@ -90,109 +90,57 @@ define amdgpu_gs void @main(<4 x i32> %arg, i32 %arg1) {
; GFX8-NEXT: s_mov_b32 m0, -1
; GFX8-NEXT: ds_write2_b32 v2, v0, v1 offset0:7 offset1:8
;
-; GFX11-TRUE16-LABEL: main:
-; GFX11-TRUE16: ; %bb.0: ; %bb
-; GFX11-TRUE16-NEXT: s_mov_b32 s4, exec_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_mov_b32 s5, s4
-; GFX11-TRUE16-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
-; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX11-TRUE16-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[0:3], 0 idxen
-; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4
-; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB0_1
-; GFX11-TRUE16-NEXT: ; %bb.2:
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s4
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v0, 16, v5
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_and_b32 v1, 0xffff, v6
-; GFX11-TRUE16-NEXT: ds_store_2addr_b32 v2, v0, v1 offset0:7 offset1:8
+; GFX11-LABEL: main:
+; GFX11: ; %bb.0: ; %bb
+; GFX11-NEXT: s_mov_b32 s4, exec_lo
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_mov_b32 s5, s4
+; GFX11-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[0:3], 0 idxen
+; GFX11-NEXT: s_and_not1_wrexec_b32 s5, s5
+; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX11-NEXT: ; implicit-def: $vgpr4
+; GFX11-NEXT: s_cbranch_execnz .LBB0_1
+; GFX11-NEXT: ; %bb.2:
+; GFX11-NEXT: s_mov_b32 exec_lo, s4
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v5
+; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_and_b32 v1, 0xffff, v6
+; GFX11-NEXT: ds_store_2addr_b32 v2, v0, v1 offset0:7 offset1:8
;
-; GFX11-FAKE16-LABEL: main:
-; GFX11-FAKE16: ; %bb.0: ; %bb
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, exec_lo
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_mov_b32 s5, s4
-; GFX11-FAKE16-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
-; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX11-FAKE16-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[0:3], 0 idxen
-; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr4
-; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB0_1
-; GFX11-FAKE16-NEXT: ; %bb.2:
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s4
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v5
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_and_b32 v1, 0xffff, v6
-; GFX11-FAKE16-NEXT: ds_store_2addr_b32 v2, v0, v1 offset0:7 offset1:8
-;
-; GFX12-TRUE16-LABEL: main:
-; GFX12-TRUE16: ; %bb.0: ; %bb
-; GFX12-TRUE16-NEXT: s_mov_b32 s4, exec_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-TRUE16-NEXT: s_mov_b32 s5, s4
-; GFX12-TRUE16-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
-; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[0:3], null idxen
-; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
-; GFX12-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX12-TRUE16-NEXT: ; implicit-def: $vgpr4
-; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB0_1
-; GFX12-TRUE16-NEXT: ; %bb.2:
-; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s4
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v0, 16, v5
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_and_b32 v1, 0xffff, v6
-; GFX12-TRUE16-NEXT: ds_store_2addr_b32 v2, v0, v1 offset0:7 offset1:8
-;
-; GFX12-FAKE16-LABEL: main:
-; GFX12-FAKE16: ; %bb.0: ; %bb
-; GFX12-FAKE16-NEXT: s_mov_b32 s4, exec_lo
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-FAKE16-NEXT: s_mov_b32 s5, s4
-; GFX12-FAKE16-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
-; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[0:3], null idxen
-; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
-; GFX12-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX12-FAKE16-NEXT: ; implicit-def: $vgpr4
-; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB0_1
-; GFX12-FAKE16-NEXT: ; %bb.2:
-; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s4
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v5
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_and_b32 v1, 0xffff, v6
-; GFX12-FAKE16-NEXT: ds_store_2addr_b32 v2, v0, v1 offset0:7 offset1:8
+; GFX12-LABEL: main:
+; GFX12: ; %bb.0: ; %bb
+; GFX12-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_mov_b32 s5, s4
+; GFX12-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-NEXT: v_readfirstlane_b32 s1, v1
+; GFX12-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[0:3], null idxen
+; GFX12-NEXT: s_and_not1_wrexec_b32 s5, s5
+; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX12-NEXT: ; implicit-def: $vgpr4
+; GFX12-NEXT: s_cbranch_execnz .LBB0_1
+; GFX12-NEXT: ; %bb.2:
+; GFX12-NEXT: s_mov_b32 exec_lo, s4
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, 16, v5
+; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_and_b32 v1, 0xffff, v6
+; GFX12-NEXT: ds_store_2addr_b32 v2, v0, v1 offset0:7 offset1:8
bb:
%i = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 poison)
%i2 = call nsz arcp <3 x half> @llvm.amdgcn.struct.buffer.load.format.v3f16(<4 x i32> %arg, i32 %arg1, i32 0, i32 0, i32 0)
@@ -217,5 +165,7 @@ declare <3 x half> @llvm.amdgcn.struct.buffer.load.format.v3f16(<4 x i32>, i32,
attributes #0 = { nounwind readnone willreturn }
attributes #1 = { nounwind readonly willreturn }
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GFX11: {{.*}}
-; GFX12: {{.*}}
+; GFX11-FAKE16: {{.*}}
+; GFX11-TRUE16: {{.*}}
+; GFX12-FAKE16: {{.*}}
+; GFX12-TRUE16: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.v3f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.v3f16.ll
index 5270e67304e8b..1db4db27e8841 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.v3f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.v3f16.ll
@@ -87,55 +87,30 @@ define amdgpu_gs void @main(ptr addrspace(8) %arg, i32 %arg1) {
; GFX8-NEXT: s_mov_b32 m0, -1
; GFX8-NEXT: ds_write2_b32 v2, v0, v1 offset0:7 offset1:8
;
-; GFX11-TRUE16-LABEL: main:
-; GFX11-TRUE16: ; %bb.0: ; %bb
-; GFX11-TRUE16-NEXT: s_mov_b32 s4, exec_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_mov_b32 s5, s4
-; GFX11-TRUE16-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
-; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX11-TRUE16-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[0:3], 0 idxen
-; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4
-; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB0_1
-; GFX11-TRUE16-NEXT: ; %bb.2:
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s4
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v0, 16, v5
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_and_b32 v1, 0xffff, v6
-; GFX11-TRUE16-NEXT: ds_store_2addr_b32 v2, v0, v1 offset0:7 offset1:8
-;
-; GFX11-FAKE16-LABEL: main:
-; GFX11-FAKE16: ; %bb.0: ; %bb
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, exec_lo
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_mov_b32 s5, s4
-; GFX11-FAKE16-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
-; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX11-FAKE16-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[0:3], 0 idxen
-; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr4
-; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB0_1
-; GFX11-FAKE16-NEXT: ; %bb.2:
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s4
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v5
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_and_b32 v1, 0xffff, v6
-; GFX11-FAKE16-NEXT: ds_store_2addr_b32 v2, v0, v1 offset0:7 offset1:8
+; GFX11-LABEL: main:
+; GFX11: ; %bb.0: ; %bb
+; GFX11-NEXT: s_mov_b32 s4, exec_lo
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_mov_b32 s5, s4
+; GFX11-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[0:3], 0 idxen
+; GFX11-NEXT: s_and_not1_wrexec_b32 s5, s5
+; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX11-NEXT: ; implicit-def: $vgpr4
+; GFX11-NEXT: s_cbranch_execnz .LBB0_1
+; GFX11-NEXT: ; %bb.2:
+; GFX11-NEXT: s_mov_b32 exec_lo, s4
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v5
+; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_and_b32 v1, 0xffff, v6
+; GFX11-NEXT: ds_store_2addr_b32 v2, v0, v1 offset0:7 offset1:8
bb:
%i = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 poison)
%i2 = call nsz arcp <3 x half> @llvm.amdgcn.struct.ptr.buffer.load.format.v3f16(ptr addrspace(8) %arg, i32 %arg1, i32 0, i32 0, i32 0)
@@ -160,4 +135,5 @@ declare <3 x half> @llvm.amdgcn.struct.ptr.buffer.load.format.v3f16(ptr addrspac
attributes #0 = { nounwind readnone willreturn }
attributes #1 = { nounwind readonly willreturn }
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GFX11: {{.*}}
+; GFX11-FAKE16: {{.*}}
+; GFX11-TRUE16: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.waitcnt.out.order.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.waitcnt.out.order.ll
index 3a3b6340b8cf5..1d8da982544b2 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.waitcnt.out.order.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.waitcnt.out.order.ll
@@ -86,7 +86,7 @@ define amdgpu_ps <3 x float> @sample_load(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX11-TRUE16-LABEL: sample_load:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v0, v1, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v2, v0.l, v1.l
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, 0
; GFX11-TRUE16-NEXT: image_msaa_load v[0:3], v[2:3], s[12:19] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm a16
; GFX11-TRUE16-NEXT: image_sample_lz v2, [v4, v4], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
@@ -105,7 +105,7 @@ define amdgpu_ps <3 x float> @sample_load(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX1150-TRUE16-LABEL: sample_load:
; GFX1150-TRUE16: ; %bb.0:
; GFX1150-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX1150-TRUE16-NEXT: v_perm_b32 v2, v0, v1, 0x5040100
+; GFX1150-TRUE16-NEXT: v_pack_b32_f16 v2, v0.l, v1.l
; GFX1150-TRUE16-NEXT: v_mov_b32_e32 v4, 0
; GFX1150-TRUE16-NEXT: image_msaa_load v[0:3], v[2:3], s[12:19] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm a16
; GFX1150-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -158,7 +158,7 @@ define amdgpu_ps <3 x float> @load_sample(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX11-TRUE16-LABEL: load_sample:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v0, v1, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v2, v0.l, v1.l
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, 0
; GFX11-TRUE16-NEXT: image_msaa_load v[0:3], v[2:3], s[12:19] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm a16
; GFX11-TRUE16-NEXT: image_sample_lz v2, [v4, v4], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
@@ -177,7 +177,7 @@ define amdgpu_ps <3 x float> @load_sample(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX1150-TRUE16-LABEL: load_sample:
; GFX1150-TRUE16: ; %bb.0:
; GFX1150-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX1150-TRUE16-NEXT: v_perm_b32 v2, v0, v1, 0x5040100
+; GFX1150-TRUE16-NEXT: v_pack_b32_f16 v2, v0.l, v1.l
; GFX1150-TRUE16-NEXT: v_mov_b32_e32 v4, 0
; GFX1150-TRUE16-NEXT: image_msaa_load v[0:3], v[2:3], s[12:19] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm a16
; GFX1150-TRUE16-NEXT: s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll
index 35dfb69d7acea..c90f93cded7c5 100644
--- a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll
@@ -3387,7 +3387,7 @@ define bfloat @local_atomic_fadd_ret_bf16(ptr addrspace(3) %ptr) nounwind {
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v3
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v0, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v3
@@ -3523,7 +3523,7 @@ define bfloat @local_atomic_fadd_ret_bf16(ptr addrspace(3) %ptr) nounwind {
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v0, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v3
@@ -3837,7 +3837,7 @@ define bfloat @local_atomic_fadd_ret_bf16__offset(ptr addrspace(3) %ptr) nounwin
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v3
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v1, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v3
@@ -3978,7 +3978,7 @@ define bfloat @local_atomic_fadd_ret_bf16__offset(ptr addrspace(3) %ptr) nounwin
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v1, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v3
@@ -4298,7 +4298,7 @@ define void @local_atomic_fadd_noret_bf16(ptr addrspace(3) %ptr) nounwind {
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v4
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v0, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v4, v2, v3, v4
@@ -4431,7 +4431,7 @@ define void @local_atomic_fadd_noret_bf16(ptr addrspace(3) %ptr) nounwind {
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v0, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v4, v2, v3, v4
@@ -4736,7 +4736,7 @@ define void @local_atomic_fadd_noret_bf16__offset(ptr addrspace(3) %ptr) nounwin
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v4
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v1, v5
; GFX12-TRUE16-NEXT: v_and_or_b32 v4, v3, v2, v4
@@ -4872,7 +4872,7 @@ define void @local_atomic_fadd_noret_bf16__offset(ptr addrspace(3) %ptr) nounwin
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v1, v5
; GFX11-TRUE16-NEXT: v_and_or_b32 v4, v3, v2, v4
@@ -5177,7 +5177,7 @@ define bfloat @local_atomic_fadd_ret_bf16__offset__align4(ptr addrspace(3) %ptr)
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v3, 16, v1
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v1, 0xffff0000, v2, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
@@ -5289,7 +5289,7 @@ define bfloat @local_atomic_fadd_ret_bf16__offset__align4(ptr addrspace(3) %ptr)
; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v3, 16, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v1, 0xffff0000, v2, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
@@ -5549,7 +5549,7 @@ define void @local_atomic_fadd_noret_bf16__offset__align4(ptr addrspace(3) %ptr)
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v3, 16, v2
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v2
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v1, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v1 offset:65534
@@ -5656,7 +5656,7 @@ define void @local_atomic_fadd_noret_bf16__offset__align4(ptr addrspace(3) %ptr)
; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v3, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v2
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v1, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v1 offset:65534
diff --git a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmax.ll
index f0b501588da28..f9851338a6165 100644
--- a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmax.ll
@@ -2979,7 +2979,7 @@ define bfloat @local_atomic_fmax_ret_bf16(ptr addrspace(3) %ptr) nounwind {
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v3
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v0, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v3
@@ -3115,7 +3115,7 @@ define bfloat @local_atomic_fmax_ret_bf16(ptr addrspace(3) %ptr) nounwind {
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v0, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v3
@@ -3431,7 +3431,7 @@ define bfloat @local_atomic_fmax_ret_bf16__offset(ptr addrspace(3) %ptr) nounwin
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v3
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v1, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v3
@@ -3572,7 +3572,7 @@ define bfloat @local_atomic_fmax_ret_bf16__offset(ptr addrspace(3) %ptr) nounwin
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v1, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v3
@@ -3894,7 +3894,7 @@ define void @local_atomic_fmax_noret_bf16(ptr addrspace(3) %ptr) nounwind {
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v4
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v0, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v4, v2, v3, v4
@@ -4027,7 +4027,7 @@ define void @local_atomic_fmax_noret_bf16(ptr addrspace(3) %ptr) nounwind {
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v0, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v4, v2, v3, v4
@@ -4334,7 +4334,7 @@ define void @local_atomic_fmax_noret_bf16__offset(ptr addrspace(3) %ptr) nounwin
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v4
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v1, v5
; GFX12-TRUE16-NEXT: v_and_or_b32 v4, v3, v2, v4
@@ -4470,7 +4470,7 @@ define void @local_atomic_fmax_noret_bf16__offset(ptr addrspace(3) %ptr) nounwin
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v1, v5
; GFX11-TRUE16-NEXT: v_and_or_b32 v4, v3, v2, v4
@@ -4777,7 +4777,7 @@ define bfloat @local_atomic_fmax_ret_bf16__offset__align4(ptr addrspace(3) %ptr)
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v3, 16, v1
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v1, 0xffff0000, v2, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
@@ -4889,7 +4889,7 @@ define bfloat @local_atomic_fmax_ret_bf16__offset__align4(ptr addrspace(3) %ptr)
; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v3, 16, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v1, 0xffff0000, v2, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
@@ -5151,7 +5151,7 @@ define void @local_atomic_fmax_noret_bf16__offset__align4(ptr addrspace(3) %ptr)
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v3, 16, v2
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v2
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v1, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v1 offset:65534
@@ -5258,7 +5258,7 @@ define void @local_atomic_fmax_noret_bf16__offset__align4(ptr addrspace(3) %ptr)
; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v3, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v2
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v1, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v1 offset:65534
diff --git a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmin.ll
index 8dc13425b2035..d7a3aa08333d0 100644
--- a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmin.ll
@@ -2979,7 +2979,7 @@ define bfloat @local_atomic_fmin_ret_bf16(ptr addrspace(3) %ptr) nounwind {
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v3
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v0, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v3
@@ -3115,7 +3115,7 @@ define bfloat @local_atomic_fmin_ret_bf16(ptr addrspace(3) %ptr) nounwind {
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v0, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v3
@@ -3431,7 +3431,7 @@ define bfloat @local_atomic_fmin_ret_bf16__offset(ptr addrspace(3) %ptr) nounwin
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v3
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v1, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v3
@@ -3572,7 +3572,7 @@ define bfloat @local_atomic_fmin_ret_bf16__offset(ptr addrspace(3) %ptr) nounwin
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v1, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v3
@@ -3894,7 +3894,7 @@ define void @local_atomic_fmin_noret_bf16(ptr addrspace(3) %ptr) nounwind {
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v4
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v0, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v4, v2, v3, v4
@@ -4027,7 +4027,7 @@ define void @local_atomic_fmin_noret_bf16(ptr addrspace(3) %ptr) nounwind {
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v0, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v4, v2, v3, v4
@@ -4334,7 +4334,7 @@ define void @local_atomic_fmin_noret_bf16__offset(ptr addrspace(3) %ptr) nounwin
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v4
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v1, v5
; GFX12-TRUE16-NEXT: v_and_or_b32 v4, v3, v2, v4
@@ -4470,7 +4470,7 @@ define void @local_atomic_fmin_noret_bf16__offset(ptr addrspace(3) %ptr) nounwin
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v1, v5
; GFX11-TRUE16-NEXT: v_and_or_b32 v4, v3, v2, v4
@@ -4777,7 +4777,7 @@ define bfloat @local_atomic_fmin_ret_bf16__offset__align4(ptr addrspace(3) %ptr)
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v3, 16, v1
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v1, 0xffff0000, v2, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
@@ -4889,7 +4889,7 @@ define bfloat @local_atomic_fmin_ret_bf16__offset__align4(ptr addrspace(3) %ptr)
; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v3, 16, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v1, 0xffff0000, v2, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
@@ -5151,7 +5151,7 @@ define void @local_atomic_fmin_noret_bf16__offset__align4(ptr addrspace(3) %ptr)
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v3, 16, v2
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v2
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v1, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v1 offset:65534
@@ -5258,7 +5258,7 @@ define void @local_atomic_fmin_noret_bf16__offset__align4(ptr addrspace(3) %ptr)
; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v3, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v2
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v1, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v1 offset:65534
diff --git a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fsub.ll b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fsub.ll
index 503b2922e0159..732a955e0d841 100644
--- a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fsub.ll
@@ -3849,7 +3849,7 @@ define bfloat @local_atomic_fsub_ret_bf16(ptr addrspace(3) %ptr) nounwind {
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v3
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v0, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v3
@@ -3985,7 +3985,7 @@ define bfloat @local_atomic_fsub_ret_bf16(ptr addrspace(3) %ptr) nounwind {
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v0, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v3
@@ -4299,7 +4299,7 @@ define bfloat @local_atomic_fsub_ret_bf16__offset(ptr addrspace(3) %ptr) nounwin
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v3
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v1, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v3
@@ -4440,7 +4440,7 @@ define bfloat @local_atomic_fsub_ret_bf16__offset(ptr addrspace(3) %ptr) nounwin
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v1, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v3
@@ -4760,7 +4760,7 @@ define void @local_atomic_fsub_noret_bf16(ptr addrspace(3) %ptr) nounwind {
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v4
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v0, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v4, v2, v3, v4
@@ -4893,7 +4893,7 @@ define void @local_atomic_fsub_noret_bf16(ptr addrspace(3) %ptr) nounwind {
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v0, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v4, v2, v3, v4
@@ -5198,7 +5198,7 @@ define void @local_atomic_fsub_noret_bf16__offset(ptr addrspace(3) %ptr) nounwin
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v4
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v1, v5
; GFX12-TRUE16-NEXT: v_and_or_b32 v4, v3, v2, v4
@@ -5334,7 +5334,7 @@ define void @local_atomic_fsub_noret_bf16__offset(ptr addrspace(3) %ptr) nounwin
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v5, 16, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v1, v5
; GFX11-TRUE16-NEXT: v_and_or_b32 v4, v3, v2, v4
@@ -5639,7 +5639,7 @@ define bfloat @local_atomic_fsub_ret_bf16__offset__align4(ptr addrspace(3) %ptr)
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v3, 16, v1
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v1, 0xffff0000, v2, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
@@ -5751,7 +5751,7 @@ define bfloat @local_atomic_fsub_ret_bf16__offset__align4(ptr addrspace(3) %ptr)
; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v3, 16, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v1, 0xffff0000, v2, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
@@ -6011,7 +6011,7 @@ define void @local_atomic_fsub_noret_bf16__offset__align4(ptr addrspace(3) %ptr)
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e64 v3, 16, v2
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v2
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v1, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v1 offset:65534
@@ -6118,7 +6118,7 @@ define void @local_atomic_fsub_noret_bf16__offset__align4(ptr addrspace(3) %ptr)
; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v3, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v2
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v1, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v1 offset:65534
diff --git a/llvm/test/CodeGen/AMDGPU/repeated-divisor.ll b/llvm/test/CodeGen/AMDGPU/repeated-divisor.ll
index cd9e40198284c..fccbcdb5cae81 100644
--- a/llvm/test/CodeGen/AMDGPU/repeated-divisor.ll
+++ b/llvm/test/CodeGen/AMDGPU/repeated-divisor.ll
@@ -981,7 +981,7 @@ define <6 x half> @v_repeat_divisor_v3f16_x2(<3 x half> %x, <3 x half> %y, <3 x
; GFX11-TRUE16-NEXT: v_pk_mul_f16 v0, v0, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v2, v3, v2, 16
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v2, v2.h, v3.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_repeat_divisor_v3f16_x2:
diff --git a/llvm/test/CodeGen/AMDGPU/scalar_to_vector.ll b/llvm/test/CodeGen/AMDGPU/scalar_to_vector.ll
index caa9c9095d7e1..682c441bf98ed 100644
--- a/llvm/test/CodeGen/AMDGPU/scalar_to_vector.ll
+++ b/llvm/test/CodeGen/AMDGPU/scalar_to_vector.ll
@@ -475,7 +475,7 @@ define i64 @bitcast_combine_scalar_to_vector_v4i16(i16 %arg) {
; GFX11-NEXT: v_lshrrev_b16 v1.l, 8, v1.l
; GFX11-NEXT: v_or_b16 v2.h, v1.l, v1.h
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_perm_b32 v1, v2, v2, 0x7060302
+; GFX11-NEXT: v_pack_b32_f16 v1, v2.h, v2.h
; GFX11-NEXT: v_mov_b32_e32 v0, v2
; GFX11-NEXT: s_setpc_b64 s[30:31]
%arg.cast = bitcast i16 %arg to <2 x i8>
diff --git a/llvm/test/CodeGen/AMDGPU/si-pre-emit-peephole-merge-v-mov-b16.mir b/llvm/test/CodeGen/AMDGPU/si-pre-emit-peephole-merge-v-mov-b16.mir
index 8a09c2c241436..fd0a89a21bbfc 100644
--- a/llvm/test/CodeGen/AMDGPU/si-pre-emit-peephole-merge-v-mov-b16.mir
+++ b/llvm/test/CodeGen/AMDGPU/si-pre-emit-peephole-merge-v-mov-b16.mir
@@ -16,7 +16,7 @@ body: |
; CHECK-LABEL: name: merge_hi_zero_lo_from_hi
; CHECK: liveins: $vgpr2
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: $vgpr0 = V_LSHRREV_B32_e64 16, $vgpr2, implicit $exec
+ ; CHECK-NEXT: $vgpr0 = V_LSHRREV_B32_e32 16, $vgpr2, implicit $exec
; CHECK-NEXT: S_ENDPGM 0, implicit $vgpr0
$vgpr0_lo16 = V_MOV_B16_t16_e32 $vgpr2_hi16, implicit $exec
$vgpr0_hi16 = V_MOV_B16_t16_e32 0, implicit $exec
@@ -35,7 +35,7 @@ body: |
; CHECK-LABEL: name: merge_lo_zero_hi_from_lo
; CHECK: liveins: $vgpr2
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: $vgpr0 = V_LSHLREV_B32_e64 16, $vgpr2, implicit $exec
+ ; CHECK-NEXT: $vgpr0 = V_LSHLREV_B32_e32 16, $vgpr2, implicit $exec
; CHECK-NEXT: S_ENDPGM 0, implicit $vgpr0
$vgpr0_lo16 = V_MOV_B16_t16_e32 0, implicit $exec
$vgpr0_hi16 = V_MOV_B16_t16_e32 $vgpr2_lo16, implicit $exec
@@ -44,7 +44,7 @@ body: |
---
# Pattern: v_mov_b16 v0.l, v2.l + v_mov_b16 v0.h, v3.l
-# => v_perm_b32 v0, v3, v2, 0x05040100
+# => v_pack_b32_f16 v0, v2.l, v3.l
name: merge_lo_from_lo_hi_from_lo
tracksRegLiveness: true
body: |
@@ -54,7 +54,7 @@ body: |
; CHECK-LABEL: name: merge_lo_from_lo_hi_from_lo
; CHECK: liveins: $vgpr2, $vgpr3
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: $vgpr0 = V_PERM_B32_e64 $vgpr2, $vgpr3, 84148480, implicit $exec
+ ; CHECK-NEXT: $vgpr0 = V_PACK_B32_F16_t16_e64 0, $vgpr2_lo16, 0, $vgpr3_lo16, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: S_ENDPGM 0, implicit $vgpr0
$vgpr0_lo16 = V_MOV_B16_t16_e32 $vgpr2_lo16, implicit $exec
$vgpr0_hi16 = V_MOV_B16_t16_e32 $vgpr3_lo16, implicit $exec
@@ -62,59 +62,59 @@ body: |
...
---
-# Pattern: v_mov_b16 v0.l, v2.l + v_mov_b16 v0.h, v3.h
-# => v_bfi_b32 v0, 0x0000ffff, v2, v3
-name: merge_lo_from_lo_hi_from_hi
+# Pattern: v_mov_b16 v0.l, v2.h + v_mov_b16 v0.h, v3.h
+# => v_pack_b32_f16 v0, v2.h, v3.h
+name: merge_lo_from_hi_hi_from_hi
tracksRegLiveness: true
body: |
bb.0:
liveins: $vgpr2, $vgpr3
- ; CHECK-LABEL: name: merge_lo_from_lo_hi_from_hi
+ ; CHECK-LABEL: name: merge_lo_from_hi_hi_from_hi
; CHECK: liveins: $vgpr2, $vgpr3
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: $vgpr0 = V_BFI_B32_e64 65535, $vgpr2, $vgpr3, implicit $exec
+ ; CHECK-NEXT: $vgpr0 = V_PACK_B32_F16_t16_e64 0, $vgpr2_hi16, 0, $vgpr3_hi16, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: S_ENDPGM 0, implicit $vgpr0
- $vgpr0_lo16 = V_MOV_B16_t16_e32 $vgpr2_lo16, implicit $exec
+ $vgpr0_lo16 = V_MOV_B16_t16_e32 $vgpr2_hi16, implicit $exec
$vgpr0_hi16 = V_MOV_B16_t16_e32 $vgpr3_hi16, implicit $exec
S_ENDPGM 0, implicit $vgpr0
...
---
-# Pattern: v_mov_b16 v0.l, v2.h + v_mov_b16 v0.h, v3.l
-# => v_perm_b32 v0, v2, v3, 0x01000706
-name: merge_lo_from_hi_hi_from_lo
+# Pattern: v_mov_b16 v0.l, v2.l + v_mov_b16 v0.h, v3.h
+# => v_pack_b32_f16 v0, v2.l, v3.h
+name: merge_lo_from_lo_hi_from_hi
tracksRegLiveness: true
body: |
bb.0:
liveins: $vgpr2, $vgpr3
- ; CHECK-LABEL: name: merge_lo_from_hi_hi_from_lo
+ ; CHECK-LABEL: name: merge_lo_from_lo_hi_from_hi
; CHECK: liveins: $vgpr2, $vgpr3
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: $vgpr0 = V_ALIGNBIT_B32_t16_e64 0, $vgpr3, 0, $vgpr2, 0, 16, 0, 0, implicit $exec
+ ; CHECK-NEXT: $vgpr0 = V_PACK_B32_F16_t16_e64 0, $vgpr2_lo16, 0, $vgpr3_hi16, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: S_ENDPGM 0, implicit $vgpr0
- $vgpr0_lo16 = V_MOV_B16_t16_e32 $vgpr2_hi16, implicit $exec
- $vgpr0_hi16 = V_MOV_B16_t16_e32 $vgpr3_lo16, implicit $exec
+ $vgpr0_lo16 = V_MOV_B16_t16_e32 $vgpr2_lo16, implicit $exec
+ $vgpr0_hi16 = V_MOV_B16_t16_e32 $vgpr3_hi16, implicit $exec
S_ENDPGM 0, implicit $vgpr0
...
---
-# Pattern: v_mov_b16 v0.l, v2.h + v_mov_b16 v0.h, v3.h
-# => v_perm_b32 v0, v2, v3, 0x03020706
-name: merge_lo_from_hi_hi_from_hi
+# Pattern: v_mov_b16 v0.l, v2.h + v_mov_b16 v0.h, v3.l
+# => v_pack_b32_f16 v0, v2.h, v3.l
+name: merge_lo_from_hi_hi_from_lo
tracksRegLiveness: true
body: |
bb.0:
liveins: $vgpr2, $vgpr3
- ; CHECK-LABEL: name: merge_lo_from_hi_hi_from_hi
+ ; CHECK-LABEL: name: merge_lo_from_hi_hi_from_lo
; CHECK: liveins: $vgpr2, $vgpr3
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: $vgpr0 = V_PERM_B32_e64 $vgpr2, $vgpr3, 117834498, implicit $exec
+ ; CHECK-NEXT: $vgpr0 = V_PACK_B32_F16_t16_e64 0, $vgpr2_hi16, 0, $vgpr3_lo16, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: S_ENDPGM 0, implicit $vgpr0
$vgpr0_lo16 = V_MOV_B16_t16_e32 $vgpr2_hi16, implicit $exec
- $vgpr0_hi16 = V_MOV_B16_t16_e32 $vgpr3_hi16, implicit $exec
+ $vgpr0_hi16 = V_MOV_B16_t16_e32 $vgpr3_lo16, implicit $exec
S_ENDPGM 0, implicit $vgpr0
...
@@ -130,7 +130,7 @@ body: |
; CHECK-LABEL: name: merge_lo_zero_hi_from_hi
; CHECK: liveins: $vgpr2
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: $vgpr0 = V_AND_B32_e64 4294901760, $vgpr2, implicit $exec
+ ; CHECK-NEXT: $vgpr0 = V_AND_B32_e32 4294901760, $vgpr2, implicit $exec
; CHECK-NEXT: S_ENDPGM 0, implicit $vgpr0
$vgpr0_lo16 = V_MOV_B16_t16_e32 0, implicit $exec
$vgpr0_hi16 = V_MOV_B16_t16_e32 $vgpr2_hi16, implicit $exec
@@ -149,7 +149,7 @@ body: |
; CHECK-LABEL: name: merge_hi_zero_lo_from_lo
; CHECK: liveins: $vgpr2
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: $vgpr0 = V_AND_B32_e64 65535, $vgpr2, implicit $exec
+ ; CHECK-NEXT: $vgpr0 = V_AND_B32_e32 65535, $vgpr2, implicit $exec
; CHECK-NEXT: S_ENDPGM 0, implicit $vgpr0
$vgpr0_lo16 = V_MOV_B16_t16_e32 $vgpr2_lo16, implicit $exec
$vgpr0_hi16 = V_MOV_B16_t16_e32 0, implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/sub.v2i16.ll b/llvm/test/CodeGen/AMDGPU/sub.v2i16.ll
index 332c31555acc8..0fca52a064214 100644
--- a/llvm/test/CodeGen/AMDGPU/sub.v2i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/sub.v2i16.ll
@@ -695,7 +695,7 @@ define amdgpu_kernel void @v_test_sub_v2i16_zext_to_v2i32(ptr addrspace(1) %out,
; GFX11-TRUE16-NEXT: v_pk_sub_i16 v2, v1, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v2
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v1, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; GFX11-TRUE16-NEXT: buffer_store_b64 v[0:1], off, s[0:3], 0
; GFX11-TRUE16-NEXT: s_endpgm
;
@@ -817,8 +817,8 @@ define amdgpu_kernel void @v_test_sub_v2i16_zext_to_v2i64(ptr addrspace(1) %out,
; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1
; GFX11-TRUE16-NEXT: v_pk_sub_i16 v0, v2, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v2, v1, v0, 16
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v2, v0.h, v1.l
; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX11-TRUE16-NEXT: buffer_store_b128 v[0:3], off, s[0:3], 0
; GFX11-TRUE16-NEXT: s_endpgm
@@ -1026,57 +1026,31 @@ define amdgpu_kernel void @v_test_sub_v2i16_sext_to_v2i64(ptr addrspace(1) %out,
; GFX10-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
; GFX10-NEXT: s_endpgm
;
-; GFX11-TRUE16-LABEL: v_test_sub_v2i16_sext_to_v2i64:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-TRUE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: global_load_b32 v1, v0, s[2:3]
-; GFX11-TRUE16-NEXT: global_load_b32 v0, v0, s[4:5]
-; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
-; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_pk_sub_i16 v0, v1, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v1, 16, v0
-; GFX11-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX11-TRUE16-NEXT: v_bfe_i32 v2, v1, 0, 16
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GFX11-TRUE16-NEXT: v_ashrrev_i32_e32 v3, 31, v2
-; GFX11-TRUE16-NEXT: buffer_store_b128 v[0:3], off, s[0:3], 0
-; GFX11-TRUE16-NEXT: s_endpgm
-;
-; GFX11-FAKE16-LABEL: v_test_sub_v2i16_sext_to_v2i64:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_clause 0x1
-; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-FAKE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_clause 0x1
-; GFX11-FAKE16-NEXT: global_load_b32 v1, v0, s[2:3]
-; GFX11-FAKE16-NEXT: global_load_b32 v0, v0, s[4:5]
-; GFX11-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
-; GFX11-FAKE16-NEXT: s_mov_b32 s2, -1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_pk_sub_i16 v0, v1, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX11-FAKE16-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX11-FAKE16-NEXT: v_bfe_i32 v2, v1, 0, 16
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GFX11-FAKE16-NEXT: v_ashrrev_i32_e32 v3, 31, v2
-; GFX11-FAKE16-NEXT: buffer_store_b128 v[0:3], off, s[0:3], 0
-; GFX11-FAKE16-NEXT: s_endpgm
+; GFX11-LABEL: v_test_sub_v2i16_sext_to_v2i64:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: global_load_b32 v1, v0, s[2:3]
+; GFX11-NEXT: global_load_b32 v0, v0, s[4:5]
+; GFX11-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-NEXT: s_mov_b32 s2, -1
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_pk_sub_i16 v0, v1, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-NEXT: v_bfe_i32 v0, v0, 0, 16
+; GFX11-NEXT: v_bfe_i32 v2, v1, 0, 16
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; GFX11-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX11-NEXT: buffer_store_b128 v[0:3], off, s[0:3], 0
+; GFX11-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%gep.out = getelementptr inbounds <2 x i64>, ptr addrspace(1) %out, i32 %tid
%gep.in0 = getelementptr inbounds <2 x i16>, ptr addrspace(1) %in0, i32 %tid
diff --git a/llvm/test/CodeGen/AMDGPU/v_swap_b16.ll b/llvm/test/CodeGen/AMDGPU/v_swap_b16.ll
index 57cf677f9e567..5f79ed74e7abf 100644
--- a/llvm/test/CodeGen/AMDGPU/v_swap_b16.ll
+++ b/llvm/test/CodeGen/AMDGPU/v_swap_b16.ll
@@ -8,7 +8,7 @@ define half @swap(half %a, half %b, i32 %i) {
; GFX11-TRUE16-LABEL: swap:
; GFX11-TRUE16: ; %bb.0: ; %entry
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v0, v1.l, v0.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: .LBB0_1: ; %loop
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -51,7 +51,7 @@ define half @swap(half %a, half %b, i32 %i) {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
+; GFX12-TRUE16-NEXT: v_pack_b32_f16 v0, v1.l, v0.l
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB0_1: ; %loop
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-and.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-and.ll
index 62bfa4b52559e..806c34bfc8400 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-and.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-and.ll
@@ -1102,71 +1102,27 @@ define i16 @test_vector_reduce_and_v3i16(<3 x i16> %v) {
; GFX10-NEXT: v_and_b32_e32 v0, v0, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_and_v3i16:
-; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
-; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e64 v2, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-SDAG-FAKE16-LABEL: test_vector_reduce_and_v3i16:
-; GFX11-SDAG-FAKE16: ; %bb.0: ; %entry
-; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-GISEL-LABEL: test_vector_reduce_and_v3i16:
-; GFX11-GISEL: ; %bb.0: ; %entry
-; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX11-GISEL-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_and_v3i16:
-; GFX12-SDAG-TRUE16: ; %bb.0: ; %entry
-; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: s_wait_expcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e64 v2, 16, v0
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-SDAG-FAKE16-LABEL: test_vector_reduce_and_v3i16:
-; GFX12-SDAG-FAKE16: ; %bb.0: ; %entry
-; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: s_wait_expcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: test_vector_reduce_and_v3i16:
+; GFX11: ; %bb.0: ; %entry
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX11-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-GISEL-LABEL: test_vector_reduce_and_v3i16:
-; GFX12-GISEL: ; %bb.0: ; %entry
-; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
-; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
-; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
-; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX12-GISEL-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX12-LABEL: test_vector_reduce_and_v3i16:
+; GFX12: ; %bb.0: ; %entry
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX12-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX12-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i16 @llvm.vector.reduce.and.v3i16(<3 x i16> %v)
ret i16 %res
@@ -1218,23 +1174,14 @@ define i16 @test_vector_reduce_and_v4i16(<4 x i16> %v) {
; GFX10-GISEL-NEXT: v_and_b32_e32 v0, v0, v1
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_and_v4i16:
-; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
-; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e64 v1, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-SDAG-FAKE16-LABEL: test_vector_reduce_and_v4i16:
-; GFX11-SDAG-FAKE16: ; %bb.0: ; %entry
-; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-SDAG-LABEL: test_vector_reduce_and_v4i16:
+; GFX11-SDAG: ; %bb.0: ; %entry
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-SDAG-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-LABEL: test_vector_reduce_and_v4i16:
; GFX11-GISEL: ; %bb.0: ; %entry
@@ -1245,31 +1192,18 @@ define i16 @test_vector_reduce_and_v4i16(<4 x i16> %v) {
; GFX11-GISEL-NEXT: v_and_b32_e32 v0, v0, v1
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_and_v4i16:
-; GFX12-SDAG-TRUE16: ; %bb.0: ; %entry
-; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: s_wait_expcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e64 v1, 16, v0
-; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-SDAG-FAKE16-LABEL: test_vector_reduce_and_v4i16:
-; GFX12-SDAG-FAKE16: ; %bb.0: ; %entry
-; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: s_wait_expcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX12-SDAG-LABEL: test_vector_reduce_and_v4i16:
+; GFX12-SDAG: ; %bb.0: ; %entry
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-SDAG-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-LABEL: test_vector_reduce_and_v4i16:
; GFX12-GISEL: ; %bb.0: ; %entry
@@ -1381,29 +1315,17 @@ define i16 @test_vector_reduce_and_v8i16(<8 x i16> %v) {
; GFX10-GISEL-NEXT: v_and_b32_e32 v0, v0, v1
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_and_v8i16:
-; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
-; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e64 v1, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-SDAG-FAKE16-LABEL: test_vector_reduce_and_v8i16:
-; GFX11-SDAG-FAKE16: ; %bb.0: ; %entry
-; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-SDAG-LABEL: test_vector_reduce_and_v8i16:
+; GFX11-SDAG: ; %bb.0: ; %entry
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX11-SDAG-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX11-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-LABEL: test_vector_reduce_and_v8i16:
; GFX11-GISEL: ; %bb.0: ; %entry
@@ -1417,37 +1339,21 @@ define i16 @test_vector_reduce_and_v8i16(<8 x i16> %v) {
; GFX11-GISEL-NEXT: v_and_b32_e32 v0, v0, v1
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_and_v8i16:
-; GFX12-SDAG-TRUE16: ; %bb.0: ; %entry
-; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: s_wait_expcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e64 v1, 16, v0
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-SDAG-FAKE16-LABEL: test_vector_reduce_and_v8i16:
-; GFX12-SDAG-FAKE16: ; %bb.0: ; %entry
-; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: s_wait_expcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX12-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX12-SDAG-LABEL: test_vector_reduce_and_v8i16:
+; GFX12-SDAG: ; %bb.0: ; %entry
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX12-SDAG-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX12-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-LABEL: test_vector_reduce_and_v8i16:
; GFX12-GISEL: ; %bb.0: ; %entry
@@ -1610,39 +1516,22 @@ define i16 @test_vector_reduce_and_v16i16(<16 x i16> %v) {
; GFX10-GISEL-NEXT: v_and_b32_e32 v0, v0, v1
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_and_v16i16:
-; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
-; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v2, v2, v6
-; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v3, v3, v7
-; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v1, v1, v5
-; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, v0, v4
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e64 v1, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-SDAG-FAKE16-LABEL: test_vector_reduce_and_v16i16:
-; GFX11-SDAG-FAKE16: ; %bb.0: ; %entry
-; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v2, v2, v6
-; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v3, v3, v7
-; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v1, v1, v5
-; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, v0, v4
-; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-SDAG-LABEL: test_vector_reduce_and_v16i16:
+; GFX11-SDAG: ; %bb.0: ; %entry
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX11-SDAG-NEXT: v_and_b32_e32 v3, v3, v7
+; GFX11-SDAG-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX11-SDAG-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX11-SDAG-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX11-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-LABEL: test_vector_reduce_and_v16i16:
; GFX11-GISEL: ; %bb.0: ; %entry
@@ -1661,47 +1550,26 @@ define i16 @test_vector_reduce_and_v16i16(<16 x i16> %v) {
; GFX11-GISEL-NEXT: v_and_b32_e32 v0, v0, v1
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_and_v16i16:
-; GFX12-SDAG-TRUE16: ; %bb.0: ; %entry
-; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: s_wait_expcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v2, v2, v6
-; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v3, v3, v7
-; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v1, v1, v5
-; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, v0, v4
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e64 v1, 16, v0
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-SDAG-FAKE16-LABEL: test_vector_reduce_and_v16i16:
-; GFX12-SDAG-FAKE16: ; %bb.0: ; %entry
-; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: s_wait_expcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v2, v2, v6
-; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v3, v3, v7
-; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v1, v1, v5
-; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, v0, v4
-; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v1, v1, v3
-; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX12-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX12-SDAG-LABEL: test_vector_reduce_and_v16i16:
+; GFX12-SDAG: ; %bb.0: ; %entry
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: v_and_b32_e32 v2, v2, v6
+; GFX12-SDAG-NEXT: v_and_b32_e32 v3, v3, v7
+; GFX12-SDAG-NEXT: v_and_b32_e32 v1, v1, v5
+; GFX12-SDAG-NEXT: v_and_b32_e32 v0, v0, v4
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-NEXT: v_and_b32_e32 v1, v1, v3
+; GFX12-SDAG-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX12-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-LABEL: test_vector_reduce_and_v16i16:
; GFX12-GISEL: ; %bb.0: ; %entry
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-or.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-or.ll
index f07115d24b870..0294a14a11266 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-or.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-or.ll
@@ -931,65 +931,25 @@ define i16 @test_vector_reduce_or_v3i16(<3 x i16> %v) {
; GFX10-NEXT: v_or3_b32 v0, v0, v2, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_or_v3i16:
-; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
-; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e64 v2, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_or3_b32 v0, v0, v2, v1
-; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-SDAG-FAKE16-LABEL: test_vector_reduce_or_v3i16:
-; GFX11-SDAG-FAKE16: ; %bb.0: ; %entry
-; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT: v_or3_b32 v0, v0, v2, v1
-; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-GISEL-LABEL: test_vector_reduce_or_v3i16:
-; GFX11-GISEL: ; %bb.0: ; %entry
-; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_or3_b32 v0, v0, v2, v1
-; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_or_v3i16:
-; GFX12-SDAG-TRUE16: ; %bb.0: ; %entry
-; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: s_wait_expcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e64 v2, 16, v0
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_or3_b32 v0, v0, v2, v1
-; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-SDAG-FAKE16-LABEL: test_vector_reduce_or_v3i16:
-; GFX12-SDAG-FAKE16: ; %bb.0: ; %entry
-; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: s_wait_expcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT: v_or3_b32 v0, v0, v2, v1
-; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: test_vector_reduce_or_v3i16:
+; GFX11: ; %bb.0: ; %entry
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_or3_b32 v0, v0, v2, v1
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-GISEL-LABEL: test_vector_reduce_or_v3i16:
-; GFX12-GISEL: ; %bb.0: ; %entry
-; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
-; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
-; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
-; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_or3_b32 v0, v0, v2, v1
-; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX12-LABEL: test_vector_reduce_or_v3i16:
+; GFX12: ; %bb.0: ; %entry
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_or3_b32 v0, v0, v2, v1
+; GFX12-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i16 @llvm.vector.reduce.or.v3i16(<3 x i16> %v)
ret i16 %res
@@ -1042,23 +1002,14 @@ define i16 @test_vector_reduce_or_v4i16(<4 x i16> %v) {
; GFX10-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_or_v4i16:
-; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
-; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e64 v1, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-SDAG-FAKE16-LABEL: test_vector_reduce_or_v4i16:
-; GFX11-SDAG-FAKE16: ; %bb.0: ; %entry
-; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-FAKE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX11-SDAG-FAKE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-SDAG-LABEL: test_vector_reduce_or_v4i16:
+; GFX11-SDAG: ; %bb.0: ; %entry
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-LABEL: test_vector_reduce_or_v4i16:
; GFX11-GISEL: ; %bb.0: ; %entry
@@ -1069,31 +1020,18 @@ define i16 @test_vector_reduce_or_v4i16(<4 x i16> %v) {
; GFX11-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_or_v4i16:
-; GFX12-SDAG-TRUE16: ; %bb.0: ; %entry
-; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: s_wait_expcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e64 v1, 16, v0
-; GFX12-SDAG-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-SDAG-FAKE16-LABEL: test_vector_reduce_or_v4i16:
-; GFX12-SDAG-FAKE16: ; %bb.0: ; %entry
-; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: s_wait_expcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX12-SDAG-FAKE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX12-SDAG-LABEL: test_vector_reduce_or_v4i16:
+; GFX12-SDAG: ; %bb.0: ; %entry
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-LABEL: test_vector_reduce_or_v4i16:
; GFX12-GISEL: ; %bb.0: ; %entry
@@ -1204,27 +1142,16 @@ define i16 @test_vector_reduce_or_v8i16(<8 x i16> %v) {
; GFX10-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_or_v8i16:
-; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
-; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_or3_b32 v0, v0, v2, v1
-; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e64 v1, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-SDAG-FAKE16-LABEL: test_vector_reduce_or_v8i16:
-; GFX11-SDAG-FAKE16: ; %bb.0: ; %entry
-; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-FAKE16-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT: v_or3_b32 v0, v0, v2, v1
-; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-SDAG-LABEL: test_vector_reduce_or_v8i16:
+; GFX11-SDAG: ; %bb.0: ; %entry
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_or3_b32 v0, v0, v2, v1
+; GFX11-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-LABEL: test_vector_reduce_or_v8i16:
; GFX11-GISEL: ; %bb.0: ; %entry
@@ -1238,35 +1165,20 @@ define i16 @test_vector_reduce_or_v8i16(<8 x i16> %v) {
; GFX11-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_or_v8i16:
-; GFX12-SDAG-TRUE16: ; %bb.0: ; %entry
-; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: s_wait_expcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_or3_b32 v0, v0, v2, v1
-; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e64 v1, 16, v0
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-SDAG-FAKE16-LABEL: test_vector_reduce_or_v8i16:
-; GFX12-SDAG-FAKE16: ; %bb.0: ; %entry
-; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: s_wait_expcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT: v_or3_b32 v0, v0, v2, v1
-; GFX12-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX12-SDAG-LABEL: test_vector_reduce_or_v8i16:
+; GFX12-SDAG: ; %bb.0: ; %entry
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_or3_b32 v0, v0, v2, v1
+; GFX12-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-LABEL: test_vector_reduce_or_v8i16:
; GFX12-GISEL: ; %bb.0: ; %entry
@@ -1426,33 +1338,19 @@ define i16 @test_vector_reduce_or_v16i16(<16 x i16> %v) {
; GFX10-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_or_v16i16:
-; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
-; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v3, v3, v7
-; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v2, v2, v6
-; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_or3_b32 v1, v1, v5, v3
-; GFX11-SDAG-TRUE16-NEXT: v_or3_b32 v0, v0, v2, v1
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e64 v1, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-SDAG-FAKE16-LABEL: test_vector_reduce_or_v16i16:
-; GFX11-SDAG-FAKE16: ; %bb.0: ; %entry
-; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-FAKE16-NEXT: v_or_b32_e32 v3, v3, v7
-; GFX11-SDAG-FAKE16-NEXT: v_or_b32_e32 v2, v2, v6
-; GFX11-SDAG-FAKE16-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT: v_or3_b32 v1, v1, v5, v3
-; GFX11-SDAG-FAKE16-NEXT: v_or3_b32 v0, v0, v2, v1
-; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX11-SDAG-FAKE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-SDAG-LABEL: test_vector_reduce_or_v16i16:
+; GFX11-SDAG: ; %bb.0: ; %entry
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX11-SDAG-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX11-SDAG-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_or3_b32 v1, v1, v5, v3
+; GFX11-SDAG-NEXT: v_or3_b32 v0, v0, v2, v1
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-LABEL: test_vector_reduce_or_v16i16:
; GFX11-GISEL: ; %bb.0: ; %entry
@@ -1471,41 +1369,23 @@ define i16 @test_vector_reduce_or_v16i16(<16 x i16> %v) {
; GFX11-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_or_v16i16:
-; GFX12-SDAG-TRUE16: ; %bb.0: ; %entry
-; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: s_wait_expcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_or_b32_e32 v3, v3, v7
-; GFX12-SDAG-TRUE16-NEXT: v_or_b32_e32 v2, v2, v6
-; GFX12-SDAG-TRUE16-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_or3_b32 v1, v1, v5, v3
-; GFX12-SDAG-TRUE16-NEXT: v_or3_b32 v0, v0, v2, v1
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e64 v1, 16, v0
-; GFX12-SDAG-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-SDAG-FAKE16-LABEL: test_vector_reduce_or_v16i16:
-; GFX12-SDAG-FAKE16: ; %bb.0: ; %entry
-; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: s_wait_expcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: v_or_b32_e32 v3, v3, v7
-; GFX12-SDAG-FAKE16-NEXT: v_or_b32_e32 v2, v2, v6
-; GFX12-SDAG-FAKE16-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT: v_or3_b32 v1, v1, v5, v3
-; GFX12-SDAG-FAKE16-NEXT: v_or3_b32 v0, v0, v2, v1
-; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX12-SDAG-FAKE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX12-SDAG-LABEL: test_vector_reduce_or_v16i16:
+; GFX12-SDAG: ; %bb.0: ; %entry
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX12-SDAG-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX12-SDAG-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_or3_b32 v1, v1, v5, v3
+; GFX12-SDAG-NEXT: v_or3_b32 v0, v0, v2, v1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-LABEL: test_vector_reduce_or_v16i16:
; GFX12-GISEL: ; %bb.0: ; %entry
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-xor.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-xor.ll
index 9cef96fb63ecc..b58b94763ccff 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-xor.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-xor.ll
@@ -1080,65 +1080,25 @@ define i16 @test_vector_reduce_xor_v3i16(<3 x i16> %v) {
; GFX10-NEXT: v_xor3_b32 v0, v0, v2, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_xor_v3i16:
-; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
-; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e64 v2, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_xor3_b32 v0, v0, v2, v1
-; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-SDAG-FAKE16-LABEL: test_vector_reduce_xor_v3i16:
-; GFX11-SDAG-FAKE16: ; %bb.0: ; %entry
-; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT: v_xor3_b32 v0, v0, v2, v1
-; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-GISEL-LABEL: test_vector_reduce_xor_v3i16:
-; GFX11-GISEL: ; %bb.0: ; %entry
-; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_xor3_b32 v0, v0, v2, v1
-; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_xor_v3i16:
-; GFX12-SDAG-TRUE16: ; %bb.0: ; %entry
-; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: s_wait_expcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e64 v2, 16, v0
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_xor3_b32 v0, v0, v2, v1
-; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-SDAG-FAKE16-LABEL: test_vector_reduce_xor_v3i16:
-; GFX12-SDAG-FAKE16: ; %bb.0: ; %entry
-; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: s_wait_expcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT: v_xor3_b32 v0, v0, v2, v1
-; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: test_vector_reduce_xor_v3i16:
+; GFX11: ; %bb.0: ; %entry
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_xor3_b32 v0, v0, v2, v1
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-GISEL-LABEL: test_vector_reduce_xor_v3i16:
-; GFX12-GISEL: ; %bb.0: ; %entry
-; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
-; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
-; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
-; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_xor3_b32 v0, v0, v2, v1
-; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX12-LABEL: test_vector_reduce_xor_v3i16:
+; GFX12: ; %bb.0: ; %entry
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_xor3_b32 v0, v0, v2, v1
+; GFX12-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i16 @llvm.vector.reduce.xor.v3i16(<3 x i16> %v)
ret i16 %res
@@ -1191,23 +1151,14 @@ define i16 @test_vector_reduce_xor_v4i16(<4 x i16> %v) {
; GFX10-GISEL-NEXT: v_xor_b32_e32 v0, v0, v1
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_xor_v4i16:
-; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
-; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_xor_b32_e32 v0, v0, v1
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e64 v1, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: v_xor_b32_e32 v0, v0, v1
-; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-SDAG-FAKE16-LABEL: test_vector_reduce_xor_v4i16:
-; GFX11-SDAG-FAKE16: ; %bb.0: ; %entry
-; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-FAKE16-NEXT: v_xor_b32_e32 v0, v0, v1
-; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX11-SDAG-FAKE16-NEXT: v_xor_b32_e32 v0, v0, v1
-; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-SDAG-LABEL: test_vector_reduce_xor_v4i16:
+; GFX11-SDAG: ; %bb.0: ; %entry
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-LABEL: test_vector_reduce_xor_v4i16:
; GFX11-GISEL: ; %bb.0: ; %entry
@@ -1218,31 +1169,18 @@ define i16 @test_vector_reduce_xor_v4i16(<4 x i16> %v) {
; GFX11-GISEL-NEXT: v_xor_b32_e32 v0, v0, v1
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_xor_v4i16:
-; GFX12-SDAG-TRUE16: ; %bb.0: ; %entry
-; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: s_wait_expcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_xor_b32_e32 v0, v0, v1
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e64 v1, 16, v0
-; GFX12-SDAG-TRUE16-NEXT: v_xor_b32_e32 v0, v0, v1
-; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-SDAG-FAKE16-LABEL: test_vector_reduce_xor_v4i16:
-; GFX12-SDAG-FAKE16: ; %bb.0: ; %entry
-; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: s_wait_expcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: v_xor_b32_e32 v0, v0, v1
-; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX12-SDAG-FAKE16-NEXT: v_xor_b32_e32 v0, v0, v1
-; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX12-SDAG-LABEL: test_vector_reduce_xor_v4i16:
+; GFX12-SDAG: ; %bb.0: ; %entry
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-LABEL: test_vector_reduce_xor_v4i16:
; GFX12-GISEL: ; %bb.0: ; %entry
@@ -1354,27 +1292,16 @@ define i16 @test_vector_reduce_xor_v8i16(<8 x i16> %v) {
; GFX10-GISEL-NEXT: v_xor_b32_e32 v0, v0, v1
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_xor_v8i16:
-; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
-; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_xor_b32_e32 v1, v1, v3
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_xor3_b32 v0, v0, v2, v1
-; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e64 v1, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_xor_b32_e32 v0, v0, v1
-; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-SDAG-FAKE16-LABEL: test_vector_reduce_xor_v8i16:
-; GFX11-SDAG-FAKE16: ; %bb.0: ; %entry
-; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-FAKE16-NEXT: v_xor_b32_e32 v1, v1, v3
-; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT: v_xor3_b32 v0, v0, v2, v1
-; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT: v_xor_b32_e32 v0, v0, v1
-; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-SDAG-LABEL: test_vector_reduce_xor_v8i16:
+; GFX11-SDAG: ; %bb.0: ; %entry
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_xor3_b32 v0, v0, v2, v1
+; GFX11-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-LABEL: test_vector_reduce_xor_v8i16:
; GFX11-GISEL: ; %bb.0: ; %entry
@@ -1388,35 +1315,20 @@ define i16 @test_vector_reduce_xor_v8i16(<8 x i16> %v) {
; GFX11-GISEL-NEXT: v_xor_b32_e32 v0, v0, v1
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_xor_v8i16:
-; GFX12-SDAG-TRUE16: ; %bb.0: ; %entry
-; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: s_wait_expcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_xor_b32_e32 v1, v1, v3
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_xor3_b32 v0, v0, v2, v1
-; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e64 v1, 16, v0
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_xor_b32_e32 v0, v0, v1
-; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-SDAG-FAKE16-LABEL: test_vector_reduce_xor_v8i16:
-; GFX12-SDAG-FAKE16: ; %bb.0: ; %entry
-; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: s_wait_expcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: v_xor_b32_e32 v1, v1, v3
-; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT: v_xor3_b32 v0, v0, v2, v1
-; GFX12-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT: v_xor_b32_e32 v0, v0, v1
-; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX12-SDAG-LABEL: test_vector_reduce_xor_v8i16:
+; GFX12-SDAG: ; %bb.0: ; %entry
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_xor3_b32 v0, v0, v2, v1
+; GFX12-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-LABEL: test_vector_reduce_xor_v8i16:
; GFX12-GISEL: ; %bb.0: ; %entry
@@ -1580,33 +1492,19 @@ define i16 @test_vector_reduce_xor_v16i16(<16 x i16> %v) {
; GFX10-GISEL-NEXT: v_xor_b32_e32 v0, v0, v1
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_xor_v16i16:
-; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
-; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_xor_b32_e32 v3, v3, v7
-; GFX11-SDAG-TRUE16-NEXT: v_xor_b32_e32 v2, v2, v6
-; GFX11-SDAG-TRUE16-NEXT: v_xor_b32_e32 v0, v0, v4
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_xor3_b32 v1, v1, v5, v3
-; GFX11-SDAG-TRUE16-NEXT: v_xor3_b32 v0, v0, v2, v1
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e64 v1, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: v_xor_b32_e32 v0, v0, v1
-; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-SDAG-FAKE16-LABEL: test_vector_reduce_xor_v16i16:
-; GFX11-SDAG-FAKE16: ; %bb.0: ; %entry
-; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-FAKE16-NEXT: v_xor_b32_e32 v3, v3, v7
-; GFX11-SDAG-FAKE16-NEXT: v_xor_b32_e32 v2, v2, v6
-; GFX11-SDAG-FAKE16-NEXT: v_xor_b32_e32 v0, v0, v4
-; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT: v_xor3_b32 v1, v1, v5, v3
-; GFX11-SDAG-FAKE16-NEXT: v_xor3_b32 v0, v0, v2, v1
-; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX11-SDAG-FAKE16-NEXT: v_xor_b32_e32 v0, v0, v1
-; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-SDAG-LABEL: test_vector_reduce_xor_v16i16:
+; GFX11-SDAG: ; %bb.0: ; %entry
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v3, v3, v7
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v0, v0, v4
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_xor3_b32 v1, v1, v5, v3
+; GFX11-SDAG-NEXT: v_xor3_b32 v0, v0, v2, v1
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-SDAG-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-LABEL: test_vector_reduce_xor_v16i16:
; GFX11-GISEL: ; %bb.0: ; %entry
@@ -1625,41 +1523,23 @@ define i16 @test_vector_reduce_xor_v16i16(<16 x i16> %v) {
; GFX11-GISEL-NEXT: v_xor_b32_e32 v0, v0, v1
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_xor_v16i16:
-; GFX12-SDAG-TRUE16: ; %bb.0: ; %entry
-; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: s_wait_expcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_xor_b32_e32 v3, v3, v7
-; GFX12-SDAG-TRUE16-NEXT: v_xor_b32_e32 v2, v2, v6
-; GFX12-SDAG-TRUE16-NEXT: v_xor_b32_e32 v0, v0, v4
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_xor3_b32 v1, v1, v5, v3
-; GFX12-SDAG-TRUE16-NEXT: v_xor3_b32 v0, v0, v2, v1
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e64 v1, 16, v0
-; GFX12-SDAG-TRUE16-NEXT: v_xor_b32_e32 v0, v0, v1
-; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-SDAG-FAKE16-LABEL: test_vector_reduce_xor_v16i16:
-; GFX12-SDAG-FAKE16: ; %bb.0: ; %entry
-; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: s_wait_expcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: v_xor_b32_e32 v3, v3, v7
-; GFX12-SDAG-FAKE16-NEXT: v_xor_b32_e32 v2, v2, v6
-; GFX12-SDAG-FAKE16-NEXT: v_xor_b32_e32 v0, v0, v4
-; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT: v_xor3_b32 v1, v1, v5, v3
-; GFX12-SDAG-FAKE16-NEXT: v_xor3_b32 v0, v0, v2, v1
-; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX12-SDAG-FAKE16-NEXT: v_xor_b32_e32 v0, v0, v1
-; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX12-SDAG-LABEL: test_vector_reduce_xor_v16i16:
+; GFX12-SDAG: ; %bb.0: ; %entry
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v3, v3, v7
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v2, v2, v6
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v0, v0, v4
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_xor3_b32 v1, v1, v5, v3
+; GFX12-SDAG-NEXT: v_xor3_b32 v0, v0, v2, v1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-SDAG-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-LABEL: test_vector_reduce_xor_v16i16:
; GFX12-GISEL: ; %bb.0: ; %entry
diff --git a/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll b/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll
index 5f19ee77bc237..cdc821bca8340 100644
--- a/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll
@@ -329,7 +329,7 @@ define <4 x half> @shuffle_v4f16_35u5(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
; GFX11-TRUE16-NEXT: global_load_b32 v1, v[2:3], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v1, 0x7060302
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v0, v0.h, v1.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -991,15 +991,25 @@ define <4 x half> @shuffle_v4f16_2356(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX10-NEXT: v_mov_b32_e32 v0, v4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: shuffle_v4f16_2356:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b64 v[2:3], v[2:3], off
-; GFX11-NEXT: global_load_b32 v0, v[0:1], off offset:4
-; GFX11-NEXT: s_waitcnt vmcnt(1)
-; GFX11-NEXT: v_alignbit_b32 v1, v3, v2, 16
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: shuffle_v4f16_2356:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: global_load_b64 v[2:3], v[2:3], off
+; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v1, v2.h, v3.l
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: shuffle_v4f16_2356:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: global_load_b64 v[2:3], v[2:3], off
+; GFX11-FAKE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
+; GFX11-FAKE16-NEXT: v_alignbit_b32 v1, v3, v2, 16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, ptr addrspace(1) %arg0
%val1 = load <4 x half>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 2, i32 3, i32 5, i32 6>
@@ -1040,15 +1050,25 @@ define <4 x half> @shuffle_v4f16_5623(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX10-NEXT: v_mov_b32_e32 v1, v4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: shuffle_v4f16_5623:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b64 v[2:3], v[2:3], off
-; GFX11-NEXT: global_load_b32 v1, v[0:1], off offset:4
-; GFX11-NEXT: s_waitcnt vmcnt(1)
-; GFX11-NEXT: v_alignbit_b32 v0, v3, v2, 16
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: shuffle_v4f16_5623:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: global_load_b64 v[2:3], v[2:3], off
+; GFX11-TRUE16-NEXT: global_load_b32 v1, v[0:1], off offset:4
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v0, v2.h, v3.l
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: shuffle_v4f16_5623:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: global_load_b64 v[2:3], v[2:3], off
+; GFX11-FAKE16-NEXT: global_load_b32 v1, v[0:1], off offset:4
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
+; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, v3, v2, 16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, ptr addrspace(1) %arg0
%val1 = load <4 x half>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 5, i32 6, i32 2, i32 3>
@@ -1084,9 +1104,9 @@ define <4 x half> @shuffle_v4f16_3456(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
; GFX11-TRUE16-NEXT: global_load_b64 v[1:2], v[2:3], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, v1, v0, 16
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v0, v0.h, v1.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v1, v2, v1, 16
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v1, v1.h, v2.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: shuffle_v4f16_3456:
@@ -1134,9 +1154,9 @@ define <4 x half> @shuffle_v4f16_5634(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
; GFX11-TRUE16-NEXT: global_load_b64 v[2:3], v[2:3], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v1, v2, v0, 16
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v1, v0.h, v2.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, v3, v2, 16
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v0, v2.h, v3.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: shuffle_v4f16_5634:
@@ -1197,7 +1217,7 @@ define <4 x half> @shuffle_v4f16_5734(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
; GFX11-TRUE16-NEXT: global_load_b64 v[2:3], v[2:3], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v1, v2, v0, 16
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v1, v0.h, v2.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -1254,15 +1274,25 @@ define <4 x i16> @shuffle_v4i16_2356(ptr addrspace(1) %arg0, ptr addrspace(1) %a
; GFX10-NEXT: v_mov_b32_e32 v0, v4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: shuffle_v4i16_2356:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b64 v[2:3], v[2:3], off
-; GFX11-NEXT: global_load_b32 v0, v[0:1], off offset:4
-; GFX11-NEXT: s_waitcnt vmcnt(1)
-; GFX11-NEXT: v_alignbit_b32 v1, v3, v2, 16
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: shuffle_v4i16_2356:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: global_load_b64 v[2:3], v[2:3], off
+; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v1, v2.h, v3.l
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: shuffle_v4i16_2356:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: global_load_b64 v[2:3], v[2:3], off
+; GFX11-FAKE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
+; GFX11-FAKE16-NEXT: v_alignbit_b32 v1, v3, v2, 16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x i16>, ptr addrspace(1) %arg0
%val1 = load <4 x i16>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x i16> %val0, <4 x i16> %val1, <4 x i32> <i32 2, i32 3, i32 5, i32 6>
@@ -1384,7 +1414,7 @@ define <4 x half> @shuffle_v4f16_1010(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b64 v[1:2], v[0:1], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, v1, v1, 16
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v0, v1.h, v1.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -1441,7 +1471,7 @@ define <4 x half> @shuffle_v4f16_1100(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b64 v[0:1], v[0:1], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v0, v0, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v1, v0.l, v0.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -1546,13 +1576,21 @@ define <4 x half> @shuffle_v4f16_2333(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX10-NEXT: v_perm_b32 v1, v0, v0, 0x7060302
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: shuffle_v4f16_2333:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b32 v0, v[0:1], off offset:4
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_perm_b32 v1, v0, v0, 0x7060302
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: shuffle_v4f16_2333:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v1, v0.h, v0.h
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: shuffle_v4f16_2333:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, v0, v0, 0x7060302
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, ptr addrspace(1) %arg0
%val1 = load <4 x half>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 2, i32 3, i32 3, i32 3>
@@ -1586,13 +1624,21 @@ define <4 x half> @shuffle_v4f16_6667(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX10-NEXT: v_perm_b32 v1, v0, v0, 0x7060302
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: shuffle_v4f16_6667:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b32 v0, v[0:1], off offset:4
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_perm_b32 v1, v0, v0, 0x7060302
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: shuffle_v4f16_6667:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v1, v0.h, v0.h
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: shuffle_v4f16_6667:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, v0, v0, 0x7060302
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, ptr addrspace(1) %arg0
%val1 = load <4 x half>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 2, i32 3, i32 3, i32 3>
@@ -1764,15 +1810,25 @@ define <4 x half> @shuffle_v8f16_13_14_2_3(ptr addrspace(1) %arg0, ptr addrspace
; GFX10-NEXT: v_mov_b32_e32 v1, v4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: shuffle_v8f16_13_14_2_3:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b64 v[2:3], v[2:3], off offset:8
-; GFX11-NEXT: global_load_b32 v1, v[0:1], off offset:4
-; GFX11-NEXT: s_waitcnt vmcnt(1)
-; GFX11-NEXT: v_alignbit_b32 v0, v3, v2, 16
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: shuffle_v8f16_13_14_2_3:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: global_load_b64 v[2:3], v[2:3], off offset:8
+; GFX11-TRUE16-NEXT: global_load_b32 v1, v[0:1], off offset:4
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v0, v2.h, v3.l
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: shuffle_v8f16_13_14_2_3:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: global_load_b64 v[2:3], v[2:3], off offset:8
+; GFX11-FAKE16-NEXT: global_load_b32 v1, v[0:1], off offset:4
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
+; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, v3, v2, 16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%val0 = load <8 x half>, ptr addrspace(1) %arg0
%val1 = load <8 x half>, ptr addrspace(1) %arg1
%shuffle = shufflevector <8 x half> %val0, <8 x half> %val1, <4 x i32> <i32 13, i32 14, i32 2, i32 3>
@@ -1844,13 +1900,21 @@ define <4 x half> @shuffle_v2f16_0122(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX10-NEXT: v_alignbit_b32 v1, v0, v0, 16
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: shuffle_v2f16_0122:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b32 v0, v[0:1], off
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_alignbit_b32 v1, v0, v0, 16
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: shuffle_v2f16_0122:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v1, v0.h, v0.l
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: shuffle_v2f16_0122:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: global_load_b32 v0, v[0:1], off
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_alignbit_b32 v1, v0, v0, 16
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%val0 = load <2 x half>, ptr addrspace(1) %arg0
%val1 = load <2 x half>, ptr addrspace(1) %arg1
%shuffle = shufflevector <2 x half> %val0, <2 x half> %val1, <4 x i32> <i32 0, i32 1, i32 1, i32 0>
@@ -2068,7 +2132,7 @@ define <4 x half> @shuffle_v4f16_0456(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-TRUE16-NEXT: global_load_b64 v[0:1], v[0:1], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v1, v3, v2, 16
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v1, v2.h, v3.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: shuffle_v4f16_0456:
@@ -2334,7 +2398,7 @@ define <2 x half> @hi16low16bits_v2bf16(ptr addrspace(1) %x0, ptr addrspace(1) %
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off
; GFX11-TRUE16-NEXT: global_load_b32 v1, v[2:3], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, v1, v0, 16
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v0, v0.h, v1.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -2489,7 +2553,7 @@ define <2 x i16> @i16_hi16low16bits(ptr addrspace(1) %x0, ptr addrspace(1) %x1)
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off
; GFX11-TRUE16-NEXT: global_load_b32 v1, v[2:3], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, v1, v0, 16
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v0, v0.h, v1.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -3537,7 +3601,7 @@ define <4 x bfloat> @shuffle_v4bf16_35u5(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
; GFX11-TRUE16-NEXT: global_load_b32 v1, v[2:3], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v1, 0x7060302
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v0, v0.h, v1.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -4199,15 +4263,25 @@ define <4 x bfloat> @shuffle_v4bf16_2356(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX10-NEXT: v_mov_b32_e32 v0, v4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: shuffle_v4bf16_2356:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b64 v[2:3], v[2:3], off
-; GFX11-NEXT: global_load_b32 v0, v[0:1], off offset:4
-; GFX11-NEXT: s_waitcnt vmcnt(1)
-; GFX11-NEXT: v_alignbit_b32 v1, v3, v2, 16
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: shuffle_v4bf16_2356:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: global_load_b64 v[2:3], v[2:3], off
+; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v1, v2.h, v3.l
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: shuffle_v4bf16_2356:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: global_load_b64 v[2:3], v[2:3], off
+; GFX11-FAKE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
+; GFX11-FAKE16-NEXT: v_alignbit_b32 v1, v3, v2, 16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <4 x bfloat>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> <i32 2, i32 3, i32 5, i32 6>
@@ -4248,15 +4322,25 @@ define <4 x bfloat> @shuffle_v4bf16_5623(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX10-NEXT: v_mov_b32_e32 v1, v4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: shuffle_v4bf16_5623:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b64 v[2:3], v[2:3], off
-; GFX11-NEXT: global_load_b32 v1, v[0:1], off offset:4
-; GFX11-NEXT: s_waitcnt vmcnt(1)
-; GFX11-NEXT: v_alignbit_b32 v0, v3, v2, 16
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: shuffle_v4bf16_5623:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: global_load_b64 v[2:3], v[2:3], off
+; GFX11-TRUE16-NEXT: global_load_b32 v1, v[0:1], off offset:4
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v0, v2.h, v3.l
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: shuffle_v4bf16_5623:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: global_load_b64 v[2:3], v[2:3], off
+; GFX11-FAKE16-NEXT: global_load_b32 v1, v[0:1], off offset:4
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
+; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, v3, v2, 16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <4 x bfloat>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> <i32 5, i32 6, i32 2, i32 3>
@@ -4292,9 +4376,9 @@ define <4 x bfloat> @shuffle_v4bf16_3456(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
; GFX11-TRUE16-NEXT: global_load_b64 v[1:2], v[2:3], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, v1, v0, 16
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v0, v0.h, v1.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v1, v2, v1, 16
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v1, v1.h, v2.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: shuffle_v4bf16_3456:
@@ -4342,9 +4426,9 @@ define <4 x bfloat> @shuffle_v4bf16_5634(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
; GFX11-TRUE16-NEXT: global_load_b64 v[2:3], v[2:3], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v1, v2, v0, 16
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v1, v0.h, v2.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, v3, v2, 16
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v0, v2.h, v3.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: shuffle_v4bf16_5634:
@@ -4405,7 +4489,7 @@ define <4 x bfloat> @shuffle_v4bf16_5734(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
; GFX11-TRUE16-NEXT: global_load_b64 v[2:3], v[2:3], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v1, v2, v0, 16
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v1, v0.h, v2.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -4507,7 +4591,7 @@ define <4 x bfloat> @shuffle_v4bf16_1010(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b64 v[1:2], v[0:1], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, v1, v1, 16
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v0, v1.h, v1.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -4564,7 +4648,7 @@ define <4 x bfloat> @shuffle_v4bf16_1100(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b64 v[0:1], v[0:1], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v0, v0, 0x5040100
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v1, v0.l, v0.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -4669,13 +4753,21 @@ define <4 x bfloat> @shuffle_v4bf16_2333(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX10-NEXT: v_perm_b32 v1, v0, v0, 0x7060302
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: shuffle_v4bf16_2333:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b32 v0, v[0:1], off offset:4
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_perm_b32 v1, v0, v0, 0x7060302
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: shuffle_v4bf16_2333:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v1, v0.h, v0.h
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: shuffle_v4bf16_2333:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, v0, v0, 0x7060302
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <4 x bfloat>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> <i32 2, i32 3, i32 3, i32 3>
@@ -4709,13 +4801,21 @@ define <4 x bfloat> @shuffle_v4bf16_6667(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX10-NEXT: v_perm_b32 v1, v0, v0, 0x7060302
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: shuffle_v4bf16_6667:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b32 v0, v[0:1], off offset:4
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_perm_b32 v1, v0, v0, 0x7060302
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: shuffle_v4bf16_6667:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v1, v0.h, v0.h
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: shuffle_v4bf16_6667:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, v0, v0, 0x7060302
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <4 x bfloat>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> <i32 2, i32 3, i32 3, i32 3>
@@ -4887,15 +4987,25 @@ define <4 x bfloat> @shuffle_v8bf16_13_14_2_3(ptr addrspace(1) %arg0, ptr addrsp
; GFX10-NEXT: v_mov_b32_e32 v1, v4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: shuffle_v8bf16_13_14_2_3:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b64 v[2:3], v[2:3], off offset:8
-; GFX11-NEXT: global_load_b32 v1, v[0:1], off offset:4
-; GFX11-NEXT: s_waitcnt vmcnt(1)
-; GFX11-NEXT: v_alignbit_b32 v0, v3, v2, 16
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: shuffle_v8bf16_13_14_2_3:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: global_load_b64 v[2:3], v[2:3], off offset:8
+; GFX11-TRUE16-NEXT: global_load_b32 v1, v[0:1], off offset:4
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v0, v2.h, v3.l
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: shuffle_v8bf16_13_14_2_3:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: global_load_b64 v[2:3], v[2:3], off offset:8
+; GFX11-FAKE16-NEXT: global_load_b32 v1, v[0:1], off offset:4
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
+; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, v3, v2, 16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%val0 = load <8 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <8 x bfloat>, ptr addrspace(1) %arg1
%shuffle = shufflevector <8 x bfloat> %val0, <8 x bfloat> %val1, <4 x i32> <i32 13, i32 14, i32 2, i32 3>
@@ -4967,13 +5077,21 @@ define <4 x bfloat> @shuffle_v2bf16_0122(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX10-NEXT: v_alignbit_b32 v1, v0, v0, 16
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: shuffle_v2bf16_0122:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b32 v0, v[0:1], off
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_alignbit_b32 v1, v0, v0, 16
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: shuffle_v2bf16_0122:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v1, v0.h, v0.l
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: shuffle_v2bf16_0122:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: global_load_b32 v0, v[0:1], off
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_alignbit_b32 v1, v0, v0, 16
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%val0 = load <2 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <2 x bfloat>, ptr addrspace(1) %arg1
%shuffle = shufflevector <2 x bfloat> %val0, <2 x bfloat> %val1, <4 x i32> <i32 0, i32 1, i32 1, i32 0>
@@ -5557,7 +5675,7 @@ define <4 x bfloat> @shuffle_v4bf16_0456(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-TRUE16-NEXT: global_load_b64 v[0:1], v[0:1], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v1, v3, v2, 16
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v1, v2.h, v3.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: shuffle_v4bf16_0456:
@@ -5765,7 +5883,7 @@ define <2 x bfloat> @hi16low16bits(ptr addrspace(1) %x0, ptr addrspace(1) %x1) {
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off
; GFX11-TRUE16-NEXT: global_load_b32 v1, v[2:3], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, v1, v0, 16
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v0, v0.h, v1.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
>From a07f87b8e7a86c3675b70d3c795860f80a2d99ee Mon Sep 17 00:00:00 2001
From: guochen2 <guochen2 at amd.com>
Date: Tue, 14 Jul 2026 11:48:01 -0400
Subject: [PATCH 3/5] address comment
---
llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp | 179 ++++++++++--------
.../AMDGPU/GlobalISel/store-local.128.ll | 19 +-
.../AMDGPU/GlobalISel/store-local.96.ll | 18 +-
llvm/test/CodeGen/AMDGPU/add.v2i16.ll | 2 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll | 26 ++-
.../CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll | 42 ++--
.../CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll | 5 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll | 7 +-
llvm/test/CodeGen/AMDGPU/bf16.ll | 32 ++--
llvm/test/CodeGen/AMDGPU/function-args.ll | 4 +-
.../AMDGPU/llvm.amdgcn.cvt.fp8.e5m3.ll | 5 +-
.../CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.ll | 60 +++---
....amdgcn.struct.buffer.load.format.v3f16.ll | 158 ++++++++++------
...gcn.struct.ptr.buffer.load.format.v3f16.ll | 76 +++++---
.../si-pre-emit-peephole-merge-v-mov-b16.mir | 176 ++++++++++++++---
llvm/test/CodeGen/AMDGPU/sub.v2i16.ll | 2 +-
16 files changed, 493 insertions(+), 318 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp b/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
index 8799f6236e0be..ed1910113f8af 100644
--- a/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
+++ b/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
@@ -44,9 +44,14 @@ class SIPreEmitPeephole {
void updateMLIBeforeRemovingEdge(MachineBasicBlock *From,
MachineBasicBlock *To) const;
bool optimizeSetGPR(MachineInstr &First, MachineInstr &MI) const;
+ void getMovB16Info(const MachineInstr &MI, const SIRegisterInfo *TRI,
+ MCRegister &SrcReg16, bool &SrcIsVGPR,
+ MCRegister &SrcReg32, bool &SrcIsHi, bool &SrcIsImm,
+ int64_t &ImmVal) const;
+
bool mergeSingleMovB16Pair(MachineInstr &Lo, MachineInstr &Hi,
bool IsHiFirst) const;
- bool mergeMovB16Pair(MachineFunction &MF) const;
+ bool mergeMovB16Pairs(MachineFunction &MF) const;
bool getBlockDestinations(MachineBasicBlock &SrcMBB,
MachineBasicBlock *&TrueMBB,
MachineBasicBlock *&FalseMBB,
@@ -767,50 +772,36 @@ MachineInstrBuilder SIPreEmitPeephole::createUnpackedMI(MachineInstr &I,
}
// Helper: extract the src operand and whether it is from the hi16 half.
-// Post-RA, both V_MOV_B16_t16_e32 and V_MOV_B16_t16_e64 use VGPR_16 physical
-// registers whose encoding already encodes hi/lo (IS_HI16 bit).
-// Returns false if the source is not a physical VGPR_16 or immediate zero.
-//
-// Operand layouts (post-RA, physical registers):
-// V_MOV_B16_t16_e32: dst(0), src0(1)
-// V_MOV_B16_t16_e64: dst(0), src0_mods(1), src0(2), op_sel(3)
-static bool getMovB16Info(const MachineInstr &MI, const SIRegisterInfo *TRI,
- MCRegister &SrcReg16, MCRegister &SrcReg32,
- bool &SrcIsHi, bool &SrcIsImm, int64_t &ImmVal) {
+// Post-RA, both V_MOV_B16_t16_e32 and V_MOV_B16_t16_e64 use VGPR_16 dst
+// physical registers whose encoding already encodes hi/lo (IS_HI16 bit).
+void SIPreEmitPeephole::getMovB16Info(const MachineInstr &MI,
+ const SIRegisterInfo *TRI,
+ MCRegister &SrcReg16, bool &SrcIsVGPR,
+ MCRegister &SrcReg32, bool &SrcIsHi,
+ bool &SrcIsImm, int64_t &ImmVal) const {
SrcIsImm = false;
SrcIsHi = false;
+ SrcIsVGPR = false;
SrcReg16 = MCRegister();
SrcReg32 = MCRegister();
- unsigned Opc = MI.getOpcode();
- const MachineOperand *SrcOp = nullptr;
-
- if (Opc == AMDGPU::V_MOV_B16_t16_e64)
- SrcOp = &MI.getOperand(2);
- else if (Opc == AMDGPU::V_MOV_B16_t16_e32)
- SrcOp = &MI.getOperand(1);
- else
- return false;
+ const MachineOperand *SrcOp = TII->getNamedOperand(MI, AMDGPU::OpName::src0);
if (SrcOp->isImm()) {
SrcIsImm = true;
ImmVal = SrcOp->getImm();
- return true;
+ return;
}
- if (!SrcOp->isReg() || !SrcOp->getReg().isPhysical())
- return false;
-
- MCRegister SrcReg = SrcOp->getReg().asMCReg();
-
- // We require the source to be a 16-bit VGPR so we can determine hi/lo.
- if (!AMDGPU::VGPR_16RegClass.contains(SrcReg))
- return false;
-
- SrcIsHi = AMDGPU::isHi16Reg(SrcReg, *TRI);
- SrcReg16 = SrcReg;
- SrcReg32 = TRI->get32BitRegister(SrcReg);
- return SrcReg32.isValid();
+ SrcReg16 = SrcOp->getReg().asMCReg();
+ SrcIsVGPR = AMDGPU::VGPR_16RegClass.contains(SrcReg16);
+ if (SrcIsVGPR) {
+ SrcIsHi = AMDGPU::isHi16Reg(SrcReg16, *TRI);
+ SrcReg32 = TRI->get32BitRegister(SrcReg16);
+ } else {
+ SrcIsHi = false;
+ SrcReg32 = SrcReg16;
+ }
}
// clang-format off
@@ -820,11 +811,11 @@ static bool getMovB16Info(const MachineInstr &MI, const SIRegisterInfo *TRI,
// Caller guarantee the pair to be two v_mov_b16 and targets the same dst32
//
// Patterns:
-// v_mov_b16 v0.h, 0 v_mov_b16 v0.l, v2.l => v_and_b32 v0,0xffff,v2
-// v_mov_b16 v0.h, 0 v_mov_b16 v0.l, v2.h => v_lshrrev_b32 v0,16,v2
-// v_mov_b16 v0.l, 0 v_mov_b16 v0.h, v2.l => v_lshlrev_b32 v0,16,v2
-// v_mov_b16 v0.l, 0 v_mov_b16 v0.h, v2.h => v_and_b32 v0,0xffff0000,v2
-// v_mov_b16 v0.l, v2.x v_mov_b16 v0.h, v3.y => v_pack_b32_f16 v0,v2.x,v3.y
+// v_mov_b16 v0.h, 0 v_mov_b16 v0.l, v2.l/s2 => v_and_b32 v0,0xffff,v2/s2
+// v_mov_b16 v0.h, 0 v_mov_b16 v0.l, v2.h => v_lshrrev_b32 v0,16,v2
+// v_mov_b16 v0.l, 0 v_mov_b16 v0.h, v2.l/s2 => v_lshlrev_b32 v0,16,v2/s2
+// v_mov_b16 v0.l, 0 v_mov_b16 v0.h, v2.h => v_and_b32 v0,0xffff0000,v2
+// v_mov_b16 v0.l, v.x/s v_mov_b16 v0.h, v.y/s => v_pack_b32_f16 v0, v/s, v/s
// clang-format on
bool SIPreEmitPeephole::mergeSingleMovB16Pair(MachineInstr &Lo,
MachineInstr &Hi,
@@ -835,43 +826,51 @@ bool SIPreEmitPeephole::mergeSingleMovB16Pair(MachineInstr &Lo,
// Extract source info for Lo and Hi.
MCRegister LoSrc16, LoSrc32, HiSrc16, HiSrc32;
- bool LoSrcIsHi, HiSrcIsHi, LoSrcIsImm, HiSrcIsImm;
+ bool LoSrcIsHi, HiSrcIsHi, LoSrcIsImm, HiSrcIsImm, LoSrcIsVGPR, HiSrcIsVGPR;
int64_t LoImm = 0, HiImm = 0;
- if (!getMovB16Info(Lo, TRI, LoSrc16, LoSrc32, LoSrcIsHi, LoSrcIsImm, LoImm))
- return false;
- if (!getMovB16Info(Hi, TRI, HiSrc16, HiSrc32, HiSrcIsHi, HiSrcIsImm, HiImm))
- return false;
+ getMovB16Info(Lo, TRI, LoSrc16, LoSrcIsVGPR, LoSrc32, LoSrcIsHi, LoSrcIsImm,
+ LoImm);
+ getMovB16Info(Hi, TRI, HiSrc16, HiSrcIsVGPR, HiSrc32, HiSrcIsHi, HiSrcIsImm,
+ HiImm);
MachineInstr &FirstMI = IsHiFirst ? Hi : Lo;
MachineInstr &SecondMI = IsHiFirst ? Lo : Hi;
- bool IsSecondImm = IsHiFirst ? LoSrcIsImm : HiSrcIsImm;
- MachineBasicBlock &MBB = *FirstMI.getParent();
- const DebugLoc &DL = FirstMI.getDebugLoc();
+ bool DataConflictOnFirst = false, DataConflictOnSecond = false;
+
+ MachineBasicBlock &MBB = *Lo.getParent();
// Check that between Lo and Hi, there are no instructions that:
- // - modify Dst32 (except through Lo/Hi themselves)
- // - modify LoSrc16 or HiSrc16 dependinig on order (data dependency)
+ // - modify Dst32
+ // - modify LoSrc16 or HiSrc16 depending on order (data dependency)
// We scan from the instruction after the first mov up to (but not including)
// the second mov.
+ MCRegister FirstSrc16 = IsHiFirst ? HiSrc16 : LoSrc16;
MCRegister SecondSrc16 = IsHiFirst ? LoSrc16 : HiSrc16;
- for (auto &It :
+ for (const MachineInstr &Scan :
drop_begin(make_range(FirstMI.getIterator(), SecondMI.getIterator()))) {
- const MachineInstr &Scan = It;
+
if (Scan.modifiesRegister(Dst32, TRI))
return false;
- if (!IsSecondImm && Scan.modifiesRegister(SecondSrc16, TRI))
+ DataConflictOnFirst |=
+ (FirstSrc16 && Scan.modifiesRegister(FirstSrc16, TRI));
+ DataConflictOnSecond |=
+ (SecondSrc16 && Scan.modifiesRegister(SecondSrc16, TRI));
+ if (DataConflictOnFirst && DataConflictOnSecond)
return false;
}
+ MachineInstr &Selected = DataConflictOnSecond ? SecondMI : FirstMI;
+ const DebugLoc &DL = Selected.getDebugLoc();
+
// Now match patterns and emit the replacement instruction.
- // Insert before the first (Lo) instruction, then remove both.
+ // Insert on Selected MI location, then remove both mov.
- // Pattern: v_mov_b16 v0.l, v2.x + v_mov_b16 v0.h, v3.y
- // => v_pack_b32_f16 v0,v2.x,v3.y
+ // Pattern: v_mov_b16 v0.l, v2.x/s2 + v_mov_b16 v0.h, v3.y/s3
+ // => v_pack_b32_f16 v0,v2.x/s2,v3.y/s3
if (!HiSrcIsImm && !LoSrcIsImm) {
- BuildMI(MBB, Lo, DL, TII->get(AMDGPU::V_PACK_B32_F16_t16_e64), Dst32)
+ BuildMI(MBB, Selected, DL, TII->get(AMDGPU::V_PACK_B32_F16_t16_e64), Dst32)
.addImm(0) // SrcMod
.addReg(LoSrc16)
.addImm(0) // SrcMod
@@ -883,10 +882,19 @@ bool SIPreEmitPeephole::mergeSingleMovB16Pair(MachineInstr &Lo,
return true;
}
- // Pattern: v_mov_b16 v0.h, 0 + v_mov_b16 v0.l, v2.l
- // => v_and_b32 v0, 0x0000ffff, v2
+ bool Usevop2 =
+ AMDGPU::VGPR_32_Lo128RegClass.contains(Dst32) &&
+ (LoSrcIsImm ||
+ (LoSrcIsVGPR && AMDGPU::VGPR_32_Lo128RegClass.contains(LoSrc32))) &&
+ (HiSrcIsImm ||
+ (HiSrcIsVGPR && AMDGPU::VGPR_32_Lo128RegClass.contains(HiSrc32)));
+
+ // Pattern: v_mov_b16 v0.h, 0 + v_mov_b16 v0.l, v2.l/s2
+ // => v_and_b32 v0, 0x0000ffff, v2/s2
if (HiSrcIsImm && HiImm == 0 && !LoSrcIsImm && !LoSrcIsHi) {
- BuildMI(MBB, Lo, DL, TII->get(AMDGPU::V_AND_B32_e32), Dst32)
+ BuildMI(MBB, Selected, DL,
+ TII->get(Usevop2 ? AMDGPU::V_AND_B32_e32 : AMDGPU::V_AND_B32_e64),
+ Dst32)
.addImm(0x0000ffff)
.addReg(LoSrc32);
Lo.eraseFromParent();
@@ -897,7 +905,10 @@ bool SIPreEmitPeephole::mergeSingleMovB16Pair(MachineInstr &Lo,
// Pattern: v_mov_b16 v0.h, 0 + v_mov_b16 v0.l, v2.h
// => v_lshrrev_b32 v0, 16, v2
if (HiSrcIsImm && HiImm == 0 && !LoSrcIsImm && LoSrcIsHi) {
- BuildMI(MBB, Lo, DL, TII->get(AMDGPU::V_LSHRREV_B32_e32), Dst32)
+ BuildMI(MBB, Selected, DL,
+ TII->get(Usevop2 ? AMDGPU::V_LSHRREV_B32_e32
+ : AMDGPU::V_LSHRREV_B32_e64),
+ Dst32)
.addImm(16)
.addReg(LoSrc32);
Lo.eraseFromParent();
@@ -905,10 +916,13 @@ bool SIPreEmitPeephole::mergeSingleMovB16Pair(MachineInstr &Lo,
return true;
}
- // Pattern: v_mov_b16 v0.l, 0 + v_mov_b16 v0.h, v2.l
- // => v_lshlrev_b32 v0, 16, v2
+ // Pattern: v_mov_b16 v0.l, 0 + v_mov_b16 v0.h, v2.l/s2
+ // => v_lshlrev_b32 v0, 16, v2/s2
if (LoSrcIsImm && LoImm == 0 && !HiSrcIsImm && !HiSrcIsHi) {
- BuildMI(MBB, Lo, DL, TII->get(AMDGPU::V_LSHLREV_B32_e32), Dst32)
+ BuildMI(MBB, Selected, DL,
+ TII->get(Usevop2 ? AMDGPU::V_LSHLREV_B32_e32
+ : AMDGPU::V_LSHLREV_B32_e64),
+ Dst32)
.addImm(16)
.addReg(HiSrc32);
Lo.eraseFromParent();
@@ -919,7 +933,9 @@ bool SIPreEmitPeephole::mergeSingleMovB16Pair(MachineInstr &Lo,
// Pattern: v_mov_b16 v0.l, 0 + v_mov_b16 v0.h, v2.h
// => v_and_b32 v0, 0xffff0000, v2
if (LoSrcIsImm && LoImm == 0 && !HiSrcIsImm && HiSrcIsHi) {
- BuildMI(MBB, Lo, DL, TII->get(AMDGPU::V_AND_B32_e32), Dst32)
+ BuildMI(MBB, Selected, DL,
+ TII->get(Usevop2 ? AMDGPU::V_AND_B32_e32 : AMDGPU::V_AND_B32_e64),
+ Dst32)
.addImm(0xffff0000)
.addReg(HiSrc32);
Lo.eraseFromParent();
@@ -932,12 +948,12 @@ bool SIPreEmitPeephole::mergeSingleMovB16Pair(MachineInstr &Lo,
// Merge pairs of v_mov_b16 targeting the lo16 and hi16 halves of the same
// VGPR into a single 32-bit instruction (true16 mode only).
-bool SIPreEmitPeephole::mergeMovB16Pair(MachineFunction &MF) const {
+bool SIPreEmitPeephole::mergeMovB16Pairs(MachineFunction &MF) const {
bool Changed = false;
for (MachineBasicBlock &MBB : MF) {
// Map from 32-bit VGPR to the pending v_mov_b16 and its age.
// Age tracks how many non-mov-b16 instructions have passed since the
- // lo16 write, used to bound the search window.
+ // 16-bit write, used to bound the search window.
struct Pending {
MachineInstr *MI;
unsigned Age; // instructions since was seen
@@ -948,14 +964,17 @@ bool SIPreEmitPeephole::mergeMovB16Pair(MachineFunction &MF) const {
SmallDenseMap<MCRegister, Pending> PendingWrites;
for (auto &MI : make_early_inc_range(MBB)) {
+ if (MI.isDebugInstr())
+ continue;
+
unsigned Opc = MI.getOpcode();
bool IsMovB16 = (Opc == AMDGPU::V_MOV_B16_t16_e32 ||
Opc == AMDGPU::V_MOV_B16_t16_e64);
if (!IsMovB16) {
- // Age all pending lo writes and invalidate stale or clobbered ones.
- for (auto &[key, value] : PendingWrites)
- value.Age++;
+ // Age all pending writes and invalidate stale or clobbered ones.
+ for (auto &[key, Value] : PendingWrites)
+ Value.Age++;
PendingWrites.remove_if([&](const auto &KV) {
return (KV.second.Age >= ScanLimit ||
@@ -968,20 +987,24 @@ bool SIPreEmitPeephole::mergeMovB16Pair(MachineFunction &MF) const {
bool DstIsHi = AMDGPU::isHi16Reg(DstReg, *TRI);
MCRegister Dst32 = TRI->get32BitRegister(DstReg);
- if (!Dst32.isValid())
- continue;
- auto It = PendingWrites.find(Dst32);
- if (It != PendingWrites.end() && It->second.IsHi != DstIsHi) {
+ auto [It, Inserted] = PendingWrites.insert({Dst32, {&MI, 0, DstIsHi}});
+ if (!Inserted) {
+ if (It->second.IsHi == DstIsHi) {
+ It->second = {&MI, 0, DstIsHi};
+ continue;
+ }
+
// Look for a matching pending write.
MachineInstr &LoMI = !DstIsHi ? MI : *It->second.MI;
MachineInstr &HiMI = DstIsHi ? MI : *It->second.MI;
bool IsHiFirst = It->second.IsHi;
- if (mergeSingleMovB16Pair(LoMI, HiMI, IsHiFirst))
+ if (mergeSingleMovB16Pair(LoMI, HiMI, IsHiFirst)) {
Changed = true;
- PendingWrites.erase(It);
- } else {
- PendingWrites[Dst32] = {&MI, 0, DstIsHi};
+ PendingWrites.erase(It);
+ } else {
+ It->second = {&MI, 0, DstIsHi};
+ }
}
}
}
@@ -1064,7 +1087,7 @@ bool SIPreEmitPeephole::run(MachineFunction &MF, MachineLoopInfo *LoopInfo) {
// Try merge B16 Pair in true16 mode
if (ST.useRealTrue16Insts())
- Changed |= mergeMovB16Pair(MF);
+ Changed |= mergeMovB16Pairs(MF);
// TODO: Fold this into previous block, if possible. Evaluate and handle any
// side effects.
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.128.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.128.ll
index f13c387042b08..a6e36f4f00428 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.128.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.128.ll
@@ -239,23 +239,21 @@ define amdgpu_kernel void @store_lds_v4i32_align1(ptr addrspace(3) %out, <4 x i3
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_and_b32 s6, 0xffff, s0
; GFX11-NEXT: s_lshr_b32 s5, s0, 16
-; GFX11-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX11-NEXT: v_pack_b32_f16 v0, s0, s1
; GFX11-NEXT: s_lshr_b32 s0, s0, 24
; GFX11-NEXT: s_and_b32 s7, 0xffff, s1
; GFX11-NEXT: s_lshr_b32 s6, s6, 8
; GFX11-NEXT: v_mov_b32_e32 v5, s4
; GFX11-NEXT: v_mov_b16_e32 v2.l, s0
; GFX11-NEXT: s_lshr_b32 s0, s7, 8
-; GFX11-NEXT: v_mov_b16_e32 v3.l, s6
; GFX11-NEXT: v_mov_b16_e32 v1.h, s5
; GFX11-NEXT: s_lshr_b32 s4, s1, 16
-; GFX11-NEXT: v_mov_b16_e32 v0.h, s1
; GFX11-NEXT: s_lshr_b32 s1, s1, 24
; GFX11-NEXT: s_and_b32 s9, 0xffff, s2
; GFX11-NEXT: v_mov_b16_e32 v4.l, s0
; GFX11-NEXT: s_lshr_b32 s0, s2, 24
; GFX11-NEXT: v_mov_b16_e32 v2.h, s4
-; GFX11-NEXT: v_mov_b16_e32 v3.h, s1
+; GFX11-NEXT: v_pack_b32_f16 v3, s6, s1
; GFX11-NEXT: s_lshr_b32 s1, s9, 8
; GFX11-NEXT: ds_store_b8 v5, v0
; GFX11-NEXT: ds_store_b8 v5, v3 offset:1
@@ -265,19 +263,16 @@ define amdgpu_kernel void @store_lds_v4i32_align1(ptr addrspace(3) %out, <4 x i3
; GFX11-NEXT: ds_store_b8 v5, v4 offset:5
; GFX11-NEXT: ds_store_b8_d16_hi v5, v2 offset:6
; GFX11-NEXT: ds_store_b8_d16_hi v5, v3 offset:7
-; GFX11-NEXT: v_mov_b16_e32 v1.h, s0
+; GFX11-NEXT: v_pack_b32_f16 v1, s2, s0
; GFX11-NEXT: s_and_b32 s0, 0xffff, s3
; GFX11-NEXT: s_lshr_b32 s8, s2, 16
-; GFX11-NEXT: v_mov_b16_e32 v1.l, s2
-; GFX11-NEXT: v_mov_b16_e32 v0.l, s1
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_pack_b32_f16 v0, s1, s8
; GFX11-NEXT: s_lshr_b32 s0, s0, 8
-; GFX11-NEXT: v_mov_b16_e32 v0.h, s8
; GFX11-NEXT: s_lshr_b32 s1, s3, 16
-; GFX11-NEXT: v_mov_b16_e32 v2.h, s0
+; GFX11-NEXT: v_pack_b32_f16 v2, s3, s0
; GFX11-NEXT: s_lshr_b32 s0, s3, 24
-; GFX11-NEXT: v_mov_b16_e32 v2.l, s3
-; GFX11-NEXT: v_mov_b16_e32 v3.l, s1
-; GFX11-NEXT: v_mov_b16_e32 v3.h, s0
+; GFX11-NEXT: v_pack_b32_f16 v3, s1, s0
; GFX11-NEXT: ds_store_b8 v5, v1 offset:8
; GFX11-NEXT: ds_store_b8 v5, v0 offset:9
; GFX11-NEXT: ds_store_b8_d16_hi v5, v0 offset:10
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.96.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.96.ll
index 8cc2985728255..b6c29129bad3d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.96.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.96.ll
@@ -225,29 +225,23 @@ define amdgpu_kernel void @store_lds_v3i32_align1(ptr addrspace(3) %out, <3 x i3
; GFX11-NEXT: s_and_b32 s5, 0xffff, s0
; GFX11-NEXT: s_lshr_b32 s4, s0, 16
; GFX11-NEXT: s_lshr_b32 s5, s5, 8
-; GFX11-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX11-NEXT: v_pack_b32_f16 v0, s0, s1
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v6, s3
; GFX11-NEXT: s_lshr_b32 s0, s0, 24
; GFX11-NEXT: s_lshr_b32 s3, s1, 16
; GFX11-NEXT: s_and_b32 s6, 0xffff, s1
-; GFX11-NEXT: v_mov_b16_e32 v0.h, s1
; GFX11-NEXT: s_lshr_b32 s1, s1, 24
; GFX11-NEXT: s_and_b32 s8, 0xffff, s2
-; GFX11-NEXT: v_mov_b16_e32 v4.l, s5
-; GFX11-NEXT: v_mov_b16_e32 v1.h, s4
+; GFX11-NEXT: v_pack_b32_f16 v1, s2, s4
; GFX11-NEXT: s_lshr_b32 s7, s2, 16
-; GFX11-NEXT: v_mov_b16_e32 v1.l, s2
; GFX11-NEXT: s_lshr_b32 s2, s2, 24
-; GFX11-NEXT: v_mov_b16_e32 v2.l, s0
+; GFX11-NEXT: v_pack_b32_f16 v2, s0, s3
; GFX11-NEXT: s_lshr_b32 s0, s6, 8
-; GFX11-NEXT: v_mov_b16_e32 v3.l, s1
+; GFX11-NEXT: v_pack_b32_f16 v3, s1, s7
; GFX11-NEXT: s_lshr_b32 s1, s8, 8
-; GFX11-NEXT: v_mov_b16_e32 v2.h, s3
-; GFX11-NEXT: v_mov_b16_e32 v3.h, s7
-; GFX11-NEXT: v_mov_b16_e32 v4.h, s2
-; GFX11-NEXT: v_mov_b16_e32 v5.l, s0
-; GFX11-NEXT: v_mov_b16_e32 v5.h, s1
+; GFX11-NEXT: v_pack_b32_f16 v4, s5, s2
+; GFX11-NEXT: v_pack_b32_f16 v5, s0, s1
; GFX11-NEXT: ds_store_b8 v6, v0
; GFX11-NEXT: ds_store_b8 v6, v4 offset:1
; GFX11-NEXT: ds_store_b8_d16_hi v6, v1 offset:2
diff --git a/llvm/test/CodeGen/AMDGPU/add.v2i16.ll b/llvm/test/CodeGen/AMDGPU/add.v2i16.ll
index a54223795c640..cc938af3ef231 100644
--- a/llvm/test/CodeGen/AMDGPU/add.v2i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/add.v2i16.ll
@@ -672,8 +672,8 @@ define amdgpu_kernel void @v_test_add_v2i16_zext_to_v2i32(ptr addrspace(1) %out,
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_pk_add_u16 v2, v1, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v2
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v2
; GFX11-TRUE16-NEXT: global_store_b64 v3, v[0:1], s[0:1]
; GFX11-TRUE16-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll
index 72a4c2376fa3f..7305af8092f9e 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll
@@ -26073,28 +26073,27 @@ define inreg <16 x i8> @bitcast_v8bf16_to_v16i8_scalar(<8 x bfloat> inreg %a, i3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
; GFX11-TRUE16-NEXT: v_add_f32_e64 v4, 0x40c00000, s0
; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s0, 0, s2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v17.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v9, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v10, v10, v9
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v10, 0x7fff, v10
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x7fff, v6
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, v8, v5
; GFX11-TRUE16-NEXT: v_add_f32_e64 v8, 0x40c00000, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v2, vcc_lo
; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v7, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v8
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.h, v6.l
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v19, v17.l, v6.l
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x7fff, v1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, 0x400000, v5
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, v3, v7
; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v8, 16, 1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo
; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v4, 16, 1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x7fff, v3
@@ -26109,30 +26108,27 @@ define inreg <16 x i8> @bitcast_v8bf16_to_v16i8_scalar(<8 x bfloat> inreg %a, i3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x7fff, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x7fff, v5
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v18, v0.l, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v7, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.h, v14.l
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v19
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 16, v1
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v10, v11, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v18
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v16.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v21, v16.l, v14.l
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v4
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v5, v12, vcc_lo
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[18:19]
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v21
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v8.l
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v5
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v21
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v19
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.h, v10.l
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v20, v8.l, v10.l
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[20:21]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v20
; GFX11-TRUE16-NEXT: s_branch .LBB109_6
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll
index b9719f62c2358..de3a67a159eca 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll
@@ -33961,21 +33961,21 @@ define <16 x i16> @bitcast_v32i8_to_v16i16(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_clause 0x1
; GFX11-TRUE16-NEXT: scratch_load_b32 v37, off, s32 offset:4
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v27, v21.l, v27.l
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v25, v15.l, v25.l
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v23, v6.l, v23.l
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v19, v13.l, v19.l
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v17, v11.l, v17.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v29.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v28.l
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v27, v21.l, v27.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v20.l
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v25, v15.l, v25.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v14.l
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v19, v13.l, v19.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v12.l
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v17, v11.l, v17.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v10.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v9.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v8.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v7.l
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v23, v6.l, v23.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v5.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v4.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v3.l
@@ -39511,21 +39511,21 @@ define <16 x half> @bitcast_v32i8_to_v16f16(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_clause 0x1
; GFX11-TRUE16-NEXT: scratch_load_b32 v37, off, s32 offset:4
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v27, v21.l, v27.l
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v25, v15.l, v25.l
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v23, v6.l, v23.l
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v19, v13.l, v19.l
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v17, v11.l, v17.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v29.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v28.l
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v27, v21.l, v27.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v20.l
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v25, v15.l, v25.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v14.l
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v19, v13.l, v19.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v12.l
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v17, v11.l, v17.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v10.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v9.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v8.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v7.l
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v23, v6.l, v23.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v5.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v4.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v3.l
@@ -42980,30 +42980,28 @@ define inreg <32 x i8> @bitcast_v16bf16_to_v32i8_scalar(<16 x bfloat> inreg %a,
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, v5, v11
; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v12
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x7fff, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.h, v30.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.h, v18.l
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v32, 16, v3
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v13, v15, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.h, v10.l
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[36:37]
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v32.l
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v51, v32.l, v30.l
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v24, 16, v7
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v5, v16, vcc_lo
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 16, v1
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[38:39]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v51
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v24.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v16.l
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v48, v16.l, v18.l
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v51
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v49
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v49
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.h, v26.l
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v50, v24.l, v26.l
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[19:20], 24, v[48:49]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v48
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v39
@@ -43949,21 +43947,21 @@ define <16 x bfloat> @bitcast_v32i8_to_v16bf16(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_clause 0x1
; GFX11-TRUE16-NEXT: scratch_load_b32 v37, off, s32 offset:4
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v27, v21.l, v27.l
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v25, v15.l, v25.l
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v23, v6.l, v23.l
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v19, v13.l, v19.l
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v17, v11.l, v17.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v29.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v28.l
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v27, v21.l, v27.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v20.l
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v25, v15.l, v25.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v14.l
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v19, v13.l, v19.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v12.l
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v17, v11.l, v17.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v10.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v9.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v8.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v7.l
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v23, v6.l, v23.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v5.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v4.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v3.l
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll
index 7b6e0a48b0ebd..18b849cdc216e 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll
@@ -90526,8 +90526,7 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v55, 16, v7
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v8, v11, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v67.h, v53.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_pack_b32_f16 v68, v55.l, v50.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v25.l
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v11, v15, v51, vcc_lo
@@ -90537,7 +90536,7 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v20.l
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v10, v10, v52, vcc_lo
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v52, 16, v11
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v67.l, v51.l
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v67, v51.l, v53.l
; GFX11-TRUE16-NEXT: v_pack_b32_f16 v3, v22.l, v24.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v54, 16, v10
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll
index c4b05da065ae3..855483b10391d 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll
@@ -8784,16 +8784,15 @@ define inreg <12 x i8> @bitcast_v6bf16_to_v12i8_scalar(<6 x bfloat> inreg %a, i3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v4
; GFX11-TRUE16-NEXT: v_pack_b32_f16 v14, v0.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v3
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v8.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[14:15]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v14
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v13
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.h, v10.l
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v11, v8.l, v10.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v11
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[11:12]
diff --git a/llvm/test/CodeGen/AMDGPU/bf16.ll b/llvm/test/CodeGen/AMDGPU/bf16.ll
index 8c24ad53dbefa..eecc5a9a6a5f8 100644
--- a/llvm/test/CodeGen/AMDGPU/bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/bf16.ll
@@ -38131,9 +38131,9 @@ define <2 x bfloat> @v_uitofp_v2i16_to_v2bf16(<2 x i16> %x) #0 {
; GFX1250TRUE16: ; %bb.0:
; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v0
; GFX1250TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v0
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX1250TRUE16-NEXT: v_cvt_f32_u32_e32 v0, v2
; GFX1250TRUE16-NEXT: v_cvt_f32_u32_e32 v1, v1
; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -41646,10 +41646,9 @@ define amdgpu_ps i32 @s_vselect_v2bf16(<2 x bfloat> inreg %a, <2 x bfloat> inreg
; GFX11TRUE16-NEXT: s_lshr_b32 s3, s0, 16
; GFX11TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX11TRUE16-NEXT: v_cmp_eq_u32_e64 s2, 0, v1
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, s3
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.h, s0
+; GFX11TRUE16-NEXT: v_pack_b32_f16 v0, s3, s0
; GFX11TRUE16-NEXT: s_lshr_b32 s0, s1, 16
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instid1(SALU_CYCLE_1)
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
; GFX11TRUE16-NEXT: v_cndmask_b16 v1.h, s0, v0.l, s2
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11TRUE16-NEXT: v_cndmask_b16 v1.l, s1, v0.h, vcc_lo
@@ -41680,10 +41679,9 @@ define amdgpu_ps i32 @s_vselect_v2bf16(<2 x bfloat> inreg %a, <2 x bfloat> inreg
; GFX1250TRUE16-NEXT: s_lshr_b32 s3, s0, 16
; GFX1250TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1250TRUE16-NEXT: v_cmp_eq_u32_e64 s2, 0, v1
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v0.l, s3
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v0.h, s0
+; GFX1250TRUE16-NEXT: v_pack_b32_f16 v0, s3, s0
; GFX1250TRUE16-NEXT: s_lshr_b32 s0, s1, 16
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instid1(SALU_CYCLE_1)
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
; GFX1250TRUE16-NEXT: v_cndmask_b16 v1.h, s0, v0.l, s2
; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250TRUE16-NEXT: v_cndmask_b16 v1.l, s1, v0.h, vcc_lo
@@ -42789,18 +42787,17 @@ define amdgpu_ps <2 x i32> @s_vselect_v4bf16(<4 x bfloat> inreg %a, <4 x bfloat>
; GFX11TRUE16-NEXT: v_cmp_eq_u32_e64 s4, 0, v1
; GFX11TRUE16-NEXT: v_cmp_eq_u32_e64 s5, 0, v2
; GFX11TRUE16-NEXT: v_cmp_eq_u32_e64 s6, 0, v3
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, s7
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.h, s9
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, s0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, s1
+; GFX11TRUE16-NEXT: v_pack_b32_f16 v0, s7, s9
+; GFX11TRUE16-NEXT: v_pack_b32_f16 v1, s0, s1
; GFX11TRUE16-NEXT: s_lshr_b32 s8, s3, 16
; GFX11TRUE16-NEXT: s_lshr_b32 s0, s2, 16
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_cndmask_b16 v2.h, s8, v0.l, s6
; GFX11TRUE16-NEXT: v_cndmask_b16 v0.h, s0, v0.h, s4
; GFX11TRUE16-NEXT: v_cndmask_b16 v0.l, s2, v1.l, vcc_lo
; GFX11TRUE16-NEXT: v_cndmask_b16 v2.l, s3, v1.h, s5
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_readfirstlane_b32 s1, v2
; GFX11TRUE16-NEXT: ; return to shader part epilog
;
@@ -42843,18 +42840,17 @@ define amdgpu_ps <2 x i32> @s_vselect_v4bf16(<4 x bfloat> inreg %a, <4 x bfloat>
; GFX1250TRUE16-NEXT: v_cmp_eq_u32_e64 s4, 0, v1
; GFX1250TRUE16-NEXT: v_cmp_eq_u32_e64 s5, 0, v2
; GFX1250TRUE16-NEXT: v_cmp_eq_u32_e64 s6, 0, v3
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v0.l, s7
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v0.h, s9
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.l, s0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.h, s1
+; GFX1250TRUE16-NEXT: v_pack_b32_f16 v0, s7, s9
+; GFX1250TRUE16-NEXT: v_pack_b32_f16 v1, s0, s1
; GFX1250TRUE16-NEXT: s_lshr_b32 s8, s3, 16
; GFX1250TRUE16-NEXT: s_lshr_b32 s0, s2, 16
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX1250TRUE16-NEXT: v_cndmask_b16 v2.h, s8, v0.l, s6
; GFX1250TRUE16-NEXT: v_cndmask_b16 v0.h, s0, v0.h, s4
; GFX1250TRUE16-NEXT: v_cndmask_b16 v0.l, s2, v1.l, vcc_lo
; GFX1250TRUE16-NEXT: v_cndmask_b16 v2.l, s3, v1.h, s5
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1250TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1250TRUE16-NEXT: v_readfirstlane_b32 s1, v2
; GFX1250TRUE16-NEXT: ; return to shader part epilog
;
diff --git a/llvm/test/CodeGen/AMDGPU/function-args.ll b/llvm/test/CodeGen/AMDGPU/function-args.ll
index 4d90d3f716ed1..1b06e392f2bb1 100644
--- a/llvm/test/CodeGen/AMDGPU/function-args.ll
+++ b/llvm/test/CodeGen/AMDGPU/function-args.ll
@@ -1800,10 +1800,10 @@ define void @void_func_v2i24(<2 x i24> %arg0) #0 {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
-; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1
; GFX11-TRUE16-NEXT: s_clause 0x1
; GFX11-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0
; GFX11-TRUE16-NEXT: buffer_store_b8 v1, off, s[0:3], 0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.e5m3.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.e5m3.ll
index 7de8136b6b1a7..c93925d4616ee 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.e5m3.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.e5m3.ll
@@ -251,9 +251,8 @@ define amdgpu_ps i32 @test_cvt_pk_fp8_f32_e5m3_word0_sss(float inreg %x, float i
; GFX1250-TRUE16-NEXT: global_wb
; GFX1250-TRUE16-NEXT: v_nop
; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.h, s2
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-TRUE16-NEXT: v_pack_b32_f16 v0, s2, s2
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_cvt_pk_fp8_f32 v0.l, s0, s1 clamp
; GFX1250-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX1250-TRUE16-NEXT: ; return to shader part epilog
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.ll
index c71ed9b81f35d..a822a17c5ae85 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.ll
@@ -1571,9 +1571,8 @@ define amdgpu_ps i32 @test_cvt_pk_bf8_f32_word0_sss(float inreg %x, float inreg
;
; GFX1170-TRUE16-LABEL: test_cvt_pk_bf8_f32_word0_sss:
; GFX1170-TRUE16: ; %bb.0:
-; GFX1170-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
-; GFX1170-TRUE16-NEXT: v_mov_b16_e32 v0.h, s2
-; GFX1170-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1170-TRUE16-NEXT: v_pack_b32_f16 v0, s2, s2
+; GFX1170-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1170-TRUE16-NEXT: v_cvt_pk_bf8_f32 v0.l, s0, s1
; GFX1170-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX1170-TRUE16-NEXT: ; return to shader part epilog
@@ -1596,9 +1595,8 @@ define amdgpu_ps i32 @test_cvt_pk_bf8_f32_word0_sss(float inreg %x, float inreg
;
; GFX12-TRUE16-LABEL: test_cvt_pk_bf8_f32_word0_sss:
; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, s2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_pack_b32_f16 v0, s2, s2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cvt_pk_bf8_f32 v0.l, s0, s1
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
@@ -1627,9 +1625,8 @@ define amdgpu_ps i32 @test_cvt_pk_bf8_f32_word0_sss(float inreg %x, float inreg
; GFX1250-TRUE16-NEXT: global_wb
; GFX1250-TRUE16-NEXT: v_nop
; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.h, s2
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-TRUE16-NEXT: v_pack_b32_f16 v0, s2, s2
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_cvt_pk_bf8_f32 v0.l, s0, s1
; GFX1250-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX1250-TRUE16-NEXT: ; return to shader part epilog
@@ -1698,9 +1695,8 @@ define amdgpu_ps i32 @test_cvt_pk_bf8_f32_word1_sss(float inreg %x, float inreg
;
; GFX1170-TRUE16-LABEL: test_cvt_pk_bf8_f32_word1_sss:
; GFX1170-TRUE16: ; %bb.0:
-; GFX1170-TRUE16-NEXT: v_mov_b16_e32 v0.h, s2
-; GFX1170-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
-; GFX1170-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1170-TRUE16-NEXT: v_pack_b32_f16 v0, s2, s2
+; GFX1170-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1170-TRUE16-NEXT: v_cvt_pk_bf8_f32 v0.h, s0, s1
; GFX1170-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX1170-TRUE16-NEXT: ; return to shader part epilog
@@ -1723,9 +1719,8 @@ define amdgpu_ps i32 @test_cvt_pk_bf8_f32_word1_sss(float inreg %x, float inreg
;
; GFX12-TRUE16-LABEL: test_cvt_pk_bf8_f32_word1_sss:
; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, s2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_pack_b32_f16 v0, s2, s2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cvt_pk_bf8_f32 v0.h, s0, s1
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
@@ -1754,9 +1749,8 @@ define amdgpu_ps i32 @test_cvt_pk_bf8_f32_word1_sss(float inreg %x, float inreg
; GFX1250-TRUE16-NEXT: global_wb
; GFX1250-TRUE16-NEXT: v_nop
; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.h, s2
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-TRUE16-NEXT: v_pack_b32_f16 v0, s2, s2
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_cvt_pk_bf8_f32 v0.h, s0, s1
; GFX1250-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX1250-TRUE16-NEXT: ; return to shader part epilog
@@ -1825,9 +1819,8 @@ define amdgpu_ps i32 @test_cvt_pk_fp8_f32_word0_sss(float inreg %x, float inreg
;
; GFX1170-TRUE16-LABEL: test_cvt_pk_fp8_f32_word0_sss:
; GFX1170-TRUE16: ; %bb.0:
-; GFX1170-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
-; GFX1170-TRUE16-NEXT: v_mov_b16_e32 v0.h, s2
-; GFX1170-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1170-TRUE16-NEXT: v_pack_b32_f16 v0, s2, s2
+; GFX1170-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1170-TRUE16-NEXT: v_cvt_pk_fp8_f32 v0.l, s0, s1
; GFX1170-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX1170-TRUE16-NEXT: ; return to shader part epilog
@@ -1850,9 +1843,8 @@ define amdgpu_ps i32 @test_cvt_pk_fp8_f32_word0_sss(float inreg %x, float inreg
;
; GFX12-TRUE16-LABEL: test_cvt_pk_fp8_f32_word0_sss:
; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, s2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_pack_b32_f16 v0, s2, s2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cvt_pk_fp8_f32 v0.l, s0, s1
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
@@ -1881,9 +1873,8 @@ define amdgpu_ps i32 @test_cvt_pk_fp8_f32_word0_sss(float inreg %x, float inreg
; GFX1250-TRUE16-NEXT: global_wb
; GFX1250-TRUE16-NEXT: v_nop
; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.h, s2
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-TRUE16-NEXT: v_pack_b32_f16 v0, s2, s2
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_cvt_pk_fp8_f32 v0.l, s0, s1
; GFX1250-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX1250-TRUE16-NEXT: ; return to shader part epilog
@@ -1952,9 +1943,8 @@ define amdgpu_ps i32 @test_cvt_pk_fp8_f32_word1_sss(float inreg %x, float inreg
;
; GFX1170-TRUE16-LABEL: test_cvt_pk_fp8_f32_word1_sss:
; GFX1170-TRUE16: ; %bb.0:
-; GFX1170-TRUE16-NEXT: v_mov_b16_e32 v0.h, s2
-; GFX1170-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
-; GFX1170-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1170-TRUE16-NEXT: v_pack_b32_f16 v0, s2, s2
+; GFX1170-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1170-TRUE16-NEXT: v_cvt_pk_fp8_f32 v0.h, s0, s1
; GFX1170-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX1170-TRUE16-NEXT: ; return to shader part epilog
@@ -1977,9 +1967,8 @@ define amdgpu_ps i32 @test_cvt_pk_fp8_f32_word1_sss(float inreg %x, float inreg
;
; GFX12-TRUE16-LABEL: test_cvt_pk_fp8_f32_word1_sss:
; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, s2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_pack_b32_f16 v0, s2, s2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cvt_pk_fp8_f32 v0.h, s0, s1
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
@@ -2008,9 +1997,8 @@ define amdgpu_ps i32 @test_cvt_pk_fp8_f32_word1_sss(float inreg %x, float inreg
; GFX1250-TRUE16-NEXT: global_wb
; GFX1250-TRUE16-NEXT: v_nop
; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.h, s2
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-TRUE16-NEXT: v_pack_b32_f16 v0, s2, s2
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_cvt_pk_fp8_f32 v0.h, s0, s1
; GFX1250-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX1250-TRUE16-NEXT: ; return to shader part epilog
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.v3f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.v3f16.ll
index 574b45472f9f1..3162e6610a331 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.v3f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.v3f16.ll
@@ -90,57 +90,109 @@ define amdgpu_gs void @main(<4 x i32> %arg, i32 %arg1) {
; GFX8-NEXT: s_mov_b32 m0, -1
; GFX8-NEXT: ds_write2_b32 v2, v0, v1 offset0:7 offset1:8
;
-; GFX11-LABEL: main:
-; GFX11: ; %bb.0: ; %bb
-; GFX11-NEXT: s_mov_b32 s4, exec_lo
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_mov_b32 s5, s4
-; GFX11-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-NEXT: v_readfirstlane_b32 s2, v2
-; GFX11-NEXT: v_readfirstlane_b32 s3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
-; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX11-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[0:3], 0 idxen
-; GFX11-NEXT: s_and_not1_wrexec_b32 s5, s5
-; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX11-NEXT: ; implicit-def: $vgpr4
-; GFX11-NEXT: s_cbranch_execnz .LBB0_1
-; GFX11-NEXT: ; %bb.2:
-; GFX11-NEXT: s_mov_b32 exec_lo, s4
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v5
-; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_and_b32 v1, 0xffff, v6
-; GFX11-NEXT: ds_store_2addr_b32 v2, v0, v1 offset0:7 offset1:8
+; GFX11-TRUE16-LABEL: main:
+; GFX11-TRUE16: ; %bb.0: ; %bb
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s4
+; GFX11-TRUE16-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-TRUE16-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[0:3], 0 idxen
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB0_1
+; GFX11-TRUE16-NEXT: ; %bb.2:
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v5
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_and_b32 v1, 0xffff, v6
+; GFX11-TRUE16-NEXT: ds_store_2addr_b32 v2, v0, v1 offset0:7 offset1:8
;
-; GFX12-LABEL: main:
-; GFX12: ; %bb.0: ; %bb
-; GFX12-NEXT: s_mov_b32 s4, exec_lo
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_mov_b32 s5, s4
-; GFX12-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: v_readfirstlane_b32 s0, v0
-; GFX12-NEXT: v_readfirstlane_b32 s1, v1
-; GFX12-NEXT: v_readfirstlane_b32 s2, v2
-; GFX12-NEXT: v_readfirstlane_b32 s3, v3
-; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
-; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[0:3], null idxen
-; GFX12-NEXT: s_and_not1_wrexec_b32 s5, s5
-; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX12-NEXT: ; implicit-def: $vgpr4
-; GFX12-NEXT: s_cbranch_execnz .LBB0_1
-; GFX12-NEXT: ; %bb.2:
-; GFX12-NEXT: s_mov_b32 exec_lo, s4
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, 16, v5
-; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_and_b32 v1, 0xffff, v6
-; GFX12-NEXT: ds_store_2addr_b32 v2, v0, v1 offset0:7 offset1:8
+; GFX11-FAKE16-LABEL: main:
+; GFX11-FAKE16: ; %bb.0: ; %bb
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s4
+; GFX11-FAKE16-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-FAKE16-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[0:3], 0 idxen
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr4
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB0_1
+; GFX11-FAKE16-NEXT: ; %bb.2:
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s4
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v5
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_and_b32 v1, 0xffff, v6
+; GFX11-FAKE16-NEXT: ds_store_2addr_b32 v2, v0, v1 offset0:7 offset1:8
+;
+; GFX12-TRUE16-LABEL: main:
+; GFX12-TRUE16: ; %bb.0: ; %bb
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s4
+; GFX12-TRUE16-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[0:3], null idxen
+; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
+; GFX12-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX12-TRUE16-NEXT: ; implicit-def: $vgpr4
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB0_1
+; GFX12-TRUE16-NEXT: ; %bb.2:
+; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s4
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v5
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_and_b32 v1, 0xffff, v6
+; GFX12-TRUE16-NEXT: ds_store_2addr_b32 v2, v0, v1 offset0:7 offset1:8
+;
+; GFX12-FAKE16-LABEL: main:
+; GFX12-FAKE16: ; %bb.0: ; %bb
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s4
+; GFX12-FAKE16-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[0:3], null idxen
+; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
+; GFX12-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX12-FAKE16-NEXT: ; implicit-def: $vgpr4
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB0_1
+; GFX12-FAKE16-NEXT: ; %bb.2:
+; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s4
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v5
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_and_b32 v1, 0xffff, v6
+; GFX12-FAKE16-NEXT: ds_store_2addr_b32 v2, v0, v1 offset0:7 offset1:8
bb:
%i = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 poison)
%i2 = call nsz arcp <3 x half> @llvm.amdgcn.struct.buffer.load.format.v3f16(<4 x i32> %arg, i32 %arg1, i32 0, i32 0, i32 0)
@@ -165,7 +217,5 @@ declare <3 x half> @llvm.amdgcn.struct.buffer.load.format.v3f16(<4 x i32>, i32,
attributes #0 = { nounwind readnone willreturn }
attributes #1 = { nounwind readonly willreturn }
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GFX11-FAKE16: {{.*}}
-; GFX11-TRUE16: {{.*}}
-; GFX12-FAKE16: {{.*}}
-; GFX12-TRUE16: {{.*}}
+; GFX11: {{.*}}
+; GFX12: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.v3f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.v3f16.ll
index 1db4db27e8841..0def045b78fc7 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.v3f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.v3f16.ll
@@ -87,30 +87,55 @@ define amdgpu_gs void @main(ptr addrspace(8) %arg, i32 %arg1) {
; GFX8-NEXT: s_mov_b32 m0, -1
; GFX8-NEXT: ds_write2_b32 v2, v0, v1 offset0:7 offset1:8
;
-; GFX11-LABEL: main:
-; GFX11: ; %bb.0: ; %bb
-; GFX11-NEXT: s_mov_b32 s4, exec_lo
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_mov_b32 s5, s4
-; GFX11-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-NEXT: v_readfirstlane_b32 s2, v2
-; GFX11-NEXT: v_readfirstlane_b32 s3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
-; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX11-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[0:3], 0 idxen
-; GFX11-NEXT: s_and_not1_wrexec_b32 s5, s5
-; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX11-NEXT: ; implicit-def: $vgpr4
-; GFX11-NEXT: s_cbranch_execnz .LBB0_1
-; GFX11-NEXT: ; %bb.2:
-; GFX11-NEXT: s_mov_b32 exec_lo, s4
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v5
-; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_and_b32 v1, 0xffff, v6
-; GFX11-NEXT: ds_store_2addr_b32 v2, v0, v1 offset0:7 offset1:8
+; GFX11-TRUE16-LABEL: main:
+; GFX11-TRUE16: ; %bb.0: ; %bb
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s4
+; GFX11-TRUE16-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-TRUE16-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[0:3], 0 idxen
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB0_1
+; GFX11-TRUE16-NEXT: ; %bb.2:
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v5
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_and_b32 v1, 0xffff, v6
+; GFX11-TRUE16-NEXT: ds_store_2addr_b32 v2, v0, v1 offset0:7 offset1:8
+;
+; GFX11-FAKE16-LABEL: main:
+; GFX11-FAKE16: ; %bb.0: ; %bb
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s4
+; GFX11-FAKE16-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-FAKE16-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[0:3], 0 idxen
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr4
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB0_1
+; GFX11-FAKE16-NEXT: ; %bb.2:
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s4
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v5
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_and_b32 v1, 0xffff, v6
+; GFX11-FAKE16-NEXT: ds_store_2addr_b32 v2, v0, v1 offset0:7 offset1:8
bb:
%i = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 poison)
%i2 = call nsz arcp <3 x half> @llvm.amdgcn.struct.ptr.buffer.load.format.v3f16(ptr addrspace(8) %arg, i32 %arg1, i32 0, i32 0, i32 0)
@@ -135,5 +160,4 @@ declare <3 x half> @llvm.amdgcn.struct.ptr.buffer.load.format.v3f16(ptr addrspac
attributes #0 = { nounwind readnone willreturn }
attributes #1 = { nounwind readonly willreturn }
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GFX11-FAKE16: {{.*}}
-; GFX11-TRUE16: {{.*}}
+; GFX11: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/si-pre-emit-peephole-merge-v-mov-b16.mir b/llvm/test/CodeGen/AMDGPU/si-pre-emit-peephole-merge-v-mov-b16.mir
index fd0a89a21bbfc..97e08bdde56a0 100644
--- a/llvm/test/CodeGen/AMDGPU/si-pre-emit-peephole-merge-v-mov-b16.mir
+++ b/llvm/test/CodeGen/AMDGPU/si-pre-emit-peephole-merge-v-mov-b16.mir
@@ -23,6 +23,23 @@ body: |
S_ENDPGM 0, implicit $vgpr0
...
+---
+name: merge_hi_zero_lo_from_hi_vop3
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr128
+
+ ; CHECK-LABEL: name: merge_hi_zero_lo_from_hi_vop3
+ ; CHECK: liveins: $vgpr128
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $vgpr0 = V_LSHRREV_B32_e64 16, $vgpr128, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit $vgpr0
+ $vgpr0_lo16 = V_MOV_B16_t16_e64 0, $vgpr128_hi16, 0, implicit $exec
+ $vgpr0_hi16 = V_MOV_B16_t16_e32 0, implicit $exec
+ S_ENDPGM 0, implicit $vgpr0
+...
+
---
# Pattern: v_mov_b16 v0.l, 0 + v_mov_b16 v0.h, v2.l
# => v_lshlrev_b32 v0, 16, v2
@@ -42,6 +59,82 @@ body: |
S_ENDPGM 0, implicit $vgpr0
...
+---
+# Pattern: v_mov_b16 v0.l, 0 + v_mov_b16 v0.h, s2
+# => v_lshlrev_b32 v0, 16, s2
+name: merge_lo_zero_hi_from_lo_sgpr
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr2
+
+ ; CHECK-LABEL: name: merge_lo_zero_hi_from_lo_sgpr
+ ; CHECK: liveins: $sgpr2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $vgpr0 = V_LSHLREV_B32_e64 16, $sgpr2, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit $vgpr0
+ $vgpr0_lo16 = V_MOV_B16_t16_e32 0, implicit $exec
+ $vgpr0_hi16 = V_MOV_B16_t16_e32 $sgpr2, implicit $exec
+ S_ENDPGM 0, implicit $vgpr0
+...
+
+---
+# Pattern: v_mov_b16 v0.l, 0 + v_mov_b16 v0.h, v2.h
+# => v_and_b32 v0, 0xffff0000, v2
+name: merge_lo_zero_hi_from_hi
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr2
+
+ ; CHECK-LABEL: name: merge_lo_zero_hi_from_hi
+ ; CHECK: liveins: $vgpr2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $vgpr0 = V_AND_B32_e32 4294901760, $vgpr2, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit $vgpr0
+ $vgpr0_lo16 = V_MOV_B16_t16_e32 0, implicit $exec
+ $vgpr0_hi16 = V_MOV_B16_t16_e32 $vgpr2_hi16, implicit $exec
+ S_ENDPGM 0, implicit $vgpr0
+...
+
+---
+# Pattern: v_mov_b16 v0.h, 0 + v_mov_b16 v0.l, v2.l
+# => v_and_b32 v0, 0x0000ffff, v2
+name: merge_hi_zero_lo_from_lo
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr2
+
+ ; CHECK-LABEL: name: merge_hi_zero_lo_from_lo
+ ; CHECK: liveins: $vgpr2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $vgpr0 = V_AND_B32_e32 65535, $vgpr2, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit $vgpr0
+ $vgpr0_lo16 = V_MOV_B16_t16_e32 $vgpr2_lo16, implicit $exec
+ $vgpr0_hi16 = V_MOV_B16_t16_e32 0, implicit $exec
+ S_ENDPGM 0, implicit $vgpr0
+...
+
+---
+# Pattern: v_mov_b16 v0.h, 0 + v_mov_b16 v0.l, s2
+# => v_and_b32 v0, 0x0000ffff, s2
+name: merge_hi_zero_lo_from_lo_sgpr
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr2
+
+ ; CHECK-LABEL: name: merge_hi_zero_lo_from_lo_sgpr
+ ; CHECK: liveins: $sgpr2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $vgpr0 = V_AND_B32_e64 65535, $sgpr2, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit $vgpr0
+ $vgpr0_lo16 = V_MOV_B16_t16_e32 $sgpr2, implicit $exec
+ $vgpr0_hi16 = V_MOV_B16_t16_e32 0, implicit $exec
+ S_ENDPGM 0, implicit $vgpr0
+...
+
---
# Pattern: v_mov_b16 v0.l, v2.l + v_mov_b16 v0.h, v3.l
# => v_pack_b32_f16 v0, v2.l, v3.l
@@ -119,81 +212,102 @@ body: |
...
---
-# Pattern: v_mov_b16 v0.l, 0 + v_mov_b16 v0.h, v2.h
-# => v_and_b32 v0, 0xffff0000, v2
-name: merge_lo_zero_hi_from_hi
+# Pattern: v_mov_b16 v0.l, s2 + v_mov_b16 v0.h, s3
+# => v_pack_b32_f16 v0, s2, s3
+name: merge_lo_from_lo_hi_from_lo_sgpr
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr2, $sgpr3
+
+ ; CHECK-LABEL: name: merge_lo_from_lo_hi_from_lo_sgpr
+ ; CHECK: liveins: $sgpr2, $sgpr3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $vgpr0 = V_PACK_B32_F16_t16_e64 0, $sgpr2, 0, $sgpr3, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit $vgpr0
+ $vgpr0_lo16 = V_MOV_B16_t16_e32 $sgpr2, implicit $exec
+ $vgpr0_hi16 = V_MOV_B16_t16_e32 $sgpr3, implicit $exec
+ S_ENDPGM 0, implicit $vgpr0
+...
+
+---
+# Negative: intervening instruction that clobbers dst — must NOT merge.
+name: no_merge_dst_clobbered
tracksRegLiveness: true
body: |
bb.0:
liveins: $vgpr2
- ; CHECK-LABEL: name: merge_lo_zero_hi_from_hi
+ ; CHECK-LABEL: name: no_merge_dst_clobbered
; CHECK: liveins: $vgpr2
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: $vgpr0 = V_AND_B32_e32 4294901760, $vgpr2, implicit $exec
+ ; CHECK-NEXT: $vgpr0_lo16 = V_MOV_B16_t16_e32 $vgpr2_lo16, implicit $exec
+ ; CHECK-NEXT: $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: $vgpr0_hi16 = V_MOV_B16_t16_e32 $vgpr2_hi16, implicit $exec
; CHECK-NEXT: S_ENDPGM 0, implicit $vgpr0
- $vgpr0_lo16 = V_MOV_B16_t16_e32 0, implicit $exec
+ $vgpr0_lo16 = V_MOV_B16_t16_e32 $vgpr2_lo16, implicit $exec
+ $vgpr0 = V_MOV_B32_e32 0, implicit $exec
$vgpr0_hi16 = V_MOV_B16_t16_e32 $vgpr2_hi16, implicit $exec
S_ENDPGM 0, implicit $vgpr0
...
---
-# Pattern: v_mov_b16 v0.h, 0 + v_mov_b16 v0.l, v2.l
-# => v_and_b32 v0, 0x0000ffff, v2
-name: merge_hi_zero_lo_from_lo
+name: merge_src_clobbered_First_Src
tracksRegLiveness: true
body: |
bb.0:
- liveins: $vgpr2
+ liveins: $vgpr2, $vgpr3
- ; CHECK-LABEL: name: merge_hi_zero_lo_from_lo
- ; CHECK: liveins: $vgpr2
+ ; CHECK-LABEL: name: merge_src_clobbered_First_Src
+ ; CHECK: liveins: $vgpr2, $vgpr3
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: $vgpr0 = V_AND_B32_e32 65535, $vgpr2, implicit $exec
+ ; CHECK-NEXT: $vgpr0 = V_PACK_B32_F16_t16_e64 0, $vgpr2_lo16, 0, $vgpr3_hi16, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: $vgpr2 = V_MOV_B32_e32 0, implicit $exec
; CHECK-NEXT: S_ENDPGM 0, implicit $vgpr0
$vgpr0_lo16 = V_MOV_B16_t16_e32 $vgpr2_lo16, implicit $exec
- $vgpr0_hi16 = V_MOV_B16_t16_e32 0, implicit $exec
+ $vgpr2 = V_MOV_B32_e32 0, implicit $exec
+ $vgpr0_hi16 = V_MOV_B16_t16_e32 $vgpr3_hi16, implicit $exec
S_ENDPGM 0, implicit $vgpr0
...
---
-# Negative: intervening instruction that clobbers dst — must NOT merge.
-name: no_merge_dst_clobbered
+name: merge_src_clobbered_Second_Src
tracksRegLiveness: true
body: |
bb.0:
- liveins: $vgpr2
+ liveins: $vgpr2, $vgpr3
- ; CHECK-LABEL: name: no_merge_dst_clobbered
- ; CHECK: liveins: $vgpr2
+ ; CHECK-LABEL: name: merge_src_clobbered_Second_Src
+ ; CHECK: liveins: $vgpr2, $vgpr3
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: $vgpr0_lo16 = V_MOV_B16_t16_e32 $vgpr2_lo16, implicit $exec
- ; CHECK-NEXT: $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; CHECK-NEXT: $vgpr0_hi16 = V_MOV_B16_t16_e32 $vgpr2_hi16, implicit $exec
+ ; CHECK-NEXT: $vgpr3 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: $vgpr0 = V_PACK_B32_F16_t16_e64 0, $vgpr2_lo16, 0, $vgpr3_hi16, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: S_ENDPGM 0, implicit $vgpr0
$vgpr0_lo16 = V_MOV_B16_t16_e32 $vgpr2_lo16, implicit $exec
- $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- $vgpr0_hi16 = V_MOV_B16_t16_e32 $vgpr2_hi16, implicit $exec
+ $vgpr3 = V_MOV_B32_e32 0, implicit $exec
+ $vgpr0_hi16 = V_MOV_B16_t16_e32 $vgpr3_hi16, implicit $exec
S_ENDPGM 0, implicit $vgpr0
...
---
-# Negative: intervening instruction that clobbers src — must NOT merge.
-name: no_merge_src_clobbered
+# Negative: intervening instruction that clobbers both src — must NOT merge.
+name: no_merge_src_clobbered_both
tracksRegLiveness: true
body: |
bb.0:
- liveins: $vgpr2
+ liveins: $vgpr2, $vgpr3
- ; CHECK-LABEL: name: no_merge_src_clobbered
- ; CHECK: liveins: $vgpr2
+ ; CHECK-LABEL: name: no_merge_src_clobbered_both
+ ; CHECK: liveins: $vgpr2, $vgpr3
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: $vgpr0_lo16 = V_MOV_B16_t16_e32 $vgpr2_lo16, implicit $exec
; CHECK-NEXT: $vgpr2 = V_MOV_B32_e32 0, implicit $exec
- ; CHECK-NEXT: $vgpr0_hi16 = V_MOV_B16_t16_e32 $vgpr2_hi16, implicit $exec
+ ; CHECK-NEXT: $vgpr3 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: $vgpr0_hi16 = V_MOV_B16_t16_e32 $vgpr3_hi16, implicit $exec
; CHECK-NEXT: S_ENDPGM 0, implicit $vgpr0
$vgpr0_lo16 = V_MOV_B16_t16_e32 $vgpr2_lo16, implicit $exec
$vgpr2 = V_MOV_B32_e32 0, implicit $exec
- $vgpr0_hi16 = V_MOV_B16_t16_e32 $vgpr2_hi16, implicit $exec
+ $vgpr3 = V_MOV_B32_e32 0, implicit $exec
+ $vgpr0_hi16 = V_MOV_B16_t16_e32 $vgpr3_hi16, implicit $exec
S_ENDPGM 0, implicit $vgpr0
...
diff --git a/llvm/test/CodeGen/AMDGPU/sub.v2i16.ll b/llvm/test/CodeGen/AMDGPU/sub.v2i16.ll
index 0fca52a064214..9dcc25b0abea8 100644
--- a/llvm/test/CodeGen/AMDGPU/sub.v2i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/sub.v2i16.ll
@@ -694,8 +694,8 @@ define amdgpu_kernel void @v_test_sub_v2i16_zext_to_v2i32(ptr addrspace(1) %out,
; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1
; GFX11-TRUE16-NEXT: v_pk_sub_i16 v2, v1, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v2
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v2
; GFX11-TRUE16-NEXT: buffer_store_b64 v[0:1], off, s[0:3], 0
; GFX11-TRUE16-NEXT: s_endpgm
;
>From 23e66e2157260ea7664a84ce27c5e1d60b3e2a11 Mon Sep 17 00:00:00 2001
From: guochen2 <guochen2 at amd.com>
Date: Wed, 15 Jul 2026 13:57:45 -0400
Subject: [PATCH 4/5] improve code
---
llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp | 72 +++++++++++--------
.../GlobalISel/merge-values-s16-true16.ll | 5 +-
.../AMDGPU/GlobalISel/store-local.128.ll | 3 +-
.../AMDGPU/GlobalISel/store-local.96.ll | 3 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll | 21 +++---
.../CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll | 18 +++--
.../CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll | 6 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll | 5 +-
.../si-pre-emit-peephole-merge-v-mov-b16.mir | 25 ++++++-
9 files changed, 100 insertions(+), 58 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp b/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
index ed1910113f8af..6a59075ae2b0f 100644
--- a/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
+++ b/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
@@ -837,7 +837,10 @@ bool SIPreEmitPeephole::mergeSingleMovB16Pair(MachineInstr &Lo,
MachineInstr &FirstMI = IsHiFirst ? Hi : Lo;
MachineInstr &SecondMI = IsHiFirst ? Lo : Hi;
- bool DataConflictOnFirst = false, DataConflictOnSecond = false;
+ // Data Conflict counter
+ auto UpperBound = SecondMI.getIterator();
+ auto LowerBound = FirstMI.getIterator();
+ unsigned LoopCnt = 0, UpperBoundCnt = UINT_MAX, LowerBoundCnt = 0;
MachineBasicBlock &MBB = *Lo.getParent();
@@ -848,20 +851,30 @@ bool SIPreEmitPeephole::mergeSingleMovB16Pair(MachineInstr &Lo,
// the second mov.
MCRegister FirstSrc16 = IsHiFirst ? HiSrc16 : LoSrc16;
MCRegister SecondSrc16 = IsHiFirst ? LoSrc16 : HiSrc16;
- for (const MachineInstr &Scan :
+ for (MachineInstr &Scan :
drop_begin(make_range(FirstMI.getIterator(), SecondMI.getIterator()))) {
-
if (Scan.modifiesRegister(Dst32, TRI))
return false;
- DataConflictOnFirst |=
- (FirstSrc16 && Scan.modifiesRegister(FirstSrc16, TRI));
- DataConflictOnSecond |=
- (SecondSrc16 && Scan.modifiesRegister(SecondSrc16, TRI));
- if (DataConflictOnFirst && DataConflictOnSecond)
- return false;
+ LoopCnt++;
+ if (FirstSrc16 && LoopCnt < UpperBoundCnt &&
+ Scan.modifiesRegister(FirstSrc16, TRI)) {
+ UpperBound = Scan.getIterator();
+ UpperBoundCnt = LoopCnt;
+ }
+ if (SecondSrc16 && LoopCnt > LowerBoundCnt &&
+ Scan.modifiesRegister(SecondSrc16, TRI)) {
+ LowerBound = Scan.getIterator();
+ LowerBoundCnt = LoopCnt;
+ }
}
- MachineInstr &Selected = DataConflictOnSecond ? SecondMI : FirstMI;
+ // No spot maintains data dependency
+ if (LowerBoundCnt >= UpperBoundCnt)
+ return false;
+
+ // Insert MI before selected. Any spots between (LowerBound, UpperBound] would
+ // work
+ MachineInstr &Selected = *(++LowerBound);
const DebugLoc &DL = Selected.getDebugLoc();
// Now match patterns and emit the replacement instruction.
@@ -955,13 +968,15 @@ bool SIPreEmitPeephole::mergeMovB16Pairs(MachineFunction &MF) const {
// Age tracks how many non-mov-b16 instructions have passed since the
// 16-bit write, used to bound the search window.
struct Pending {
+ MCRegister Dst32;
MachineInstr *MI;
- unsigned Age; // instructions since was seen
unsigned IsHi;
};
// Search window size
const unsigned ScanLimit = 16;
- SmallDenseMap<MCRegister, Pending> PendingWrites;
+ std::array<Pending, ScanLimit> CirBuf = {{{MCRegister(), nullptr, false}}};
+ SmallDenseMap<MCRegister, unsigned> PendingWrites;
+ unsigned Head = 0;
for (auto &MI : make_early_inc_range(MBB)) {
if (MI.isDebugInstr())
@@ -971,39 +986,40 @@ bool SIPreEmitPeephole::mergeMovB16Pairs(MachineFunction &MF) const {
bool IsMovB16 = (Opc == AMDGPU::V_MOV_B16_t16_e32 ||
Opc == AMDGPU::V_MOV_B16_t16_e64);
+ if (++Head == ScanLimit)
+ Head = 0;
+
+ // Expire the last one
+ PendingWrites.erase(CirBuf[Head].Dst32);
+
if (!IsMovB16) {
- // Age all pending writes and invalidate stale or clobbered ones.
- for (auto &[key, Value] : PendingWrites)
- Value.Age++;
-
- PendingWrites.remove_if([&](const auto &KV) {
- return (KV.second.Age >= ScanLimit ||
- MI.modifiesRegister(KV.first, TRI));
- });
+ CirBuf[Head] = {MCRegister(), nullptr, false};
continue;
}
MCRegister DstReg = MI.getOperand(0).getReg().asMCReg();
-
bool DstIsHi = AMDGPU::isHi16Reg(DstReg, *TRI);
MCRegister Dst32 = TRI->get32BitRegister(DstReg);
- auto [It, Inserted] = PendingWrites.insert({Dst32, {&MI, 0, DstIsHi}});
+ // Insert new one
+ CirBuf[Head] = {Dst32, &MI, DstIsHi};
+
+ auto [It, Inserted] = PendingWrites.insert({Dst32, Head});
if (!Inserted) {
- if (It->second.IsHi == DstIsHi) {
- It->second = {&MI, 0, DstIsHi};
+ if (CirBuf[It->second].IsHi == DstIsHi) {
+ It->second = Head;
continue;
}
// Look for a matching pending write.
- MachineInstr &LoMI = !DstIsHi ? MI : *It->second.MI;
- MachineInstr &HiMI = DstIsHi ? MI : *It->second.MI;
- bool IsHiFirst = It->second.IsHi;
+ MachineInstr &LoMI = !DstIsHi ? MI : *CirBuf[It->second].MI;
+ MachineInstr &HiMI = DstIsHi ? MI : *CirBuf[It->second].MI;
+ bool IsHiFirst = CirBuf[It->second].IsHi;
if (mergeSingleMovB16Pair(LoMI, HiMI, IsHiFirst)) {
Changed = true;
PendingWrites.erase(It);
} else {
- It->second = {&MI, 0, DstIsHi};
+ It->second = Head;
}
}
}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/merge-values-s16-true16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/merge-values-s16-true16.ll
index fc0df6181b7d9..4a11593ecab1f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/merge-values-s16-true16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/merge-values-s16-true16.ll
@@ -36,9 +36,8 @@ define amdgpu_kernel void @store_i136_divergent(ptr %p) {
; GFX1150-NEXT: v_mov_b16_e32 v0.h, 0
; GFX1150-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX1150-NEXT: v_lshlrev_b16 v4.l, 8, v1.l
-; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1150-NEXT: v_mov_b16_e32 v1.l, v0.h
-; GFX1150-NEXT: v_mov_b16_e32 v1.h, v0.h
+; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1150-NEXT: v_pack_b32_f16 v1, v0.h, v0.h
; GFX1150-NEXT: v_or_b16 v0.l, v0.l, v4.l
; GFX1150-NEXT: s_waitcnt lgkmcnt(0)
; GFX1150-NEXT: v_dual_mov_b32 v5, s1 :: v_dual_mov_b32 v4, s0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.128.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.128.ll
index a6e36f4f00428..b8a91df80de99 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.128.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.128.ll
@@ -249,11 +249,12 @@ define amdgpu_kernel void @store_lds_v4i32_align1(ptr addrspace(3) %out, <4 x i3
; GFX11-NEXT: v_mov_b16_e32 v1.h, s5
; GFX11-NEXT: s_lshr_b32 s4, s1, 16
; GFX11-NEXT: s_lshr_b32 s1, s1, 24
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: v_pack_b32_f16 v3, s6, s1
; GFX11-NEXT: s_and_b32 s9, 0xffff, s2
; GFX11-NEXT: v_mov_b16_e32 v4.l, s0
; GFX11-NEXT: s_lshr_b32 s0, s2, 24
; GFX11-NEXT: v_mov_b16_e32 v2.h, s4
-; GFX11-NEXT: v_pack_b32_f16 v3, s6, s1
; GFX11-NEXT: s_lshr_b32 s1, s9, 8
; GFX11-NEXT: ds_store_b8 v5, v0
; GFX11-NEXT: ds_store_b8 v5, v3 offset:1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.96.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.96.ll
index b6c29129bad3d..eadfb7c68f35f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.96.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.96.ll
@@ -236,11 +236,12 @@ define amdgpu_kernel void @store_lds_v3i32_align1(ptr addrspace(3) %out, <3 x i3
; GFX11-NEXT: v_pack_b32_f16 v1, s2, s4
; GFX11-NEXT: s_lshr_b32 s7, s2, 16
; GFX11-NEXT: s_lshr_b32 s2, s2, 24
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_pack_b32_f16 v4, s5, s2
; GFX11-NEXT: v_pack_b32_f16 v2, s0, s3
; GFX11-NEXT: s_lshr_b32 s0, s6, 8
; GFX11-NEXT: v_pack_b32_f16 v3, s1, s7
; GFX11-NEXT: s_lshr_b32 s1, s8, 8
-; GFX11-NEXT: v_pack_b32_f16 v4, s5, s2
; GFX11-NEXT: v_pack_b32_f16 v5, s0, s1
; GFX11-NEXT: ds_store_b8 v6, v0
; GFX11-NEXT: ds_store_b8 v6, v4 offset:1
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll
index 7305af8092f9e..f322bf61f5186 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll
@@ -26076,18 +26076,18 @@ define inreg <16 x i8> @bitcast_v8bf16_to_v16i8_scalar(<8 x bfloat> inreg %a, i3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v9, 16, 1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v10, v10, v9
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v10, 0x7fff, v10
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x7fff, v6
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v19, v17.l, v6.l
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, v8, v5
; GFX11-TRUE16-NEXT: v_add_f32_e64 v8, 0x40c00000, s0
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v2, vcc_lo
; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v7, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v8
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v19, v17.l, v6.l
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x7fff, v1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, 0x400000, v5
@@ -26102,33 +26102,34 @@ define inreg <16 x i8> @bitcast_v8bf16_to_v16i8_scalar(<8 x bfloat> inreg %a, i3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, v5, v8
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, v1, v4
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v18, v0.l, v2.l
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v11, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v9
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x7fff, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x7fff, v5
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v3
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v18, v0.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v7, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v19
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v21, v16.l, v14.l
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v10, v11, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v18
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v21, v16.l, v14.l
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v4
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v5, v12, vcc_lo
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[18:19]
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v21
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v5
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v20, v8.l, v10.l
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v21
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v19
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v20, v8.l, v10.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[20:21]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v20
; GFX11-TRUE16-NEXT: s_branch .LBB109_6
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll
index de3a67a159eca..52e28165907ff 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll
@@ -33963,7 +33963,7 @@ define <16 x i16> @bitcast_v32i8_to_v16i16(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32
; GFX11-TRUE16-NEXT: v_pack_b32_f16 v27, v21.l, v27.l
; GFX11-TRUE16-NEXT: v_pack_b32_f16 v25, v15.l, v25.l
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v23, v6.l, v23.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.h, v23.l
; GFX11-TRUE16-NEXT: v_pack_b32_f16 v19, v13.l, v19.l
; GFX11-TRUE16-NEXT: v_pack_b32_f16 v17, v11.l, v17.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v29.l
@@ -33976,6 +33976,7 @@ define <16 x i16> @bitcast_v32i8_to_v16i16(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v9.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v8.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v6.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v5.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v4.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v3.l
@@ -39513,7 +39514,7 @@ define <16 x half> @bitcast_v32i8_to_v16f16(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32
; GFX11-TRUE16-NEXT: v_pack_b32_f16 v27, v21.l, v27.l
; GFX11-TRUE16-NEXT: v_pack_b32_f16 v25, v15.l, v25.l
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v23, v6.l, v23.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.h, v23.l
; GFX11-TRUE16-NEXT: v_pack_b32_f16 v19, v13.l, v19.l
; GFX11-TRUE16-NEXT: v_pack_b32_f16 v17, v11.l, v17.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v29.l
@@ -39526,6 +39527,7 @@ define <16 x half> @bitcast_v32i8_to_v16f16(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v9.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v8.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v6.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v5.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v4.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v3.l
@@ -42985,23 +42987,24 @@ define inreg <32 x i8> @bitcast_v16bf16_to_v32i8_scalar(<16 x bfloat> inreg %a,
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x7fff, v5
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v32, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v51, v32.l, v30.l
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v13, v15, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.h, v10.l
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[36:37]
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v51, v32.l, v30.l
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v24, 16, v7
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v5, v16, vcc_lo
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v48, v16.l, v18.l
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[38:39]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v51
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v5
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v48, v16.l, v18.l
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v50, v24.l, v26.l
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v51
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v49
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v49
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v50, v24.l, v26.l
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[19:20], 24, v[48:49]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v48
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v39
@@ -43949,7 +43952,7 @@ define <16 x bfloat> @bitcast_v32i8_to_v16bf16(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32
; GFX11-TRUE16-NEXT: v_pack_b32_f16 v27, v21.l, v27.l
; GFX11-TRUE16-NEXT: v_pack_b32_f16 v25, v15.l, v25.l
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v23, v6.l, v23.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.h, v23.l
; GFX11-TRUE16-NEXT: v_pack_b32_f16 v19, v13.l, v19.l
; GFX11-TRUE16-NEXT: v_pack_b32_f16 v17, v11.l, v17.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v29.l
@@ -43962,6 +43965,7 @@ define <16 x bfloat> @bitcast_v32i8_to_v16bf16(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v9.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v8.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v6.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v5.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v4.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v3.l
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll
index 18b849cdc216e..aa5f4372c3886 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll
@@ -90526,19 +90526,19 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v55, 16, v7
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v8, v11, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_pack_b32_f16 v68, v55.l, v50.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v25.l
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v11, v15, v51, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v51, 16, v8
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v67, v51.l, v53.l
; GFX11-TRUE16-NEXT: v_pack_b32_f16 v5, v26.l, v27.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v20.l
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v10, v10, v52, vcc_lo
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v52, 16, v11
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v67, v51.l, v53.l
; GFX11-TRUE16-NEXT: v_pack_b32_f16 v3, v22.l, v24.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v54, 16, v10
; GFX11-TRUE16-NEXT: v_pack_b32_f16 v66, v52.l, v54.l
; GFX11-TRUE16-NEXT: v_pack_b32_f16 v1, v18.l, v19.l
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll
index 855483b10391d..e34a1edd10282 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll
@@ -8788,12 +8788,11 @@ define inreg <12 x i8> @bitcast_v6bf16_to_v12i8_scalar(<6 x bfloat> inreg %a, i3
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v3
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v10, vcc_lo
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v11, v8.l, v10.l
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[14:15]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v14
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v13
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v11, v8.l, v10.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v11
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[11:12]
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/si-pre-emit-peephole-merge-v-mov-b16.mir b/llvm/test/CodeGen/AMDGPU/si-pre-emit-peephole-merge-v-mov-b16.mir
index 97e08bdde56a0..31b6ec8691dbb 100644
--- a/llvm/test/CodeGen/AMDGPU/si-pre-emit-peephole-merge-v-mov-b16.mir
+++ b/llvm/test/CodeGen/AMDGPU/si-pre-emit-peephole-merge-v-mov-b16.mir
@@ -289,15 +289,36 @@ body: |
S_ENDPGM 0, implicit $vgpr0
...
+---
+name: merge_src_clobbered_both_accepted
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr2, $vgpr3
+
+ ; CHECK-LABEL: name: merge_src_clobbered_both_accepted
+ ; CHECK: liveins: $vgpr2, $vgpr3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $vgpr3 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: $vgpr0 = V_PACK_B32_F16_t16_e64 0, $vgpr2_lo16, 0, $vgpr3_hi16, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: $vgpr2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit $vgpr0
+ $vgpr0_lo16 = V_MOV_B16_t16_e32 $vgpr2_lo16, implicit $exec
+ $vgpr3 = V_MOV_B32_e32 0, implicit $exec
+ $vgpr2 = V_MOV_B32_e32 0, implicit $exec
+ $vgpr0_hi16 = V_MOV_B16_t16_e32 $vgpr3_hi16, implicit $exec
+ S_ENDPGM 0, implicit $vgpr0
+...
+
---
# Negative: intervening instruction that clobbers both src — must NOT merge.
-name: no_merge_src_clobbered_both
+name: no_merge_src_clobbered_both_declined
tracksRegLiveness: true
body: |
bb.0:
liveins: $vgpr2, $vgpr3
- ; CHECK-LABEL: name: no_merge_src_clobbered_both
+ ; CHECK-LABEL: name: no_merge_src_clobbered_both_declined
; CHECK: liveins: $vgpr2, $vgpr3
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: $vgpr0_lo16 = V_MOV_B16_t16_e32 $vgpr2_lo16, implicit $exec
>From 0af856b422b65605c00ced400172be90499a9b94 Mon Sep 17 00:00:00 2001
From: guochen2 <guochen2 at amd.com>
Date: Thu, 16 Jul 2026 10:53:15 -0400
Subject: [PATCH 5/5] remove auto
---
llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp | 8 +-
.../GlobalISel/llvm.amdgcn.intersect_ray.ll | 8 +-
.../AMDGPU/GlobalISel/store-local.128.ll | 7 +-
.../AMDGPU/GlobalISel/store-local.96.ll | 7 +-
llvm/test/CodeGen/AMDGPU/add.v2i16.ll | 2 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll | 17 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll | 21 ++-
.../CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll | 38 ++---
.../CodeGen/AMDGPU/amdgcn.bitcast.48bit.ll | 9 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll | 19 ++-
.../CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll | 6 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll | 11 +-
llvm/test/CodeGen/AMDGPU/bf16.ll | 4 +-
llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll | 12 +-
llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll | 4 +-
llvm/test/CodeGen/AMDGPU/function-args.ll | 4 +-
.../llvm.amdgcn.image.sample.g16.encode.ll | 16 +-
.../AMDGPU/llvm.amdgcn.image.sample.g16.ll | 4 +-
....amdgcn.struct.buffer.load.format.v3f16.ll | 158 ++++++------------
...gcn.struct.ptr.buffer.load.format.v3f16.ll | 76 +++------
llvm/test/CodeGen/AMDGPU/sub.v2i16.ll | 2 +-
.../CodeGen/AMDGPU/vector_shuffle.packed.ll | 26 +--
22 files changed, 193 insertions(+), 266 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp b/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
index 6a59075ae2b0f..cd1e4856ec924 100644
--- a/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
+++ b/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
@@ -838,8 +838,8 @@ bool SIPreEmitPeephole::mergeSingleMovB16Pair(MachineInstr &Lo,
MachineInstr &SecondMI = IsHiFirst ? Lo : Hi;
// Data Conflict counter
- auto UpperBound = SecondMI.getIterator();
- auto LowerBound = FirstMI.getIterator();
+ MachineBasicBlock::iterator UpperBound = SecondMI.getIterator();
+ MachineBasicBlock::iterator LowerBound = FirstMI.getIterator();
unsigned LoopCnt = 0, UpperBoundCnt = UINT_MAX, LowerBoundCnt = 0;
MachineBasicBlock &MBB = *Lo.getParent();
@@ -874,7 +874,7 @@ bool SIPreEmitPeephole::mergeSingleMovB16Pair(MachineInstr &Lo,
// Insert MI before selected. Any spots between (LowerBound, UpperBound] would
// work
- MachineInstr &Selected = *(++LowerBound);
+ MachineInstr &Selected = *UpperBound;
const DebugLoc &DL = Selected.getDebugLoc();
// Now match patterns and emit the replacement instruction.
@@ -978,7 +978,7 @@ bool SIPreEmitPeephole::mergeMovB16Pairs(MachineFunction &MF) const {
SmallDenseMap<MCRegister, unsigned> PendingWrites;
unsigned Head = 0;
- for (auto &MI : make_early_inc_range(MBB)) {
+ for (MachineInstr &MI : make_early_inc_range(MBB)) {
if (MI.isDebugInstr())
continue;
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.intersect_ray.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.intersect_ray.ll
index 8131248d1fbf4..959bd1384bcee 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.intersect_ray.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.intersect_ray.ll
@@ -60,9 +60,9 @@ define amdgpu_ps <4 x float> @image_bvh_intersect_ray_a16(i32 %node_ptr, float %
; GFX11-TRUE16-LABEL: image_bvh_intersect_ray_a16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v9, v5 :: v_dual_mov_b32 v10, v8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v8, v7.l, v9.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.h, v6.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v8, v7.l, v9.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v7.h
; GFX11-TRUE16-NEXT: image_bvh_intersect_ray v[0:3], [v0, v1, v[2:4], v[8:10]], s[0:3] a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -125,9 +125,9 @@ define amdgpu_ps <4 x float> @image_bvh64_intersect_ray_a16(i64 %node_ptr, float
; GFX11-TRUE16-LABEL: image_bvh64_intersect_ray_a16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, v6 :: v_dual_mov_b32 v11, v9
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v9, v8.l, v10.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.h, v7.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v9, v8.l, v10.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v8.h
; GFX11-TRUE16-NEXT: image_bvh64_intersect_ray v[0:3], [v[0:1], v2, v[3:5], v[9:11]], s[0:3] a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.128.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.128.ll
index b8a91df80de99..efd9baad8fa62 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.128.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.128.ll
@@ -249,12 +249,11 @@ define amdgpu_kernel void @store_lds_v4i32_align1(ptr addrspace(3) %out, <4 x i3
; GFX11-NEXT: v_mov_b16_e32 v1.h, s5
; GFX11-NEXT: s_lshr_b32 s4, s1, 16
; GFX11-NEXT: s_lshr_b32 s1, s1, 24
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: v_pack_b32_f16 v3, s6, s1
; GFX11-NEXT: s_and_b32 s9, 0xffff, s2
; GFX11-NEXT: v_mov_b16_e32 v4.l, s0
; GFX11-NEXT: s_lshr_b32 s0, s2, 24
; GFX11-NEXT: v_mov_b16_e32 v2.h, s4
+; GFX11-NEXT: v_pack_b32_f16 v3, s6, s1
; GFX11-NEXT: s_lshr_b32 s1, s9, 8
; GFX11-NEXT: ds_store_b8 v5, v0
; GFX11-NEXT: ds_store_b8 v5, v3 offset:1
@@ -267,12 +266,12 @@ define amdgpu_kernel void @store_lds_v4i32_align1(ptr addrspace(3) %out, <4 x i3
; GFX11-NEXT: v_pack_b32_f16 v1, s2, s0
; GFX11-NEXT: s_and_b32 s0, 0xffff, s3
; GFX11-NEXT: s_lshr_b32 s8, s2, 16
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_pack_b32_f16 v0, s1, s8
; GFX11-NEXT: s_lshr_b32 s0, s0, 8
+; GFX11-NEXT: v_pack_b32_f16 v0, s1, s8
; GFX11-NEXT: s_lshr_b32 s1, s3, 16
; GFX11-NEXT: v_pack_b32_f16 v2, s3, s0
; GFX11-NEXT: s_lshr_b32 s0, s3, 24
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_pack_b32_f16 v3, s1, s0
; GFX11-NEXT: ds_store_b8 v5, v1 offset:8
; GFX11-NEXT: ds_store_b8 v5, v0 offset:9
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.96.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.96.ll
index eadfb7c68f35f..382183386d646 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.96.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.96.ll
@@ -225,23 +225,22 @@ define amdgpu_kernel void @store_lds_v3i32_align1(ptr addrspace(3) %out, <3 x i3
; GFX11-NEXT: s_and_b32 s5, 0xffff, s0
; GFX11-NEXT: s_lshr_b32 s4, s0, 16
; GFX11-NEXT: s_lshr_b32 s5, s5, 8
-; GFX11-NEXT: v_pack_b32_f16 v0, s0, s1
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v6, s3
+; GFX11-NEXT: v_pack_b32_f16 v0, s0, s1
; GFX11-NEXT: s_lshr_b32 s0, s0, 24
; GFX11-NEXT: s_lshr_b32 s3, s1, 16
; GFX11-NEXT: s_and_b32 s6, 0xffff, s1
; GFX11-NEXT: s_lshr_b32 s1, s1, 24
; GFX11-NEXT: s_and_b32 s8, 0xffff, s2
-; GFX11-NEXT: v_pack_b32_f16 v1, s2, s4
; GFX11-NEXT: s_lshr_b32 s7, s2, 16
+; GFX11-NEXT: v_pack_b32_f16 v1, s2, s4
; GFX11-NEXT: s_lshr_b32 s2, s2, 24
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_pack_b32_f16 v4, s5, s2
; GFX11-NEXT: v_pack_b32_f16 v2, s0, s3
; GFX11-NEXT: s_lshr_b32 s0, s6, 8
; GFX11-NEXT: v_pack_b32_f16 v3, s1, s7
; GFX11-NEXT: s_lshr_b32 s1, s8, 8
+; GFX11-NEXT: v_pack_b32_f16 v4, s5, s2
; GFX11-NEXT: v_pack_b32_f16 v5, s0, s1
; GFX11-NEXT: ds_store_b8 v6, v0
; GFX11-NEXT: ds_store_b8 v6, v4 offset:1
diff --git a/llvm/test/CodeGen/AMDGPU/add.v2i16.ll b/llvm/test/CodeGen/AMDGPU/add.v2i16.ll
index cc938af3ef231..a54223795c640 100644
--- a/llvm/test/CodeGen/AMDGPU/add.v2i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/add.v2i16.ll
@@ -672,8 +672,8 @@ define amdgpu_kernel void @v_test_add_v2i16_zext_to_v2i32(ptr addrspace(1) %out,
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_pk_add_u16 v2, v1, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; GFX11-TRUE16-NEXT: global_store_b64 v3, v[0:1], s[0:1]
; GFX11-TRUE16-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
index f1f93bc72f946..27a9c4ed17b70 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
@@ -163412,7 +163412,7 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX11-TRUE16-NEXT: v_bfe_u32 v22, v23, 16, 1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v103, 16, v13
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v100, 16, v11
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v21, v99.l, v112.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v16, v16, v17, vcc_lo
; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v18, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
@@ -163425,7 +163425,7 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX11-TRUE16-NEXT: v_add_f32_e64 v22, 0x40c00000, s0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
; GFX11-TRUE16-NEXT: s_and_b32 s0, s3, 0xffff0000
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v21, v99.l, v112.l
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v113, 16, v15
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v15, 0x7fff, v16
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v16, 0x7fff, v17
@@ -163452,13 +163452,14 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX11-TRUE16-NEXT: v_bfe_u32 v22, v23, 16, 1
; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s2, 16
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.h, v101.l
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v13, v98.l, v103.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v16, v16, v17, vcc_lo
; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v18, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
; GFX11-TRUE16-NEXT: v_add_f32_e64 v19, 0x40c00000, s0
; GFX11-TRUE16-NEXT: s_and_b32 s0, s17, 0xffff0000
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v115, 16, v16
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v13, v98.l, v103.l
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v15, v15, v24, vcc_lo
; GFX11-TRUE16-NEXT: v_or_b32_e32 v24, 0x400000, v18
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v16, v17, v18
@@ -163529,12 +163530,12 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v24
; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s26, 16
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v84.l
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v12, v81.l, v97.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v16, v16, v17, vcc_lo
; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v18, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v24, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v12, v81.l, v97.l
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v130, 16, v16
; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v15, v15, v23 :: v_dual_add_nc_u32 v16, v17, v18
; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v18
@@ -163706,17 +163707,17 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v18, 0x7fff, v18
; GFX11-TRUE16-NEXT: v_pack_b32_f16 v40, v150.l, v163.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v162, 16, v16
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v19, v25, v26, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
; GFX11-TRUE16-NEXT: v_pack_b32_f16 v47, v134.l, v149.l
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v46, v146.l, v160.l
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v165, 16, v19
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v18, v18, v27, vcc_lo
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v46, v146.l, v160.l
; GFX11-TRUE16-NEXT: v_mov_b16_e64 v45.h, v145.l
; GFX11-TRUE16-NEXT: v_pack_b32_f16 v44, v135.l, v148.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v177, 16, v18
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.h, v100.l
; GFX11-TRUE16-NEXT: v_pack_b32_f16 v29, v114.l, v117.l
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll
index f322bf61f5186..7305af8092f9e 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll
@@ -26076,18 +26076,18 @@ define inreg <16 x i8> @bitcast_v8bf16_to_v16i8_scalar(<8 x bfloat> inreg %a, i3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v9, 16, 1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v10, v10, v9
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v10, 0x7fff, v10
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x7fff, v6
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v1
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v19, v17.l, v6.l
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, v8, v5
; GFX11-TRUE16-NEXT: v_add_f32_e64 v8, 0x40c00000, s0
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v2, vcc_lo
; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v7, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v8
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v19, v17.l, v6.l
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x7fff, v1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, 0x400000, v5
@@ -26102,34 +26102,33 @@ define inreg <16 x i8> @bitcast_v8bf16_to_v16i8_scalar(<8 x bfloat> inreg %a, i3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, v5, v8
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, v1, v4
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v18, v0.l, v2.l
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v11, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v9
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x7fff, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x7fff, v5
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v18, v0.l, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v7, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v19
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v21, v16.l, v14.l
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v10, v11, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v18
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v21, v16.l, v14.l
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v4
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v5, v12, vcc_lo
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[18:19]
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v21
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v5
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v20, v8.l, v10.l
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v21
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v19
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v20, v8.l, v10.l
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[20:21]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v20
; GFX11-TRUE16-NEXT: s_branch .LBB109_6
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll
index 52e28165907ff..76dabcd5e7b4c 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll
@@ -33961,17 +33961,17 @@ define <16 x i16> @bitcast_v32i8_to_v16i16(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_clause 0x1
; GFX11-TRUE16-NEXT: scratch_load_b32 v37, off, s32 offset:4
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v27, v21.l, v27.l
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v25, v15.l, v25.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.h, v23.l
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v19, v13.l, v19.l
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v17, v11.l, v17.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v29.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v28.l
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v27, v21.l, v27.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v20.l
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v25, v15.l, v25.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v14.l
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v19, v13.l, v19.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v12.l
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v17, v11.l, v17.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v10.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v9.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v8.l
@@ -39512,17 +39512,17 @@ define <16 x half> @bitcast_v32i8_to_v16f16(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_clause 0x1
; GFX11-TRUE16-NEXT: scratch_load_b32 v37, off, s32 offset:4
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v27, v21.l, v27.l
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v25, v15.l, v25.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.h, v23.l
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v19, v13.l, v19.l
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v17, v11.l, v17.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v29.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v28.l
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v27, v21.l, v27.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v20.l
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v25, v15.l, v25.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v14.l
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v19, v13.l, v19.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v12.l
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v17, v11.l, v17.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v10.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v9.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v8.l
@@ -42974,37 +42974,37 @@ define inreg <32 x i8> @bitcast_v16bf16_to_v32i8_scalar(<16 x bfloat> inreg %a,
; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v11, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v11
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, v3, v9
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v49, v33.l, v22.l
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v7, v15, vcc_lo
; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v9
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x7fff, v3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, v5, v11
; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v12
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v49, v33.l, v22.l
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x7fff, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v32, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v51, v32.l, v30.l
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v13, v15, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.h, v10.l
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[36:37]
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v51, v32.l, v30.l
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v24, 16, v7
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v5, v16, vcc_lo
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v48, v16.l, v18.l
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[38:39]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v51
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v5
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v50, v24.l, v26.l
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v48, v16.l, v18.l
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v51
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v49
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v49
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v50, v24.l, v26.l
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[19:20], 24, v[48:49]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v48
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v39
@@ -43950,17 +43950,17 @@ define <16 x bfloat> @bitcast_v32i8_to_v16bf16(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_clause 0x1
; GFX11-TRUE16-NEXT: scratch_load_b32 v37, off, s32 offset:4
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v27, v21.l, v27.l
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v25, v15.l, v25.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.h, v23.l
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v19, v13.l, v19.l
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v17, v11.l, v17.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v29.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v28.l
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v27, v21.l, v27.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v20.l
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v25, v15.l, v25.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v14.l
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v19, v13.l, v19.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v12.l
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v17, v11.l, v17.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v10.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v9.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v8.l
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.48bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.48bit.ll
index a8407e50594e1..8478218071a80 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.48bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.48bit.ll
@@ -930,15 +930,16 @@ define <3 x i16> @bitcast_v3bf16_to_v3i16(<3 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v1, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v3, v7, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, 0x7fc0
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v1, v1.h, v3.l
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v6, vcc_lo
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v1, v1.h, v3.l
; GFX11-TRUE16-NEXT: .LBB4_2: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -1194,10 +1195,10 @@ define inreg <3 x i16> @bitcast_v3bf16_to_v3i16_scalar(<3 x bfloat> inreg %a, i3
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v6, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, 0x7fc0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v1, v1.h, v4.l
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v1, v1.h, v4.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB5_4:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll
index aa5f4372c3886..d8d89c2d726b8 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll
@@ -90464,16 +90464,16 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v13, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 0x7fff, v7
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v81, v34.l, v37.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v48, 16, v8
; GFX11-TRUE16-NEXT: v_add_f32_e64 v8, 0x40c00000, s0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v9, v9, v13
; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s24, 16
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v80, v48.l, v38.l
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v81, v34.l, v37.l
; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v8, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v9, 0x7fff, v9
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v80, v48.l, v38.l
; GFX11-TRUE16-NEXT: v_pack_b32_f16 v84, v30.l, v33.l
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v11, v11, v8
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v10, vcc_lo
@@ -90490,9 +90490,9 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; GFX11-TRUE16-NEXT: s_and_b32 s0, s27, 0xffff0000
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v69.h, v39.l
; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v7, 16, 1
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v85, v31.l, v32.l
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v10, v11, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v85, v31.l, v32.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v10, v12, v7
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v49, 16, v9
@@ -90526,23 +90526,24 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v55, 16, v7
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v8, v11, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_pack_b32_f16 v68, v55.l, v50.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v25.l
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v11, v15, v51, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v51, 16, v8
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v67, v51.l, v53.l
; GFX11-TRUE16-NEXT: v_pack_b32_f16 v5, v26.l, v27.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v20.l
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v10, v10, v52, vcc_lo
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v52, 16, v11
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v67, v51.l, v53.l
; GFX11-TRUE16-NEXT: v_pack_b32_f16 v3, v22.l, v24.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v54, 16, v10
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v66, v52.l, v54.l
; GFX11-TRUE16-NEXT: v_pack_b32_f16 v1, v18.l, v19.l
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v65, 24, v67
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v66, v52.l, v54.l
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v70, 24, v81
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v71, 8, v80
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v82, 24, v84
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll
index 21f0d838d316a..b4b9fb12ee368 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll
@@ -18096,10 +18096,10 @@ define inreg <8 x i8> @bitcast_v4bf16_to_v8i8_scalar(<4 x bfloat> inreg %a, i32
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_pack_b32_f16 v9, v0.l, v2.l
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v10, v8.l, v6.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v9
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v10, v8.l, v6.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[9:10]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v10
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v10
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll
index e34a1edd10282..ba0739f2e6d9c 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll
@@ -8778,21 +8778,22 @@ define inreg <12 x i8> @bitcast_v6bf16_to_v12i8_scalar(<6 x bfloat> inreg %a, i3
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v7, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v15
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v4
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v14, v0.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v3
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v10, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v14, v0.l, v2.l
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v11, v8.l, v10.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[14:15]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v14
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v13
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v11, v8.l, v10.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v11
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[11:12]
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/bf16.ll b/llvm/test/CodeGen/AMDGPU/bf16.ll
index eecc5a9a6a5f8..7321c609a446a 100644
--- a/llvm/test/CodeGen/AMDGPU/bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/bf16.ll
@@ -38131,9 +38131,9 @@ define <2 x bfloat> @v_uitofp_v2i16_to_v2bf16(<2 x i16> %x) #0 {
; GFX1250TRUE16: ; %bb.0:
; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; GFX1250TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v0
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1250TRUE16-NEXT: v_cvt_f32_u32_e32 v0, v2
; GFX1250TRUE16-NEXT: v_cvt_f32_u32_e32 v1, v1
; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
diff --git a/llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll b/llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll
index e11b0dc0cbddb..f1fb4975556ae 100644
--- a/llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll
@@ -6114,29 +6114,29 @@ define <4 x half> @v_copysign_out_v4f16_mag_v4f64_sign_v4f16(<4 x double> %mag,
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 8, v3
; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v3, 20, 11
; GFX11-TRUE16-NEXT: v_bfe_u32 v20, v1, 20, 11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 16, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_and_or_b32 v6, 0xffe, v11, v6
; GFX11-TRUE16-NEXT: v_med3_i32 v11, v12, 0, 13
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 16, v7
; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x1000, v6
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, v11, v12
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, v11, v12
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, v11, v14
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v11, v12
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v12, 8, v1
; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v14, v11
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v10, 0xfffffc10, v10
; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc_lo
; GFX11-TRUE16-NEXT: v_sub_nc_u32_e32 v14, 0x3f1, v17
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v21, v10, 12, v6
; GFX11-TRUE16-NEXT: v_cmp_gt_i32_e32 vcc_lo, 1, v10
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_and_or_b32 v4, 0xffe, v16, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_med3_i32 v14, v14, 0, 13
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v11, v21, v11, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v2
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v13, 0x7e00
diff --git a/llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll b/llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll
index 47a6e97b16167..73713f33f5107 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll
@@ -2916,8 +2916,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zero_hi(ptr inreg %sbase,
; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-TRUE16-NEXT: flat_load_u16 v1, v0, s[2:3]
; GFX1250-SDAG-TRUE16-NEXT: s_wait_xcnt 0x0
-; GFX1250-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
;
; GFX1250-NOECC-SDAG-TRUE16-LABEL: flat_load_saddr_i16_d16hi_zero_hi:
@@ -2977,8 +2977,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zero_hi_immneg128(ptr inr
; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-TRUE16-NEXT: flat_load_u16 v1, v0, s[2:3] offset:-128
; GFX1250-SDAG-TRUE16-NEXT: s_wait_xcnt 0x0
-; GFX1250-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
;
; GFX1250-NOECC-SDAG-TRUE16-LABEL: flat_load_saddr_i16_d16hi_zero_hi_immneg128:
diff --git a/llvm/test/CodeGen/AMDGPU/function-args.ll b/llvm/test/CodeGen/AMDGPU/function-args.ll
index 1b06e392f2bb1..4d90d3f716ed1 100644
--- a/llvm/test/CodeGen/AMDGPU/function-args.ll
+++ b/llvm/test/CodeGen/AMDGPU/function-args.ll
@@ -1800,10 +1800,10 @@ define void @void_func_v2i24(<2 x i24> %arg0) #0 {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; GFX11-TRUE16-NEXT: s_clause 0x1
; GFX11-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0
; GFX11-TRUE16-NEXT: buffer_store_b8 v1, off, s[0:3], 0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.encode.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.encode.ll
index a73b8f2f17e95..c6b31dce6cd46 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.encode.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.encode.ll
@@ -512,9 +512,9 @@ define amdgpu_ps float @sample_c_d_o_2darray_V1(<8 x i32> inreg %rsrc, <4 x i32>
; GFX12-TRUE16-LABEL: sample_c_d_o_2darray_V1:
; GFX12-TRUE16: ; %bb.0: ; %main_body
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v9, v5 ; encoding: [0x05,0x03,0x12,0x7e]
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; encoding: [0x01,0x00,0x87,0xbf]
-; GFX12-TRUE16-NEXT: v_pack_b32_f16 v5, v4.l, v9.l ; encoding: [0x05,0x00,0x11,0xd7,0x04,0x13,0x02,0x02]
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l ; encoding: [0x03,0x39,0x04,0x7f]
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) ; encoding: [0x02,0x00,0x87,0xbf]
+; GFX12-TRUE16-NEXT: v_pack_b32_f16 v5, v4.l, v9.l ; encoding: [0x05,0x00,0x11,0xd7,0x04,0x13,0x02,0x02]
; GFX12-TRUE16-NEXT: image_sample_c_d_o_g16 v0, [v0, v1, v2, v[5:8]], s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_2D_ARRAY ; encoding: [0x05,0x00,0x0f,0xe5,0x00,0x00,0x00,0x04,0x00,0x01,0x02,0x05]
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; encoding: [0x00,0x00,0xc2,0xbf]
; GFX12-TRUE16-NEXT: ; return to shader part epilog
@@ -530,9 +530,9 @@ define amdgpu_ps float @sample_c_d_o_2darray_V1(<8 x i32> inreg %rsrc, <4 x i32>
; GFX13-TRUE16-LABEL: sample_c_d_o_2darray_V1:
; GFX13-TRUE16: ; %bb.0: ; %main_body
; GFX13-TRUE16-NEXT: v_mov_b32_e32 v9, v5 ; encoding: [0x05,0x03,0x12,0x7e]
-; GFX13-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; encoding: [0x01,0x00,0xae,0xbf]
-; GFX13-TRUE16-NEXT: v_pack_b32_f16 v5, v4.l, v9.l ; encoding: [0x05,0x00,0x11,0xd7,0x04,0x13,0x02,0x02]
; GFX13-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l ; encoding: [0x03,0x39,0x04,0x7f]
+; GFX13-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) ; encoding: [0x02,0x00,0xae,0xbf]
+; GFX13-TRUE16-NEXT: v_pack_b32_f16 v5, v4.l, v9.l ; encoding: [0x05,0x00,0x11,0xd7,0x04,0x13,0x02,0x02]
; GFX13-TRUE16-NEXT: image_sample_c_d_o_g16 v0, [v0, v1, v2, v[5:8]], s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_2D_ARRAY ; encoding: [0x05,0x80,0x2e,0xe5,0x00,0x00,0x00,0x04,0x00,0x01,0x02,0x05]
; GFX13-TRUE16-NEXT: s_wait_samplecnt 0x0 ; encoding: [0x00,0x00,0xc2,0xbf]
; GFX13-TRUE16-NEXT: ; return to shader part epilog
@@ -581,9 +581,9 @@ define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4
; GFX12-TRUE16-LABEL: sample_c_d_o_2darray_V2:
; GFX12-TRUE16: ; %bb.0: ; %main_body
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v9, v5 ; encoding: [0x05,0x03,0x12,0x7e]
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; encoding: [0x01,0x00,0x87,0xbf]
-; GFX12-TRUE16-NEXT: v_pack_b32_f16 v5, v4.l, v9.l ; encoding: [0x05,0x00,0x11,0xd7,0x04,0x13,0x02,0x02]
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l ; encoding: [0x03,0x39,0x04,0x7f]
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) ; encoding: [0x02,0x00,0x87,0xbf]
+; GFX12-TRUE16-NEXT: v_pack_b32_f16 v5, v4.l, v9.l ; encoding: [0x05,0x00,0x11,0xd7,0x04,0x13,0x02,0x02]
; GFX12-TRUE16-NEXT: image_sample_c_d_o_g16 v[0:1], [v0, v1, v2, v[5:8]], s[0:7], s[8:11] dmask:0x6 dim:SQ_RSRC_IMG_2D_ARRAY ; encoding: [0x05,0x00,0x8f,0xe5,0x00,0x00,0x00,0x04,0x00,0x01,0x02,0x05]
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; encoding: [0x00,0x00,0xc2,0xbf]
; GFX12-TRUE16-NEXT: ; return to shader part epilog
@@ -599,9 +599,9 @@ define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4
; GFX13-TRUE16-LABEL: sample_c_d_o_2darray_V2:
; GFX13-TRUE16: ; %bb.0: ; %main_body
; GFX13-TRUE16-NEXT: v_mov_b32_e32 v9, v5 ; encoding: [0x05,0x03,0x12,0x7e]
-; GFX13-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; encoding: [0x01,0x00,0xae,0xbf]
-; GFX13-TRUE16-NEXT: v_pack_b32_f16 v5, v4.l, v9.l ; encoding: [0x05,0x00,0x11,0xd7,0x04,0x13,0x02,0x02]
; GFX13-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l ; encoding: [0x03,0x39,0x04,0x7f]
+; GFX13-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) ; encoding: [0x02,0x00,0xae,0xbf]
+; GFX13-TRUE16-NEXT: v_pack_b32_f16 v5, v4.l, v9.l ; encoding: [0x05,0x00,0x11,0xd7,0x04,0x13,0x02,0x02]
; GFX13-TRUE16-NEXT: image_sample_c_d_o_g16 v[0:1], [v0, v1, v2, v[5:8]], s[0:7], s[8:11] dmask:0x6 dim:SQ_RSRC_IMG_2D_ARRAY ; encoding: [0x05,0x80,0xae,0xe5,0x00,0x00,0x00,0x04,0x00,0x01,0x02,0x05]
; GFX13-TRUE16-NEXT: s_wait_samplecnt 0x0 ; encoding: [0x00,0x00,0xc2,0xbf]
; GFX13-TRUE16-NEXT: ; return to shader part epilog
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.ll
index eff5f0ff42f45..e02c3b92bf7ce 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.ll
@@ -360,8 +360,8 @@ define amdgpu_ps float @sample_c_d_o_2darray_V1(<8 x i32> inreg %rsrc, <4 x i32>
; GFX12PLUS-TRUE16-LABEL: sample_c_d_o_2darray_V1:
; GFX12PLUS-TRUE16: ; %bb.0: ; %main_body
; GFX12PLUS-TRUE16-NEXT: v_mov_b32_e32 v9, v5
-; GFX12PLUS-TRUE16-NEXT: v_pack_b32_f16 v5, v4.l, v9.l
; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
+; GFX12PLUS-TRUE16-NEXT: v_pack_b32_f16 v5, v4.l, v9.l
; GFX12PLUS-TRUE16-NEXT: image_sample_c_d_o_g16 v0, [v0, v1, v2, v[5:8]], s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_2D_ARRAY
; GFX12PLUS-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12PLUS-TRUE16-NEXT: ; return to shader part epilog
@@ -410,8 +410,8 @@ define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4
; GFX12PLUS-TRUE16-LABEL: sample_c_d_o_2darray_V2:
; GFX12PLUS-TRUE16: ; %bb.0: ; %main_body
; GFX12PLUS-TRUE16-NEXT: v_mov_b32_e32 v9, v5
-; GFX12PLUS-TRUE16-NEXT: v_pack_b32_f16 v5, v4.l, v9.l
; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
+; GFX12PLUS-TRUE16-NEXT: v_pack_b32_f16 v5, v4.l, v9.l
; GFX12PLUS-TRUE16-NEXT: image_sample_c_d_o_g16 v[0:1], [v0, v1, v2, v[5:8]], s[0:7], s[8:11] dmask:0x6 dim:SQ_RSRC_IMG_2D_ARRAY
; GFX12PLUS-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12PLUS-TRUE16-NEXT: ; return to shader part epilog
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.v3f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.v3f16.ll
index 3162e6610a331..574b45472f9f1 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.v3f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.v3f16.ll
@@ -90,109 +90,57 @@ define amdgpu_gs void @main(<4 x i32> %arg, i32 %arg1) {
; GFX8-NEXT: s_mov_b32 m0, -1
; GFX8-NEXT: ds_write2_b32 v2, v0, v1 offset0:7 offset1:8
;
-; GFX11-TRUE16-LABEL: main:
-; GFX11-TRUE16: ; %bb.0: ; %bb
-; GFX11-TRUE16-NEXT: s_mov_b32 s4, exec_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_mov_b32 s5, s4
-; GFX11-TRUE16-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
-; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX11-TRUE16-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[0:3], 0 idxen
-; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4
-; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB0_1
-; GFX11-TRUE16-NEXT: ; %bb.2:
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v5
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_and_b32 v1, 0xffff, v6
-; GFX11-TRUE16-NEXT: ds_store_2addr_b32 v2, v0, v1 offset0:7 offset1:8
+; GFX11-LABEL: main:
+; GFX11: ; %bb.0: ; %bb
+; GFX11-NEXT: s_mov_b32 s4, exec_lo
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_mov_b32 s5, s4
+; GFX11-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[0:3], 0 idxen
+; GFX11-NEXT: s_and_not1_wrexec_b32 s5, s5
+; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX11-NEXT: ; implicit-def: $vgpr4
+; GFX11-NEXT: s_cbranch_execnz .LBB0_1
+; GFX11-NEXT: ; %bb.2:
+; GFX11-NEXT: s_mov_b32 exec_lo, s4
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v5
+; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_and_b32 v1, 0xffff, v6
+; GFX11-NEXT: ds_store_2addr_b32 v2, v0, v1 offset0:7 offset1:8
;
-; GFX11-FAKE16-LABEL: main:
-; GFX11-FAKE16: ; %bb.0: ; %bb
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, exec_lo
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_mov_b32 s5, s4
-; GFX11-FAKE16-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
-; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX11-FAKE16-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[0:3], 0 idxen
-; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr4
-; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB0_1
-; GFX11-FAKE16-NEXT: ; %bb.2:
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s4
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v5
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_and_b32 v1, 0xffff, v6
-; GFX11-FAKE16-NEXT: ds_store_2addr_b32 v2, v0, v1 offset0:7 offset1:8
-;
-; GFX12-TRUE16-LABEL: main:
-; GFX12-TRUE16: ; %bb.0: ; %bb
-; GFX12-TRUE16-NEXT: s_mov_b32 s4, exec_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-TRUE16-NEXT: s_mov_b32 s5, s4
-; GFX12-TRUE16-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
-; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[0:3], null idxen
-; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
-; GFX12-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX12-TRUE16-NEXT: ; implicit-def: $vgpr4
-; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB0_1
-; GFX12-TRUE16-NEXT: ; %bb.2:
-; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s4
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v5
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_and_b32 v1, 0xffff, v6
-; GFX12-TRUE16-NEXT: ds_store_2addr_b32 v2, v0, v1 offset0:7 offset1:8
-;
-; GFX12-FAKE16-LABEL: main:
-; GFX12-FAKE16: ; %bb.0: ; %bb
-; GFX12-FAKE16-NEXT: s_mov_b32 s4, exec_lo
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-FAKE16-NEXT: s_mov_b32 s5, s4
-; GFX12-FAKE16-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
-; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[0:3], null idxen
-; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
-; GFX12-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX12-FAKE16-NEXT: ; implicit-def: $vgpr4
-; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB0_1
-; GFX12-FAKE16-NEXT: ; %bb.2:
-; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s4
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v5
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_and_b32 v1, 0xffff, v6
-; GFX12-FAKE16-NEXT: ds_store_2addr_b32 v2, v0, v1 offset0:7 offset1:8
+; GFX12-LABEL: main:
+; GFX12: ; %bb.0: ; %bb
+; GFX12-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_mov_b32 s5, s4
+; GFX12-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-NEXT: v_readfirstlane_b32 s1, v1
+; GFX12-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[0:3], null idxen
+; GFX12-NEXT: s_and_not1_wrexec_b32 s5, s5
+; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX12-NEXT: ; implicit-def: $vgpr4
+; GFX12-NEXT: s_cbranch_execnz .LBB0_1
+; GFX12-NEXT: ; %bb.2:
+; GFX12-NEXT: s_mov_b32 exec_lo, s4
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, 16, v5
+; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_and_b32 v1, 0xffff, v6
+; GFX12-NEXT: ds_store_2addr_b32 v2, v0, v1 offset0:7 offset1:8
bb:
%i = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 poison)
%i2 = call nsz arcp <3 x half> @llvm.amdgcn.struct.buffer.load.format.v3f16(<4 x i32> %arg, i32 %arg1, i32 0, i32 0, i32 0)
@@ -217,5 +165,7 @@ declare <3 x half> @llvm.amdgcn.struct.buffer.load.format.v3f16(<4 x i32>, i32,
attributes #0 = { nounwind readnone willreturn }
attributes #1 = { nounwind readonly willreturn }
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GFX11: {{.*}}
-; GFX12: {{.*}}
+; GFX11-FAKE16: {{.*}}
+; GFX11-TRUE16: {{.*}}
+; GFX12-FAKE16: {{.*}}
+; GFX12-TRUE16: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.v3f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.v3f16.ll
index 0def045b78fc7..1db4db27e8841 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.v3f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.v3f16.ll
@@ -87,55 +87,30 @@ define amdgpu_gs void @main(ptr addrspace(8) %arg, i32 %arg1) {
; GFX8-NEXT: s_mov_b32 m0, -1
; GFX8-NEXT: ds_write2_b32 v2, v0, v1 offset0:7 offset1:8
;
-; GFX11-TRUE16-LABEL: main:
-; GFX11-TRUE16: ; %bb.0: ; %bb
-; GFX11-TRUE16-NEXT: s_mov_b32 s4, exec_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_mov_b32 s5, s4
-; GFX11-TRUE16-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
-; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX11-TRUE16-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[0:3], 0 idxen
-; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4
-; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB0_1
-; GFX11-TRUE16-NEXT: ; %bb.2:
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v5
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_and_b32 v1, 0xffff, v6
-; GFX11-TRUE16-NEXT: ds_store_2addr_b32 v2, v0, v1 offset0:7 offset1:8
-;
-; GFX11-FAKE16-LABEL: main:
-; GFX11-FAKE16: ; %bb.0: ; %bb
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, exec_lo
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_mov_b32 s5, s4
-; GFX11-FAKE16-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
-; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX11-FAKE16-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[0:3], 0 idxen
-; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr4
-; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB0_1
-; GFX11-FAKE16-NEXT: ; %bb.2:
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s4
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v5
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_and_b32 v1, 0xffff, v6
-; GFX11-FAKE16-NEXT: ds_store_2addr_b32 v2, v0, v1 offset0:7 offset1:8
+; GFX11-LABEL: main:
+; GFX11: ; %bb.0: ; %bb
+; GFX11-NEXT: s_mov_b32 s4, exec_lo
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_mov_b32 s5, s4
+; GFX11-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[0:3], 0 idxen
+; GFX11-NEXT: s_and_not1_wrexec_b32 s5, s5
+; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX11-NEXT: ; implicit-def: $vgpr4
+; GFX11-NEXT: s_cbranch_execnz .LBB0_1
+; GFX11-NEXT: ; %bb.2:
+; GFX11-NEXT: s_mov_b32 exec_lo, s4
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v5
+; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_and_b32 v1, 0xffff, v6
+; GFX11-NEXT: ds_store_2addr_b32 v2, v0, v1 offset0:7 offset1:8
bb:
%i = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 poison)
%i2 = call nsz arcp <3 x half> @llvm.amdgcn.struct.ptr.buffer.load.format.v3f16(ptr addrspace(8) %arg, i32 %arg1, i32 0, i32 0, i32 0)
@@ -160,4 +135,5 @@ declare <3 x half> @llvm.amdgcn.struct.ptr.buffer.load.format.v3f16(ptr addrspac
attributes #0 = { nounwind readnone willreturn }
attributes #1 = { nounwind readonly willreturn }
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GFX11: {{.*}}
+; GFX11-FAKE16: {{.*}}
+; GFX11-TRUE16: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/sub.v2i16.ll b/llvm/test/CodeGen/AMDGPU/sub.v2i16.ll
index 9dcc25b0abea8..0fca52a064214 100644
--- a/llvm/test/CodeGen/AMDGPU/sub.v2i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/sub.v2i16.ll
@@ -694,8 +694,8 @@ define amdgpu_kernel void @v_test_sub_v2i16_zext_to_v2i32(ptr addrspace(1) %out,
; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1
; GFX11-TRUE16-NEXT: v_pk_sub_i16 v2, v1, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; GFX11-TRUE16-NEXT: buffer_store_b64 v[0:1], off, s[0:3], 0
; GFX11-TRUE16-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll b/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll
index cdc821bca8340..70ef8c6e7cd94 100644
--- a/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll
@@ -329,8 +329,8 @@ define <4 x half> @shuffle_v4f16_35u5(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
; GFX11-TRUE16-NEXT: global_load_b32 v1, v[2:3], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v0, v0.h, v1.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v0, v0.h, v1.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: shuffle_v4f16_35u5:
@@ -1104,8 +1104,8 @@ define <4 x half> @shuffle_v4f16_3456(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
; GFX11-TRUE16-NEXT: global_load_b64 v[1:2], v[2:3], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v0, v0.h, v1.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v0, v0.h, v1.l
; GFX11-TRUE16-NEXT: v_pack_b32_f16 v1, v1.h, v2.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -1154,8 +1154,8 @@ define <4 x half> @shuffle_v4f16_5634(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
; GFX11-TRUE16-NEXT: global_load_b64 v[2:3], v[2:3], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v1, v0.h, v2.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v1, v0.h, v2.l
; GFX11-TRUE16-NEXT: v_pack_b32_f16 v0, v2.h, v3.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -1217,10 +1217,10 @@ define <4 x half> @shuffle_v4f16_5734(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
; GFX11-TRUE16-NEXT: global_load_b64 v[2:3], v[2:3], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v1, v0.h, v2.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v1, v0.h, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -2398,8 +2398,8 @@ define <2 x half> @hi16low16bits_v2bf16(ptr addrspace(1) %x0, ptr addrspace(1) %
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off
; GFX11-TRUE16-NEXT: global_load_b32 v1, v[2:3], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v0, v0.h, v1.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v0, v0.h, v1.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: hi16low16bits_v2bf16:
@@ -2553,8 +2553,8 @@ define <2 x i16> @i16_hi16low16bits(ptr addrspace(1) %x0, ptr addrspace(1) %x1)
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off
; GFX11-TRUE16-NEXT: global_load_b32 v1, v[2:3], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v0, v0.h, v1.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v0, v0.h, v1.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: i16_hi16low16bits:
@@ -3601,8 +3601,8 @@ define <4 x bfloat> @shuffle_v4bf16_35u5(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
; GFX11-TRUE16-NEXT: global_load_b32 v1, v[2:3], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v0, v0.h, v1.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v0, v0.h, v1.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: shuffle_v4bf16_35u5:
@@ -4376,8 +4376,8 @@ define <4 x bfloat> @shuffle_v4bf16_3456(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
; GFX11-TRUE16-NEXT: global_load_b64 v[1:2], v[2:3], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v0, v0.h, v1.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v0, v0.h, v1.l
; GFX11-TRUE16-NEXT: v_pack_b32_f16 v1, v1.h, v2.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -4426,8 +4426,8 @@ define <4 x bfloat> @shuffle_v4bf16_5634(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
; GFX11-TRUE16-NEXT: global_load_b64 v[2:3], v[2:3], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v1, v0.h, v2.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v1, v0.h, v2.l
; GFX11-TRUE16-NEXT: v_pack_b32_f16 v0, v2.h, v3.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -4489,10 +4489,10 @@ define <4 x bfloat> @shuffle_v4bf16_5734(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
; GFX11-TRUE16-NEXT: global_load_b64 v[2:3], v[2:3], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v1, v0.h, v2.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v1, v0.h, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -5883,8 +5883,8 @@ define <2 x bfloat> @hi16low16bits(ptr addrspace(1) %x0, ptr addrspace(1) %x1) {
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off
; GFX11-TRUE16-NEXT: global_load_b32 v1, v[2:3], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_pack_b32_f16 v0, v0.h, v1.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_pack_b32_f16 v0, v0.h, v1.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: hi16low16bits:
More information about the llvm-commits
mailing list