[llvm] [AMDGPU] Re-fold masked hi16 pack into v_or_b32_sdwa (PR #210735)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Jul 20 08:10:29 PDT 2026
https://github.com/michaelselehov created https://github.com/llvm/llvm-project/pull/210735
Follow-up to #206058, which selects the (hi << 16) | (z & 0xffff) high-half pack to a fused v_lshl_or_b32. When z's high bits are not known zero this needs an explicit v_and_b32 0xffff, and since v_lshl_or_b32 has no SDWA form the mask can no longer fold into a single v_or_b32_sdwa (src1_sel:WORD_0) as it did before -- a small but real perf hole (~5% on a VALU-bound char4 kernel on gfx942).
Add a SIPeepholeSDWA rewrite of the masked pack back into v_lshlrev_b32 + v_or_b32_sdwa, erasing the dead v_and. It fires only on the masked form, so the mask-free #206058 win is preserved, and it never increases the instruction count.
Assisted-by: Claude Opus
>From 330ebfe3f6b7d8d21192e92c37cb9ddbe661523d Mon Sep 17 00:00:00 2001
From: Michael Selehov <michael.selehov at amd.com>
Date: Mon, 20 Jul 2026 09:44:28 -0500
Subject: [PATCH] [AMDGPU] Re-fold masked hi16 pack into v_or_b32_sdwa
Follow-up to #206058, which selects the (hi << 16) | (z & 0xffff)
high-half pack to a fused v_lshl_or_b32. When z's high bits are not
known zero this needs an explicit v_and_b32 0xffff, and since
v_lshl_or_b32 has no SDWA form the mask can no longer fold into a
single v_or_b32_sdwa (src1_sel:WORD_0) as it did before -- a small but
real perf hole (~5% on a VALU-bound char4 kernel on gfx942).
Add a SIPeepholeSDWA rewrite of the masked pack back into
v_lshlrev_b32 + v_or_b32_sdwa, erasing the dead v_and. It fires only
on the masked form, so the mask-free #206058 win is preserved, and it
never increases the instruction count.
Assisted-by: Claude Opus
---
llvm/lib/Target/AMDGPU/SIPeepholeSDWA.cpp | 109 +
.../CodeGen/AMDGPU/GlobalISel/add.vni16.ll | 192 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/fpow.ll | 38 +-
.../CodeGen/AMDGPU/GlobalISel/llvm.abs.ll | 8 +-
.../llvm.amdgcn.image.atomic.dim.a16.ll | 902 +--
.../llvm.amdgcn.image.gather4.a16.dim.ll | 438 +-
.../llvm.amdgcn.image.getresinfo.a16.ll | 160 +-
.../llvm.amdgcn.image.load.2darraymsaa.a16.ll | 94 +-
.../llvm.amdgcn.image.load.3d.a16.ll | 106 +-
.../llvm.amdgcn.image.sample.cd.g16.ll | 61 +-
.../llvm.amdgcn.image.sample.g16.ll | 104 +-
.../test/CodeGen/AMDGPU/GlobalISel/saddsat.ll | 88 +-
.../AMDGPU/GlobalISel/select-to-fmin-fmax.ll | 48 +-
.../test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll | 88 +-
.../test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll | 88 +-
.../test/CodeGen/AMDGPU/GlobalISel/usubsat.ll | 88 +-
.../AMDGPU/GlobalISel/vni8-across-blocks.ll | 8 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll | 5434 ++++++++---------
.../CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll | 377 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll | 964 ++-
.../CodeGen/AMDGPU/amdgcn.bitcast.320bit.ll | 310 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.32bit.ll | 32 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.48bit.ll | 18 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll | 1949 +++---
.../CodeGen/AMDGPU/amdgcn.bitcast.576bit.ll | 432 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.640bit.ll | 480 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll | 153 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.704bit.ll | 528 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.768bit.ll | 576 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.832bit.ll | 624 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.896bit.ll | 672 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.960bit.ll | 720 +--
.../CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll | 159 +-
.../AMDGPU/amdgpu-codegenprepare-idiv.ll | 160 +-
.../buffer-fat-pointer-atomicrmw-fadd.ll | 404 +-
.../buffer-fat-pointer-atomicrmw-fmax.ll | 152 +-
.../buffer-fat-pointer-atomicrmw-fmin.ll | 152 +-
...ffer-fat-pointers-contents-legalization.ll | 10 +-
.../build-vector-packed-partial-undef.ll | 24 +-
.../AMDGPU/constant-address-space-32bit.ll | 7 +-
llvm/test/CodeGen/AMDGPU/ctlz_zero_poison.ll | 34 +-
llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll | 26 +-
llvm/test/CodeGen/AMDGPU/fmaximum.ll | 45 +-
llvm/test/CodeGen/AMDGPU/fminimum.ll | 45 +-
llvm/test/CodeGen/AMDGPU/fmul-to-ldexp.ll | 50 +-
llvm/test/CodeGen/AMDGPU/fneg-fabs.bf16.ll | 17 +-
llvm/test/CodeGen/AMDGPU/fneg.bf16.ll | 37 +-
llvm/test/CodeGen/AMDGPU/fptosi-sat-vector.ll | 55 +-
llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll | 62 +-
llvm/test/CodeGen/AMDGPU/freeze.ll | 20 +-
llvm/test/CodeGen/AMDGPU/frem.ll | 32 +-
llvm/test/CodeGen/AMDGPU/idot4u.ll | 33 +-
llvm/test/CodeGen/AMDGPU/idot8s.ll | 260 +-
llvm/test/CodeGen/AMDGPU/idot8u.ll | 202 +-
.../CodeGen/AMDGPU/integer-mad-patterns.ll | 8 +-
.../llvm.amdgcn.image.sample.g16.a16.dim.ll | 313 +-
.../AMDGPU/llvm.amdgcn.image.sample.noret.ll | 7 +-
.../test/CodeGen/AMDGPU/llvm.fptrunc.round.ll | 36 +-
llvm/test/CodeGen/AMDGPU/llvm.maximum.f16.ll | 12 +-
llvm/test/CodeGen/AMDGPU/llvm.minimum.f16.ll | 12 +-
llvm/test/CodeGen/AMDGPU/llvm.modf.ll | 10 +-
llvm/test/CodeGen/AMDGPU/mad-mix-lo.ll | 16 +-
llvm/test/CodeGen/AMDGPU/packed-op-sel.ll | 12 +-
llvm/test/CodeGen/AMDGPU/permute_i8.ll | 561 +-
llvm/test/CodeGen/AMDGPU/scalar_to_vector.ll | 9 +-
.../AMDGPU/sdwa-peephole-lshl-or-hi16pack.mir | 156 +
llvm/test/CodeGen/AMDGPU/sdwa-peephole.ll | 79 +-
llvm/test/CodeGen/AMDGPU/sminmax.v2i16.ll | 12 +-
llvm/test/CodeGen/AMDGPU/strict_fsub.f16.ll | 14 +-
llvm/test/CodeGen/AMDGPU/v_sat_pk_u8_i16.ll | 22 +-
70 files changed, 8977 insertions(+), 10177 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/sdwa-peephole-lshl-or-hi16pack.mir
diff --git a/llvm/lib/Target/AMDGPU/SIPeepholeSDWA.cpp b/llvm/lib/Target/AMDGPU/SIPeepholeSDWA.cpp
index 8c45017cd4c43..9262b004aefb6 100644
--- a/llvm/lib/Target/AMDGPU/SIPeepholeSDWA.cpp
+++ b/llvm/lib/Target/AMDGPU/SIPeepholeSDWA.cpp
@@ -66,6 +66,7 @@ class SIPeepholeSDWA {
MachineInstr *createSDWAVersion(MachineInstr &MI);
bool convertToSDWA(MachineInstr &MI, const SDWAOperandsVector &SDWAOperands);
void legalizeScalarOperands(MachineInstr &MI, const GCNSubtarget &ST) const;
+ bool splitLshlOrForSDWA(MachineBasicBlock &MBB);
public:
bool run(MachineFunction &MF);
@@ -1359,6 +1360,108 @@ void SIPeepholeSDWA::legalizeScalarOperands(MachineInstr &MI,
}
}
+// Rewrite the high-half packing idiom
+// %m = V_AND_B32 0xffff, %z ; (single use)
+// %d = V_LSHL_OR_B32 %hi, 16, %m ; (%hi << 16) | (%z & 0xffff)
+// into
+// %s = V_LSHLREV_B32 16, %hi
+// %d = V_OR_B32_sdwa %s, %z src1_sel:WORD_0
+// and erase the now-dead V_AND. The fused V_LSHL_OR_B32 has no SDWA form
+// (getSDWAOp == -1), so once ISel emits it the regular peephole can no longer
+// fold the 0xffff mask into V_OR_B32_sdwa; this restores that fold.
+//
+// The rewrite is done atomically (the sdwa OR is built directly and the V_AND
+// erased) rather than by splitting and hoping the generic matcher re-folds:
+// if the V_AND were left in place a competing fold (e.g. folding a source
+// V_LSHRREV into the V_AND) could win and leave the extra V_LSHLREV behind,
+// increasing the instruction count. Built this way the count is never worse
+// (2 ops -> 2 ops in isolation; the following MachineCSE shares the %hi<<16
+// shift when it already exists, and the generic matcher folds any source
+// select of %z into the new sdwa OR, giving the pre-#206058 codegen).
+bool SIPeepholeSDWA::splitLshlOrForSDWA(MachineBasicBlock &MBB) {
+ struct Candidate {
+ MachineInstr *LshlOr;
+ MachineInstr *AndMI;
+ MachineOperand *Hi; // V_LSHL_OR_B32 src0 (high half)
+ MachineOperand *ValSrc; // the non-0xffff operand of the V_AND
+ };
+ SmallVector<Candidate, 4> Candidates;
+
+ for (MachineInstr &MI : MBB) {
+ if (MI.getOpcode() != AMDGPU::V_LSHL_OR_B32_e64)
+ continue;
+
+ // Shift amount must be 16 (packing into the high 16 bits).
+ MachineOperand *Shift = TII->getNamedOperand(MI, AMDGPU::OpName::src1);
+ std::optional<int64_t> ShiftImm = foldToImm(*Shift);
+ if (!ShiftImm || *ShiftImm != 16)
+ continue;
+
+ // vdst is copied into the new V_OR_B32_sdwa; it must be a virtual VGPR
+ // (guaranteed pre-RA, checked for consistency with the SDWA matchers).
+ MachineOperand *Dst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst);
+ if (!Dst->isReg() || Dst->getReg().isPhysical())
+ continue;
+
+ MachineOperand *Hi = TII->getNamedOperand(MI, AMDGPU::OpName::src0);
+ MachineOperand *Src2 = TII->getNamedOperand(MI, AMDGPU::OpName::src2);
+ if (!Hi->isReg() || Hi->getReg().isPhysical() || !Src2->isReg() ||
+ Src2->getReg().isPhysical())
+ continue;
+
+ // Src2 must be (V_AND_B32 0xffff, %z), single-use, so it can be replaced
+ // by reading %z's WORD_0 and the V_AND removed.
+ if (!MRI->hasOneNonDBGUse(Src2->getReg()))
+ continue;
+ MachineInstr *AndMI = MRI->getVRegDef(Src2->getReg());
+ if (!AndMI || (AndMI->getOpcode() != AMDGPU::V_AND_B32_e32 &&
+ AndMI->getOpcode() != AMDGPU::V_AND_B32_e64))
+ continue;
+ MachineOperand *A0 = TII->getNamedOperand(*AndMI, AMDGPU::OpName::src0);
+ MachineOperand *A1 = TII->getNamedOperand(*AndMI, AMDGPU::OpName::src1);
+ std::optional<int64_t> M0 = foldToImm(*A0);
+ std::optional<int64_t> M1 = foldToImm(*A1);
+ MachineOperand *ValSrc =
+ (M0 && *M0 == 0xffff) ? A1 : ((M1 && *M1 == 0xffff) ? A0 : nullptr);
+ if (!ValSrc || !ValSrc->isReg() || ValSrc->getReg().isPhysical() ||
+ !TRI->isVGPR(*MRI, ValSrc->getReg()))
+ continue;
+
+ Candidates.push_back({&MI, AndMI, Hi, ValSrc});
+ }
+
+ for (const Candidate &C : Candidates) {
+ MachineOperand *Dst = TII->getNamedOperand(*C.LshlOr, AMDGPU::OpName::vdst);
+
+ Register ShiftReg = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
+ BuildMI(*C.LshlOr->getParent(), *C.LshlOr, C.LshlOr->getDebugLoc(),
+ TII->get(AMDGPU::V_LSHLREV_B32_e64), ShiftReg)
+ .addImm(16)
+ .add(*C.Hi);
+
+ // V_OR_B32_sdwa %d, 0, %s, 0, %z, clamp, dst_sel:DWORD, UNUSED_PAD,
+ // src0_sel:DWORD, src1_sel:WORD_0
+ BuildMI(*C.LshlOr->getParent(), *C.LshlOr, C.LshlOr->getDebugLoc(),
+ TII->get(AMDGPU::V_OR_B32_sdwa))
+ .add(*Dst)
+ .addImm(0)
+ .addReg(ShiftReg)
+ .addImm(0)
+ .add(*C.ValSrc)
+ .addImm(0)
+ .addImm(DWORD)
+ .addImm(UNUSED_PAD)
+ .addImm(DWORD)
+ .addImm(WORD_0);
+
+ MRI->clearKillFlags(C.ValSrc->getReg());
+ C.LshlOr->eraseFromParent();
+ C.AndMI->eraseFromParent();
+ }
+
+ return !Candidates.empty();
+}
+
bool SIPeepholeSDWALegacy::runOnMachineFunction(MachineFunction &MF) {
if (skipFunction(MF.getFunction()))
return false;
@@ -1381,6 +1484,12 @@ bool SIPeepholeSDWA::run(MachineFunction &MF) {
for (MachineBasicBlock &MBB : MF) {
bool Changed = false;
do {
+ // Split fused V_LSHL_OR_B32 high-half packs whose low operand is a
+ // maskable (V_AND 0xffff) value, so the mask can be folded into a
+ // single V_OR_B32_sdwa below.
+ if (splitLshlOrForSDWA(MBB))
+ Ret = true;
+
// Preprocess the ADD/SUB pairs so they could be SDWA'ed.
// Look for a possible ADD or SUB that resulted from a previously lowered
// V_{ADD|SUB}_U64_PSEUDO. The function pseudoOpConvertToVOP2
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/add.vni16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/add.vni16.ll
index 3a8d2a90a7a84..d3eed3e7b4fd9 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/add.vni16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/add.vni16.ll
@@ -43,31 +43,30 @@ define void @add_v3i16(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb, ptr addrs
; GFX9-LABEL: add_v3i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_ushort v6, v[0:1], off
+; GFX9-NEXT: global_load_ushort v6, v[0:1], off offset:2
; GFX9-NEXT: global_load_ushort v7, v[0:1], off offset:4
-; GFX9-NEXT: global_load_ushort v8, v[2:3], off
+; GFX9-NEXT: global_load_ushort v8, v[2:3], off offset:2
; GFX9-NEXT: global_load_ushort v9, v[2:3], off offset:4
-; GFX9-NEXT: global_load_ushort v10, v[0:1], off offset:2
-; GFX9-NEXT: global_load_ushort v11, v[2:3], off offset:2
+; GFX9-NEXT: global_load_ushort v10, v[0:1], off
+; GFX9-NEXT: global_load_ushort v11, v[2:3], off
+; GFX9-NEXT: v_lshlrev_b32_e64 v0, 16, s4
; GFX9-NEXT: s_waitcnt vmcnt(5)
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v6
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v6
; GFX9-NEXT: s_waitcnt vmcnt(4)
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v7
+; GFX9-NEXT: v_or_b32_sdwa v2, v0, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(3)
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v8
; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v9
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_lshl_or_b32 v0, v10, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_lshl_or_b32 v2, v11, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v1, s4, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v3, s4, 16, v3
-; GFX9-NEXT: v_pk_add_u16 v0, v0, v2
+; GFX9-NEXT: v_or_b32_sdwa v3, v3, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_pk_add_u16 v1, v1, v3
-; GFX9-NEXT: global_store_short v[4:5], v0, off
-; GFX9-NEXT: global_store_short_d16_hi v[4:5], v0, off offset:2
-; GFX9-NEXT: global_store_short v[4:5], v1, off offset:4
+; GFX9-NEXT: v_pk_add_u16 v0, v2, v0
+; GFX9-NEXT: global_store_short v[4:5], v1, off
+; GFX9-NEXT: global_store_short_d16_hi v[4:5], v1, off offset:2
+; GFX9-NEXT: global_store_short v[4:5], v0, off offset:4
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
%a = load <3 x i16>, ptr addrspace(1) %ptra, align 4
@@ -213,46 +212,45 @@ define void @add_v5i16(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb, ptr addrs
; GFX9-LABEL: add_v5i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_ushort v6, v[0:1], off
-; GFX9-NEXT: global_load_ushort v7, v[0:1], off offset:4
+; GFX9-NEXT: global_load_ushort v6, v[0:1], off offset:2
+; GFX9-NEXT: global_load_ushort v7, v[0:1], off offset:6
; GFX9-NEXT: global_load_ushort v8, v[0:1], off offset:8
-; GFX9-NEXT: global_load_ushort v9, v[2:3], off
-; GFX9-NEXT: global_load_ushort v10, v[2:3], off offset:4
+; GFX9-NEXT: global_load_ushort v9, v[2:3], off offset:2
+; GFX9-NEXT: global_load_ushort v10, v[2:3], off offset:6
; GFX9-NEXT: global_load_ushort v11, v[2:3], off offset:8
-; GFX9-NEXT: global_load_ushort v12, v[0:1], off offset:2
-; GFX9-NEXT: global_load_ushort v13, v[0:1], off offset:6
-; GFX9-NEXT: global_load_ushort v14, v[2:3], off offset:2
-; GFX9-NEXT: global_load_ushort v15, v[2:3], off offset:6
+; GFX9-NEXT: global_load_ushort v12, v[0:1], off
+; GFX9-NEXT: global_load_ushort v13, v[0:1], off offset:4
+; GFX9-NEXT: global_load_ushort v14, v[2:3], off
+; GFX9-NEXT: global_load_ushort v15, v[2:3], off offset:4
+; GFX9-NEXT: v_lshlrev_b32_e64 v0, 16, s4
; GFX9-NEXT: s_waitcnt vmcnt(9)
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v6
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v6
; GFX9-NEXT: s_waitcnt vmcnt(8)
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v7
; GFX9-NEXT: s_waitcnt vmcnt(7)
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v8
+; GFX9-NEXT: v_or_b32_sdwa v3, v0, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(6)
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v9
+; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v9
; GFX9-NEXT: s_waitcnt vmcnt(5)
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v10
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v10
; GFX9-NEXT: s_waitcnt vmcnt(4)
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v11
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(3)
-; GFX9-NEXT: v_lshl_or_b32 v0, v12, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_lshl_or_b32 v1, v13, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v2, v2, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_lshl_or_b32 v3, v14, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v2, s4, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v6, v6, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_lshl_or_b32 v6, v15, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, s4, 16, v7
-; GFX9-NEXT: v_pk_add_u16 v0, v0, v3
+; GFX9-NEXT: v_or_b32_sdwa v7, v7, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_pk_add_u16 v1, v1, v6
+; GFX9-NEXT: v_pk_add_u16 v0, v3, v0
; GFX9-NEXT: v_pk_add_u16 v2, v2, v7
-; GFX9-NEXT: global_store_short v[4:5], v0, off
-; GFX9-NEXT: global_store_short_d16_hi v[4:5], v0, off offset:2
-; GFX9-NEXT: global_store_short v[4:5], v1, off offset:4
-; GFX9-NEXT: global_store_short_d16_hi v[4:5], v1, off offset:6
-; GFX9-NEXT: global_store_short v[4:5], v2, off offset:8
+; GFX9-NEXT: global_store_short v[4:5], v1, off
+; GFX9-NEXT: global_store_short_d16_hi v[4:5], v1, off offset:2
+; GFX9-NEXT: global_store_short v[4:5], v2, off offset:4
+; GFX9-NEXT: global_store_short_d16_hi v[4:5], v2, off offset:6
+; GFX9-NEXT: global_store_short v[4:5], v0, off offset:8
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
%a = load <5 x i16>, ptr addrspace(1) %ptra, align 4
@@ -432,61 +430,60 @@ define void @addv_7i16(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb, ptr addrs
; GFX9-LABEL: addv_7i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_ushort v6, v[0:1], off
-; GFX9-NEXT: global_load_ushort v7, v[0:1], off offset:4
-; GFX9-NEXT: global_load_ushort v8, v[0:1], off offset:8
+; GFX9-NEXT: global_load_ushort v6, v[0:1], off offset:2
+; GFX9-NEXT: global_load_ushort v7, v[0:1], off offset:6
+; GFX9-NEXT: global_load_ushort v8, v[0:1], off offset:10
; GFX9-NEXT: global_load_ushort v9, v[0:1], off offset:12
-; GFX9-NEXT: global_load_ushort v10, v[2:3], off
-; GFX9-NEXT: global_load_ushort v11, v[2:3], off offset:4
-; GFX9-NEXT: global_load_ushort v12, v[2:3], off offset:8
+; GFX9-NEXT: global_load_ushort v10, v[2:3], off offset:2
+; GFX9-NEXT: global_load_ushort v11, v[2:3], off offset:6
+; GFX9-NEXT: global_load_ushort v12, v[2:3], off offset:10
; GFX9-NEXT: global_load_ushort v13, v[2:3], off offset:12
-; GFX9-NEXT: global_load_ushort v14, v[0:1], off offset:2
-; GFX9-NEXT: global_load_ushort v15, v[0:1], off offset:6
-; GFX9-NEXT: global_load_ushort v16, v[0:1], off offset:10
-; GFX9-NEXT: global_load_ushort v17, v[2:3], off offset:2
-; GFX9-NEXT: global_load_ushort v18, v[2:3], off offset:6
-; GFX9-NEXT: global_load_ushort v19, v[2:3], off offset:10
+; GFX9-NEXT: global_load_ushort v14, v[0:1], off
+; GFX9-NEXT: global_load_ushort v15, v[0:1], off offset:4
+; GFX9-NEXT: global_load_ushort v16, v[0:1], off offset:8
+; GFX9-NEXT: global_load_ushort v17, v[2:3], off
+; GFX9-NEXT: global_load_ushort v18, v[2:3], off offset:4
+; GFX9-NEXT: global_load_ushort v19, v[2:3], off offset:8
+; GFX9-NEXT: v_lshlrev_b32_e64 v0, 16, s4
; GFX9-NEXT: s_waitcnt vmcnt(13)
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v6
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v6
; GFX9-NEXT: s_waitcnt vmcnt(12)
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v7
; GFX9-NEXT: s_waitcnt vmcnt(11)
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v8
; GFX9-NEXT: s_waitcnt vmcnt(10)
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v9
+; GFX9-NEXT: v_or_b32_sdwa v6, v0, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(9)
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v10
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v10
; GFX9-NEXT: s_waitcnt vmcnt(8)
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v11
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v11
; GFX9-NEXT: s_waitcnt vmcnt(7)
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v12
+; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v12
; GFX9-NEXT: s_waitcnt vmcnt(6)
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v13
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(5)
-; GFX9-NEXT: v_lshl_or_b32 v0, v14, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(4)
-; GFX9-NEXT: v_lshl_or_b32 v1, v15, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v2, v2, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(3)
-; GFX9-NEXT: v_lshl_or_b32 v2, v16, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v3, v3, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_lshl_or_b32 v6, v17, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v3, s4, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v7, v7, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_lshl_or_b32 v7, v18, 16, v7
+; GFX9-NEXT: v_or_b32_sdwa v8, v8, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_lshl_or_b32 v8, v19, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, s4, 16, v9
-; GFX9-NEXT: v_pk_add_u16 v0, v0, v6
+; GFX9-NEXT: v_or_b32_sdwa v9, v9, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_pk_add_u16 v1, v1, v7
+; GFX9-NEXT: v_pk_add_u16 v0, v6, v0
; GFX9-NEXT: v_pk_add_u16 v2, v2, v8
; GFX9-NEXT: v_pk_add_u16 v3, v3, v9
-; GFX9-NEXT: global_store_short v[4:5], v0, off
-; GFX9-NEXT: global_store_short_d16_hi v[4:5], v0, off offset:2
-; GFX9-NEXT: global_store_short v[4:5], v1, off offset:4
-; GFX9-NEXT: global_store_short_d16_hi v[4:5], v1, off offset:6
-; GFX9-NEXT: global_store_short v[4:5], v2, off offset:8
-; GFX9-NEXT: global_store_short_d16_hi v[4:5], v2, off offset:10
-; GFX9-NEXT: global_store_short v[4:5], v3, off offset:12
+; GFX9-NEXT: global_store_short v[4:5], v1, off
+; GFX9-NEXT: global_store_short_d16_hi v[4:5], v1, off offset:2
+; GFX9-NEXT: global_store_short v[4:5], v2, off offset:4
+; GFX9-NEXT: global_store_short_d16_hi v[4:5], v2, off offset:6
+; GFX9-NEXT: global_store_short v[4:5], v3, off offset:8
+; GFX9-NEXT: global_store_short_d16_hi v[4:5], v3, off offset:10
+; GFX9-NEXT: global_store_short v[4:5], v0, off offset:12
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
%a = load <7 x i16>, ptr addrspace(1) %ptra, align 4
@@ -565,18 +562,17 @@ define void @add_v9i16(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb, ptr addrs
; GFX9-NEXT: global_load_ushort v14, v[0:1], off offset:16
; GFX9-NEXT: global_load_ushort v15, v[2:3], off offset:16
; GFX9-NEXT: global_load_dwordx4 v[10:13], v[2:3], off
+; GFX9-NEXT: v_lshlrev_b32_e64 v0, 16, s4
; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
+; GFX9-NEXT: v_or_b32_sdwa v14, v0, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
+; GFX9-NEXT: v_or_b32_sdwa v15, v0, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_pk_add_u16 v0, v6, v10
; GFX9-NEXT: v_pk_add_u16 v1, v7, v11
; GFX9-NEXT: v_pk_add_u16 v2, v8, v12
; GFX9-NEXT: v_pk_add_u16 v3, v9, v13
-; GFX9-NEXT: v_lshl_or_b32 v6, s4, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v7, s4, 16, v15
-; GFX9-NEXT: v_pk_add_u16 v6, v6, v7
+; GFX9-NEXT: v_pk_add_u16 v6, v14, v15
; GFX9-NEXT: global_store_dwordx4 v[4:5], v[0:3], off
; GFX9-NEXT: global_store_short v[4:5], v6, off offset:16
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -740,40 +736,38 @@ define void @add_v11i16(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb, ptr addr
; GFX9-LABEL: add_v11i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_ushort v14, v[0:1], off offset:16
+; GFX9-NEXT: global_load_ushort v14, v[0:1], off offset:18
; GFX9-NEXT: global_load_ushort v15, v[0:1], off offset:20
; GFX9-NEXT: global_load_dwordx4 v[6:9], v[2:3], off
-; GFX9-NEXT: global_load_ushort v16, v[2:3], off offset:16
+; GFX9-NEXT: global_load_ushort v16, v[2:3], off offset:18
; GFX9-NEXT: global_load_ushort v17, v[2:3], off offset:20
; GFX9-NEXT: global_load_dwordx4 v[10:13], v[0:1], off
-; GFX9-NEXT: global_load_ushort v18, v[0:1], off offset:18
-; GFX9-NEXT: global_load_ushort v19, v[2:3], off offset:18
+; GFX9-NEXT: global_load_ushort v18, v[0:1], off offset:16
+; GFX9-NEXT: global_load_ushort v19, v[2:3], off offset:16
+; GFX9-NEXT: v_lshlrev_b32_e64 v0, 16, s4
; GFX9-NEXT: s_waitcnt vmcnt(7)
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
+; GFX9-NEXT: v_lshlrev_b32_e32 v14, 16, v14
; GFX9-NEXT: s_waitcnt vmcnt(6)
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
+; GFX9-NEXT: v_or_b32_sdwa v15, v0, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(4)
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
+; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v16
; GFX9-NEXT: s_waitcnt vmcnt(3)
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
+; GFX9-NEXT: v_or_b32_sdwa v17, v0, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(2)
; GFX9-NEXT: v_pk_add_u16 v0, v10, v6
; GFX9-NEXT: v_pk_add_u16 v1, v11, v7
; GFX9-NEXT: v_pk_add_u16 v2, v12, v8
; GFX9-NEXT: v_pk_add_u16 v3, v13, v9
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_lshl_or_b32 v6, v18, 16, v14
+; GFX9-NEXT: v_or_b32_sdwa v6, v14, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_lshl_or_b32 v8, v19, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v7, s4, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v9, s4, 16, v17
+; GFX9-NEXT: v_or_b32_sdwa v7, v16, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: global_store_dwordx4 v[4:5], v[0:3], off
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: v_pk_add_u16 v0, v6, v8
-; GFX9-NEXT: v_pk_add_u16 v1, v7, v9
+; GFX9-NEXT: v_pk_add_u16 v8, v15, v17
+; GFX9-NEXT: v_pk_add_u16 v0, v6, v7
; GFX9-NEXT: global_store_short v[4:5], v0, off offset:16
; GFX9-NEXT: global_store_short_d16_hi v[4:5], v0, off offset:18
-; GFX9-NEXT: global_store_short v[4:5], v1, off offset:20
+; GFX9-NEXT: global_store_short v[4:5], v8, off offset:20
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
%a = load <11 x i16>, ptr addrspace(1) %ptra, align 4
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fpow.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fpow.ll
index fc7c4a57fe32e..07111699de4b3 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fpow.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fpow.ll
@@ -282,12 +282,11 @@ define <2 x half> @v_pow_v2f16(<2 x half> %x, <2 x half> %y) {
; GFX10-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX10-NEXT: v_mul_legacy_f32_e32 v2, v2, v3
; GFX10-NEXT: v_mul_legacy_f32_e32 v0, v0, v1
-; GFX10-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX10-NEXT: v_cvt_f16_f32_e32 v1, v2
; GFX10-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX10-NEXT: v_exp_f16_e32 v1, v2
-; GFX10-NEXT: v_exp_f16_e32 v0, v0
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_lshl_or_b32 v0, v0, 16, v1
+; GFX10-NEXT: v_exp_f16_e32 v1, v1
+; GFX10-NEXT: v_exp_f16_sdwa v0, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: v_pow_v2f16:
@@ -429,12 +428,11 @@ define <2 x half> @v_pow_v2f16_fneg_lhs(<2 x half> %x, <2 x half> %y) {
; GFX10-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX10-NEXT: v_mul_legacy_f32_e32 v2, v2, v3
; GFX10-NEXT: v_mul_legacy_f32_e32 v0, v0, v1
-; GFX10-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX10-NEXT: v_cvt_f16_f32_e32 v1, v2
; GFX10-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX10-NEXT: v_exp_f16_e32 v1, v2
-; GFX10-NEXT: v_exp_f16_e32 v0, v0
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_lshl_or_b32 v0, v0, 16, v1
+; GFX10-NEXT: v_exp_f16_e32 v1, v1
+; GFX10-NEXT: v_exp_f16_sdwa v0, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: v_pow_v2f16_fneg_lhs:
@@ -573,20 +571,19 @@ define <2 x half> @v_pow_v2f16_fneg_rhs(<2 x half> %x, <2 x half> %y) {
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_log_f16_e32 v2, v0
-; GFX10-NEXT: v_xor_b32_e32 v1, 0x80008000, v1
; GFX10-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX10-NEXT: v_xor_b32_e32 v1, 0x80008000, v1
; GFX10-NEXT: v_cvt_f32_f16_e32 v3, v1
; GFX10-NEXT: v_cvt_f32_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
; GFX10-NEXT: v_cvt_f32_f16_e32 v2, v2
; GFX10-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX10-NEXT: v_mul_legacy_f32_e32 v2, v2, v3
; GFX10-NEXT: v_mul_legacy_f32_e32 v0, v0, v1
-; GFX10-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX10-NEXT: v_cvt_f16_f32_e32 v1, v2
; GFX10-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX10-NEXT: v_exp_f16_e32 v1, v2
-; GFX10-NEXT: v_exp_f16_e32 v0, v0
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_lshl_or_b32 v0, v0, 16, v1
+; GFX10-NEXT: v_exp_f16_e32 v1, v1
+; GFX10-NEXT: v_exp_f16_sdwa v0, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: v_pow_v2f16_fneg_rhs:
@@ -736,12 +733,11 @@ define <2 x half> @v_pow_v2f16_fneg_lhs_rhs(<2 x half> %x, <2 x half> %y) {
; GFX10-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX10-NEXT: v_mul_legacy_f32_e32 v2, v2, v3
; GFX10-NEXT: v_mul_legacy_f32_e32 v0, v0, v1
-; GFX10-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX10-NEXT: v_cvt_f16_f32_e32 v1, v2
; GFX10-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX10-NEXT: v_exp_f16_e32 v1, v2
-; GFX10-NEXT: v_exp_f16_e32 v0, v0
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_lshl_or_b32 v0, v0, 16, v1
+; GFX10-NEXT: v_exp_f16_e32 v1, v1
+; GFX10-NEXT: v_exp_f16_sdwa v0, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: v_pow_v2f16_fneg_lhs_rhs:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.abs.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.abs.ll
index ecc0f74b2efef..0982a8f6fa755 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.abs.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.abs.ll
@@ -655,11 +655,11 @@ define <3 x i16> @abs_vgpr_v3i16(<3 x i16> %arg) {
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_sub_nc_u16 v2, 0, v1
+; GFX10-NEXT: v_pk_sub_i16 v3, 0, v0
; GFX10-NEXT: v_max_i16 v1, v1, v2
-; GFX10-NEXT: v_pk_sub_i16 v2, 0, v0
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_pk_max_i16 v0, v0, v2
-; GFX10-NEXT: v_lshl_or_b32 v1, s4, 16, v1
+; GFX10-NEXT: v_lshlrev_b32_e64 v2, 16, s4
+; GFX10-NEXT: v_pk_max_i16 v0, v0, v3
+; GFX10-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-FAKE16-LABEL: abs_vgpr_v3i16:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.atomic.dim.a16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.atomic.dim.a16.ll
index 89f73ba07d58d..4c0eb9c776045 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.atomic.dim.a16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.atomic.dim.a16.ll
@@ -8,7 +8,7 @@ define amdgpu_ps float @atomic_swap_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX9-LABEL: atomic_swap_i32_1d:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT: v_lshlrev_b32_e64 v2, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -16,27 +16,11 @@ define amdgpu_ps float @atomic_swap_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_swap v0, v1, s[0:7] dmask:0x1 unorm glc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: atomic_swap_i32_1d:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: s_mov_b32 s0, s2
-; GFX10-NEXT: s_mov_b32 s1, s3
-; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v1, s0, 16, v1
-; GFX10-NEXT: s_mov_b32 s4, s6
-; GFX10-NEXT: s_mov_b32 s5, s7
-; GFX10-NEXT: s_mov_b32 s6, s8
-; GFX10-NEXT: s_mov_b32 s7, s9
-; GFX10-NEXT: image_atomic_swap v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: atomic_swap_i32_1d:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v1, 0xffff, v1
@@ -62,7 +46,7 @@ define amdgpu_ps float @atomic_add_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX9-LABEL: atomic_add_i32_1d:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT: v_lshlrev_b32_e64 v2, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -70,27 +54,11 @@ define amdgpu_ps float @atomic_add_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_add v0, v1, s[0:7] dmask:0x1 unorm glc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: atomic_add_i32_1d:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: s_mov_b32 s0, s2
-; GFX10-NEXT: s_mov_b32 s1, s3
-; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v1, s0, 16, v1
-; GFX10-NEXT: s_mov_b32 s4, s6
-; GFX10-NEXT: s_mov_b32 s5, s7
-; GFX10-NEXT: s_mov_b32 s6, s8
-; GFX10-NEXT: s_mov_b32 s7, s9
-; GFX10-NEXT: image_atomic_add v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: atomic_add_i32_1d:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v1, 0xffff, v1
@@ -116,7 +84,7 @@ define amdgpu_ps float @atomic_sub_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX9-LABEL: atomic_sub_i32_1d:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT: v_lshlrev_b32_e64 v2, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -124,27 +92,11 @@ define amdgpu_ps float @atomic_sub_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_sub v0, v1, s[0:7] dmask:0x1 unorm glc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: atomic_sub_i32_1d:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: s_mov_b32 s0, s2
-; GFX10-NEXT: s_mov_b32 s1, s3
-; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v1, s0, 16, v1
-; GFX10-NEXT: s_mov_b32 s4, s6
-; GFX10-NEXT: s_mov_b32 s5, s7
-; GFX10-NEXT: s_mov_b32 s6, s8
-; GFX10-NEXT: s_mov_b32 s7, s9
-; GFX10-NEXT: image_atomic_sub v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: atomic_sub_i32_1d:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v1, 0xffff, v1
@@ -170,7 +122,7 @@ define amdgpu_ps float @atomic_smin_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX9-LABEL: atomic_smin_i32_1d:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT: v_lshlrev_b32_e64 v2, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -178,27 +130,11 @@ define amdgpu_ps float @atomic_smin_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_smin v0, v1, s[0:7] dmask:0x1 unorm glc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: atomic_smin_i32_1d:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: s_mov_b32 s0, s2
-; GFX10-NEXT: s_mov_b32 s1, s3
-; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v1, s0, 16, v1
-; GFX10-NEXT: s_mov_b32 s4, s6
-; GFX10-NEXT: s_mov_b32 s5, s7
-; GFX10-NEXT: s_mov_b32 s6, s8
-; GFX10-NEXT: s_mov_b32 s7, s9
-; GFX10-NEXT: image_atomic_smin v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: atomic_smin_i32_1d:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v1, 0xffff, v1
@@ -224,7 +160,7 @@ define amdgpu_ps float @atomic_umin_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX9-LABEL: atomic_umin_i32_1d:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT: v_lshlrev_b32_e64 v2, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -232,27 +168,11 @@ define amdgpu_ps float @atomic_umin_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_umin v0, v1, s[0:7] dmask:0x1 unorm glc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: atomic_umin_i32_1d:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: s_mov_b32 s0, s2
-; GFX10-NEXT: s_mov_b32 s1, s3
-; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v1, s0, 16, v1
-; GFX10-NEXT: s_mov_b32 s4, s6
-; GFX10-NEXT: s_mov_b32 s5, s7
-; GFX10-NEXT: s_mov_b32 s6, s8
-; GFX10-NEXT: s_mov_b32 s7, s9
-; GFX10-NEXT: image_atomic_umin v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: atomic_umin_i32_1d:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v1, 0xffff, v1
@@ -278,7 +198,7 @@ define amdgpu_ps float @atomic_smax_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX9-LABEL: atomic_smax_i32_1d:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT: v_lshlrev_b32_e64 v2, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -286,27 +206,11 @@ define amdgpu_ps float @atomic_smax_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_smax v0, v1, s[0:7] dmask:0x1 unorm glc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: atomic_smax_i32_1d:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: s_mov_b32 s0, s2
-; GFX10-NEXT: s_mov_b32 s1, s3
-; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v1, s0, 16, v1
-; GFX10-NEXT: s_mov_b32 s4, s6
-; GFX10-NEXT: s_mov_b32 s5, s7
-; GFX10-NEXT: s_mov_b32 s6, s8
-; GFX10-NEXT: s_mov_b32 s7, s9
-; GFX10-NEXT: image_atomic_smax v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: atomic_smax_i32_1d:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v1, 0xffff, v1
@@ -332,7 +236,7 @@ define amdgpu_ps float @atomic_umax_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX9-LABEL: atomic_umax_i32_1d:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT: v_lshlrev_b32_e64 v2, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -340,27 +244,11 @@ define amdgpu_ps float @atomic_umax_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_umax v0, v1, s[0:7] dmask:0x1 unorm glc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: atomic_umax_i32_1d:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: s_mov_b32 s0, s2
-; GFX10-NEXT: s_mov_b32 s1, s3
-; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v1, s0, 16, v1
-; GFX10-NEXT: s_mov_b32 s4, s6
-; GFX10-NEXT: s_mov_b32 s5, s7
-; GFX10-NEXT: s_mov_b32 s6, s8
-; GFX10-NEXT: s_mov_b32 s7, s9
-; GFX10-NEXT: image_atomic_umax v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: atomic_umax_i32_1d:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v1, 0xffff, v1
@@ -386,7 +274,7 @@ define amdgpu_ps float @atomic_and_i321d(<8 x i32> inreg %rsrc, i32 %data, i16 %
; GFX9-LABEL: atomic_and_i321d:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT: v_lshlrev_b32_e64 v2, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -394,27 +282,11 @@ define amdgpu_ps float @atomic_and_i321d(<8 x i32> inreg %rsrc, i32 %data, i16 %
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_and v0, v1, s[0:7] dmask:0x1 unorm glc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: atomic_and_i321d:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: s_mov_b32 s0, s2
-; GFX10-NEXT: s_mov_b32 s1, s3
-; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v1, s0, 16, v1
-; GFX10-NEXT: s_mov_b32 s4, s6
-; GFX10-NEXT: s_mov_b32 s5, s7
-; GFX10-NEXT: s_mov_b32 s6, s8
-; GFX10-NEXT: s_mov_b32 s7, s9
-; GFX10-NEXT: image_atomic_and v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: atomic_and_i321d:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v1, 0xffff, v1
@@ -440,7 +312,7 @@ define amdgpu_ps float @atomic_or_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %
; GFX9-LABEL: atomic_or_i32_1d:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT: v_lshlrev_b32_e64 v2, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -448,27 +320,11 @@ define amdgpu_ps float @atomic_or_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_or v0, v1, s[0:7] dmask:0x1 unorm glc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: atomic_or_i32_1d:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: s_mov_b32 s0, s2
-; GFX10-NEXT: s_mov_b32 s1, s3
-; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v1, s0, 16, v1
-; GFX10-NEXT: s_mov_b32 s4, s6
-; GFX10-NEXT: s_mov_b32 s5, s7
-; GFX10-NEXT: s_mov_b32 s6, s8
-; GFX10-NEXT: s_mov_b32 s7, s9
-; GFX10-NEXT: image_atomic_or v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: atomic_or_i32_1d:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v1, 0xffff, v1
@@ -494,7 +350,7 @@ define amdgpu_ps float @atomic_xor_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX9-LABEL: atomic_xor_i32_1d:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT: v_lshlrev_b32_e64 v2, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -502,27 +358,11 @@ define amdgpu_ps float @atomic_xor_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_xor v0, v1, s[0:7] dmask:0x1 unorm glc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: atomic_xor_i32_1d:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: s_mov_b32 s0, s2
-; GFX10-NEXT: s_mov_b32 s1, s3
-; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v1, s0, 16, v1
-; GFX10-NEXT: s_mov_b32 s4, s6
-; GFX10-NEXT: s_mov_b32 s5, s7
-; GFX10-NEXT: s_mov_b32 s6, s8
-; GFX10-NEXT: s_mov_b32 s7, s9
-; GFX10-NEXT: image_atomic_xor v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: atomic_xor_i32_1d:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v1, 0xffff, v1
@@ -548,7 +388,7 @@ define amdgpu_ps float @atomic_inc_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX9-LABEL: atomic_inc_i32_1d:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT: v_lshlrev_b32_e64 v2, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -556,27 +396,11 @@ define amdgpu_ps float @atomic_inc_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_inc v0, v1, s[0:7] dmask:0x1 unorm glc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: atomic_inc_i32_1d:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: s_mov_b32 s0, s2
-; GFX10-NEXT: s_mov_b32 s1, s3
-; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v1, s0, 16, v1
-; GFX10-NEXT: s_mov_b32 s4, s6
-; GFX10-NEXT: s_mov_b32 s5, s7
-; GFX10-NEXT: s_mov_b32 s6, s8
-; GFX10-NEXT: s_mov_b32 s7, s9
-; GFX10-NEXT: image_atomic_inc v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: atomic_inc_i32_1d:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v1, 0xffff, v1
@@ -602,7 +426,7 @@ define amdgpu_ps float @atomic_dec_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX9-LABEL: atomic_dec_i32_1d:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT: v_lshlrev_b32_e64 v2, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -610,27 +434,11 @@ define amdgpu_ps float @atomic_dec_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_dec v0, v1, s[0:7] dmask:0x1 unorm glc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: atomic_dec_i32_1d:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: s_mov_b32 s0, s2
-; GFX10-NEXT: s_mov_b32 s1, s3
-; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v1, s0, 16, v1
-; GFX10-NEXT: s_mov_b32 s4, s6
-; GFX10-NEXT: s_mov_b32 s5, s7
-; GFX10-NEXT: s_mov_b32 s6, s8
-; GFX10-NEXT: s_mov_b32 s7, s9
-; GFX10-NEXT: image_atomic_dec v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: atomic_dec_i32_1d:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v1, 0xffff, v1
@@ -656,7 +464,7 @@ define amdgpu_ps float @atomic_cmpswap_i32_1d(<8 x i32> inreg %rsrc, i32 %cmp, i
; GFX9-LABEL: atomic_cmpswap_i32_1d:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX9-NEXT: v_lshlrev_b32_e64 v3, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -664,27 +472,11 @@ define amdgpu_ps float @atomic_cmpswap_i32_1d(<8 x i32> inreg %rsrc, i32 %cmp, i
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_cmpswap v[0:1], v2, s[0:7] dmask:0x3 unorm glc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: atomic_cmpswap_i32_1d:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10-NEXT: s_mov_b32 s0, s2
-; GFX10-NEXT: s_mov_b32 s1, s3
-; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v2, s0, 16, v2
-; GFX10-NEXT: s_mov_b32 s4, s6
-; GFX10-NEXT: s_mov_b32 s5, s7
-; GFX10-NEXT: s_mov_b32 s6, s8
-; GFX10-NEXT: s_mov_b32 s7, s9
-; GFX10-NEXT: image_atomic_cmpswap v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: atomic_cmpswap_i32_1d:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -709,7 +501,7 @@ main_body:
define amdgpu_ps float @atomic_add_i32_2d(<8 x i32> inreg %rsrc, i32 %data, i16 %s, i16 %t) {
; GFX9-LABEL: atomic_add_i32_2d:
; GFX9: ; %bb.0: ; %main_body
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX9-NEXT: s_mov_b32 s0, s2
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
@@ -718,27 +510,11 @@ define amdgpu_ps float @atomic_add_i32_2d(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_add v0, v1, s[0:7] dmask:0x1 unorm glc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: atomic_add_i32_2d:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: s_mov_b32 s0, s2
-; GFX10-NEXT: s_mov_b32 s1, s3
-; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX10-NEXT: s_mov_b32 s4, s6
-; GFX10-NEXT: s_mov_b32 s5, s7
-; GFX10-NEXT: s_mov_b32 s6, s8
-; GFX10-NEXT: s_mov_b32 s7, s9
-; GFX10-NEXT: image_atomic_add v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D unorm glc a16
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: atomic_add_i32_2d:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v1, 0xffff, v1
@@ -763,10 +539,10 @@ main_body:
define amdgpu_ps float @atomic_add_i32_3d(<8 x i32> inreg %rsrc, i32 %data, i16 %s, i16 %t, i16 %r) {
; GFX9-LABEL: atomic_add_i32_3d:
; GFX9: ; %bb.0: ; %main_body
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v3
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e64 v2, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -774,29 +550,11 @@ define amdgpu_ps float @atomic_add_i32_3d(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 unorm glc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: atomic_add_i32_3d:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX10-NEXT: s_mov_b32 s0, s2
-; GFX10-NEXT: s_mov_b32 s1, s3
-; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX10-NEXT: v_lshl_or_b32 v2, s0, 16, v3
-; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: s_mov_b32 s4, s6
-; GFX10-NEXT: s_mov_b32 s5, s7
-; GFX10-NEXT: s_mov_b32 s6, s8
-; GFX10-NEXT: s_mov_b32 s7, s9
-; GFX10-NEXT: image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_3D unorm glc a16
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: atomic_add_i32_3d:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v1, 0xffff, v1
@@ -823,10 +581,10 @@ main_body:
define amdgpu_ps float @atomic_add_i32_cube(<8 x i32> inreg %rsrc, i32 %data, i16 %s, i16 %t, i16 %face) {
; GFX9-LABEL: atomic_add_i32_cube:
; GFX9: ; %bb.0: ; %main_body
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v3
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e64 v2, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -834,29 +592,11 @@ define amdgpu_ps float @atomic_add_i32_cube(<8 x i32> inreg %rsrc, i32 %data, i1
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 unorm glc a16 da
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: atomic_add_i32_cube:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX10-NEXT: s_mov_b32 s0, s2
-; GFX10-NEXT: s_mov_b32 s1, s3
-; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX10-NEXT: v_lshl_or_b32 v2, s0, 16, v3
-; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: s_mov_b32 s4, s6
-; GFX10-NEXT: s_mov_b32 s5, s7
-; GFX10-NEXT: s_mov_b32 s6, s8
-; GFX10-NEXT: s_mov_b32 s7, s9
-; GFX10-NEXT: image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_CUBE unorm glc a16
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: atomic_add_i32_cube:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v1, 0xffff, v1
@@ -883,7 +623,7 @@ main_body:
define amdgpu_ps float @atomic_add_i32_1darray(<8 x i32> inreg %rsrc, i32 %data, i16 %s, i16 %slice) {
; GFX9-LABEL: atomic_add_i32_1darray:
; GFX9: ; %bb.0: ; %main_body
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX9-NEXT: s_mov_b32 s0, s2
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
@@ -892,27 +632,11 @@ define amdgpu_ps float @atomic_add_i32_1darray(<8 x i32> inreg %rsrc, i32 %data,
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_add v0, v1, s[0:7] dmask:0x1 unorm glc a16 da
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: atomic_add_i32_1darray:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: s_mov_b32 s0, s2
-; GFX10-NEXT: s_mov_b32 s1, s3
-; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX10-NEXT: s_mov_b32 s4, s6
-; GFX10-NEXT: s_mov_b32 s5, s7
-; GFX10-NEXT: s_mov_b32 s6, s8
-; GFX10-NEXT: s_mov_b32 s7, s9
-; GFX10-NEXT: image_atomic_add v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D_ARRAY unorm glc a16
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: atomic_add_i32_1darray:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v1, 0xffff, v1
@@ -937,10 +661,10 @@ main_body:
define amdgpu_ps float @atomic_add_i32_2darray(<8 x i32> inreg %rsrc, i32 %data, i16 %s, i16 %t, i16 %slice) {
; GFX9-LABEL: atomic_add_i32_2darray:
; GFX9: ; %bb.0: ; %main_body
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v3
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e64 v2, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -948,29 +672,11 @@ define amdgpu_ps float @atomic_add_i32_2darray(<8 x i32> inreg %rsrc, i32 %data,
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 unorm glc a16 da
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: atomic_add_i32_2darray:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX10-NEXT: s_mov_b32 s0, s2
-; GFX10-NEXT: s_mov_b32 s1, s3
-; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX10-NEXT: v_lshl_or_b32 v2, s0, 16, v3
-; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: s_mov_b32 s4, s6
-; GFX10-NEXT: s_mov_b32 s5, s7
-; GFX10-NEXT: s_mov_b32 s6, s8
-; GFX10-NEXT: s_mov_b32 s7, s9
-; GFX10-NEXT: image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY unorm glc a16
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: atomic_add_i32_2darray:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v1, 0xffff, v1
@@ -997,10 +703,10 @@ main_body:
define amdgpu_ps float @atomic_add_i32_2dmsaa(<8 x i32> inreg %rsrc, i32 %data, i16 %s, i16 %t, i16 %fragid) {
; GFX9-LABEL: atomic_add_i32_2dmsaa:
; GFX9: ; %bb.0: ; %main_body
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v3
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e64 v2, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -1008,29 +714,11 @@ define amdgpu_ps float @atomic_add_i32_2dmsaa(<8 x i32> inreg %rsrc, i32 %data,
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 unorm glc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: atomic_add_i32_2dmsaa:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX10-NEXT: s_mov_b32 s0, s2
-; GFX10-NEXT: s_mov_b32 s1, s3
-; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX10-NEXT: v_lshl_or_b32 v2, s0, 16, v3
-; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: s_mov_b32 s4, s6
-; GFX10-NEXT: s_mov_b32 s5, s7
-; GFX10-NEXT: s_mov_b32 s6, s8
-; GFX10-NEXT: s_mov_b32 s7, s9
-; GFX10-NEXT: image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm glc a16
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: atomic_add_i32_2dmsaa:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v1, 0xffff, v1
@@ -1057,9 +745,9 @@ main_body:
define amdgpu_ps float @atomic_add_i32_2darraymsaa(<8 x i32> inreg %rsrc, i32 %data, i16 %s, i16 %t, i16 %slice, i16 %fragid) {
; GFX9-LABEL: atomic_add_i32_2darraymsaa:
; GFX9: ; %bb.0: ; %main_body
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v3
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v4
; GFX9-NEXT: s_mov_b32 s0, s2
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
@@ -1068,29 +756,11 @@ define amdgpu_ps float @atomic_add_i32_2darraymsaa(<8 x i32> inreg %rsrc, i32 %d
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v2, v4, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 unorm glc a16 da
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: atomic_add_i32_2darraymsaa:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX10-NEXT: s_mov_b32 s0, s2
-; GFX10-NEXT: s_mov_b32 s1, s3
-; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX10-NEXT: v_lshl_or_b32 v2, v4, 16, v3
-; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: s_mov_b32 s4, s6
-; GFX10-NEXT: s_mov_b32 s5, s7
-; GFX10-NEXT: s_mov_b32 s6, s8
-; GFX10-NEXT: s_mov_b32 s7, s9
-; GFX10-NEXT: image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm glc a16
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: atomic_add_i32_2darraymsaa:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v1, 0xffff, v1
@@ -1118,7 +788,7 @@ define amdgpu_ps float @atomic_add_i32_1d_slc(<8 x i32> inreg %rsrc, i32 %data,
; GFX9-LABEL: atomic_add_i32_1d_slc:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT: v_lshlrev_b32_e64 v2, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -1126,27 +796,11 @@ define amdgpu_ps float @atomic_add_i32_1d_slc(<8 x i32> inreg %rsrc, i32 %data,
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_add v0, v1, s[0:7] dmask:0x1 unorm glc slc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: atomic_add_i32_1d_slc:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: s_mov_b32 s0, s2
-; GFX10-NEXT: s_mov_b32 s1, s3
-; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v1, s0, 16, v1
-; GFX10-NEXT: s_mov_b32 s4, s6
-; GFX10-NEXT: s_mov_b32 s5, s7
-; GFX10-NEXT: s_mov_b32 s6, s8
-; GFX10-NEXT: s_mov_b32 s7, s9
-; GFX10-NEXT: image_atomic_add v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc slc a16
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: atomic_add_i32_1d_slc:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v1, 0xffff, v1
@@ -1172,7 +826,7 @@ define amdgpu_ps <2 x float> @atomic_swap_i64_1d(<8 x i32> inreg %rsrc, i64 %dat
; GFX9-LABEL: atomic_swap_i64_1d:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX9-NEXT: v_lshlrev_b32_e64 v3, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -1180,27 +834,11 @@ define amdgpu_ps <2 x float> @atomic_swap_i64_1d(<8 x i32> inreg %rsrc, i64 %dat
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_swap v[0:1], v2, s[0:7] dmask:0x3 unorm glc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: atomic_swap_i64_1d:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10-NEXT: s_mov_b32 s0, s2
-; GFX10-NEXT: s_mov_b32 s1, s3
-; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v2, s0, 16, v2
-; GFX10-NEXT: s_mov_b32 s4, s6
-; GFX10-NEXT: s_mov_b32 s5, s7
-; GFX10-NEXT: s_mov_b32 s6, s8
-; GFX10-NEXT: s_mov_b32 s7, s9
-; GFX10-NEXT: image_atomic_swap v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: atomic_swap_i64_1d:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -1226,7 +864,7 @@ define amdgpu_ps <2 x float> @atomic_add_i64_1d(<8 x i32> inreg %rsrc, i64 %data
; GFX9-LABEL: atomic_add_i64_1d:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX9-NEXT: v_lshlrev_b32_e64 v3, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -1234,27 +872,11 @@ define amdgpu_ps <2 x float> @atomic_add_i64_1d(<8 x i32> inreg %rsrc, i64 %data
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_add v[0:1], v2, s[0:7] dmask:0x3 unorm glc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: atomic_add_i64_1d:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10-NEXT: s_mov_b32 s0, s2
-; GFX10-NEXT: s_mov_b32 s1, s3
-; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v2, s0, 16, v2
-; GFX10-NEXT: s_mov_b32 s4, s6
-; GFX10-NEXT: s_mov_b32 s5, s7
-; GFX10-NEXT: s_mov_b32 s6, s8
-; GFX10-NEXT: s_mov_b32 s7, s9
-; GFX10-NEXT: image_atomic_add v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: atomic_add_i64_1d:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -1280,7 +902,7 @@ define amdgpu_ps <2 x float> @atomic_sub_i64_1d(<8 x i32> inreg %rsrc, i64 %data
; GFX9-LABEL: atomic_sub_i64_1d:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX9-NEXT: v_lshlrev_b32_e64 v3, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -1288,27 +910,11 @@ define amdgpu_ps <2 x float> @atomic_sub_i64_1d(<8 x i32> inreg %rsrc, i64 %data
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_sub v[0:1], v2, s[0:7] dmask:0x3 unorm glc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: atomic_sub_i64_1d:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10-NEXT: s_mov_b32 s0, s2
-; GFX10-NEXT: s_mov_b32 s1, s3
-; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v2, s0, 16, v2
-; GFX10-NEXT: s_mov_b32 s4, s6
-; GFX10-NEXT: s_mov_b32 s5, s7
-; GFX10-NEXT: s_mov_b32 s6, s8
-; GFX10-NEXT: s_mov_b32 s7, s9
-; GFX10-NEXT: image_atomic_sub v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: atomic_sub_i64_1d:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -1334,7 +940,7 @@ define amdgpu_ps <2 x float> @atomic_smin_i64_1d(<8 x i32> inreg %rsrc, i64 %dat
; GFX9-LABEL: atomic_smin_i64_1d:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX9-NEXT: v_lshlrev_b32_e64 v3, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -1342,27 +948,11 @@ define amdgpu_ps <2 x float> @atomic_smin_i64_1d(<8 x i32> inreg %rsrc, i64 %dat
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_smin v[0:1], v2, s[0:7] dmask:0x3 unorm glc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: atomic_smin_i64_1d:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10-NEXT: s_mov_b32 s0, s2
-; GFX10-NEXT: s_mov_b32 s1, s3
-; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v2, s0, 16, v2
-; GFX10-NEXT: s_mov_b32 s4, s6
-; GFX10-NEXT: s_mov_b32 s5, s7
-; GFX10-NEXT: s_mov_b32 s6, s8
-; GFX10-NEXT: s_mov_b32 s7, s9
-; GFX10-NEXT: image_atomic_smin v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: atomic_smin_i64_1d:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -1388,7 +978,7 @@ define amdgpu_ps <2 x float> @atomic_umin_i64_1d(<8 x i32> inreg %rsrc, i64 %dat
; GFX9-LABEL: atomic_umin_i64_1d:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX9-NEXT: v_lshlrev_b32_e64 v3, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -1396,27 +986,11 @@ define amdgpu_ps <2 x float> @atomic_umin_i64_1d(<8 x i32> inreg %rsrc, i64 %dat
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_umin v[0:1], v2, s[0:7] dmask:0x3 unorm glc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: atomic_umin_i64_1d:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10-NEXT: s_mov_b32 s0, s2
-; GFX10-NEXT: s_mov_b32 s1, s3
-; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v2, s0, 16, v2
-; GFX10-NEXT: s_mov_b32 s4, s6
-; GFX10-NEXT: s_mov_b32 s5, s7
-; GFX10-NEXT: s_mov_b32 s6, s8
-; GFX10-NEXT: s_mov_b32 s7, s9
-; GFX10-NEXT: image_atomic_umin v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: atomic_umin_i64_1d:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -1442,7 +1016,7 @@ define amdgpu_ps <2 x float> @atomic_smax_i64_1d(<8 x i32> inreg %rsrc, i64 %dat
; GFX9-LABEL: atomic_smax_i64_1d:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX9-NEXT: v_lshlrev_b32_e64 v3, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -1450,27 +1024,11 @@ define amdgpu_ps <2 x float> @atomic_smax_i64_1d(<8 x i32> inreg %rsrc, i64 %dat
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_smax v[0:1], v2, s[0:7] dmask:0x3 unorm glc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: atomic_smax_i64_1d:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10-NEXT: s_mov_b32 s0, s2
-; GFX10-NEXT: s_mov_b32 s1, s3
-; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v2, s0, 16, v2
-; GFX10-NEXT: s_mov_b32 s4, s6
-; GFX10-NEXT: s_mov_b32 s5, s7
-; GFX10-NEXT: s_mov_b32 s6, s8
-; GFX10-NEXT: s_mov_b32 s7, s9
-; GFX10-NEXT: image_atomic_smax v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: atomic_smax_i64_1d:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -1496,7 +1054,7 @@ define amdgpu_ps <2 x float> @atomic_umax_i64_1d(<8 x i32> inreg %rsrc, i64 %dat
; GFX9-LABEL: atomic_umax_i64_1d:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX9-NEXT: v_lshlrev_b32_e64 v3, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -1504,27 +1062,11 @@ define amdgpu_ps <2 x float> @atomic_umax_i64_1d(<8 x i32> inreg %rsrc, i64 %dat
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_umax v[0:1], v2, s[0:7] dmask:0x3 unorm glc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: atomic_umax_i64_1d:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10-NEXT: s_mov_b32 s0, s2
-; GFX10-NEXT: s_mov_b32 s1, s3
-; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v2, s0, 16, v2
-; GFX10-NEXT: s_mov_b32 s4, s6
-; GFX10-NEXT: s_mov_b32 s5, s7
-; GFX10-NEXT: s_mov_b32 s6, s8
-; GFX10-NEXT: s_mov_b32 s7, s9
-; GFX10-NEXT: image_atomic_umax v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: atomic_umax_i64_1d:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -1550,7 +1092,7 @@ define amdgpu_ps <2 x float> @atomic_and_i64_1d(<8 x i32> inreg %rsrc, i64 %data
; GFX9-LABEL: atomic_and_i64_1d:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX9-NEXT: v_lshlrev_b32_e64 v3, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -1558,27 +1100,11 @@ define amdgpu_ps <2 x float> @atomic_and_i64_1d(<8 x i32> inreg %rsrc, i64 %data
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_and v[0:1], v2, s[0:7] dmask:0x3 unorm glc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: atomic_and_i64_1d:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10-NEXT: s_mov_b32 s0, s2
-; GFX10-NEXT: s_mov_b32 s1, s3
-; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v2, s0, 16, v2
-; GFX10-NEXT: s_mov_b32 s4, s6
-; GFX10-NEXT: s_mov_b32 s5, s7
-; GFX10-NEXT: s_mov_b32 s6, s8
-; GFX10-NEXT: s_mov_b32 s7, s9
-; GFX10-NEXT: image_atomic_and v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: atomic_and_i64_1d:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -1604,7 +1130,7 @@ define amdgpu_ps <2 x float> @atomic_or_i64_1d(<8 x i32> inreg %rsrc, i64 %data,
; GFX9-LABEL: atomic_or_i64_1d:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX9-NEXT: v_lshlrev_b32_e64 v3, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -1612,27 +1138,11 @@ define amdgpu_ps <2 x float> @atomic_or_i64_1d(<8 x i32> inreg %rsrc, i64 %data,
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_or v[0:1], v2, s[0:7] dmask:0x3 unorm glc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: atomic_or_i64_1d:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10-NEXT: s_mov_b32 s0, s2
-; GFX10-NEXT: s_mov_b32 s1, s3
-; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v2, s0, 16, v2
-; GFX10-NEXT: s_mov_b32 s4, s6
-; GFX10-NEXT: s_mov_b32 s5, s7
-; GFX10-NEXT: s_mov_b32 s6, s8
-; GFX10-NEXT: s_mov_b32 s7, s9
-; GFX10-NEXT: image_atomic_or v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: atomic_or_i64_1d:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -1658,7 +1168,7 @@ define amdgpu_ps <2 x float> @atomic_xor_i64_1d(<8 x i32> inreg %rsrc, i64 %data
; GFX9-LABEL: atomic_xor_i64_1d:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX9-NEXT: v_lshlrev_b32_e64 v3, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -1666,27 +1176,11 @@ define amdgpu_ps <2 x float> @atomic_xor_i64_1d(<8 x i32> inreg %rsrc, i64 %data
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_xor v[0:1], v2, s[0:7] dmask:0x3 unorm glc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: atomic_xor_i64_1d:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10-NEXT: s_mov_b32 s0, s2
-; GFX10-NEXT: s_mov_b32 s1, s3
-; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v2, s0, 16, v2
-; GFX10-NEXT: s_mov_b32 s4, s6
-; GFX10-NEXT: s_mov_b32 s5, s7
-; GFX10-NEXT: s_mov_b32 s6, s8
-; GFX10-NEXT: s_mov_b32 s7, s9
-; GFX10-NEXT: image_atomic_xor v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: atomic_xor_i64_1d:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -1712,7 +1206,7 @@ define amdgpu_ps <2 x float> @atomic_inc_i64_1d(<8 x i32> inreg %rsrc, i64 %data
; GFX9-LABEL: atomic_inc_i64_1d:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX9-NEXT: v_lshlrev_b32_e64 v3, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -1720,27 +1214,11 @@ define amdgpu_ps <2 x float> @atomic_inc_i64_1d(<8 x i32> inreg %rsrc, i64 %data
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_inc v[0:1], v2, s[0:7] dmask:0x3 unorm glc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: atomic_inc_i64_1d:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10-NEXT: s_mov_b32 s0, s2
-; GFX10-NEXT: s_mov_b32 s1, s3
-; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v2, s0, 16, v2
-; GFX10-NEXT: s_mov_b32 s4, s6
-; GFX10-NEXT: s_mov_b32 s5, s7
-; GFX10-NEXT: s_mov_b32 s6, s8
-; GFX10-NEXT: s_mov_b32 s7, s9
-; GFX10-NEXT: image_atomic_inc v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: atomic_inc_i64_1d:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -1766,7 +1244,7 @@ define amdgpu_ps <2 x float> @atomic_dec_i64_1d(<8 x i32> inreg %rsrc, i64 %data
; GFX9-LABEL: atomic_dec_i64_1d:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX9-NEXT: v_lshlrev_b32_e64 v3, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -1774,27 +1252,11 @@ define amdgpu_ps <2 x float> @atomic_dec_i64_1d(<8 x i32> inreg %rsrc, i64 %data
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_dec v[0:1], v2, s[0:7] dmask:0x3 unorm glc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: atomic_dec_i64_1d:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10-NEXT: s_mov_b32 s0, s2
-; GFX10-NEXT: s_mov_b32 s1, s3
-; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v2, s0, 16, v2
-; GFX10-NEXT: s_mov_b32 s4, s6
-; GFX10-NEXT: s_mov_b32 s5, s7
-; GFX10-NEXT: s_mov_b32 s6, s8
-; GFX10-NEXT: s_mov_b32 s7, s9
-; GFX10-NEXT: image_atomic_dec v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: atomic_dec_i64_1d:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -1820,7 +1282,7 @@ define amdgpu_ps <2 x float> @atomic_cmpswap_i64_1d(<8 x i32> inreg %rsrc, i64 %
; GFX9-LABEL: atomic_cmpswap_i64_1d:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX9-NEXT: v_lshlrev_b32_e64 v5, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -1828,27 +1290,11 @@ define amdgpu_ps <2 x float> @atomic_cmpswap_i64_1d(<8 x i32> inreg %rsrc, i64 %
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v4, s0, 16, v4
+; GFX9-NEXT: v_or_b32_sdwa v4, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_cmpswap v[0:3], v4, s[0:7] dmask:0xf unorm glc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: atomic_cmpswap_i64_1d:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX10-NEXT: s_mov_b32 s0, s2
-; GFX10-NEXT: s_mov_b32 s1, s3
-; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v4, s0, 16, v4
-; GFX10-NEXT: s_mov_b32 s4, s6
-; GFX10-NEXT: s_mov_b32 s5, s7
-; GFX10-NEXT: s_mov_b32 s6, s8
-; GFX10-NEXT: s_mov_b32 s7, s9
-; GFX10-NEXT: image_atomic_cmpswap v[0:3], v4, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm glc a16
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: atomic_cmpswap_i64_1d:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v4, 0xffff, v4
@@ -1873,7 +1319,7 @@ main_body:
define amdgpu_ps <2 x float> @atomic_add_i64_2d(<8 x i32> inreg %rsrc, i64 %data, i16 %s, i16 %t) {
; GFX9-LABEL: atomic_add_i64_2d:
; GFX9: ; %bb.0: ; %main_body
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: s_mov_b32 s0, s2
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
@@ -1882,27 +1328,11 @@ define amdgpu_ps <2 x float> @atomic_add_i64_2d(<8 x i32> inreg %rsrc, i64 %data
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_add v[0:1], v2, s[0:7] dmask:0x3 unorm glc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: atomic_add_i64_2d:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10-NEXT: s_mov_b32 s0, s2
-; GFX10-NEXT: s_mov_b32 s1, s3
-; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v2, v3, 16, v2
-; GFX10-NEXT: s_mov_b32 s4, s6
-; GFX10-NEXT: s_mov_b32 s5, s7
-; GFX10-NEXT: s_mov_b32 s6, s8
-; GFX10-NEXT: s_mov_b32 s7, s9
-; GFX10-NEXT: image_atomic_add v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D unorm glc a16
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: atomic_add_i64_2d:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -1927,10 +1357,10 @@ main_body:
define amdgpu_ps <2 x float> @atomic_add_i64_3d(<8 x i32> inreg %rsrc, i64 %data, i16 %s, i16 %t, i16 %r) {
; GFX9-LABEL: atomic_add_i64_3d:
; GFX9: ; %bb.0: ; %main_body
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_lshl_or_b32 v2, v3, 16, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v4
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e64 v3, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -1938,29 +1368,11 @@ define amdgpu_ps <2 x float> @atomic_add_i64_3d(<8 x i32> inreg %rsrc, i64 %data
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v3, s0, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_add v[0:1], v[2:3], s[0:7] dmask:0x3 unorm glc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: atomic_add_i64_3d:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX10-NEXT: s_mov_b32 s0, s2
-; GFX10-NEXT: s_mov_b32 s1, s3
-; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: v_lshl_or_b32 v2, v3, 16, v2
-; GFX10-NEXT: v_lshl_or_b32 v3, s0, 16, v4
-; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: s_mov_b32 s4, s6
-; GFX10-NEXT: s_mov_b32 s5, s7
-; GFX10-NEXT: s_mov_b32 s6, s8
-; GFX10-NEXT: s_mov_b32 s7, s9
-; GFX10-NEXT: image_atomic_add v[0:1], v[2:3], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_3D unorm glc a16
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: atomic_add_i64_3d:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -1987,10 +1399,10 @@ main_body:
define amdgpu_ps <2 x float> @atomic_add_i64_cube(<8 x i32> inreg %rsrc, i64 %data, i16 %s, i16 %t, i16 %face) {
; GFX9-LABEL: atomic_add_i64_cube:
; GFX9: ; %bb.0: ; %main_body
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_lshl_or_b32 v2, v3, 16, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v4
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e64 v3, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -1998,29 +1410,11 @@ define amdgpu_ps <2 x float> @atomic_add_i64_cube(<8 x i32> inreg %rsrc, i64 %da
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v3, s0, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_add v[0:1], v[2:3], s[0:7] dmask:0x3 unorm glc a16 da
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: atomic_add_i64_cube:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX10-NEXT: s_mov_b32 s0, s2
-; GFX10-NEXT: s_mov_b32 s1, s3
-; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: v_lshl_or_b32 v2, v3, 16, v2
-; GFX10-NEXT: v_lshl_or_b32 v3, s0, 16, v4
-; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: s_mov_b32 s4, s6
-; GFX10-NEXT: s_mov_b32 s5, s7
-; GFX10-NEXT: s_mov_b32 s6, s8
-; GFX10-NEXT: s_mov_b32 s7, s9
-; GFX10-NEXT: image_atomic_add v[0:1], v[2:3], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_CUBE unorm glc a16
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: atomic_add_i64_cube:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -2047,7 +1441,7 @@ main_body:
define amdgpu_ps <2 x float> @atomic_add_i64_1darray(<8 x i32> inreg %rsrc, i64 %data, i16 %s, i16 %slice) {
; GFX9-LABEL: atomic_add_i64_1darray:
; GFX9: ; %bb.0: ; %main_body
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: s_mov_b32 s0, s2
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
@@ -2056,27 +1450,11 @@ define amdgpu_ps <2 x float> @atomic_add_i64_1darray(<8 x i32> inreg %rsrc, i64
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_add v[0:1], v2, s[0:7] dmask:0x3 unorm glc a16 da
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: atomic_add_i64_1darray:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10-NEXT: s_mov_b32 s0, s2
-; GFX10-NEXT: s_mov_b32 s1, s3
-; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v2, v3, 16, v2
-; GFX10-NEXT: s_mov_b32 s4, s6
-; GFX10-NEXT: s_mov_b32 s5, s7
-; GFX10-NEXT: s_mov_b32 s6, s8
-; GFX10-NEXT: s_mov_b32 s7, s9
-; GFX10-NEXT: image_atomic_add v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D_ARRAY unorm glc a16
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: atomic_add_i64_1darray:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -2101,10 +1479,10 @@ main_body:
define amdgpu_ps <2 x float> @atomic_add_i64_2darray(<8 x i32> inreg %rsrc, i64 %data, i16 %s, i16 %t, i16 %slice) {
; GFX9-LABEL: atomic_add_i64_2darray:
; GFX9: ; %bb.0: ; %main_body
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_lshl_or_b32 v2, v3, 16, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v4
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e64 v3, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -2112,29 +1490,11 @@ define amdgpu_ps <2 x float> @atomic_add_i64_2darray(<8 x i32> inreg %rsrc, i64
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v3, s0, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_add v[0:1], v[2:3], s[0:7] dmask:0x3 unorm glc a16 da
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: atomic_add_i64_2darray:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX10-NEXT: s_mov_b32 s0, s2
-; GFX10-NEXT: s_mov_b32 s1, s3
-; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: v_lshl_or_b32 v2, v3, 16, v2
-; GFX10-NEXT: v_lshl_or_b32 v3, s0, 16, v4
-; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: s_mov_b32 s4, s6
-; GFX10-NEXT: s_mov_b32 s5, s7
-; GFX10-NEXT: s_mov_b32 s6, s8
-; GFX10-NEXT: s_mov_b32 s7, s9
-; GFX10-NEXT: image_atomic_add v[0:1], v[2:3], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D_ARRAY unorm glc a16
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: atomic_add_i64_2darray:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -2161,10 +1521,10 @@ main_body:
define amdgpu_ps <2 x float> @atomic_add_i64_2dmsaa(<8 x i32> inreg %rsrc, i64 %data, i16 %s, i16 %t, i16 %fragid) {
; GFX9-LABEL: atomic_add_i64_2dmsaa:
; GFX9: ; %bb.0: ; %main_body
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_lshl_or_b32 v2, v3, 16, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v4
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e64 v3, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -2172,29 +1532,11 @@ define amdgpu_ps <2 x float> @atomic_add_i64_2dmsaa(<8 x i32> inreg %rsrc, i64 %
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v3, s0, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_add v[0:1], v[2:3], s[0:7] dmask:0x3 unorm glc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: atomic_add_i64_2dmsaa:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX10-NEXT: s_mov_b32 s0, s2
-; GFX10-NEXT: s_mov_b32 s1, s3
-; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: v_lshl_or_b32 v2, v3, 16, v2
-; GFX10-NEXT: v_lshl_or_b32 v3, s0, 16, v4
-; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: s_mov_b32 s4, s6
-; GFX10-NEXT: s_mov_b32 s5, s7
-; GFX10-NEXT: s_mov_b32 s6, s8
-; GFX10-NEXT: s_mov_b32 s7, s9
-; GFX10-NEXT: image_atomic_add v[0:1], v[2:3], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D_MSAA unorm glc a16
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: atomic_add_i64_2dmsaa:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -2221,9 +1563,9 @@ main_body:
define amdgpu_ps <2 x float> @atomic_add_i64_2darraymsaa(<8 x i32> inreg %rsrc, i64 %data, i16 %s, i16 %t, i16 %slice, i16 %fragid) {
; GFX9-LABEL: atomic_add_i64_2darraymsaa:
; GFX9: ; %bb.0: ; %main_body
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_lshl_or_b32 v2, v3, 16, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v4
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v5
; GFX9-NEXT: s_mov_b32 s0, s2
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
@@ -2232,29 +1574,11 @@ define amdgpu_ps <2 x float> @atomic_add_i64_2darraymsaa(<8 x i32> inreg %rsrc,
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v3, v5, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_add v[0:1], v[2:3], s[0:7] dmask:0x3 unorm glc a16 da
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: atomic_add_i64_2darraymsaa:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX10-NEXT: s_mov_b32 s0, s2
-; GFX10-NEXT: s_mov_b32 s1, s3
-; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: v_lshl_or_b32 v2, v3, 16, v2
-; GFX10-NEXT: v_lshl_or_b32 v3, v5, 16, v4
-; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: s_mov_b32 s4, s6
-; GFX10-NEXT: s_mov_b32 s5, s7
-; GFX10-NEXT: s_mov_b32 s6, s8
-; GFX10-NEXT: s_mov_b32 s7, s9
-; GFX10-NEXT: image_atomic_add v[0:1], v[2:3], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm glc a16
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: atomic_add_i64_2darraymsaa:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -2282,7 +1606,7 @@ define amdgpu_ps <2 x float> @atomic_add_i64_1d_slc(<8 x i32> inreg %rsrc, i64 %
; GFX9-LABEL: atomic_add_i64_1d_slc:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX9-NEXT: v_lshlrev_b32_e64 v3, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -2290,27 +1614,11 @@ define amdgpu_ps <2 x float> @atomic_add_i64_1d_slc(<8 x i32> inreg %rsrc, i64 %
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_add v[0:1], v2, s[0:7] dmask:0x3 unorm glc slc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: atomic_add_i64_1d_slc:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10-NEXT: s_mov_b32 s0, s2
-; GFX10-NEXT: s_mov_b32 s1, s3
-; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v2, s0, 16, v2
-; GFX10-NEXT: s_mov_b32 s4, s6
-; GFX10-NEXT: s_mov_b32 s5, s7
-; GFX10-NEXT: s_mov_b32 s6, s8
-; GFX10-NEXT: s_mov_b32 s7, s9
-; GFX10-NEXT: image_atomic_add v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc slc a16
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: atomic_add_i64_1d_slc:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -2375,3 +1683,5 @@ declare i64 @llvm.amdgcn.image.atomic.add.2dmsaa.i64.i16(i64, i16, i16, i16, <8
declare i64 @llvm.amdgcn.image.atomic.add.2darraymsaa.i64.i16(i64, i16, i16, i16, i16, <8 x i32>, i32 immarg, i32 immarg) #0
attributes #0 = { nounwind }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX10: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.gather4.a16.dim.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.gather4.a16.dim.ll
index 72645e5d624fb..38f3b6ad56991 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.gather4.a16.dim.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.gather4.a16.dim.ll
@@ -10,7 +10,7 @@ define amdgpu_ps <4 x float> @gather4_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX9-NEXT: s_mov_b64 s[14:15], exec
; GFX9-NEXT: s_mov_b32 s0, s2
; GFX9-NEXT: s_wqm_b64 exec, exec
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -22,35 +22,12 @@ define amdgpu_ps <4 x float> @gather4_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX9-NEXT: s_mov_b32 s9, s11
; GFX9-NEXT: s_mov_b32 s10, s12
; GFX9-NEXT: s_mov_b32 s11, s13
-; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_and_b64 exec, exec, s[14:15]
; GFX9-NEXT: image_gather4 v[0:3], v0, s[0:7], s[8:11] dmask:0x1 a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10NSA-LABEL: gather4_2d:
-; GFX10NSA: ; %bb.0: ; %main_body
-; GFX10NSA-NEXT: s_mov_b32 s14, exec_lo
-; GFX10NSA-NEXT: s_mov_b32 s0, s2
-; GFX10NSA-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX10NSA-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10NSA-NEXT: s_mov_b32 s1, s3
-; GFX10NSA-NEXT: s_mov_b32 s2, s4
-; GFX10NSA-NEXT: s_mov_b32 s3, s5
-; GFX10NSA-NEXT: s_mov_b32 s4, s6
-; GFX10NSA-NEXT: s_mov_b32 s5, s7
-; GFX10NSA-NEXT: s_mov_b32 s6, s8
-; GFX10NSA-NEXT: s_mov_b32 s7, s9
-; GFX10NSA-NEXT: s_mov_b32 s8, s10
-; GFX10NSA-NEXT: s_mov_b32 s9, s11
-; GFX10NSA-NEXT: s_mov_b32 s10, s12
-; GFX10NSA-NEXT: s_mov_b32 s11, s13
-; GFX10NSA-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX10NSA-NEXT: s_and_b32 exec_lo, exec_lo, s14
-; GFX10NSA-NEXT: image_gather4 v[0:3], v0, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
-; GFX10NSA-NEXT: s_waitcnt vmcnt(0)
-; GFX10NSA-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: gather4_2d:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: s_mov_b32 s14, exec_lo
@@ -84,9 +61,9 @@ define amdgpu_ps <4 x float> @gather4_cube(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX9-NEXT: s_mov_b64 s[14:15], exec
; GFX9-NEXT: s_mov_b32 s0, s2
; GFX9-NEXT: s_wqm_b64 exec, exec
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e64 v1, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -98,37 +75,12 @@ define amdgpu_ps <4 x float> @gather4_cube(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX9-NEXT: s_mov_b32 s9, s11
; GFX9-NEXT: s_mov_b32 s10, s12
; GFX9-NEXT: s_mov_b32 s11, s13
-; GFX9-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_and_b64 exec, exec, s[14:15]
; GFX9-NEXT: image_gather4 v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 a16 da
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10NSA-LABEL: gather4_cube:
-; GFX10NSA: ; %bb.0: ; %main_body
-; GFX10NSA-NEXT: s_mov_b32 s14, exec_lo
-; GFX10NSA-NEXT: s_mov_b32 s0, s2
-; GFX10NSA-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX10NSA-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10NSA-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10NSA-NEXT: s_mov_b32 s1, s3
-; GFX10NSA-NEXT: s_mov_b32 s2, s4
-; GFX10NSA-NEXT: s_mov_b32 s3, s5
-; GFX10NSA-NEXT: s_mov_b32 s4, s6
-; GFX10NSA-NEXT: s_mov_b32 s5, s7
-; GFX10NSA-NEXT: s_mov_b32 s6, s8
-; GFX10NSA-NEXT: s_mov_b32 s7, s9
-; GFX10NSA-NEXT: s_mov_b32 s8, s10
-; GFX10NSA-NEXT: s_mov_b32 s9, s11
-; GFX10NSA-NEXT: s_mov_b32 s10, s12
-; GFX10NSA-NEXT: s_mov_b32 s11, s13
-; GFX10NSA-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX10NSA-NEXT: v_lshl_or_b32 v1, s0, 16, v2
-; GFX10NSA-NEXT: s_and_b32 exec_lo, exec_lo, s14
-; GFX10NSA-NEXT: image_gather4 v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_CUBE a16
-; GFX10NSA-NEXT: s_waitcnt vmcnt(0)
-; GFX10NSA-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: gather4_cube:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: s_mov_b32 s14, exec_lo
@@ -164,9 +116,9 @@ define amdgpu_ps <4 x float> @gather4_2darray(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX9-NEXT: s_mov_b64 s[14:15], exec
; GFX9-NEXT: s_mov_b32 s0, s2
; GFX9-NEXT: s_wqm_b64 exec, exec
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e64 v1, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -178,37 +130,12 @@ define amdgpu_ps <4 x float> @gather4_2darray(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX9-NEXT: s_mov_b32 s9, s11
; GFX9-NEXT: s_mov_b32 s10, s12
; GFX9-NEXT: s_mov_b32 s11, s13
-; GFX9-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_and_b64 exec, exec, s[14:15]
; GFX9-NEXT: image_gather4 v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 a16 da
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10NSA-LABEL: gather4_2darray:
-; GFX10NSA: ; %bb.0: ; %main_body
-; GFX10NSA-NEXT: s_mov_b32 s14, exec_lo
-; GFX10NSA-NEXT: s_mov_b32 s0, s2
-; GFX10NSA-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX10NSA-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10NSA-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10NSA-NEXT: s_mov_b32 s1, s3
-; GFX10NSA-NEXT: s_mov_b32 s2, s4
-; GFX10NSA-NEXT: s_mov_b32 s3, s5
-; GFX10NSA-NEXT: s_mov_b32 s4, s6
-; GFX10NSA-NEXT: s_mov_b32 s5, s7
-; GFX10NSA-NEXT: s_mov_b32 s6, s8
-; GFX10NSA-NEXT: s_mov_b32 s7, s9
-; GFX10NSA-NEXT: s_mov_b32 s8, s10
-; GFX10NSA-NEXT: s_mov_b32 s9, s11
-; GFX10NSA-NEXT: s_mov_b32 s10, s12
-; GFX10NSA-NEXT: s_mov_b32 s11, s13
-; GFX10NSA-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX10NSA-NEXT: v_lshl_or_b32 v1, s0, 16, v2
-; GFX10NSA-NEXT: s_and_b32 exec_lo, exec_lo, s14
-; GFX10NSA-NEXT: image_gather4 v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY a16
-; GFX10NSA-NEXT: s_waitcnt vmcnt(0)
-; GFX10NSA-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: gather4_2darray:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: s_mov_b32 s14, exec_lo
@@ -244,7 +171,7 @@ define amdgpu_ps <4 x float> @gather4_c_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX9-NEXT: s_mov_b64 s[14:15], exec
; GFX9-NEXT: s_mov_b32 s0, s2
; GFX9-NEXT: s_wqm_b64 exec, exec
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -256,35 +183,12 @@ define amdgpu_ps <4 x float> @gather4_c_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX9-NEXT: s_mov_b32 s9, s11
; GFX9-NEXT: s_mov_b32 s10, s12
; GFX9-NEXT: s_mov_b32 s11, s13
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_and_b64 exec, exec, s[14:15]
; GFX9-NEXT: image_gather4_c v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10NSA-LABEL: gather4_c_2d:
-; GFX10NSA: ; %bb.0: ; %main_body
-; GFX10NSA-NEXT: s_mov_b32 s14, exec_lo
-; GFX10NSA-NEXT: s_mov_b32 s0, s2
-; GFX10NSA-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX10NSA-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10NSA-NEXT: s_mov_b32 s1, s3
-; GFX10NSA-NEXT: s_mov_b32 s2, s4
-; GFX10NSA-NEXT: s_mov_b32 s3, s5
-; GFX10NSA-NEXT: s_mov_b32 s4, s6
-; GFX10NSA-NEXT: s_mov_b32 s5, s7
-; GFX10NSA-NEXT: s_mov_b32 s6, s8
-; GFX10NSA-NEXT: s_mov_b32 s7, s9
-; GFX10NSA-NEXT: s_mov_b32 s8, s10
-; GFX10NSA-NEXT: s_mov_b32 s9, s11
-; GFX10NSA-NEXT: s_mov_b32 s10, s12
-; GFX10NSA-NEXT: s_mov_b32 s11, s13
-; GFX10NSA-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX10NSA-NEXT: s_and_b32 exec_lo, exec_lo, s14
-; GFX10NSA-NEXT: image_gather4_c v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
-; GFX10NSA-NEXT: s_waitcnt vmcnt(0)
-; GFX10NSA-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: gather4_c_2d:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: s_mov_b32 s14, exec_lo
@@ -318,9 +222,9 @@ define amdgpu_ps <4 x float> @gather4_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX9-NEXT: s_mov_b64 s[14:15], exec
; GFX9-NEXT: s_mov_b32 s0, s2
; GFX9-NEXT: s_wqm_b64 exec, exec
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e64 v1, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -332,37 +236,12 @@ define amdgpu_ps <4 x float> @gather4_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX9-NEXT: s_mov_b32 s9, s11
; GFX9-NEXT: s_mov_b32 s10, s12
; GFX9-NEXT: s_mov_b32 s11, s13
-; GFX9-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_and_b64 exec, exec, s[14:15]
; GFX9-NEXT: image_gather4_cl v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10NSA-LABEL: gather4_cl_2d:
-; GFX10NSA: ; %bb.0: ; %main_body
-; GFX10NSA-NEXT: s_mov_b32 s14, exec_lo
-; GFX10NSA-NEXT: s_mov_b32 s0, s2
-; GFX10NSA-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX10NSA-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10NSA-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10NSA-NEXT: s_mov_b32 s1, s3
-; GFX10NSA-NEXT: s_mov_b32 s2, s4
-; GFX10NSA-NEXT: s_mov_b32 s3, s5
-; GFX10NSA-NEXT: s_mov_b32 s4, s6
-; GFX10NSA-NEXT: s_mov_b32 s5, s7
-; GFX10NSA-NEXT: s_mov_b32 s6, s8
-; GFX10NSA-NEXT: s_mov_b32 s7, s9
-; GFX10NSA-NEXT: s_mov_b32 s8, s10
-; GFX10NSA-NEXT: s_mov_b32 s9, s11
-; GFX10NSA-NEXT: s_mov_b32 s10, s12
-; GFX10NSA-NEXT: s_mov_b32 s11, s13
-; GFX10NSA-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX10NSA-NEXT: v_lshl_or_b32 v1, s0, 16, v2
-; GFX10NSA-NEXT: s_and_b32 exec_lo, exec_lo, s14
-; GFX10NSA-NEXT: image_gather4_cl v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
-; GFX10NSA-NEXT: s_waitcnt vmcnt(0)
-; GFX10NSA-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: gather4_cl_2d:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: s_mov_b32 s14, exec_lo
@@ -398,9 +277,9 @@ define amdgpu_ps <4 x float> @gather4_c_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX9-NEXT: s_mov_b64 s[14:15], exec
; GFX9-NEXT: s_mov_b32 s0, s2
; GFX9-NEXT: s_wqm_b64 exec, exec
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v3
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e64 v2, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -412,37 +291,12 @@ define amdgpu_ps <4 x float> @gather4_c_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX9-NEXT: s_mov_b32 s9, s11
; GFX9-NEXT: s_mov_b32 s10, s12
; GFX9-NEXT: s_mov_b32 s11, s13
-; GFX9-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_and_b64 exec, exec, s[14:15]
; GFX9-NEXT: image_gather4_c_cl v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1 a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10NSA-LABEL: gather4_c_cl_2d:
-; GFX10NSA: ; %bb.0: ; %main_body
-; GFX10NSA-NEXT: s_mov_b32 s14, exec_lo
-; GFX10NSA-NEXT: s_mov_b32 s0, s2
-; GFX10NSA-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX10NSA-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10NSA-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX10NSA-NEXT: s_mov_b32 s1, s3
-; GFX10NSA-NEXT: s_mov_b32 s2, s4
-; GFX10NSA-NEXT: s_mov_b32 s3, s5
-; GFX10NSA-NEXT: s_mov_b32 s4, s6
-; GFX10NSA-NEXT: s_mov_b32 s5, s7
-; GFX10NSA-NEXT: s_mov_b32 s6, s8
-; GFX10NSA-NEXT: s_mov_b32 s7, s9
-; GFX10NSA-NEXT: s_mov_b32 s8, s10
-; GFX10NSA-NEXT: s_mov_b32 s9, s11
-; GFX10NSA-NEXT: s_mov_b32 s10, s12
-; GFX10NSA-NEXT: s_mov_b32 s11, s13
-; GFX10NSA-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX10NSA-NEXT: v_lshl_or_b32 v2, s0, 16, v3
-; GFX10NSA-NEXT: s_and_b32 exec_lo, exec_lo, s14
-; GFX10NSA-NEXT: image_gather4_c_cl v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
-; GFX10NSA-NEXT: s_waitcnt vmcnt(0)
-; GFX10NSA-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: gather4_c_cl_2d:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: s_mov_b32 s14, exec_lo
@@ -478,8 +332,8 @@ define amdgpu_ps <4 x float> @gather4_b_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX9-NEXT: s_mov_b64 s[14:15], exec
; GFX9-NEXT: s_mov_b32 s0, s2
; GFX9-NEXT: s_wqm_b64 exec, exec
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT: v_lshlrev_b32_e64 v3, 16, s0
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -491,38 +345,13 @@ define amdgpu_ps <4 x float> @gather4_b_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX9-NEXT: s_mov_b32 s9, s11
; GFX9-NEXT: s_mov_b32 s10, s12
; GFX9-NEXT: s_mov_b32 s11, s13
-; GFX9-NEXT: v_lshl_or_b32 v0, s0, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_and_b64 exec, exec, s[14:15]
; GFX9-NEXT: image_gather4_b v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10NSA-LABEL: gather4_b_2d:
-; GFX10NSA: ; %bb.0: ; %main_body
-; GFX10NSA-NEXT: s_mov_b32 s14, exec_lo
-; GFX10NSA-NEXT: s_mov_b32 s0, s2
-; GFX10NSA-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX10NSA-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10NSA-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10NSA-NEXT: s_mov_b32 s1, s3
-; GFX10NSA-NEXT: s_mov_b32 s2, s4
-; GFX10NSA-NEXT: s_mov_b32 s3, s5
-; GFX10NSA-NEXT: s_mov_b32 s4, s6
-; GFX10NSA-NEXT: s_mov_b32 s5, s7
-; GFX10NSA-NEXT: s_mov_b32 s6, s8
-; GFX10NSA-NEXT: s_mov_b32 s7, s9
-; GFX10NSA-NEXT: s_mov_b32 s8, s10
-; GFX10NSA-NEXT: s_mov_b32 s9, s11
-; GFX10NSA-NEXT: s_mov_b32 s10, s12
-; GFX10NSA-NEXT: s_mov_b32 s11, s13
-; GFX10NSA-NEXT: v_lshl_or_b32 v0, s0, 16, v0
-; GFX10NSA-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX10NSA-NEXT: s_and_b32 exec_lo, exec_lo, s14
-; GFX10NSA-NEXT: image_gather4_b v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
-; GFX10NSA-NEXT: s_waitcnt vmcnt(0)
-; GFX10NSA-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: gather4_b_2d:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: s_mov_b32 s14, exec_lo
@@ -558,8 +387,8 @@ define amdgpu_ps <4 x float> @gather4_c_b_2d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX9-NEXT: s_mov_b64 s[14:15], exec
; GFX9-NEXT: s_mov_b32 s0, s2
; GFX9-NEXT: s_wqm_b64 exec, exec
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX9-NEXT: v_lshlrev_b32_e64 v4, 16, s0
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -571,38 +400,13 @@ define amdgpu_ps <4 x float> @gather4_c_b_2d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX9-NEXT: s_mov_b32 s9, s11
; GFX9-NEXT: s_mov_b32 s10, s12
; GFX9-NEXT: s_mov_b32 s11, s13
-; GFX9-NEXT: v_lshl_or_b32 v0, s0, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_and_b64 exec, exec, s[14:15]
; GFX9-NEXT: image_gather4_c_b v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1 a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10NSA-LABEL: gather4_c_b_2d:
-; GFX10NSA: ; %bb.0: ; %main_body
-; GFX10NSA-NEXT: s_mov_b32 s14, exec_lo
-; GFX10NSA-NEXT: s_mov_b32 s0, s2
-; GFX10NSA-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX10NSA-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10NSA-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10NSA-NEXT: s_mov_b32 s1, s3
-; GFX10NSA-NEXT: s_mov_b32 s2, s4
-; GFX10NSA-NEXT: s_mov_b32 s3, s5
-; GFX10NSA-NEXT: s_mov_b32 s4, s6
-; GFX10NSA-NEXT: s_mov_b32 s5, s7
-; GFX10NSA-NEXT: s_mov_b32 s6, s8
-; GFX10NSA-NEXT: s_mov_b32 s7, s9
-; GFX10NSA-NEXT: s_mov_b32 s8, s10
-; GFX10NSA-NEXT: s_mov_b32 s9, s11
-; GFX10NSA-NEXT: s_mov_b32 s10, s12
-; GFX10NSA-NEXT: s_mov_b32 s11, s13
-; GFX10NSA-NEXT: v_lshl_or_b32 v0, s0, 16, v0
-; GFX10NSA-NEXT: v_lshl_or_b32 v2, v3, 16, v2
-; GFX10NSA-NEXT: s_and_b32 exec_lo, exec_lo, s14
-; GFX10NSA-NEXT: image_gather4_c_b v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
-; GFX10NSA-NEXT: s_waitcnt vmcnt(0)
-; GFX10NSA-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: gather4_c_b_2d:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: s_mov_b32 s14, exec_lo
@@ -638,10 +442,8 @@ define amdgpu_ps <4 x float> @gather4_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX9-NEXT: s_mov_b64 s[14:15], exec
; GFX9-NEXT: s_mov_b32 s0, s2
; GFX9-NEXT: s_wqm_b64 exec, exec
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v3
+; GFX9-NEXT: v_lshlrev_b32_e64 v4, 16, s0
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -653,40 +455,14 @@ define amdgpu_ps <4 x float> @gather4_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX9-NEXT: s_mov_b32 s9, s11
; GFX9-NEXT: s_mov_b32 s10, s12
; GFX9-NEXT: s_mov_b32 s11, s13
-; GFX9-NEXT: v_lshl_or_b32 v0, s0, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_and_b64 exec, exec, s[14:15]
; GFX9-NEXT: image_gather4_b_cl v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1 a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10NSA-LABEL: gather4_b_cl_2d:
-; GFX10NSA: ; %bb.0: ; %main_body
-; GFX10NSA-NEXT: s_mov_b32 s14, exec_lo
-; GFX10NSA-NEXT: s_mov_b32 s0, s2
-; GFX10NSA-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX10NSA-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10NSA-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10NSA-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX10NSA-NEXT: s_mov_b32 s1, s3
-; GFX10NSA-NEXT: s_mov_b32 s2, s4
-; GFX10NSA-NEXT: s_mov_b32 s3, s5
-; GFX10NSA-NEXT: s_mov_b32 s4, s6
-; GFX10NSA-NEXT: s_mov_b32 s5, s7
-; GFX10NSA-NEXT: s_mov_b32 s6, s8
-; GFX10NSA-NEXT: s_mov_b32 s7, s9
-; GFX10NSA-NEXT: s_mov_b32 s8, s10
-; GFX10NSA-NEXT: s_mov_b32 s9, s11
-; GFX10NSA-NEXT: s_mov_b32 s10, s12
-; GFX10NSA-NEXT: s_mov_b32 s11, s13
-; GFX10NSA-NEXT: v_lshl_or_b32 v0, s0, 16, v0
-; GFX10NSA-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX10NSA-NEXT: v_lshl_or_b32 v2, s0, 16, v3
-; GFX10NSA-NEXT: s_and_b32 exec_lo, exec_lo, s14
-; GFX10NSA-NEXT: image_gather4_b_cl v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
-; GFX10NSA-NEXT: s_waitcnt vmcnt(0)
-; GFX10NSA-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: gather4_b_cl_2d:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: s_mov_b32 s14, exec_lo
@@ -724,10 +500,8 @@ define amdgpu_ps <4 x float> @gather4_c_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX9-NEXT: s_mov_b64 s[14:15], exec
; GFX9-NEXT: s_mov_b32 s0, s2
; GFX9-NEXT: s_wqm_b64 exec, exec
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v2, v3, 16, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v4
+; GFX9-NEXT: v_lshlrev_b32_e64 v5, 16, s0
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -739,40 +513,14 @@ define amdgpu_ps <4 x float> @gather4_c_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX9-NEXT: s_mov_b32 s9, s11
; GFX9-NEXT: s_mov_b32 s10, s12
; GFX9-NEXT: s_mov_b32 s11, s13
-; GFX9-NEXT: v_lshl_or_b32 v0, s0, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v3, s0, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v0, v5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_and_b64 exec, exec, s[14:15]
; GFX9-NEXT: image_gather4_c_b_cl v[0:3], v[0:3], s[0:7], s[8:11] dmask:0x1 a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10NSA-LABEL: gather4_c_b_cl_2d:
-; GFX10NSA: ; %bb.0: ; %main_body
-; GFX10NSA-NEXT: s_mov_b32 s14, exec_lo
-; GFX10NSA-NEXT: s_mov_b32 s0, s2
-; GFX10NSA-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX10NSA-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10NSA-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10NSA-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX10NSA-NEXT: s_mov_b32 s1, s3
-; GFX10NSA-NEXT: s_mov_b32 s2, s4
-; GFX10NSA-NEXT: s_mov_b32 s3, s5
-; GFX10NSA-NEXT: s_mov_b32 s4, s6
-; GFX10NSA-NEXT: s_mov_b32 s5, s7
-; GFX10NSA-NEXT: s_mov_b32 s6, s8
-; GFX10NSA-NEXT: s_mov_b32 s7, s9
-; GFX10NSA-NEXT: s_mov_b32 s8, s10
-; GFX10NSA-NEXT: s_mov_b32 s9, s11
-; GFX10NSA-NEXT: s_mov_b32 s10, s12
-; GFX10NSA-NEXT: s_mov_b32 s11, s13
-; GFX10NSA-NEXT: v_lshl_or_b32 v0, s0, 16, v0
-; GFX10NSA-NEXT: v_lshl_or_b32 v2, v3, 16, v2
-; GFX10NSA-NEXT: v_lshl_or_b32 v3, s0, 16, v4
-; GFX10NSA-NEXT: s_and_b32 exec_lo, exec_lo, s14
-; GFX10NSA-NEXT: image_gather4_c_b_cl v[0:3], v[0:3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
-; GFX10NSA-NEXT: s_waitcnt vmcnt(0)
-; GFX10NSA-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: gather4_c_b_cl_2d:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: s_mov_b32 s14, exec_lo
@@ -807,10 +555,10 @@ main_body:
define amdgpu_ps <4 x float> @gather4_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %s, half %t, half %lod) {
; GFX9-LABEL: gather4_l_2d:
; GFX9: ; %bb.0: ; %main_body
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e64 v1, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -822,33 +570,11 @@ define amdgpu_ps <4 x float> @gather4_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX9-NEXT: s_mov_b32 s9, s11
; GFX9-NEXT: s_mov_b32 s10, s12
; GFX9-NEXT: s_mov_b32 s11, s13
-; GFX9-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_gather4_l v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10NSA-LABEL: gather4_l_2d:
-; GFX10NSA: ; %bb.0: ; %main_body
-; GFX10NSA-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10NSA-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10NSA-NEXT: s_mov_b32 s0, s2
-; GFX10NSA-NEXT: s_mov_b32 s1, s3
-; GFX10NSA-NEXT: s_mov_b32 s2, s4
-; GFX10NSA-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX10NSA-NEXT: v_lshl_or_b32 v1, s0, 16, v2
-; GFX10NSA-NEXT: s_mov_b32 s3, s5
-; GFX10NSA-NEXT: s_mov_b32 s4, s6
-; GFX10NSA-NEXT: s_mov_b32 s5, s7
-; GFX10NSA-NEXT: s_mov_b32 s6, s8
-; GFX10NSA-NEXT: s_mov_b32 s7, s9
-; GFX10NSA-NEXT: s_mov_b32 s8, s10
-; GFX10NSA-NEXT: s_mov_b32 s9, s11
-; GFX10NSA-NEXT: s_mov_b32 s10, s12
-; GFX10NSA-NEXT: s_mov_b32 s11, s13
-; GFX10NSA-NEXT: image_gather4_l v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
-; GFX10NSA-NEXT: s_waitcnt vmcnt(0)
-; GFX10NSA-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: gather4_l_2d:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v0, 0xffff, v0
@@ -878,10 +604,10 @@ main_body:
define amdgpu_ps <4 x float> @gather4_c_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, half %s, half %t, half %lod) {
; GFX9-LABEL: gather4_c_l_2d:
; GFX9: ; %bb.0: ; %main_body
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v3
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e64 v2, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -893,33 +619,11 @@ define amdgpu_ps <4 x float> @gather4_c_l_2d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX9-NEXT: s_mov_b32 s9, s11
; GFX9-NEXT: s_mov_b32 s10, s12
; GFX9-NEXT: s_mov_b32 s11, s13
-; GFX9-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_gather4_c_l v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1 a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10NSA-LABEL: gather4_c_l_2d:
-; GFX10NSA: ; %bb.0: ; %main_body
-; GFX10NSA-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10NSA-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX10NSA-NEXT: s_mov_b32 s0, s2
-; GFX10NSA-NEXT: s_mov_b32 s1, s3
-; GFX10NSA-NEXT: s_mov_b32 s2, s4
-; GFX10NSA-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX10NSA-NEXT: v_lshl_or_b32 v2, s0, 16, v3
-; GFX10NSA-NEXT: s_mov_b32 s3, s5
-; GFX10NSA-NEXT: s_mov_b32 s4, s6
-; GFX10NSA-NEXT: s_mov_b32 s5, s7
-; GFX10NSA-NEXT: s_mov_b32 s6, s8
-; GFX10NSA-NEXT: s_mov_b32 s7, s9
-; GFX10NSA-NEXT: s_mov_b32 s8, s10
-; GFX10NSA-NEXT: s_mov_b32 s9, s11
-; GFX10NSA-NEXT: s_mov_b32 s10, s12
-; GFX10NSA-NEXT: s_mov_b32 s11, s13
-; GFX10NSA-NEXT: image_gather4_c_l v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
-; GFX10NSA-NEXT: s_waitcnt vmcnt(0)
-; GFX10NSA-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: gather4_c_l_2d:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v1, 0xffff, v1
@@ -949,7 +653,7 @@ main_body:
define amdgpu_ps <4 x float> @gather4_lz_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %s, half %t) {
; GFX9-LABEL: gather4_lz_2d:
; GFX9: ; %bb.0: ; %main_body
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_mov_b32 s0, s2
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
@@ -962,31 +666,11 @@ define amdgpu_ps <4 x float> @gather4_lz_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX9-NEXT: s_mov_b32 s9, s11
; GFX9-NEXT: s_mov_b32 s10, s12
; GFX9-NEXT: s_mov_b32 s11, s13
-; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_gather4_lz v[0:3], v0, s[0:7], s[8:11] dmask:0x1 a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10NSA-LABEL: gather4_lz_2d:
-; GFX10NSA: ; %bb.0: ; %main_body
-; GFX10NSA-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10NSA-NEXT: s_mov_b32 s0, s2
-; GFX10NSA-NEXT: s_mov_b32 s1, s3
-; GFX10NSA-NEXT: s_mov_b32 s2, s4
-; GFX10NSA-NEXT: s_mov_b32 s3, s5
-; GFX10NSA-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX10NSA-NEXT: s_mov_b32 s4, s6
-; GFX10NSA-NEXT: s_mov_b32 s5, s7
-; GFX10NSA-NEXT: s_mov_b32 s6, s8
-; GFX10NSA-NEXT: s_mov_b32 s7, s9
-; GFX10NSA-NEXT: s_mov_b32 s8, s10
-; GFX10NSA-NEXT: s_mov_b32 s9, s11
-; GFX10NSA-NEXT: s_mov_b32 s10, s12
-; GFX10NSA-NEXT: s_mov_b32 s11, s13
-; GFX10NSA-NEXT: image_gather4_lz v[0:3], v0, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
-; GFX10NSA-NEXT: s_waitcnt vmcnt(0)
-; GFX10NSA-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: gather4_lz_2d:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v0, 0xffff, v0
@@ -1014,7 +698,7 @@ main_body:
define amdgpu_ps <4 x float> @gather4_c_lz_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, half %s, half %t) {
; GFX9-LABEL: gather4_c_lz_2d:
; GFX9: ; %bb.0: ; %main_body
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX9-NEXT: s_mov_b32 s0, s2
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
@@ -1027,31 +711,11 @@ define amdgpu_ps <4 x float> @gather4_c_lz_2d(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX9-NEXT: s_mov_b32 s9, s11
; GFX9-NEXT: s_mov_b32 s10, s12
; GFX9-NEXT: s_mov_b32 s11, s13
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_gather4_c_lz v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10NSA-LABEL: gather4_c_lz_2d:
-; GFX10NSA: ; %bb.0: ; %main_body
-; GFX10NSA-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10NSA-NEXT: s_mov_b32 s0, s2
-; GFX10NSA-NEXT: s_mov_b32 s1, s3
-; GFX10NSA-NEXT: s_mov_b32 s2, s4
-; GFX10NSA-NEXT: s_mov_b32 s3, s5
-; GFX10NSA-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX10NSA-NEXT: s_mov_b32 s4, s6
-; GFX10NSA-NEXT: s_mov_b32 s5, s7
-; GFX10NSA-NEXT: s_mov_b32 s6, s8
-; GFX10NSA-NEXT: s_mov_b32 s7, s9
-; GFX10NSA-NEXT: s_mov_b32 s8, s10
-; GFX10NSA-NEXT: s_mov_b32 s9, s11
-; GFX10NSA-NEXT: s_mov_b32 s10, s12
-; GFX10NSA-NEXT: s_mov_b32 s11, s13
-; GFX10NSA-NEXT: image_gather4_c_lz v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
-; GFX10NSA-NEXT: s_waitcnt vmcnt(0)
-; GFX10NSA-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: gather4_c_lz_2d:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v1, 0xffff, v1
@@ -1092,3 +756,5 @@ declare <4 x float> @llvm.amdgcn.image.gather4.c.l.2d.v4f32.f16(i32 immarg, floa
declare <4 x float> @llvm.amdgcn.image.gather4.c.lz.2d.v4f32.f16(i32 immarg, float, half, half, <8 x i32>, <4 x i32>, i1 immarg, i32 immarg, i32 immarg) #0
attributes #0 = { nounwind readonly }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX10NSA: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.getresinfo.a16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.getresinfo.a16.ll
index 0b24be357ac5a..6cf7e721605a3 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.getresinfo.a16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.getresinfo.a16.ll
@@ -8,7 +8,7 @@ define amdgpu_ps <4 x float> @getresinfo_1d(<8 x i32> inreg %rsrc, i16 %mip) {
; GFX9-LABEL: getresinfo_1d:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: v_lshlrev_b32_e64 v1, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -16,27 +16,11 @@ define amdgpu_ps <4 x float> @getresinfo_1d(<8 x i32> inreg %rsrc, i16 %mip) {
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v0, s0, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf unorm a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: getresinfo_1d:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: s_mov_b32 s0, s2
-; GFX10-NEXT: s_mov_b32 s1, s3
-; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v0, s0, 16, v0
-; GFX10-NEXT: s_mov_b32 s4, s6
-; GFX10-NEXT: s_mov_b32 s5, s7
-; GFX10-NEXT: s_mov_b32 s6, s8
-; GFX10-NEXT: s_mov_b32 s7, s9
-; GFX10-NEXT: image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm a16
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: getresinfo_1d:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v0, 0xffff, v0
@@ -61,7 +45,7 @@ define amdgpu_ps <4 x float> @getresinfo_2d(<8 x i32> inreg %rsrc, i16 %mip) {
; GFX9-LABEL: getresinfo_2d:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: v_lshlrev_b32_e64 v1, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -69,27 +53,11 @@ define amdgpu_ps <4 x float> @getresinfo_2d(<8 x i32> inreg %rsrc, i16 %mip) {
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v0, s0, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf unorm a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: getresinfo_2d:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: s_mov_b32 s0, s2
-; GFX10-NEXT: s_mov_b32 s1, s3
-; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v0, s0, 16, v0
-; GFX10-NEXT: s_mov_b32 s4, s6
-; GFX10-NEXT: s_mov_b32 s5, s7
-; GFX10-NEXT: s_mov_b32 s6, s8
-; GFX10-NEXT: s_mov_b32 s7, s9
-; GFX10-NEXT: image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D unorm a16
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: getresinfo_2d:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v0, 0xffff, v0
@@ -114,7 +82,7 @@ define amdgpu_ps <4 x float> @getresinfo_3d(<8 x i32> inreg %rsrc, i16 %mip) {
; GFX9-LABEL: getresinfo_3d:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: v_lshlrev_b32_e64 v1, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -122,27 +90,11 @@ define amdgpu_ps <4 x float> @getresinfo_3d(<8 x i32> inreg %rsrc, i16 %mip) {
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v0, s0, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf unorm a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: getresinfo_3d:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: s_mov_b32 s0, s2
-; GFX10-NEXT: s_mov_b32 s1, s3
-; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v0, s0, 16, v0
-; GFX10-NEXT: s_mov_b32 s4, s6
-; GFX10-NEXT: s_mov_b32 s5, s7
-; GFX10-NEXT: s_mov_b32 s6, s8
-; GFX10-NEXT: s_mov_b32 s7, s9
-; GFX10-NEXT: image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D unorm a16
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: getresinfo_3d:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v0, 0xffff, v0
@@ -167,7 +119,7 @@ define amdgpu_ps <4 x float> @getresinfo_cube(<8 x i32> inreg %rsrc, i16 %mip) {
; GFX9-LABEL: getresinfo_cube:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: v_lshlrev_b32_e64 v1, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -175,27 +127,11 @@ define amdgpu_ps <4 x float> @getresinfo_cube(<8 x i32> inreg %rsrc, i16 %mip) {
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v0, s0, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf unorm a16 da
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: getresinfo_cube:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: s_mov_b32 s0, s2
-; GFX10-NEXT: s_mov_b32 s1, s3
-; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v0, s0, 16, v0
-; GFX10-NEXT: s_mov_b32 s4, s6
-; GFX10-NEXT: s_mov_b32 s5, s7
-; GFX10-NEXT: s_mov_b32 s6, s8
-; GFX10-NEXT: s_mov_b32 s7, s9
-; GFX10-NEXT: image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_CUBE unorm a16
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: getresinfo_cube:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v0, 0xffff, v0
@@ -220,7 +156,7 @@ define amdgpu_ps <4 x float> @getresinfo_1darray(<8 x i32> inreg %rsrc, i16 %mip
; GFX9-LABEL: getresinfo_1darray:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: v_lshlrev_b32_e64 v1, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -228,27 +164,11 @@ define amdgpu_ps <4 x float> @getresinfo_1darray(<8 x i32> inreg %rsrc, i16 %mip
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v0, s0, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf unorm a16 da
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: getresinfo_1darray:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: s_mov_b32 s0, s2
-; GFX10-NEXT: s_mov_b32 s1, s3
-; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v0, s0, 16, v0
-; GFX10-NEXT: s_mov_b32 s4, s6
-; GFX10-NEXT: s_mov_b32 s5, s7
-; GFX10-NEXT: s_mov_b32 s6, s8
-; GFX10-NEXT: s_mov_b32 s7, s9
-; GFX10-NEXT: image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D_ARRAY unorm a16
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: getresinfo_1darray:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v0, 0xffff, v0
@@ -273,7 +193,7 @@ define amdgpu_ps <4 x float> @getresinfo_2darray(<8 x i32> inreg %rsrc, i16 %mip
; GFX9-LABEL: getresinfo_2darray:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: v_lshlrev_b32_e64 v1, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -281,27 +201,11 @@ define amdgpu_ps <4 x float> @getresinfo_2darray(<8 x i32> inreg %rsrc, i16 %mip
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v0, s0, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf unorm a16 da
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: getresinfo_2darray:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: s_mov_b32 s0, s2
-; GFX10-NEXT: s_mov_b32 s1, s3
-; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v0, s0, 16, v0
-; GFX10-NEXT: s_mov_b32 s4, s6
-; GFX10-NEXT: s_mov_b32 s5, s7
-; GFX10-NEXT: s_mov_b32 s6, s8
-; GFX10-NEXT: s_mov_b32 s7, s9
-; GFX10-NEXT: image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_ARRAY unorm a16
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: getresinfo_2darray:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v0, 0xffff, v0
@@ -326,7 +230,7 @@ define amdgpu_ps <4 x float> @getresinfo_2dmsaa(<8 x i32> inreg %rsrc, i16 %mip)
; GFX9-LABEL: getresinfo_2dmsaa:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: v_lshlrev_b32_e64 v1, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -334,27 +238,11 @@ define amdgpu_ps <4 x float> @getresinfo_2dmsaa(<8 x i32> inreg %rsrc, i16 %mip)
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v0, s0, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf unorm a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: getresinfo_2dmsaa:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: s_mov_b32 s0, s2
-; GFX10-NEXT: s_mov_b32 s1, s3
-; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v0, s0, 16, v0
-; GFX10-NEXT: s_mov_b32 s4, s6
-; GFX10-NEXT: s_mov_b32 s5, s7
-; GFX10-NEXT: s_mov_b32 s6, s8
-; GFX10-NEXT: s_mov_b32 s7, s9
-; GFX10-NEXT: image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA unorm a16
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: getresinfo_2dmsaa:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v0, 0xffff, v0
@@ -379,7 +267,7 @@ define amdgpu_ps <4 x float> @getresinfo_2darraymsaa(<8 x i32> inreg %rsrc, i16
; GFX9-LABEL: getresinfo_2darraymsaa:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: v_lshlrev_b32_e64 v1, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -387,27 +275,11 @@ define amdgpu_ps <4 x float> @getresinfo_2darraymsaa(<8 x i32> inreg %rsrc, i16
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v0, s0, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf unorm a16 da
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: getresinfo_2darraymsaa:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: s_mov_b32 s0, s2
-; GFX10-NEXT: s_mov_b32 s1, s3
-; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v0, s0, 16, v0
-; GFX10-NEXT: s_mov_b32 s4, s6
-; GFX10-NEXT: s_mov_b32 s5, s7
-; GFX10-NEXT: s_mov_b32 s6, s8
-; GFX10-NEXT: s_mov_b32 s7, s9
-; GFX10-NEXT: image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm a16
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: getresinfo_2darraymsaa:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v0, 0xffff, v0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.2darraymsaa.a16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.2darraymsaa.a16.ll
index 4d1c0341c782c..2fd665643d26f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.2darraymsaa.a16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.2darraymsaa.a16.ll
@@ -7,9 +7,9 @@
define amdgpu_ps <4 x float> @load_2darraymsaa_v4f32_xyzw(<8 x i32> inreg %rsrc, i16 %s, i16 %t, i16 %slice, i16 %fragid) {
; GFX9-LABEL: load_2darraymsaa_v4f32_xyzw:
; GFX9: ; %bb.0:
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v3
; GFX9-NEXT: s_mov_b32 s0, s2
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
@@ -18,28 +18,46 @@ define amdgpu_ps <4 x float> @load_2darraymsaa_v4f32_xyzw(<8 x i32> inreg %rsrc,
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v1, v3, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_load v[0:3], v[0:1], s[0:7] dmask:0xf unorm a16 da
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10PLUS-LABEL: load_2darraymsaa_v4f32_xyzw:
-; GFX10PLUS: ; %bb.0:
-; GFX10PLUS-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10PLUS-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10PLUS-NEXT: s_mov_b32 s0, s2
-; GFX10PLUS-NEXT: s_mov_b32 s1, s3
-; GFX10PLUS-NEXT: s_mov_b32 s2, s4
-; GFX10PLUS-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX10PLUS-NEXT: v_lshl_or_b32 v1, v3, 16, v2
-; GFX10PLUS-NEXT: s_mov_b32 s3, s5
-; GFX10PLUS-NEXT: s_mov_b32 s4, s6
-; GFX10PLUS-NEXT: s_mov_b32 s5, s7
-; GFX10PLUS-NEXT: s_mov_b32 s6, s8
-; GFX10PLUS-NEXT: s_mov_b32 s7, s9
-; GFX10PLUS-NEXT: image_load v[0:3], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm a16
-; GFX10PLUS-NEXT: s_waitcnt vmcnt(0)
-; GFX10PLUS-NEXT: ; return to shader part epilog
+; GFX10-LABEL: load_2darraymsaa_v4f32_xyzw:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX10-NEXT: s_mov_b32 s0, s2
+; GFX10-NEXT: s_mov_b32 s1, s3
+; GFX10-NEXT: s_mov_b32 s2, s4
+; GFX10-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v1, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: s_mov_b32 s3, s5
+; GFX10-NEXT: s_mov_b32 s4, s6
+; GFX10-NEXT: s_mov_b32 s5, s7
+; GFX10-NEXT: s_mov_b32 s6, s8
+; GFX10-NEXT: s_mov_b32 s7, s9
+; GFX10-NEXT: image_load v[0:3], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm a16
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-LABEL: load_2darraymsaa_v4f32_xyzw:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-NEXT: s_mov_b32 s0, s2
+; GFX11-NEXT: s_mov_b32 s1, s3
+; GFX11-NEXT: s_mov_b32 s2, s4
+; GFX11-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX11-NEXT: v_lshl_or_b32 v1, v3, 16, v2
+; GFX11-NEXT: s_mov_b32 s3, s5
+; GFX11-NEXT: s_mov_b32 s4, s6
+; GFX11-NEXT: s_mov_b32 s5, s7
+; GFX11-NEXT: s_mov_b32 s6, s8
+; GFX11-NEXT: s_mov_b32 s7, s9
+; GFX11-NEXT: image_load v[0:3], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm a16
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: load_2darraymsaa_v4f32_xyzw:
; GFX12: ; %bb.0:
@@ -65,10 +83,10 @@ define amdgpu_ps <4 x float> @load_2darraymsaa_v4f32_xyzw(<8 x i32> inreg %rsrc,
define amdgpu_ps <4 x float> @load_2darraymsaa_v4f32_xyzw_tfe(<8 x i32> inreg %rsrc, ptr addrspace(1) inreg %out, i16 %s, i16 %t, i16 %slice, i16 %fragid) {
; GFX9-LABEL: load_2darraymsaa_v4f32_xyzw_tfe:
; GFX9: ; %bb.0:
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_mov_b32_e32 v5, 0
-; GFX9-NEXT: v_lshl_or_b32 v10, v1, 16, v0
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v2
+; GFX9-NEXT: v_or_b32_sdwa v10, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX9-NEXT: v_mov_b32_e32 v6, v5
; GFX9-NEXT: v_mov_b32_e32 v7, v5
; GFX9-NEXT: v_mov_b32_e32 v8, v5
@@ -81,7 +99,7 @@ define amdgpu_ps <4 x float> @load_2darraymsaa_v4f32_xyzw_tfe(<8 x i32> inreg %r
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v11, v3, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v11, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v0, v5
; GFX9-NEXT: v_mov_b32_e32 v1, v6
; GFX9-NEXT: v_mov_b32_e32 v2, v7
@@ -96,16 +114,16 @@ define amdgpu_ps <4 x float> @load_2darraymsaa_v4f32_xyzw_tfe(<8 x i32> inreg %r
; GFX10-LABEL: load_2darraymsaa_v4f32_xyzw_tfe:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_mov_b32_e32 v5, 0
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX10-NEXT: s_mov_b32 s0, s2
; GFX10-NEXT: s_mov_b32 s1, s3
; GFX10-NEXT: v_mov_b32_e32 v6, v5
; GFX10-NEXT: v_mov_b32_e32 v7, v5
; GFX10-NEXT: v_mov_b32_e32 v8, v5
; GFX10-NEXT: v_mov_b32_e32 v9, v5
-; GFX10-NEXT: v_lshl_or_b32 v10, v1, 16, v0
-; GFX10-NEXT: v_lshl_or_b32 v11, v3, 16, v2
+; GFX10-NEXT: v_or_b32_sdwa v10, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v11, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: s_mov_b32 s2, s4
; GFX10-NEXT: s_mov_b32 s3, s5
; GFX10-NEXT: s_mov_b32 s4, s6
@@ -189,10 +207,10 @@ define amdgpu_ps <4 x float> @load_2darraymsaa_v4f32_xyzw_tfe(<8 x i32> inreg %r
define amdgpu_ps <4 x float> @load_2darraymsaa_v4f32_xyzw_tfe_lwe(<8 x i32> inreg %rsrc, ptr addrspace(1) inreg %out, i16 %s, i16 %t, i16 %slice, i16 %fragid) {
; GFX9-LABEL: load_2darraymsaa_v4f32_xyzw_tfe_lwe:
; GFX9: ; %bb.0:
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_mov_b32_e32 v5, 0
-; GFX9-NEXT: v_lshl_or_b32 v10, v1, 16, v0
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v2
+; GFX9-NEXT: v_or_b32_sdwa v10, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX9-NEXT: v_mov_b32_e32 v6, v5
; GFX9-NEXT: v_mov_b32_e32 v7, v5
; GFX9-NEXT: v_mov_b32_e32 v8, v5
@@ -205,7 +223,7 @@ define amdgpu_ps <4 x float> @load_2darraymsaa_v4f32_xyzw_tfe_lwe(<8 x i32> inre
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v11, v3, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v11, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v0, v5
; GFX9-NEXT: v_mov_b32_e32 v1, v6
; GFX9-NEXT: v_mov_b32_e32 v2, v7
@@ -220,16 +238,16 @@ define amdgpu_ps <4 x float> @load_2darraymsaa_v4f32_xyzw_tfe_lwe(<8 x i32> inre
; GFX10-LABEL: load_2darraymsaa_v4f32_xyzw_tfe_lwe:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_mov_b32_e32 v5, 0
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX10-NEXT: s_mov_b32 s0, s2
; GFX10-NEXT: s_mov_b32 s1, s3
; GFX10-NEXT: v_mov_b32_e32 v6, v5
; GFX10-NEXT: v_mov_b32_e32 v7, v5
; GFX10-NEXT: v_mov_b32_e32 v8, v5
; GFX10-NEXT: v_mov_b32_e32 v9, v5
-; GFX10-NEXT: v_lshl_or_b32 v10, v1, 16, v0
-; GFX10-NEXT: v_lshl_or_b32 v11, v3, 16, v2
+; GFX10-NEXT: v_or_b32_sdwa v10, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v11, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: s_mov_b32 s2, s4
; GFX10-NEXT: s_mov_b32 s3, s5
; GFX10-NEXT: s_mov_b32 s4, s6
@@ -314,3 +332,5 @@ declare <4 x float> @llvm.amdgcn.image.load.2darraymsaa.v4f32.i16(i32 immarg, i1
declare { <4 x float>, i32 } @llvm.amdgcn.image.load.2darraymsaa.sl_v4f32i32s.i16(i32 immarg, i16, i16, i16, i16, <8 x i32>, i32 immarg, i32 immarg) #0
attributes #0 = { nounwind readonly }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX10PLUS: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.3d.a16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.3d.a16.ll
index e0b7c32d24993..2a13f083e5ff3 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.3d.a16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.3d.a16.ll
@@ -7,10 +7,10 @@
define amdgpu_ps <4 x float> @load_3d_v4f32_xyzw(<8 x i32> inreg %rsrc, i16 %s, i16 %t, i16 %r) {
; GFX9-LABEL: load_3d_v4f32_xyzw:
; GFX9: ; %bb.0:
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e64 v1, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -18,28 +18,46 @@ define amdgpu_ps <4 x float> @load_3d_v4f32_xyzw(<8 x i32> inreg %rsrc, i16 %s,
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_load v[0:3], v[0:1], s[0:7] dmask:0xf unorm a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10PLUS-LABEL: load_3d_v4f32_xyzw:
-; GFX10PLUS: ; %bb.0:
-; GFX10PLUS-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10PLUS-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10PLUS-NEXT: s_mov_b32 s0, s2
-; GFX10PLUS-NEXT: s_mov_b32 s1, s3
-; GFX10PLUS-NEXT: s_mov_b32 s2, s4
-; GFX10PLUS-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX10PLUS-NEXT: v_lshl_or_b32 v1, s0, 16, v2
-; GFX10PLUS-NEXT: s_mov_b32 s3, s5
-; GFX10PLUS-NEXT: s_mov_b32 s4, s6
-; GFX10PLUS-NEXT: s_mov_b32 s5, s7
-; GFX10PLUS-NEXT: s_mov_b32 s6, s8
-; GFX10PLUS-NEXT: s_mov_b32 s7, s9
-; GFX10PLUS-NEXT: image_load v[0:3], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D unorm a16
-; GFX10PLUS-NEXT: s_waitcnt vmcnt(0)
-; GFX10PLUS-NEXT: ; return to shader part epilog
+; GFX10-LABEL: load_3d_v4f32_xyzw:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_mov_b32 s0, s2
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-NEXT: v_lshlrev_b32_e64 v3, 16, s0
+; GFX10-NEXT: s_mov_b32 s1, s3
+; GFX10-NEXT: s_mov_b32 s2, s4
+; GFX10-NEXT: s_mov_b32 s3, s5
+; GFX10-NEXT: v_or_b32_sdwa v1, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: s_mov_b32 s4, s6
+; GFX10-NEXT: s_mov_b32 s5, s7
+; GFX10-NEXT: s_mov_b32 s6, s8
+; GFX10-NEXT: s_mov_b32 s7, s9
+; GFX10-NEXT: image_load v[0:3], v[1:2], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D unorm a16
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-LABEL: load_3d_v4f32_xyzw:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-NEXT: s_mov_b32 s0, s2
+; GFX11-NEXT: s_mov_b32 s1, s3
+; GFX11-NEXT: s_mov_b32 s2, s4
+; GFX11-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX11-NEXT: v_lshl_or_b32 v1, s0, 16, v2
+; GFX11-NEXT: s_mov_b32 s3, s5
+; GFX11-NEXT: s_mov_b32 s4, s6
+; GFX11-NEXT: s_mov_b32 s5, s7
+; GFX11-NEXT: s_mov_b32 s6, s8
+; GFX11-NEXT: s_mov_b32 s7, s9
+; GFX11-NEXT: image_load v[0:3], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D unorm a16
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: load_3d_v4f32_xyzw:
; GFX12: ; %bb.0:
@@ -65,11 +83,11 @@ define amdgpu_ps <4 x float> @load_3d_v4f32_xyzw(<8 x i32> inreg %rsrc, i16 %s,
define amdgpu_ps <4 x float> @load_3d_v4f32_xyzw_tfe(<8 x i32> inreg %rsrc, ptr addrspace(1) inreg %out, i16 %s, i16 %t, i16 %r) {
; GFX9-LABEL: load_3d_v4f32_xyzw_tfe:
; GFX9: ; %bb.0:
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_mov_b32_e32 v5, 0
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_lshl_or_b32 v10, v1, 16, v0
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_mov_b32_e32 v5, 0
+; GFX9-NEXT: v_or_b32_sdwa v10, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e64 v0, 16, s0
; GFX9-NEXT: v_mov_b32_e32 v6, v5
; GFX9-NEXT: v_mov_b32_e32 v7, v5
; GFX9-NEXT: v_mov_b32_e32 v8, v5
@@ -81,7 +99,7 @@ define amdgpu_ps <4 x float> @load_3d_v4f32_xyzw_tfe(<8 x i32> inreg %rsrc, ptr
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v11, s0, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v11, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v0, v5
; GFX9-NEXT: v_mov_b32_e32 v1, v6
; GFX9-NEXT: v_mov_b32_e32 v2, v7
@@ -95,18 +113,18 @@ define amdgpu_ps <4 x float> @load_3d_v4f32_xyzw_tfe(<8 x i32> inreg %rsrc, ptr
;
; GFX10-LABEL: load_3d_v4f32_xyzw_tfe:
; GFX10: ; %bb.0:
-; GFX10-NEXT: v_mov_b32_e32 v5, 0
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX10-NEXT: s_mov_b32 s0, s2
+; GFX10-NEXT: v_mov_b32_e32 v5, 0
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-NEXT: v_lshlrev_b32_e64 v3, 16, s0
; GFX10-NEXT: s_mov_b32 s1, s3
+; GFX10-NEXT: s_mov_b32 s2, s4
; GFX10-NEXT: v_mov_b32_e32 v6, v5
; GFX10-NEXT: v_mov_b32_e32 v7, v5
; GFX10-NEXT: v_mov_b32_e32 v8, v5
; GFX10-NEXT: v_mov_b32_e32 v9, v5
-; GFX10-NEXT: v_lshl_or_b32 v10, v1, 16, v0
-; GFX10-NEXT: v_lshl_or_b32 v11, s0, 16, v2
-; GFX10-NEXT: s_mov_b32 s2, s4
+; GFX10-NEXT: v_or_b32_sdwa v10, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v11, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: s_mov_b32 s3, s5
; GFX10-NEXT: s_mov_b32 s4, s6
; GFX10-NEXT: s_mov_b32 s5, s7
@@ -183,11 +201,11 @@ define amdgpu_ps <4 x float> @load_3d_v4f32_xyzw_tfe(<8 x i32> inreg %rsrc, ptr
define amdgpu_ps <4 x float> @load_3d_v4f32_xyzw_tfe_lwe(<8 x i32> inreg %rsrc, ptr addrspace(1) inreg %out, i16 %s, i16 %t, i16 %r) {
; GFX9-LABEL: load_3d_v4f32_xyzw_tfe_lwe:
; GFX9: ; %bb.0:
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_mov_b32_e32 v5, 0
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_lshl_or_b32 v10, v1, 16, v0
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_mov_b32_e32 v5, 0
+; GFX9-NEXT: v_or_b32_sdwa v10, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e64 v0, 16, s0
; GFX9-NEXT: v_mov_b32_e32 v6, v5
; GFX9-NEXT: v_mov_b32_e32 v7, v5
; GFX9-NEXT: v_mov_b32_e32 v8, v5
@@ -199,7 +217,7 @@ define amdgpu_ps <4 x float> @load_3d_v4f32_xyzw_tfe_lwe(<8 x i32> inreg %rsrc,
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v11, s0, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v11, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v0, v5
; GFX9-NEXT: v_mov_b32_e32 v1, v6
; GFX9-NEXT: v_mov_b32_e32 v2, v7
@@ -213,18 +231,18 @@ define amdgpu_ps <4 x float> @load_3d_v4f32_xyzw_tfe_lwe(<8 x i32> inreg %rsrc,
;
; GFX10-LABEL: load_3d_v4f32_xyzw_tfe_lwe:
; GFX10: ; %bb.0:
-; GFX10-NEXT: v_mov_b32_e32 v5, 0
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX10-NEXT: s_mov_b32 s0, s2
+; GFX10-NEXT: v_mov_b32_e32 v5, 0
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-NEXT: v_lshlrev_b32_e64 v3, 16, s0
; GFX10-NEXT: s_mov_b32 s1, s3
+; GFX10-NEXT: s_mov_b32 s2, s4
; GFX10-NEXT: v_mov_b32_e32 v6, v5
; GFX10-NEXT: v_mov_b32_e32 v7, v5
; GFX10-NEXT: v_mov_b32_e32 v8, v5
; GFX10-NEXT: v_mov_b32_e32 v9, v5
-; GFX10-NEXT: v_lshl_or_b32 v10, v1, 16, v0
-; GFX10-NEXT: v_lshl_or_b32 v11, s0, 16, v2
-; GFX10-NEXT: s_mov_b32 s2, s4
+; GFX10-NEXT: v_or_b32_sdwa v10, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v11, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: s_mov_b32 s3, s5
; GFX10-NEXT: s_mov_b32 s4, s6
; GFX10-NEXT: s_mov_b32 s5, s7
@@ -302,3 +320,5 @@ declare <4 x float> @llvm.amdgcn.image.load.3d.v4f32.i16(i32 immarg, i16, i16, i
declare { <4 x float>, i32 } @llvm.amdgcn.image.load.3d.sl_v4f32i32s.i16(i32 immarg, i16, i16, i16, <8 x i32>, i32 immarg, i32 immarg) #0
attributes #0 = { nounwind readonly }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX10PLUS: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.sample.cd.g16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.sample.cd.g16.ll
index c417d090efdbe..5657cc175db68 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.sample.cd.g16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.sample.cd.g16.ll
@@ -4,10 +4,9 @@
define amdgpu_ps <4 x float> @sample_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %dsdh, half %dsdv, float %s) {
; GFX10-LABEL: sample_cd_1d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_lshl_or_b32 v0, s0, 16, v0
-; GFX10-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX10-NEXT: v_lshlrev_b32_e64 v3, 16, s0
+; GFX10-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: image_sample_cd_g16 v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
@@ -19,10 +18,10 @@ main_body:
define amdgpu_ps <4 x float> @sample_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %dsdh, half %dtdh, half %dsdv, half %dtdv, float %s, float %t) {
; GFX10-LABEL: sample_cd_2d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX10-NEXT: v_lshl_or_b32 v1, v3, 16, v2
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX10-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v1, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: image_sample_cd_g16 v[0:3], [v0, v1, v4, v5], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
@@ -34,10 +33,9 @@ main_body:
define amdgpu_ps <4 x float> @sample_c_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, half %dsdh, half %dsdv, float %s) {
; GFX10-LABEL: sample_c_cd_1d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10-NEXT: v_lshl_or_b32 v1, s0, 16, v1
-; GFX10-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX10-NEXT: v_lshlrev_b32_e64 v4, 16, s0
+; GFX10-NEXT: v_or_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: image_sample_c_cd_g16 v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
@@ -49,10 +47,10 @@ main_body:
define amdgpu_ps <4 x float> @sample_c_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, half %dsdh, half %dtdh, half %dsdv, half %dtdv, float %s, float %t) {
; GFX10-LABEL: sample_c_cd_2d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX10-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX10-NEXT: v_lshl_or_b32 v2, v4, 16, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX10-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v2, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: image_sample_c_cd_g16 v[0:3], [v0, v1, v2, v5, v6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
@@ -64,10 +62,9 @@ main_body:
define amdgpu_ps <4 x float> @sample_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %dsdh, half %dsdv, float %s, float %clamp) {
; GFX10-LABEL: sample_cd_cl_1d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_lshl_or_b32 v0, s0, 16, v0
-; GFX10-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX10-NEXT: v_lshlrev_b32_e64 v4, 16, s0
+; GFX10-NEXT: v_or_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: image_sample_cd_cl_g16 v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
@@ -79,10 +76,10 @@ main_body:
define amdgpu_ps <4 x float> @sample_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %dsdh, half %dtdh, half %dsdv, half %dtdv, float %s, float %t, float %clamp) {
; GFX10-LABEL: sample_cd_cl_2d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX10-NEXT: v_lshl_or_b32 v1, v3, 16, v2
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX10-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v1, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: image_sample_cd_cl_g16 v[0:3], [v0, v1, v4, v5, v6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
@@ -94,10 +91,9 @@ main_body:
define amdgpu_ps <4 x float> @sample_c_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, half %dsdh, half %dsdv, float %s, float %clamp) {
; GFX10-LABEL: sample_c_cd_cl_1d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10-NEXT: v_lshl_or_b32 v1, s0, 16, v1
-; GFX10-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX10-NEXT: v_lshlrev_b32_e64 v5, 16, s0
+; GFX10-NEXT: v_or_b32_sdwa v1, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v2, v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: image_sample_c_cd_cl_g16 v[0:3], v[0:4], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
@@ -110,11 +106,12 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX10-LABEL: sample_c_cd_cl_2d:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: v_mov_b32_e32 v8, v2
+; GFX10-NEXT: v_mov_b32_e32 v9, v3
; GFX10-NEXT: v_mov_b32_e32 v2, v0
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v1
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v3
-; GFX10-NEXT: v_lshl_or_b32 v3, v8, 16, v0
-; GFX10-NEXT: v_lshl_or_b32 v4, v4, 16, v1
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v8
+; GFX10-NEXT: v_or_b32_sdwa v4, v4, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v3, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: image_sample_c_cd_cl_g16 v[0:3], v[2:7], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.sample.g16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.sample.g16.ll
index 9225426521a11..0618940be43d5 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.sample.g16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.sample.g16.ll
@@ -6,10 +6,9 @@
define amdgpu_ps <4 x float> @sample_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %dsdh, half %dsdv, float %s) {
; GFX10-LABEL: sample_d_1d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_lshl_or_b32 v0, s0, 16, v0
-; GFX10-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX10-NEXT: v_lshlrev_b32_e64 v3, 16, s0
+; GFX10-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: image_sample_d_g16 v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
@@ -41,10 +40,10 @@ main_body:
define amdgpu_ps <4 x float> @sample_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %dsdh, half %dtdh, half %dsdv, half %dtdv, float %s, float %t) {
; GFX10-LABEL: sample_d_2d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX10-NEXT: v_lshl_or_b32 v1, v3, 16, v2
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX10-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v1, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: image_sample_d_g16 v[0:3], [v0, v1, v4, v5], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
@@ -76,14 +75,15 @@ main_body:
define amdgpu_ps <4 x float> @sample_d_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %dsdh, half %dtdh, half %drdh, half %dsdv, half %dtdv, half %drdv, float %s, float %t, float %r) {
; GFX10-LABEL: sample_d_3d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: v_and_b32_e32 v9, 0xffff, v2
-; GFX10-NEXT: v_and_b32_e32 v10, 0xffff, v3
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX10-NEXT: v_lshl_or_b32 v2, v1, 16, v0
-; GFX10-NEXT: v_lshl_or_b32 v3, s0, 16, v9
-; GFX10-NEXT: v_lshl_or_b32 v4, v4, 16, v10
-; GFX10-NEXT: v_lshl_or_b32 v5, s0, 16, v5
+; GFX10-NEXT: v_mov_b32_e32 v9, v2
+; GFX10-NEXT: v_mov_b32_e32 v10, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-NEXT: v_lshlrev_b32_e64 v11, 16, s0
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX10-NEXT: v_or_b32_sdwa v2, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v3, v11, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v4, v4, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v5, v11, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: image_sample_d_g16 v[0:3], v[2:8], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_3D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
@@ -123,10 +123,9 @@ main_body:
define amdgpu_ps <4 x float> @sample_c_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, half %dsdh, half %dsdv, float %s) {
; GFX10-LABEL: sample_c_d_1d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10-NEXT: v_lshl_or_b32 v1, s0, 16, v1
-; GFX10-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX10-NEXT: v_lshlrev_b32_e64 v4, 16, s0
+; GFX10-NEXT: v_or_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: image_sample_c_d_g16 v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
@@ -158,10 +157,10 @@ main_body:
define amdgpu_ps <4 x float> @sample_c_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, half %dsdh, half %dtdh, half %dsdv, half %dtdv, float %s, float %t) {
; GFX10-LABEL: sample_c_d_2d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX10-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX10-NEXT: v_lshl_or_b32 v2, v4, 16, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX10-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v2, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: image_sample_c_d_g16 v[0:3], [v0, v1, v2, v5, v6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
@@ -193,10 +192,9 @@ main_body:
define amdgpu_ps <4 x float> @sample_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %dsdh, half %dsdv, float %s, float %clamp) {
; GFX10-LABEL: sample_d_cl_1d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_lshl_or_b32 v0, s0, 16, v0
-; GFX10-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX10-NEXT: v_lshlrev_b32_e64 v4, 16, s0
+; GFX10-NEXT: v_or_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: image_sample_d_cl_g16 v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
@@ -228,10 +226,10 @@ main_body:
define amdgpu_ps <4 x float> @sample_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %dsdh, half %dtdh, half %dsdv, half %dtdv, float %s, float %t, float %clamp) {
; GFX10-LABEL: sample_d_cl_2d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX10-NEXT: v_lshl_or_b32 v1, v3, 16, v2
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX10-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v1, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: image_sample_d_cl_g16 v[0:3], [v0, v1, v4, v5, v6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
@@ -263,10 +261,9 @@ main_body:
define amdgpu_ps <4 x float> @sample_c_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, half %dsdh, half %dsdv, float %s, float %clamp) {
; GFX10-LABEL: sample_c_d_cl_1d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10-NEXT: v_lshl_or_b32 v1, s0, 16, v1
-; GFX10-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX10-NEXT: v_lshlrev_b32_e64 v5, 16, s0
+; GFX10-NEXT: v_or_b32_sdwa v1, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v2, v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: image_sample_c_d_cl_g16 v[0:3], v[0:4], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
@@ -299,11 +296,12 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX10-LABEL: sample_c_d_cl_2d:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: v_mov_b32_e32 v8, v2
+; GFX10-NEXT: v_mov_b32_e32 v9, v3
; GFX10-NEXT: v_mov_b32_e32 v2, v0
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v1
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v3
-; GFX10-NEXT: v_lshl_or_b32 v3, v8, 16, v0
-; GFX10-NEXT: v_lshl_or_b32 v4, v4, 16, v1
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v8
+; GFX10-NEXT: v_or_b32_sdwa v4, v4, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v3, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: image_sample_c_d_cl_g16 v[0:3], v[2:7], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
@@ -335,14 +333,15 @@ main_body:
define amdgpu_ps float @sample_c_d_o_2darray_V1(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, i32 %offset, float %zcompare, half %dsdh, half %dtdh, half %dsdv, half %dtdv, float %s, float %t, float %slice) {
; GFX10-LABEL: sample_c_d_o_2darray_V1:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_mov_b32_e32 v9, v2
-; GFX10-NEXT: v_mov_b32_e32 v10, v3
+; GFX10-NEXT: v_mov_b32_e32 v9, v3
+; GFX10-NEXT: v_mov_b32_e32 v10, v2
+; GFX10-NEXT: v_mov_b32_e32 v11, v4
; GFX10-NEXT: v_mov_b32_e32 v2, v0
; GFX10-NEXT: v_mov_b32_e32 v3, v1
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v4
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v9
-; GFX10-NEXT: v_lshl_or_b32 v5, v5, 16, v1
-; GFX10-NEXT: v_lshl_or_b32 v4, v10, 16, v0
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v9
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v5
+; GFX10-NEXT: v_or_b32_sdwa v4, v0, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v5, v1, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: image_sample_c_d_o_g16 v0, v[2:8], s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_2D_ARRAY
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
@@ -374,14 +373,15 @@ main_body:
define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, i32 %offset, float %zcompare, half %dsdh, half %dtdh, half %dsdv, half %dtdv, float %s, float %t, float %slice) {
; GFX10-LABEL: sample_c_d_o_2darray_V2:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_mov_b32_e32 v9, v2
-; GFX10-NEXT: v_mov_b32_e32 v10, v3
+; GFX10-NEXT: v_mov_b32_e32 v9, v3
+; GFX10-NEXT: v_mov_b32_e32 v10, v2
+; GFX10-NEXT: v_mov_b32_e32 v11, v4
; GFX10-NEXT: v_mov_b32_e32 v2, v0
; GFX10-NEXT: v_mov_b32_e32 v3, v1
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v4
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v9
-; GFX10-NEXT: v_lshl_or_b32 v5, v5, 16, v1
-; GFX10-NEXT: v_lshl_or_b32 v4, v10, 16, v0
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v9
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v5
+; GFX10-NEXT: v_or_b32_sdwa v4, v0, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v5, v1, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: image_sample_c_d_o_g16 v[0:1], v[2:8], s[0:7], s[8:11] dmask:0x6 dim:SQ_RSRC_IMG_2D_ARRAY
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/saddsat.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/saddsat.ll
index 72d1e056bade2..9a43ae888f4e5 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/saddsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/saddsat.ll
@@ -370,12 +370,11 @@ define i16 @v_saddsat_v2i8(i16 %lhs.arg, i16 %rhs.arg) {
; GFX9-LABEL: v_saddsat_v2i8:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_lshrrev_b32_e32 v2, 8, v0
-; GFX9-NEXT: v_lshrrev_b32_e32 v3, 8, v1
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_lshl_or_b32 v0, v2, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v3, 16, v1
+; GFX9-NEXT: v_mov_b32_e32 v2, 8
+; GFX9-NEXT: v_lshrrev_b32_sdwa v3, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_lshrrev_b32_sdwa v2, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
; GFX9-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
; GFX9-NEXT: v_pk_add_i16 v0, v0, v1 clamp
@@ -388,12 +387,11 @@ define i16 @v_saddsat_v2i8(i16 %lhs.arg, i16 %rhs.arg) {
; GFX10-LABEL: v_saddsat_v2i8:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_e32 v2, 8, v0
-; GFX10-NEXT: v_lshrrev_b32_e32 v3, 8, v1
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_lshl_or_b32 v0, v2, 16, v0
-; GFX10-NEXT: v_lshl_or_b32 v1, v3, 16, v1
+; GFX10-NEXT: v_mov_b32_e32 v2, 8
+; GFX10-NEXT: v_lshrrev_b32_sdwa v3, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_lshrrev_b32_sdwa v2, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
; GFX10-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
; GFX10-NEXT: v_pk_add_i16 v0, v0, v1 clamp
@@ -709,28 +707,26 @@ define i32 @v_saddsat_v4i8(i32 %lhs.arg, i32 %rhs.arg) {
; GFX9-LABEL: v_saddsat_v4i8:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_lshrrev_b32_e32 v3, 24, v0
-; GFX9-NEXT: v_lshrrev_b32_e32 v2, 8, v0
-; GFX9-NEXT: v_lshrrev_b32_e32 v4, 8, v1
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v0
-; GFX9-NEXT: v_alignbit_b32 v0, v3, v0, 16
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v1
-; GFX9-NEXT: v_lshrrev_b32_e32 v5, 24, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v2, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v3, v4, 16, v3
-; GFX9-NEXT: v_alignbit_b32 v1, v5, v1, 16
-; GFX9-NEXT: v_pk_lshlrev_b16 v2, 8, v2 op_sel_hi:[0,1]
+; GFX9-NEXT: v_mov_b32_e32 v2, 8
+; GFX9-NEXT: v_lshrrev_b32_sdwa v3, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 24, v0
+; GFX9-NEXT: v_lshrrev_b32_sdwa v5, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_lshrrev_b32_e32 v6, 24, v1
+; GFX9-NEXT: v_or_b32_sdwa v3, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_alignbit_b32 v0, v4, v0, 16
+; GFX9-NEXT: v_or_b32_sdwa v4, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_alignbit_b32 v1, v6, v1, 16
; GFX9-NEXT: v_pk_lshlrev_b16 v3, 8, v3 op_sel_hi:[0,1]
+; GFX9-NEXT: v_pk_lshlrev_b16 v4, 8, v4 op_sel_hi:[0,1]
; GFX9-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
; GFX9-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
-; GFX9-NEXT: v_pk_add_i16 v2, v2, v3 clamp
+; GFX9-NEXT: v_pk_add_i16 v3, v3, v4 clamp
; GFX9-NEXT: v_pk_add_i16 v0, v0, v1 clamp
-; GFX9-NEXT: v_pk_ashrrev_i16 v1, 8, v2 op_sel_hi:[0,1]
-; GFX9-NEXT: v_mov_b32_e32 v3, 8
+; GFX9-NEXT: v_pk_ashrrev_i16 v1, 8, v3 op_sel_hi:[0,1]
; GFX9-NEXT: v_pk_ashrrev_i16 v0, 8, v0 op_sel_hi:[0,1]
-; GFX9-NEXT: v_mov_b32_e32 v2, 0xff
-; GFX9-NEXT: v_lshlrev_b32_sdwa v3, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
-; GFX9-NEXT: v_and_or_b32 v1, v1, v2, v3
+; GFX9-NEXT: v_mov_b32_e32 v3, 0xff
+; GFX9-NEXT: v_lshlrev_b32_sdwa v2, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
+; GFX9-NEXT: v_and_or_b32 v1, v1, v3, v2
; GFX9-NEXT: v_and_b32_e32 v2, 0xff, v0
; GFX9-NEXT: v_mov_b32_e32 v3, 24
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
@@ -741,30 +737,28 @@ define i32 @v_saddsat_v4i8(i32 %lhs.arg, i32 %rhs.arg) {
; GFX10-LABEL: v_saddsat_v4i8:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_e32 v2, 8, v0
-; GFX10-NEXT: v_lshrrev_b32_e32 v3, 24, v0
-; GFX10-NEXT: v_and_b32_e32 v4, 0xffff, v0
-; GFX10-NEXT: v_lshrrev_b32_e32 v5, 8, v1
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffff, v1
-; GFX10-NEXT: v_lshrrev_b32_e32 v7, 24, v1
-; GFX10-NEXT: v_alignbit_b32 v0, v3, v0, 16
-; GFX10-NEXT: v_lshl_or_b32 v2, v2, 16, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, 24
-; GFX10-NEXT: v_lshl_or_b32 v3, v5, 16, v6
-; GFX10-NEXT: v_alignbit_b32 v1, v7, v1, 16
-; GFX10-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
-; GFX10-NEXT: v_pk_lshlrev_b16 v2, 8, v2 op_sel_hi:[0,1]
+; GFX10-NEXT: v_mov_b32_e32 v2, 8
+; GFX10-NEXT: v_lshrrev_b32_e32 v4, 24, v0
+; GFX10-NEXT: v_lshrrev_b32_e32 v6, 24, v1
+; GFX10-NEXT: v_lshrrev_b32_sdwa v3, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_lshrrev_b32_sdwa v5, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v3, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_alignbit_b32 v0, v4, v0, 16
+; GFX10-NEXT: v_or_b32_sdwa v4, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_alignbit_b32 v1, v6, v1, 16
; GFX10-NEXT: v_pk_lshlrev_b16 v3, 8, v3 op_sel_hi:[0,1]
+; GFX10-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX10-NEXT: v_pk_lshlrev_b16 v4, 8, v4 op_sel_hi:[0,1]
; GFX10-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
-; GFX10-NEXT: v_pk_add_i16 v2, v2, v3 clamp
+; GFX10-NEXT: v_pk_add_i16 v3, v3, v4 clamp
; GFX10-NEXT: v_pk_add_i16 v0, v0, v1 clamp
-; GFX10-NEXT: v_mov_b32_e32 v1, 8
-; GFX10-NEXT: v_pk_ashrrev_i16 v2, 8, v2 op_sel_hi:[0,1]
+; GFX10-NEXT: v_mov_b32_e32 v4, 24
+; GFX10-NEXT: v_pk_ashrrev_i16 v1, 8, v3 op_sel_hi:[0,1]
; GFX10-NEXT: v_pk_ashrrev_i16 v0, 8, v0 op_sel_hi:[0,1]
-; GFX10-NEXT: v_lshlrev_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
+; GFX10-NEXT: v_lshlrev_b32_sdwa v2, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
; GFX10-NEXT: v_and_b32_e32 v3, 0xff, v0
; GFX10-NEXT: v_lshlrev_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
-; GFX10-NEXT: v_and_or_b32 v1, 0xff, v2, v1
+; GFX10-NEXT: v_and_or_b32 v1, 0xff, v1, v2
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
; GFX10-NEXT: v_or3_b32 v0, v1, v2, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/select-to-fmin-fmax.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/select-to-fmin-fmax.ll
index 66e6e1d75015f..ec149479868b5 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/select-to-fmin-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/select-to-fmin-fmax.ll
@@ -46,20 +46,20 @@ define <4 x half> @test_v4s16(<4 x half> %a) #0 {
; GCN: ; %bb.0: ; %entry
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mov_b32_e32 v4, 0
-; GCN-NEXT: v_cmp_gt_f16_e32 vcc, 0, v0
; GCN-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GCN-NEXT: v_cndmask_b32_e64 v5, v0, 0, vcc
+; GCN-NEXT: v_cmp_gt_f16_e32 vcc, 0, v0
; GCN-NEXT: v_cmp_lt_f16_sdwa s[4:5], v0, v4 src0_sel:WORD_1 src1_sel:DWORD
-; GCN-NEXT: v_cmp_gt_f16_e32 vcc, 0, v1
; GCN-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GCN-NEXT: v_cndmask_b32_e64 v5, v0, 0, vcc
; GCN-NEXT: v_cndmask_b32_e64 v0, v2, 0, s[4:5]
-; GCN-NEXT: v_cndmask_b32_e64 v2, v1, 0, vcc
+; GCN-NEXT: v_cmp_gt_f16_e32 vcc, 0, v1
; GCN-NEXT: v_cmp_lt_f16_sdwa s[4:5], v1, v4 src0_sel:WORD_1 src1_sel:DWORD
+; GCN-NEXT: v_cndmask_b32_e64 v2, v1, 0, vcc
; GCN-NEXT: v_cndmask_b32_e64 v1, v3, 0, s[4:5]
-; GCN-NEXT: v_and_b32_e32 v3, 0xffff, v5
-; GCN-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GCN-NEXT: v_lshl_or_b32 v0, v0, 16, v3
-; GCN-NEXT: v_lshl_or_b32 v1, v1, 16, v2
+; GCN-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GCN-NEXT: v_or_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GCN-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GCN-NEXT: s_setpc_b64 s[30:31]
entry:
%fcmp = fcmp olt <4 x half> %a, zeroinitializer
@@ -72,34 +72,34 @@ define <8 x half> @test_v8s16(<8 x half> %a) #0 {
; GCN: ; %bb.0: ; %entry
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mov_b32_e32 v8, 0
-; GCN-NEXT: v_cmp_gt_f16_e32 vcc, 0, v0
; GCN-NEXT: v_lshrrev_b32_e32 v4, 16, v0
-; GCN-NEXT: v_cndmask_b32_e64 v9, v0, 0, vcc
+; GCN-NEXT: v_cmp_gt_f16_e32 vcc, 0, v0
; GCN-NEXT: v_cmp_lt_f16_sdwa s[4:5], v0, v8 src0_sel:WORD_1 src1_sel:DWORD
-; GCN-NEXT: v_cmp_gt_f16_e32 vcc, 0, v1
; GCN-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GCN-NEXT: v_cndmask_b32_e64 v9, v0, 0, vcc
; GCN-NEXT: v_cndmask_b32_e64 v0, v4, 0, s[4:5]
-; GCN-NEXT: v_cndmask_b32_e64 v4, v1, 0, vcc
+; GCN-NEXT: v_cmp_gt_f16_e32 vcc, 0, v1
; GCN-NEXT: v_cmp_lt_f16_sdwa s[4:5], v1, v8 src0_sel:WORD_1 src1_sel:DWORD
-; GCN-NEXT: v_cmp_gt_f16_e32 vcc, 0, v2
; GCN-NEXT: v_lshrrev_b32_e32 v6, 16, v2
+; GCN-NEXT: v_cndmask_b32_e64 v4, v1, 0, vcc
; GCN-NEXT: v_cndmask_b32_e64 v1, v5, 0, s[4:5]
-; GCN-NEXT: v_cndmask_b32_e64 v5, v2, 0, vcc
+; GCN-NEXT: v_cmp_gt_f16_e32 vcc, 0, v2
; GCN-NEXT: v_cmp_lt_f16_sdwa s[4:5], v2, v8 src0_sel:WORD_1 src1_sel:DWORD
-; GCN-NEXT: v_cmp_gt_f16_e32 vcc, 0, v3
-; GCN-NEXT: v_and_b32_e32 v4, 0xffff, v4
; GCN-NEXT: v_lshrrev_b32_e32 v7, 16, v3
+; GCN-NEXT: v_cndmask_b32_e64 v5, v2, 0, vcc
; GCN-NEXT: v_cndmask_b32_e64 v2, v6, 0, s[4:5]
-; GCN-NEXT: v_cndmask_b32_e64 v6, v3, 0, vcc
+; GCN-NEXT: v_cmp_gt_f16_e32 vcc, 0, v3
; GCN-NEXT: v_cmp_lt_f16_sdwa s[4:5], v3, v8 src0_sel:WORD_1 src1_sel:DWORD
-; GCN-NEXT: v_lshl_or_b32 v1, v1, 16, v4
-; GCN-NEXT: v_and_b32_e32 v4, 0xffff, v5
+; GCN-NEXT: v_cndmask_b32_e64 v6, v3, 0, vcc
; GCN-NEXT: v_cndmask_b32_e64 v3, v7, 0, s[4:5]
-; GCN-NEXT: v_and_b32_e32 v7, 0xffff, v9
-; GCN-NEXT: v_lshl_or_b32 v2, v2, 16, v4
-; GCN-NEXT: v_and_b32_e32 v4, 0xffff, v6
-; GCN-NEXT: v_lshl_or_b32 v0, v0, 16, v7
-; GCN-NEXT: v_lshl_or_b32 v3, v3, 16, v4
+; GCN-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GCN-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GCN-NEXT: v_or_b32_sdwa v0, v0, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GCN-NEXT: v_or_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GCN-NEXT: v_or_b32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GCN-NEXT: v_or_b32_sdwa v3, v3, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GCN-NEXT: s_setpc_b64 s[30:31]
entry:
%fcmp = fcmp olt <8 x half> %a, zeroinitializer
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll
index 83d3cde9879f4..f5056d6247023 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll
@@ -370,12 +370,11 @@ define i16 @v_ssubsat_v2i8(i16 %lhs.arg, i16 %rhs.arg) {
; GFX9-LABEL: v_ssubsat_v2i8:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_lshrrev_b32_e32 v2, 8, v0
-; GFX9-NEXT: v_lshrrev_b32_e32 v3, 8, v1
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_lshl_or_b32 v0, v2, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v3, 16, v1
+; GFX9-NEXT: v_mov_b32_e32 v2, 8
+; GFX9-NEXT: v_lshrrev_b32_sdwa v3, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_lshrrev_b32_sdwa v2, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
; GFX9-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
; GFX9-NEXT: v_pk_sub_i16 v0, v0, v1 clamp
@@ -388,12 +387,11 @@ define i16 @v_ssubsat_v2i8(i16 %lhs.arg, i16 %rhs.arg) {
; GFX10-LABEL: v_ssubsat_v2i8:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_e32 v2, 8, v0
-; GFX10-NEXT: v_lshrrev_b32_e32 v3, 8, v1
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_lshl_or_b32 v0, v2, 16, v0
-; GFX10-NEXT: v_lshl_or_b32 v1, v3, 16, v1
+; GFX10-NEXT: v_mov_b32_e32 v2, 8
+; GFX10-NEXT: v_lshrrev_b32_sdwa v3, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_lshrrev_b32_sdwa v2, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
; GFX10-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
; GFX10-NEXT: v_pk_sub_i16 v0, v0, v1 clamp
@@ -709,28 +707,26 @@ define i32 @v_ssubsat_v4i8(i32 %lhs.arg, i32 %rhs.arg) {
; GFX9-LABEL: v_ssubsat_v4i8:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_lshrrev_b32_e32 v3, 24, v0
-; GFX9-NEXT: v_lshrrev_b32_e32 v2, 8, v0
-; GFX9-NEXT: v_lshrrev_b32_e32 v4, 8, v1
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v0
-; GFX9-NEXT: v_alignbit_b32 v0, v3, v0, 16
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v1
-; GFX9-NEXT: v_lshrrev_b32_e32 v5, 24, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v2, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v3, v4, 16, v3
-; GFX9-NEXT: v_alignbit_b32 v1, v5, v1, 16
-; GFX9-NEXT: v_pk_lshlrev_b16 v2, 8, v2 op_sel_hi:[0,1]
+; GFX9-NEXT: v_mov_b32_e32 v2, 8
+; GFX9-NEXT: v_lshrrev_b32_sdwa v3, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 24, v0
+; GFX9-NEXT: v_lshrrev_b32_sdwa v5, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_lshrrev_b32_e32 v6, 24, v1
+; GFX9-NEXT: v_or_b32_sdwa v3, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_alignbit_b32 v0, v4, v0, 16
+; GFX9-NEXT: v_or_b32_sdwa v4, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_alignbit_b32 v1, v6, v1, 16
; GFX9-NEXT: v_pk_lshlrev_b16 v3, 8, v3 op_sel_hi:[0,1]
+; GFX9-NEXT: v_pk_lshlrev_b16 v4, 8, v4 op_sel_hi:[0,1]
; GFX9-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
; GFX9-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
-; GFX9-NEXT: v_pk_sub_i16 v2, v2, v3 clamp
+; GFX9-NEXT: v_pk_sub_i16 v3, v3, v4 clamp
; GFX9-NEXT: v_pk_sub_i16 v0, v0, v1 clamp
-; GFX9-NEXT: v_pk_ashrrev_i16 v1, 8, v2 op_sel_hi:[0,1]
-; GFX9-NEXT: v_mov_b32_e32 v3, 8
+; GFX9-NEXT: v_pk_ashrrev_i16 v1, 8, v3 op_sel_hi:[0,1]
; GFX9-NEXT: v_pk_ashrrev_i16 v0, 8, v0 op_sel_hi:[0,1]
-; GFX9-NEXT: v_mov_b32_e32 v2, 0xff
-; GFX9-NEXT: v_lshlrev_b32_sdwa v3, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
-; GFX9-NEXT: v_and_or_b32 v1, v1, v2, v3
+; GFX9-NEXT: v_mov_b32_e32 v3, 0xff
+; GFX9-NEXT: v_lshlrev_b32_sdwa v2, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
+; GFX9-NEXT: v_and_or_b32 v1, v1, v3, v2
; GFX9-NEXT: v_and_b32_e32 v2, 0xff, v0
; GFX9-NEXT: v_mov_b32_e32 v3, 24
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
@@ -741,30 +737,28 @@ define i32 @v_ssubsat_v4i8(i32 %lhs.arg, i32 %rhs.arg) {
; GFX10-LABEL: v_ssubsat_v4i8:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_e32 v2, 8, v0
-; GFX10-NEXT: v_lshrrev_b32_e32 v3, 24, v0
-; GFX10-NEXT: v_and_b32_e32 v4, 0xffff, v0
-; GFX10-NEXT: v_lshrrev_b32_e32 v5, 8, v1
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffff, v1
-; GFX10-NEXT: v_lshrrev_b32_e32 v7, 24, v1
-; GFX10-NEXT: v_alignbit_b32 v0, v3, v0, 16
-; GFX10-NEXT: v_lshl_or_b32 v2, v2, 16, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, 24
-; GFX10-NEXT: v_lshl_or_b32 v3, v5, 16, v6
-; GFX10-NEXT: v_alignbit_b32 v1, v7, v1, 16
-; GFX10-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
-; GFX10-NEXT: v_pk_lshlrev_b16 v2, 8, v2 op_sel_hi:[0,1]
+; GFX10-NEXT: v_mov_b32_e32 v2, 8
+; GFX10-NEXT: v_lshrrev_b32_e32 v4, 24, v0
+; GFX10-NEXT: v_lshrrev_b32_e32 v6, 24, v1
+; GFX10-NEXT: v_lshrrev_b32_sdwa v3, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_lshrrev_b32_sdwa v5, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v3, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_alignbit_b32 v0, v4, v0, 16
+; GFX10-NEXT: v_or_b32_sdwa v4, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_alignbit_b32 v1, v6, v1, 16
; GFX10-NEXT: v_pk_lshlrev_b16 v3, 8, v3 op_sel_hi:[0,1]
+; GFX10-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX10-NEXT: v_pk_lshlrev_b16 v4, 8, v4 op_sel_hi:[0,1]
; GFX10-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
-; GFX10-NEXT: v_pk_sub_i16 v2, v2, v3 clamp
+; GFX10-NEXT: v_pk_sub_i16 v3, v3, v4 clamp
; GFX10-NEXT: v_pk_sub_i16 v0, v0, v1 clamp
-; GFX10-NEXT: v_mov_b32_e32 v1, 8
-; GFX10-NEXT: v_pk_ashrrev_i16 v2, 8, v2 op_sel_hi:[0,1]
+; GFX10-NEXT: v_mov_b32_e32 v4, 24
+; GFX10-NEXT: v_pk_ashrrev_i16 v1, 8, v3 op_sel_hi:[0,1]
; GFX10-NEXT: v_pk_ashrrev_i16 v0, 8, v0 op_sel_hi:[0,1]
-; GFX10-NEXT: v_lshlrev_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
+; GFX10-NEXT: v_lshlrev_b32_sdwa v2, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
; GFX10-NEXT: v_and_b32_e32 v3, 0xff, v0
; GFX10-NEXT: v_lshlrev_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
-; GFX10-NEXT: v_and_or_b32 v1, 0xff, v2, v1
+; GFX10-NEXT: v_and_or_b32 v1, 0xff, v1, v2
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
; GFX10-NEXT: v_or3_b32 v0, v1, v2, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll
index 54f7339ab4c32..749fe014ff706 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll
@@ -296,12 +296,11 @@ define i16 @v_uaddsat_v2i8(i16 %lhs.arg, i16 %rhs.arg) {
; GFX9-LABEL: v_uaddsat_v2i8:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_lshrrev_b32_e32 v2, 8, v0
-; GFX9-NEXT: v_lshrrev_b32_e32 v3, 8, v1
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_lshl_or_b32 v0, v2, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v3, 16, v1
+; GFX9-NEXT: v_mov_b32_e32 v2, 8
+; GFX9-NEXT: v_lshrrev_b32_sdwa v3, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_lshrrev_b32_sdwa v2, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
; GFX9-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
; GFX9-NEXT: v_pk_add_u16 v0, v0, v1 clamp
@@ -314,12 +313,11 @@ define i16 @v_uaddsat_v2i8(i16 %lhs.arg, i16 %rhs.arg) {
; GFX10-LABEL: v_uaddsat_v2i8:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_e32 v2, 8, v0
-; GFX10-NEXT: v_lshrrev_b32_e32 v3, 8, v1
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_lshl_or_b32 v0, v2, 16, v0
-; GFX10-NEXT: v_lshl_or_b32 v1, v3, 16, v1
+; GFX10-NEXT: v_mov_b32_e32 v2, 8
+; GFX10-NEXT: v_lshrrev_b32_sdwa v3, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_lshrrev_b32_sdwa v2, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
; GFX10-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
; GFX10-NEXT: v_pk_add_u16 v0, v0, v1 clamp
@@ -604,28 +602,26 @@ define i32 @v_uaddsat_v4i8(i32 %lhs.arg, i32 %rhs.arg) {
; GFX9-LABEL: v_uaddsat_v4i8:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_lshrrev_b32_e32 v3, 24, v0
-; GFX9-NEXT: v_lshrrev_b32_e32 v2, 8, v0
-; GFX9-NEXT: v_lshrrev_b32_e32 v4, 8, v1
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v0
-; GFX9-NEXT: v_alignbit_b32 v0, v3, v0, 16
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v1
-; GFX9-NEXT: v_lshrrev_b32_e32 v5, 24, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v2, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v3, v4, 16, v3
-; GFX9-NEXT: v_alignbit_b32 v1, v5, v1, 16
-; GFX9-NEXT: v_pk_lshlrev_b16 v2, 8, v2 op_sel_hi:[0,1]
+; GFX9-NEXT: v_mov_b32_e32 v2, 8
+; GFX9-NEXT: v_lshrrev_b32_sdwa v3, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 24, v0
+; GFX9-NEXT: v_lshrrev_b32_sdwa v5, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_lshrrev_b32_e32 v6, 24, v1
+; GFX9-NEXT: v_or_b32_sdwa v3, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_alignbit_b32 v0, v4, v0, 16
+; GFX9-NEXT: v_or_b32_sdwa v4, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_alignbit_b32 v1, v6, v1, 16
; GFX9-NEXT: v_pk_lshlrev_b16 v3, 8, v3 op_sel_hi:[0,1]
+; GFX9-NEXT: v_pk_lshlrev_b16 v4, 8, v4 op_sel_hi:[0,1]
; GFX9-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
; GFX9-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
-; GFX9-NEXT: v_pk_add_u16 v2, v2, v3 clamp
+; GFX9-NEXT: v_pk_add_u16 v3, v3, v4 clamp
; GFX9-NEXT: v_pk_add_u16 v0, v0, v1 clamp
-; GFX9-NEXT: v_pk_lshrrev_b16 v1, 8, v2 op_sel_hi:[0,1]
-; GFX9-NEXT: v_mov_b32_e32 v3, 8
+; GFX9-NEXT: v_pk_lshrrev_b16 v1, 8, v3 op_sel_hi:[0,1]
; GFX9-NEXT: v_pk_lshrrev_b16 v0, 8, v0 op_sel_hi:[0,1]
-; GFX9-NEXT: v_mov_b32_e32 v2, 0xff
-; GFX9-NEXT: v_lshlrev_b32_sdwa v3, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
-; GFX9-NEXT: v_and_or_b32 v1, v1, v2, v3
+; GFX9-NEXT: v_mov_b32_e32 v3, 0xff
+; GFX9-NEXT: v_lshlrev_b32_sdwa v2, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
+; GFX9-NEXT: v_and_or_b32 v1, v1, v3, v2
; GFX9-NEXT: v_and_b32_e32 v2, 0xff, v0
; GFX9-NEXT: v_mov_b32_e32 v3, 24
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
@@ -636,30 +632,28 @@ define i32 @v_uaddsat_v4i8(i32 %lhs.arg, i32 %rhs.arg) {
; GFX10-LABEL: v_uaddsat_v4i8:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_e32 v2, 8, v0
-; GFX10-NEXT: v_lshrrev_b32_e32 v3, 24, v0
-; GFX10-NEXT: v_and_b32_e32 v4, 0xffff, v0
-; GFX10-NEXT: v_lshrrev_b32_e32 v5, 8, v1
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffff, v1
-; GFX10-NEXT: v_lshrrev_b32_e32 v7, 24, v1
-; GFX10-NEXT: v_alignbit_b32 v0, v3, v0, 16
-; GFX10-NEXT: v_lshl_or_b32 v2, v2, 16, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, 24
-; GFX10-NEXT: v_lshl_or_b32 v3, v5, 16, v6
-; GFX10-NEXT: v_alignbit_b32 v1, v7, v1, 16
-; GFX10-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
-; GFX10-NEXT: v_pk_lshlrev_b16 v2, 8, v2 op_sel_hi:[0,1]
+; GFX10-NEXT: v_mov_b32_e32 v2, 8
+; GFX10-NEXT: v_lshrrev_b32_e32 v4, 24, v0
+; GFX10-NEXT: v_lshrrev_b32_e32 v6, 24, v1
+; GFX10-NEXT: v_lshrrev_b32_sdwa v3, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_lshrrev_b32_sdwa v5, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v3, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_alignbit_b32 v0, v4, v0, 16
+; GFX10-NEXT: v_or_b32_sdwa v4, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_alignbit_b32 v1, v6, v1, 16
; GFX10-NEXT: v_pk_lshlrev_b16 v3, 8, v3 op_sel_hi:[0,1]
+; GFX10-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX10-NEXT: v_pk_lshlrev_b16 v4, 8, v4 op_sel_hi:[0,1]
; GFX10-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
-; GFX10-NEXT: v_pk_add_u16 v2, v2, v3 clamp
+; GFX10-NEXT: v_pk_add_u16 v3, v3, v4 clamp
; GFX10-NEXT: v_pk_add_u16 v0, v0, v1 clamp
-; GFX10-NEXT: v_mov_b32_e32 v1, 8
-; GFX10-NEXT: v_pk_lshrrev_b16 v2, 8, v2 op_sel_hi:[0,1]
+; GFX10-NEXT: v_mov_b32_e32 v4, 24
+; GFX10-NEXT: v_pk_lshrrev_b16 v1, 8, v3 op_sel_hi:[0,1]
; GFX10-NEXT: v_pk_lshrrev_b16 v0, 8, v0 op_sel_hi:[0,1]
-; GFX10-NEXT: v_lshlrev_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
+; GFX10-NEXT: v_lshlrev_b32_sdwa v2, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
; GFX10-NEXT: v_and_b32_e32 v3, 0xff, v0
; GFX10-NEXT: v_lshlrev_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
-; GFX10-NEXT: v_and_or_b32 v1, 0xff, v2, v1
+; GFX10-NEXT: v_and_or_b32 v1, 0xff, v1, v2
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
; GFX10-NEXT: v_or3_b32 v0, v1, v2, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/usubsat.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/usubsat.ll
index fc4b7bd4c9cfd..2e76abae2ff4f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/usubsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/usubsat.ll
@@ -290,12 +290,11 @@ define i16 @v_usubsat_v2i8(i16 %lhs.arg, i16 %rhs.arg) {
; GFX9-LABEL: v_usubsat_v2i8:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_lshrrev_b32_e32 v2, 8, v0
-; GFX9-NEXT: v_lshrrev_b32_e32 v3, 8, v1
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_lshl_or_b32 v0, v2, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v3, 16, v1
+; GFX9-NEXT: v_mov_b32_e32 v2, 8
+; GFX9-NEXT: v_lshrrev_b32_sdwa v3, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_lshrrev_b32_sdwa v2, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
; GFX9-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
; GFX9-NEXT: v_pk_sub_u16 v0, v0, v1 clamp
@@ -308,12 +307,11 @@ define i16 @v_usubsat_v2i8(i16 %lhs.arg, i16 %rhs.arg) {
; GFX10-LABEL: v_usubsat_v2i8:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_e32 v2, 8, v0
-; GFX10-NEXT: v_lshrrev_b32_e32 v3, 8, v1
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_lshl_or_b32 v0, v2, 16, v0
-; GFX10-NEXT: v_lshl_or_b32 v1, v3, 16, v1
+; GFX10-NEXT: v_mov_b32_e32 v2, 8
+; GFX10-NEXT: v_lshrrev_b32_sdwa v3, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_lshrrev_b32_sdwa v2, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
; GFX10-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
; GFX10-NEXT: v_pk_sub_u16 v0, v0, v1 clamp
@@ -592,28 +590,26 @@ define i32 @v_usubsat_v4i8(i32 %lhs.arg, i32 %rhs.arg) {
; GFX9-LABEL: v_usubsat_v4i8:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_lshrrev_b32_e32 v3, 24, v0
-; GFX9-NEXT: v_lshrrev_b32_e32 v2, 8, v0
-; GFX9-NEXT: v_lshrrev_b32_e32 v4, 8, v1
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v0
-; GFX9-NEXT: v_alignbit_b32 v0, v3, v0, 16
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v1
-; GFX9-NEXT: v_lshrrev_b32_e32 v5, 24, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v2, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v3, v4, 16, v3
-; GFX9-NEXT: v_alignbit_b32 v1, v5, v1, 16
-; GFX9-NEXT: v_pk_lshlrev_b16 v2, 8, v2 op_sel_hi:[0,1]
+; GFX9-NEXT: v_mov_b32_e32 v2, 8
+; GFX9-NEXT: v_lshrrev_b32_sdwa v3, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 24, v0
+; GFX9-NEXT: v_lshrrev_b32_sdwa v5, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_lshrrev_b32_e32 v6, 24, v1
+; GFX9-NEXT: v_or_b32_sdwa v3, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_alignbit_b32 v0, v4, v0, 16
+; GFX9-NEXT: v_or_b32_sdwa v4, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_alignbit_b32 v1, v6, v1, 16
; GFX9-NEXT: v_pk_lshlrev_b16 v3, 8, v3 op_sel_hi:[0,1]
+; GFX9-NEXT: v_pk_lshlrev_b16 v4, 8, v4 op_sel_hi:[0,1]
; GFX9-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
; GFX9-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
-; GFX9-NEXT: v_pk_sub_u16 v2, v2, v3 clamp
+; GFX9-NEXT: v_pk_sub_u16 v3, v3, v4 clamp
; GFX9-NEXT: v_pk_sub_u16 v0, v0, v1 clamp
-; GFX9-NEXT: v_pk_lshrrev_b16 v1, 8, v2 op_sel_hi:[0,1]
-; GFX9-NEXT: v_mov_b32_e32 v3, 8
+; GFX9-NEXT: v_pk_lshrrev_b16 v1, 8, v3 op_sel_hi:[0,1]
; GFX9-NEXT: v_pk_lshrrev_b16 v0, 8, v0 op_sel_hi:[0,1]
-; GFX9-NEXT: v_mov_b32_e32 v2, 0xff
-; GFX9-NEXT: v_lshlrev_b32_sdwa v3, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
-; GFX9-NEXT: v_and_or_b32 v1, v1, v2, v3
+; GFX9-NEXT: v_mov_b32_e32 v3, 0xff
+; GFX9-NEXT: v_lshlrev_b32_sdwa v2, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
+; GFX9-NEXT: v_and_or_b32 v1, v1, v3, v2
; GFX9-NEXT: v_and_b32_e32 v2, 0xff, v0
; GFX9-NEXT: v_mov_b32_e32 v3, 24
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
@@ -624,30 +620,28 @@ define i32 @v_usubsat_v4i8(i32 %lhs.arg, i32 %rhs.arg) {
; GFX10-LABEL: v_usubsat_v4i8:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_e32 v2, 8, v0
-; GFX10-NEXT: v_lshrrev_b32_e32 v3, 24, v0
-; GFX10-NEXT: v_and_b32_e32 v4, 0xffff, v0
-; GFX10-NEXT: v_lshrrev_b32_e32 v5, 8, v1
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffff, v1
-; GFX10-NEXT: v_lshrrev_b32_e32 v7, 24, v1
-; GFX10-NEXT: v_alignbit_b32 v0, v3, v0, 16
-; GFX10-NEXT: v_lshl_or_b32 v2, v2, 16, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, 24
-; GFX10-NEXT: v_lshl_or_b32 v3, v5, 16, v6
-; GFX10-NEXT: v_alignbit_b32 v1, v7, v1, 16
-; GFX10-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
-; GFX10-NEXT: v_pk_lshlrev_b16 v2, 8, v2 op_sel_hi:[0,1]
+; GFX10-NEXT: v_mov_b32_e32 v2, 8
+; GFX10-NEXT: v_lshrrev_b32_e32 v4, 24, v0
+; GFX10-NEXT: v_lshrrev_b32_e32 v6, 24, v1
+; GFX10-NEXT: v_lshrrev_b32_sdwa v3, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_lshrrev_b32_sdwa v5, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v3, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_alignbit_b32 v0, v4, v0, 16
+; GFX10-NEXT: v_or_b32_sdwa v4, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_alignbit_b32 v1, v6, v1, 16
; GFX10-NEXT: v_pk_lshlrev_b16 v3, 8, v3 op_sel_hi:[0,1]
+; GFX10-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX10-NEXT: v_pk_lshlrev_b16 v4, 8, v4 op_sel_hi:[0,1]
; GFX10-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
-; GFX10-NEXT: v_pk_sub_u16 v2, v2, v3 clamp
+; GFX10-NEXT: v_pk_sub_u16 v3, v3, v4 clamp
; GFX10-NEXT: v_pk_sub_u16 v0, v0, v1 clamp
-; GFX10-NEXT: v_mov_b32_e32 v1, 8
-; GFX10-NEXT: v_pk_lshrrev_b16 v2, 8, v2 op_sel_hi:[0,1]
+; GFX10-NEXT: v_mov_b32_e32 v4, 24
+; GFX10-NEXT: v_pk_lshrrev_b16 v1, 8, v3 op_sel_hi:[0,1]
; GFX10-NEXT: v_pk_lshrrev_b16 v0, 8, v0 op_sel_hi:[0,1]
-; GFX10-NEXT: v_lshlrev_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
+; GFX10-NEXT: v_lshlrev_b32_sdwa v2, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
; GFX10-NEXT: v_and_b32_e32 v3, 0xff, v0
; GFX10-NEXT: v_lshlrev_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
-; GFX10-NEXT: v_and_or_b32 v1, 0xff, v2, v1
+; GFX10-NEXT: v_and_or_b32 v1, 0xff, v1, v2
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
; GFX10-NEXT: v_or3_b32 v0, v1, v2, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/vni8-across-blocks.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/vni8-across-blocks.ll
index c819ef7a7f6dd..2530e82e6e015 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/vni8-across-blocks.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/vni8-across-blocks.ll
@@ -31,12 +31,12 @@ define amdgpu_kernel void @v3i8_liveout(ptr addrspace(1) %src1, ptr addrspace(1)
; GFX906-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX906-NEXT: v_lshrrev_b32_e32 v0, 8, v4
; GFX906-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX906-NEXT: v_lshlrev_b16_e32 v0, 8, v0
-; GFX906-NEXT: v_or_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX906-NEXT: v_and_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX906-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX906-NEXT: v_lshlrev_b16_e32 v0, 8, v0
; GFX906-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX906-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX906-NEXT: v_or_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX906-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX906-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX906-NEXT: v_mov_b32_e32 v1, 0
; GFX906-NEXT: global_store_short v1, v0, s[6:7]
; GFX906-NEXT: global_store_byte_d16_hi v1, v0, s[6:7] offset:2
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
index 68699bc5679d0..ba0064ef7f0d7 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
@@ -27735,24 +27735,24 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
-; GFX9-NEXT: buffer_store_dword v36, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v35, off, s[0:3], s32 ; 4-byte Folded Spill
; GFX9-NEXT: s_mov_b64 exec, s[4:5]
-; GFX9-NEXT: v_writelane_b32 v36, s36, 0
-; GFX9-NEXT: v_writelane_b32 v36, s37, 1
-; GFX9-NEXT: v_writelane_b32 v36, s38, 2
-; GFX9-NEXT: v_writelane_b32 v36, s39, 3
-; GFX9-NEXT: v_writelane_b32 v36, s48, 4
-; GFX9-NEXT: v_writelane_b32 v36, s49, 5
-; GFX9-NEXT: v_writelane_b32 v36, s50, 6
-; GFX9-NEXT: v_writelane_b32 v36, s51, 7
-; GFX9-NEXT: v_writelane_b32 v36, s52, 8
-; GFX9-NEXT: v_writelane_b32 v36, s53, 9
-; GFX9-NEXT: v_writelane_b32 v36, s54, 10
-; GFX9-NEXT: v_writelane_b32 v36, s55, 11
-; GFX9-NEXT: v_writelane_b32 v36, s64, 12
-; GFX9-NEXT: v_writelane_b32 v36, s65, 13
-; GFX9-NEXT: v_writelane_b32 v36, s66, 14
-; GFX9-NEXT: v_writelane_b32 v36, s67, 15
+; GFX9-NEXT: v_writelane_b32 v35, s36, 0
+; GFX9-NEXT: v_writelane_b32 v35, s37, 1
+; GFX9-NEXT: v_writelane_b32 v35, s38, 2
+; GFX9-NEXT: v_writelane_b32 v35, s39, 3
+; GFX9-NEXT: v_writelane_b32 v35, s48, 4
+; GFX9-NEXT: v_writelane_b32 v35, s49, 5
+; GFX9-NEXT: v_writelane_b32 v35, s50, 6
+; GFX9-NEXT: v_writelane_b32 v35, s51, 7
+; GFX9-NEXT: v_writelane_b32 v35, s52, 8
+; GFX9-NEXT: v_writelane_b32 v35, s53, 9
+; GFX9-NEXT: v_writelane_b32 v35, s54, 10
+; GFX9-NEXT: v_writelane_b32 v35, s55, 11
+; GFX9-NEXT: v_writelane_b32 v35, s64, 12
+; GFX9-NEXT: v_writelane_b32 v35, s65, 13
+; GFX9-NEXT: v_writelane_b32 v35, s66, 14
+; GFX9-NEXT: v_writelane_b32 v35, s67, 15
; GFX9-NEXT: v_readfirstlane_b32 s4, v18
; GFX9-NEXT: s_mov_b32 s49, s29
; GFX9-NEXT: s_mov_b32 s48, s28
@@ -27824,25 +27824,24 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: s_lshl_b32 s4, s50, 16
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_add_f32_e32 v3, s4, v16
-; GFX9-NEXT: v_mov_b32_e32 v32, 0xffff
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_bfe_u32 v4, v3, 16, 1
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_add_u32_e32 v4, v4, v3
; GFX9-NEXT: s_and_b32 s4, s49, 0xffff0000
+; GFX9-NEXT: v_add_u32_e32 v4, v4, v3
+; GFX9-NEXT: v_or_b32_sdwa v15, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v3
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX9-NEXT: v_lshl_or_b32 v15, v0, 16, v1
-; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX9-NEXT: v_and_b32_sdwa v3, v32, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
-; GFX9-NEXT: v_lshl_or_b32 v14, v2, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v14, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_u32_e32 v1, 0x7fff, v1
; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
@@ -27854,11 +27853,11 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s48, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v13, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v13, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -27873,11 +27872,11 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s47, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v12, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v12, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -27892,11 +27891,11 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s46, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v11, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v11, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -27911,11 +27910,11 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s45, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v10, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v10, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -27930,11 +27929,11 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s44, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v9, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v9, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -27949,11 +27948,11 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s43, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v8, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v8, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -27968,11 +27967,11 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s42, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v7, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v7, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -27987,11 +27986,11 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s41, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v6, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v6, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -28006,11 +28005,11 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s40, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v5, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v5, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -28025,11 +28024,11 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s39, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v4, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v4, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -28044,11 +28043,11 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s38, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v3, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v3, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -28063,11 +28062,11 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v17, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v17, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s37, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v2, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v2, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -28082,11 +28081,11 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v17, 0x7fff, v17
; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v17, v18, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v17, v18, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s36, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v1, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v17, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v17, v17, v0
@@ -28101,11 +28100,11 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v18, 0x7fff, v18
; GFX9-NEXT: v_or_b32_e32 v19, 0x400000, v17
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
-; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v19, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v17, v32, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v19, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s67, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v17
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -28120,11 +28119,11 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s66, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v31, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v31, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -28139,11 +28138,11 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s65, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v30, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v30, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -28158,11 +28157,11 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s64, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v29, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v29, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -28177,11 +28176,11 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s63, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v28, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v28, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -28196,11 +28195,11 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s62, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v27, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v27, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -28215,11 +28214,11 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s61, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v26, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v26, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -28234,11 +28233,11 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s60, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v25, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v25, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -28253,11 +28252,11 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s59, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v24, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v24, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -28272,11 +28271,11 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s58, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v23, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v23, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -28291,11 +28290,11 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s57, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v22, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v22, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -28310,11 +28309,11 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s56, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v21, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v21, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -28329,11 +28328,11 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s55, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v20, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v20, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -28346,69 +28345,69 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_bfe_u32 v19, v18, 16, 1
; GFX9-NEXT: v_add_u32_e32 v19, v19, v18
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
-; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v18
+; GFX9-NEXT: v_or_b32_e32 v32, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v33, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v32, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s54, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v19, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v19, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
; GFX9-NEXT: v_add_u32_e32 v18, 0x7fff, v18
-; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v17
+; GFX9-NEXT: v_or_b32_e32 v32, 0x400000, v17
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
; GFX9-NEXT: s_lshl_b32 s4, s54, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v33, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v32, vcc
; GFX9-NEXT: v_add_f32_e32 v18, s4, v16
-; GFX9-NEXT: v_bfe_u32 v33, v18, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v33, v33, v18
-; GFX9-NEXT: v_add_u32_e32 v33, 0x7fff, v33
-; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v18
+; GFX9-NEXT: v_bfe_u32 v32, v18, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v32, v32, v18
+; GFX9-NEXT: v_add_u32_e32 v32, 0x7fff, v32
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v33, v34, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v32, v33, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s53, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v18, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v18, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
-; GFX9-NEXT: v_bfe_u32 v33, v17, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v33, v33, v17
-; GFX9-NEXT: v_add_u32_e32 v33, 0x7fff, v33
-; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v17
+; GFX9-NEXT: v_bfe_u32 v32, v17, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v32, v32, v17
+; GFX9-NEXT: v_add_u32_e32 v32, 0x7fff, v32
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v17
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
; GFX9-NEXT: s_lshl_b32 s4, s53, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v17, v33, v34, vcc
-; GFX9-NEXT: v_add_f32_e32 v33, s4, v16
-; GFX9-NEXT: v_bfe_u32 v34, v33, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v34, v34, v33
-; GFX9-NEXT: v_add_u32_e32 v34, 0x7fff, v34
-; GFX9-NEXT: v_or_b32_e32 v35, 0x400000, v33
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v33, v33
-; GFX9-NEXT: v_cndmask_b32_e32 v33, v34, v35, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v17, v32, v33, vcc
+; GFX9-NEXT: v_add_f32_e32 v32, s4, v16
+; GFX9-NEXT: v_bfe_u32 v33, v32, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v33, v33, v32
+; GFX9-NEXT: v_add_u32_e32 v33, 0x7fff, v33
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v32
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v32, v32
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v33, v32, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v32, v33, v34, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s52, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v17, v17, 16, v33
-; GFX9-NEXT: v_add_f32_e32 v33, s4, v16
-; GFX9-NEXT: v_bfe_u32 v34, v33, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v34, v34, v33
+; GFX9-NEXT: v_or_b32_sdwa v17, v17, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f32_e32 v32, s4, v16
+; GFX9-NEXT: v_bfe_u32 v33, v32, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v33, v33, v32
; GFX9-NEXT: s_lshl_b32 s4, s52, 16
-; GFX9-NEXT: v_add_u32_e32 v34, 0x7fff, v34
-; GFX9-NEXT: v_or_b32_e32 v35, 0x400000, v33
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v33, v33
+; GFX9-NEXT: v_add_u32_e32 v33, 0x7fff, v33
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v32
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v32, v32
; GFX9-NEXT: v_add_f32_e32 v16, s4, v16
-; GFX9-NEXT: v_cndmask_b32_e32 v33, v34, v35, vcc
-; GFX9-NEXT: v_bfe_u32 v34, v16, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v34, v34, v16
-; GFX9-NEXT: v_add_u32_e32 v34, 0x7fff, v34
-; GFX9-NEXT: v_or_b32_e32 v35, 0x400000, v16
+; GFX9-NEXT: v_cndmask_b32_e32 v32, v33, v34, vcc
+; GFX9-NEXT: v_bfe_u32 v33, v16, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v33, v33, v16
+; GFX9-NEXT: v_add_u32_e32 v33, 0x7fff, v33
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v16
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v16, v16
-; GFX9-NEXT: v_cndmask_b32_e32 v16, v34, v35, vcc
-; GFX9-NEXT: v_and_b32_sdwa v16, v32, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshrrev_b32_e32 v32, 16, v33
-; GFX9-NEXT: v_lshl_or_b32 v16, v32, 16, v16
+; GFX9-NEXT: v_lshrrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_cndmask_b32_e32 v16, v33, v34, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_or_b32_sdwa v16, v32, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_branch .LBB19_6
; GFX9-NEXT: .LBB19_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
@@ -28444,24 +28443,24 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
; GFX9-NEXT: .LBB19_6: ; %end
-; GFX9-NEXT: v_readlane_b32 s67, v36, 15
-; GFX9-NEXT: v_readlane_b32 s66, v36, 14
-; GFX9-NEXT: v_readlane_b32 s65, v36, 13
-; GFX9-NEXT: v_readlane_b32 s64, v36, 12
-; GFX9-NEXT: v_readlane_b32 s55, v36, 11
-; GFX9-NEXT: v_readlane_b32 s54, v36, 10
-; GFX9-NEXT: v_readlane_b32 s53, v36, 9
-; GFX9-NEXT: v_readlane_b32 s52, v36, 8
-; GFX9-NEXT: v_readlane_b32 s51, v36, 7
-; GFX9-NEXT: v_readlane_b32 s50, v36, 6
-; GFX9-NEXT: v_readlane_b32 s49, v36, 5
-; GFX9-NEXT: v_readlane_b32 s48, v36, 4
-; GFX9-NEXT: v_readlane_b32 s39, v36, 3
-; GFX9-NEXT: v_readlane_b32 s38, v36, 2
-; GFX9-NEXT: v_readlane_b32 s37, v36, 1
-; GFX9-NEXT: v_readlane_b32 s36, v36, 0
+; GFX9-NEXT: v_readlane_b32 s67, v35, 15
+; GFX9-NEXT: v_readlane_b32 s66, v35, 14
+; GFX9-NEXT: v_readlane_b32 s65, v35, 13
+; GFX9-NEXT: v_readlane_b32 s64, v35, 12
+; GFX9-NEXT: v_readlane_b32 s55, v35, 11
+; GFX9-NEXT: v_readlane_b32 s54, v35, 10
+; GFX9-NEXT: v_readlane_b32 s53, v35, 9
+; GFX9-NEXT: v_readlane_b32 s52, v35, 8
+; GFX9-NEXT: v_readlane_b32 s51, v35, 7
+; GFX9-NEXT: v_readlane_b32 s50, v35, 6
+; GFX9-NEXT: v_readlane_b32 s49, v35, 5
+; GFX9-NEXT: v_readlane_b32 s48, v35, 4
+; GFX9-NEXT: v_readlane_b32 s39, v35, 3
+; GFX9-NEXT: v_readlane_b32 s38, v35, 2
+; GFX9-NEXT: v_readlane_b32 s37, v35, 1
+; GFX9-NEXT: v_readlane_b32 s36, v35, 0
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
-; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v35, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-NEXT: s_mov_b64 exec, s[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -63880,24 +63879,24 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
-; GFX9-NEXT: buffer_store_dword v36, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v35, off, s[0:3], s32 ; 4-byte Folded Spill
; GFX9-NEXT: s_mov_b64 exec, s[4:5]
-; GFX9-NEXT: v_writelane_b32 v36, s36, 0
-; GFX9-NEXT: v_writelane_b32 v36, s37, 1
-; GFX9-NEXT: v_writelane_b32 v36, s38, 2
-; GFX9-NEXT: v_writelane_b32 v36, s39, 3
-; GFX9-NEXT: v_writelane_b32 v36, s48, 4
-; GFX9-NEXT: v_writelane_b32 v36, s49, 5
-; GFX9-NEXT: v_writelane_b32 v36, s50, 6
-; GFX9-NEXT: v_writelane_b32 v36, s51, 7
-; GFX9-NEXT: v_writelane_b32 v36, s52, 8
-; GFX9-NEXT: v_writelane_b32 v36, s53, 9
-; GFX9-NEXT: v_writelane_b32 v36, s54, 10
-; GFX9-NEXT: v_writelane_b32 v36, s55, 11
-; GFX9-NEXT: v_writelane_b32 v36, s64, 12
-; GFX9-NEXT: v_writelane_b32 v36, s65, 13
-; GFX9-NEXT: v_writelane_b32 v36, s66, 14
-; GFX9-NEXT: v_writelane_b32 v36, s67, 15
+; GFX9-NEXT: v_writelane_b32 v35, s36, 0
+; GFX9-NEXT: v_writelane_b32 v35, s37, 1
+; GFX9-NEXT: v_writelane_b32 v35, s38, 2
+; GFX9-NEXT: v_writelane_b32 v35, s39, 3
+; GFX9-NEXT: v_writelane_b32 v35, s48, 4
+; GFX9-NEXT: v_writelane_b32 v35, s49, 5
+; GFX9-NEXT: v_writelane_b32 v35, s50, 6
+; GFX9-NEXT: v_writelane_b32 v35, s51, 7
+; GFX9-NEXT: v_writelane_b32 v35, s52, 8
+; GFX9-NEXT: v_writelane_b32 v35, s53, 9
+; GFX9-NEXT: v_writelane_b32 v35, s54, 10
+; GFX9-NEXT: v_writelane_b32 v35, s55, 11
+; GFX9-NEXT: v_writelane_b32 v35, s64, 12
+; GFX9-NEXT: v_writelane_b32 v35, s65, 13
+; GFX9-NEXT: v_writelane_b32 v35, s66, 14
+; GFX9-NEXT: v_writelane_b32 v35, s67, 15
; GFX9-NEXT: v_readfirstlane_b32 s4, v18
; GFX9-NEXT: s_mov_b32 s49, s29
; GFX9-NEXT: s_mov_b32 s48, s28
@@ -63969,25 +63968,24 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: s_lshl_b32 s4, s50, 16
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_add_f32_e32 v3, s4, v16
-; GFX9-NEXT: v_mov_b32_e32 v32, 0xffff
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_bfe_u32 v4, v3, 16, 1
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_add_u32_e32 v4, v4, v3
; GFX9-NEXT: s_and_b32 s4, s49, 0xffff0000
+; GFX9-NEXT: v_add_u32_e32 v4, v4, v3
+; GFX9-NEXT: v_or_b32_sdwa v15, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v3
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX9-NEXT: v_lshl_or_b32 v15, v0, 16, v1
-; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX9-NEXT: v_and_b32_sdwa v3, v32, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
-; GFX9-NEXT: v_lshl_or_b32 v14, v2, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v14, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_u32_e32 v1, 0x7fff, v1
; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
@@ -63999,11 +63997,11 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s48, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v13, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v13, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -64018,11 +64016,11 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s47, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v12, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v12, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -64037,11 +64035,11 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s46, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v11, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v11, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -64056,11 +64054,11 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s45, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v10, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v10, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -64075,11 +64073,11 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s44, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v9, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v9, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -64094,11 +64092,11 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s43, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v8, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v8, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -64113,11 +64111,11 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s42, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v7, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v7, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -64132,11 +64130,11 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s41, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v6, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v6, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -64151,11 +64149,11 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s40, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v5, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v5, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -64170,11 +64168,11 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s39, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v4, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v4, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -64189,11 +64187,11 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s38, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v3, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v3, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -64208,11 +64206,11 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v17, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v17, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s37, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v2, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v2, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -64227,11 +64225,11 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v17, 0x7fff, v17
; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v17, v18, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v17, v18, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s36, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v1, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v17, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v17, v17, v0
@@ -64246,11 +64244,11 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v18, 0x7fff, v18
; GFX9-NEXT: v_or_b32_e32 v19, 0x400000, v17
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
-; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v19, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v17, v32, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v19, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s67, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v17
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -64265,11 +64263,11 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s66, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v31, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v31, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -64284,11 +64282,11 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s65, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v30, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v30, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -64303,11 +64301,11 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s64, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v29, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v29, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -64322,11 +64320,11 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s63, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v28, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v28, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -64341,11 +64339,11 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s62, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v27, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v27, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -64360,11 +64358,11 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s61, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v26, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v26, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -64379,11 +64377,11 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s60, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v25, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v25, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -64398,11 +64396,11 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s59, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v24, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v24, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -64417,11 +64415,11 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s58, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v23, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v23, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -64436,11 +64434,11 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s57, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v22, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v22, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -64455,11 +64453,11 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s56, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v21, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v21, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -64474,11 +64472,11 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s55, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v20, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v20, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -64491,69 +64489,69 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_bfe_u32 v19, v18, 16, 1
; GFX9-NEXT: v_add_u32_e32 v19, v19, v18
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
-; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v18
+; GFX9-NEXT: v_or_b32_e32 v32, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v33, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v32, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s54, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v19, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v19, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
; GFX9-NEXT: v_add_u32_e32 v18, 0x7fff, v18
-; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v17
+; GFX9-NEXT: v_or_b32_e32 v32, 0x400000, v17
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
; GFX9-NEXT: s_lshl_b32 s4, s54, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v33, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v32, vcc
; GFX9-NEXT: v_add_f32_e32 v18, s4, v16
-; GFX9-NEXT: v_bfe_u32 v33, v18, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v33, v33, v18
-; GFX9-NEXT: v_add_u32_e32 v33, 0x7fff, v33
-; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v18
+; GFX9-NEXT: v_bfe_u32 v32, v18, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v32, v32, v18
+; GFX9-NEXT: v_add_u32_e32 v32, 0x7fff, v32
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v33, v34, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v32, v33, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s53, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v18, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v18, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
-; GFX9-NEXT: v_bfe_u32 v33, v17, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v33, v33, v17
-; GFX9-NEXT: v_add_u32_e32 v33, 0x7fff, v33
-; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v17
+; GFX9-NEXT: v_bfe_u32 v32, v17, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v32, v32, v17
+; GFX9-NEXT: v_add_u32_e32 v32, 0x7fff, v32
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v17
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
; GFX9-NEXT: s_lshl_b32 s4, s53, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v17, v33, v34, vcc
-; GFX9-NEXT: v_add_f32_e32 v33, s4, v16
-; GFX9-NEXT: v_bfe_u32 v34, v33, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v34, v34, v33
-; GFX9-NEXT: v_add_u32_e32 v34, 0x7fff, v34
-; GFX9-NEXT: v_or_b32_e32 v35, 0x400000, v33
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v33, v33
-; GFX9-NEXT: v_cndmask_b32_e32 v33, v34, v35, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v17, v32, v33, vcc
+; GFX9-NEXT: v_add_f32_e32 v32, s4, v16
+; GFX9-NEXT: v_bfe_u32 v33, v32, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v33, v33, v32
+; GFX9-NEXT: v_add_u32_e32 v33, 0x7fff, v33
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v32
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v32, v32
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v33, v32, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v32, v33, v34, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s52, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v17, v17, 16, v33
-; GFX9-NEXT: v_add_f32_e32 v33, s4, v16
-; GFX9-NEXT: v_bfe_u32 v34, v33, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v34, v34, v33
+; GFX9-NEXT: v_or_b32_sdwa v17, v17, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f32_e32 v32, s4, v16
+; GFX9-NEXT: v_bfe_u32 v33, v32, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v33, v33, v32
; GFX9-NEXT: s_lshl_b32 s4, s52, 16
-; GFX9-NEXT: v_add_u32_e32 v34, 0x7fff, v34
-; GFX9-NEXT: v_or_b32_e32 v35, 0x400000, v33
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v33, v33
+; GFX9-NEXT: v_add_u32_e32 v33, 0x7fff, v33
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v32
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v32, v32
; GFX9-NEXT: v_add_f32_e32 v16, s4, v16
-; GFX9-NEXT: v_cndmask_b32_e32 v33, v34, v35, vcc
-; GFX9-NEXT: v_bfe_u32 v34, v16, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v34, v34, v16
-; GFX9-NEXT: v_add_u32_e32 v34, 0x7fff, v34
-; GFX9-NEXT: v_or_b32_e32 v35, 0x400000, v16
+; GFX9-NEXT: v_cndmask_b32_e32 v32, v33, v34, vcc
+; GFX9-NEXT: v_bfe_u32 v33, v16, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v33, v33, v16
+; GFX9-NEXT: v_add_u32_e32 v33, 0x7fff, v33
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v16
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v16, v16
-; GFX9-NEXT: v_cndmask_b32_e32 v16, v34, v35, vcc
-; GFX9-NEXT: v_and_b32_sdwa v16, v32, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshrrev_b32_e32 v32, 16, v33
-; GFX9-NEXT: v_lshl_or_b32 v16, v32, 16, v16
+; GFX9-NEXT: v_lshrrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_cndmask_b32_e32 v16, v33, v34, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_or_b32_sdwa v16, v32, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_branch .LBB43_6
; GFX9-NEXT: .LBB43_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
@@ -64589,24 +64587,24 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
; GFX9-NEXT: .LBB43_6: ; %end
-; GFX9-NEXT: v_readlane_b32 s67, v36, 15
-; GFX9-NEXT: v_readlane_b32 s66, v36, 14
-; GFX9-NEXT: v_readlane_b32 s65, v36, 13
-; GFX9-NEXT: v_readlane_b32 s64, v36, 12
-; GFX9-NEXT: v_readlane_b32 s55, v36, 11
-; GFX9-NEXT: v_readlane_b32 s54, v36, 10
-; GFX9-NEXT: v_readlane_b32 s53, v36, 9
-; GFX9-NEXT: v_readlane_b32 s52, v36, 8
-; GFX9-NEXT: v_readlane_b32 s51, v36, 7
-; GFX9-NEXT: v_readlane_b32 s50, v36, 6
-; GFX9-NEXT: v_readlane_b32 s49, v36, 5
-; GFX9-NEXT: v_readlane_b32 s48, v36, 4
-; GFX9-NEXT: v_readlane_b32 s39, v36, 3
-; GFX9-NEXT: v_readlane_b32 s38, v36, 2
-; GFX9-NEXT: v_readlane_b32 s37, v36, 1
-; GFX9-NEXT: v_readlane_b32 s36, v36, 0
+; GFX9-NEXT: v_readlane_b32 s67, v35, 15
+; GFX9-NEXT: v_readlane_b32 s66, v35, 14
+; GFX9-NEXT: v_readlane_b32 s65, v35, 13
+; GFX9-NEXT: v_readlane_b32 s64, v35, 12
+; GFX9-NEXT: v_readlane_b32 s55, v35, 11
+; GFX9-NEXT: v_readlane_b32 s54, v35, 10
+; GFX9-NEXT: v_readlane_b32 s53, v35, 9
+; GFX9-NEXT: v_readlane_b32 s52, v35, 8
+; GFX9-NEXT: v_readlane_b32 s51, v35, 7
+; GFX9-NEXT: v_readlane_b32 s50, v35, 6
+; GFX9-NEXT: v_readlane_b32 s49, v35, 5
+; GFX9-NEXT: v_readlane_b32 s48, v35, 4
+; GFX9-NEXT: v_readlane_b32 s39, v35, 3
+; GFX9-NEXT: v_readlane_b32 s38, v35, 2
+; GFX9-NEXT: v_readlane_b32 s37, v35, 1
+; GFX9-NEXT: v_readlane_b32 s36, v35, 0
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
-; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v35, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-NEXT: s_mov_b64 exec, s[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -97615,24 +97613,24 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
-; GFX9-NEXT: buffer_store_dword v36, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v35, off, s[0:3], s32 ; 4-byte Folded Spill
; GFX9-NEXT: s_mov_b64 exec, s[4:5]
-; GFX9-NEXT: v_writelane_b32 v36, s36, 0
-; GFX9-NEXT: v_writelane_b32 v36, s37, 1
-; GFX9-NEXT: v_writelane_b32 v36, s38, 2
-; GFX9-NEXT: v_writelane_b32 v36, s39, 3
-; GFX9-NEXT: v_writelane_b32 v36, s48, 4
-; GFX9-NEXT: v_writelane_b32 v36, s49, 5
-; GFX9-NEXT: v_writelane_b32 v36, s50, 6
-; GFX9-NEXT: v_writelane_b32 v36, s51, 7
-; GFX9-NEXT: v_writelane_b32 v36, s52, 8
-; GFX9-NEXT: v_writelane_b32 v36, s53, 9
-; GFX9-NEXT: v_writelane_b32 v36, s54, 10
-; GFX9-NEXT: v_writelane_b32 v36, s55, 11
-; GFX9-NEXT: v_writelane_b32 v36, s64, 12
-; GFX9-NEXT: v_writelane_b32 v36, s65, 13
-; GFX9-NEXT: v_writelane_b32 v36, s66, 14
-; GFX9-NEXT: v_writelane_b32 v36, s67, 15
+; GFX9-NEXT: v_writelane_b32 v35, s36, 0
+; GFX9-NEXT: v_writelane_b32 v35, s37, 1
+; GFX9-NEXT: v_writelane_b32 v35, s38, 2
+; GFX9-NEXT: v_writelane_b32 v35, s39, 3
+; GFX9-NEXT: v_writelane_b32 v35, s48, 4
+; GFX9-NEXT: v_writelane_b32 v35, s49, 5
+; GFX9-NEXT: v_writelane_b32 v35, s50, 6
+; GFX9-NEXT: v_writelane_b32 v35, s51, 7
+; GFX9-NEXT: v_writelane_b32 v35, s52, 8
+; GFX9-NEXT: v_writelane_b32 v35, s53, 9
+; GFX9-NEXT: v_writelane_b32 v35, s54, 10
+; GFX9-NEXT: v_writelane_b32 v35, s55, 11
+; GFX9-NEXT: v_writelane_b32 v35, s64, 12
+; GFX9-NEXT: v_writelane_b32 v35, s65, 13
+; GFX9-NEXT: v_writelane_b32 v35, s66, 14
+; GFX9-NEXT: v_writelane_b32 v35, s67, 15
; GFX9-NEXT: v_readfirstlane_b32 s4, v18
; GFX9-NEXT: s_mov_b32 s49, s29
; GFX9-NEXT: s_mov_b32 s48, s28
@@ -97704,25 +97702,24 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: s_lshl_b32 s4, s50, 16
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_add_f32_e32 v3, s4, v16
-; GFX9-NEXT: v_mov_b32_e32 v32, 0xffff
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_bfe_u32 v4, v3, 16, 1
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_add_u32_e32 v4, v4, v3
; GFX9-NEXT: s_and_b32 s4, s49, 0xffff0000
+; GFX9-NEXT: v_add_u32_e32 v4, v4, v3
+; GFX9-NEXT: v_or_b32_sdwa v15, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v3
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX9-NEXT: v_lshl_or_b32 v15, v0, 16, v1
-; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX9-NEXT: v_and_b32_sdwa v3, v32, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
-; GFX9-NEXT: v_lshl_or_b32 v14, v2, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v14, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_u32_e32 v1, 0x7fff, v1
; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
@@ -97734,11 +97731,11 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s48, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v13, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v13, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -97753,11 +97750,11 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s47, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v12, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v12, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -97772,11 +97769,11 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s46, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v11, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v11, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -97791,11 +97788,11 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s45, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v10, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v10, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -97810,11 +97807,11 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s44, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v9, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v9, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -97829,11 +97826,11 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s43, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v8, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v8, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -97848,11 +97845,11 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s42, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v7, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v7, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -97867,11 +97864,11 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s41, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v6, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v6, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -97886,11 +97883,11 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s40, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v5, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v5, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -97905,11 +97902,11 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s39, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v4, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v4, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -97924,11 +97921,11 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s38, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v3, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v3, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -97943,11 +97940,11 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v17, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v17, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s37, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v2, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v2, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -97962,11 +97959,11 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v17, 0x7fff, v17
; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v17, v18, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v17, v18, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s36, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v1, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v17, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v17, v17, v0
@@ -97981,11 +97978,11 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v18, 0x7fff, v18
; GFX9-NEXT: v_or_b32_e32 v19, 0x400000, v17
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
-; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v19, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v17, v32, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v19, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s67, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v17
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -98000,11 +97997,11 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s66, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v31, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v31, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -98019,11 +98016,11 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s65, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v30, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v30, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -98038,11 +98035,11 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s64, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v29, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v29, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -98057,11 +98054,11 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s63, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v28, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v28, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -98076,11 +98073,11 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s62, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v27, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v27, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -98095,11 +98092,11 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s61, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v26, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v26, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -98114,11 +98111,11 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s60, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v25, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v25, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -98133,11 +98130,11 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s59, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v24, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v24, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -98152,11 +98149,11 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s58, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v23, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v23, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -98171,11 +98168,11 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s57, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v22, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v22, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -98190,11 +98187,11 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s56, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v21, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v21, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -98209,11 +98206,11 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s55, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v20, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v20, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -98226,69 +98223,69 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_bfe_u32 v19, v18, 16, 1
; GFX9-NEXT: v_add_u32_e32 v19, v19, v18
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
-; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v18
+; GFX9-NEXT: v_or_b32_e32 v32, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v33, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v32, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s54, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v19, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v19, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
; GFX9-NEXT: v_add_u32_e32 v18, 0x7fff, v18
-; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v17
+; GFX9-NEXT: v_or_b32_e32 v32, 0x400000, v17
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
; GFX9-NEXT: s_lshl_b32 s4, s54, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v33, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v32, vcc
; GFX9-NEXT: v_add_f32_e32 v18, s4, v16
-; GFX9-NEXT: v_bfe_u32 v33, v18, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v33, v33, v18
-; GFX9-NEXT: v_add_u32_e32 v33, 0x7fff, v33
-; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v18
+; GFX9-NEXT: v_bfe_u32 v32, v18, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v32, v32, v18
+; GFX9-NEXT: v_add_u32_e32 v32, 0x7fff, v32
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v33, v34, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v32, v33, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s53, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v18, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v18, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
-; GFX9-NEXT: v_bfe_u32 v33, v17, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v33, v33, v17
-; GFX9-NEXT: v_add_u32_e32 v33, 0x7fff, v33
-; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v17
+; GFX9-NEXT: v_bfe_u32 v32, v17, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v32, v32, v17
+; GFX9-NEXT: v_add_u32_e32 v32, 0x7fff, v32
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v17
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
; GFX9-NEXT: s_lshl_b32 s4, s53, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v17, v33, v34, vcc
-; GFX9-NEXT: v_add_f32_e32 v33, s4, v16
-; GFX9-NEXT: v_bfe_u32 v34, v33, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v34, v34, v33
-; GFX9-NEXT: v_add_u32_e32 v34, 0x7fff, v34
-; GFX9-NEXT: v_or_b32_e32 v35, 0x400000, v33
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v33, v33
-; GFX9-NEXT: v_cndmask_b32_e32 v33, v34, v35, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v17, v32, v33, vcc
+; GFX9-NEXT: v_add_f32_e32 v32, s4, v16
+; GFX9-NEXT: v_bfe_u32 v33, v32, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v33, v33, v32
+; GFX9-NEXT: v_add_u32_e32 v33, 0x7fff, v33
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v32
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v32, v32
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v33, v32, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v32, v33, v34, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s52, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v17, v17, 16, v33
-; GFX9-NEXT: v_add_f32_e32 v33, s4, v16
-; GFX9-NEXT: v_bfe_u32 v34, v33, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v34, v34, v33
+; GFX9-NEXT: v_or_b32_sdwa v17, v17, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f32_e32 v32, s4, v16
+; GFX9-NEXT: v_bfe_u32 v33, v32, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v33, v33, v32
; GFX9-NEXT: s_lshl_b32 s4, s52, 16
-; GFX9-NEXT: v_add_u32_e32 v34, 0x7fff, v34
-; GFX9-NEXT: v_or_b32_e32 v35, 0x400000, v33
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v33, v33
+; GFX9-NEXT: v_add_u32_e32 v33, 0x7fff, v33
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v32
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v32, v32
; GFX9-NEXT: v_add_f32_e32 v16, s4, v16
-; GFX9-NEXT: v_cndmask_b32_e32 v33, v34, v35, vcc
-; GFX9-NEXT: v_bfe_u32 v34, v16, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v34, v34, v16
-; GFX9-NEXT: v_add_u32_e32 v34, 0x7fff, v34
-; GFX9-NEXT: v_or_b32_e32 v35, 0x400000, v16
+; GFX9-NEXT: v_cndmask_b32_e32 v32, v33, v34, vcc
+; GFX9-NEXT: v_bfe_u32 v33, v16, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v33, v33, v16
+; GFX9-NEXT: v_add_u32_e32 v33, 0x7fff, v33
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v16
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v16, v16
-; GFX9-NEXT: v_cndmask_b32_e32 v16, v34, v35, vcc
-; GFX9-NEXT: v_and_b32_sdwa v16, v32, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshrrev_b32_e32 v32, 16, v33
-; GFX9-NEXT: v_lshl_or_b32 v16, v32, 16, v16
+; GFX9-NEXT: v_lshrrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_cndmask_b32_e32 v16, v33, v34, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_or_b32_sdwa v16, v32, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_branch .LBB63_6
; GFX9-NEXT: .LBB63_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
@@ -98324,24 +98321,24 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
; GFX9-NEXT: .LBB63_6: ; %end
-; GFX9-NEXT: v_readlane_b32 s67, v36, 15
-; GFX9-NEXT: v_readlane_b32 s66, v36, 14
-; GFX9-NEXT: v_readlane_b32 s65, v36, 13
-; GFX9-NEXT: v_readlane_b32 s64, v36, 12
-; GFX9-NEXT: v_readlane_b32 s55, v36, 11
-; GFX9-NEXT: v_readlane_b32 s54, v36, 10
-; GFX9-NEXT: v_readlane_b32 s53, v36, 9
-; GFX9-NEXT: v_readlane_b32 s52, v36, 8
-; GFX9-NEXT: v_readlane_b32 s51, v36, 7
-; GFX9-NEXT: v_readlane_b32 s50, v36, 6
-; GFX9-NEXT: v_readlane_b32 s49, v36, 5
-; GFX9-NEXT: v_readlane_b32 s48, v36, 4
-; GFX9-NEXT: v_readlane_b32 s39, v36, 3
-; GFX9-NEXT: v_readlane_b32 s38, v36, 2
-; GFX9-NEXT: v_readlane_b32 s37, v36, 1
-; GFX9-NEXT: v_readlane_b32 s36, v36, 0
+; GFX9-NEXT: v_readlane_b32 s67, v35, 15
+; GFX9-NEXT: v_readlane_b32 s66, v35, 14
+; GFX9-NEXT: v_readlane_b32 s65, v35, 13
+; GFX9-NEXT: v_readlane_b32 s64, v35, 12
+; GFX9-NEXT: v_readlane_b32 s55, v35, 11
+; GFX9-NEXT: v_readlane_b32 s54, v35, 10
+; GFX9-NEXT: v_readlane_b32 s53, v35, 9
+; GFX9-NEXT: v_readlane_b32 s52, v35, 8
+; GFX9-NEXT: v_readlane_b32 s51, v35, 7
+; GFX9-NEXT: v_readlane_b32 s50, v35, 6
+; GFX9-NEXT: v_readlane_b32 s49, v35, 5
+; GFX9-NEXT: v_readlane_b32 s48, v35, 4
+; GFX9-NEXT: v_readlane_b32 s39, v35, 3
+; GFX9-NEXT: v_readlane_b32 s38, v35, 2
+; GFX9-NEXT: v_readlane_b32 s37, v35, 1
+; GFX9-NEXT: v_readlane_b32 s36, v35, 0
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
-; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v35, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-NEXT: s_mov_b64 exec, s[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -130664,24 +130661,24 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
-; GFX9-NEXT: buffer_store_dword v36, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v35, off, s[0:3], s32 ; 4-byte Folded Spill
; GFX9-NEXT: s_mov_b64 exec, s[4:5]
-; GFX9-NEXT: v_writelane_b32 v36, s36, 0
-; GFX9-NEXT: v_writelane_b32 v36, s37, 1
-; GFX9-NEXT: v_writelane_b32 v36, s38, 2
-; GFX9-NEXT: v_writelane_b32 v36, s39, 3
-; GFX9-NEXT: v_writelane_b32 v36, s48, 4
-; GFX9-NEXT: v_writelane_b32 v36, s49, 5
-; GFX9-NEXT: v_writelane_b32 v36, s50, 6
-; GFX9-NEXT: v_writelane_b32 v36, s51, 7
-; GFX9-NEXT: v_writelane_b32 v36, s52, 8
-; GFX9-NEXT: v_writelane_b32 v36, s53, 9
-; GFX9-NEXT: v_writelane_b32 v36, s54, 10
-; GFX9-NEXT: v_writelane_b32 v36, s55, 11
-; GFX9-NEXT: v_writelane_b32 v36, s64, 12
-; GFX9-NEXT: v_writelane_b32 v36, s65, 13
-; GFX9-NEXT: v_writelane_b32 v36, s66, 14
-; GFX9-NEXT: v_writelane_b32 v36, s67, 15
+; GFX9-NEXT: v_writelane_b32 v35, s36, 0
+; GFX9-NEXT: v_writelane_b32 v35, s37, 1
+; GFX9-NEXT: v_writelane_b32 v35, s38, 2
+; GFX9-NEXT: v_writelane_b32 v35, s39, 3
+; GFX9-NEXT: v_writelane_b32 v35, s48, 4
+; GFX9-NEXT: v_writelane_b32 v35, s49, 5
+; GFX9-NEXT: v_writelane_b32 v35, s50, 6
+; GFX9-NEXT: v_writelane_b32 v35, s51, 7
+; GFX9-NEXT: v_writelane_b32 v35, s52, 8
+; GFX9-NEXT: v_writelane_b32 v35, s53, 9
+; GFX9-NEXT: v_writelane_b32 v35, s54, 10
+; GFX9-NEXT: v_writelane_b32 v35, s55, 11
+; GFX9-NEXT: v_writelane_b32 v35, s64, 12
+; GFX9-NEXT: v_writelane_b32 v35, s65, 13
+; GFX9-NEXT: v_writelane_b32 v35, s66, 14
+; GFX9-NEXT: v_writelane_b32 v35, s67, 15
; GFX9-NEXT: v_readfirstlane_b32 s4, v18
; GFX9-NEXT: s_mov_b32 s49, s29
; GFX9-NEXT: s_mov_b32 s48, s28
@@ -130753,25 +130750,24 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: s_lshl_b32 s4, s50, 16
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_add_f32_e32 v3, s4, v16
-; GFX9-NEXT: v_mov_b32_e32 v32, 0xffff
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_bfe_u32 v4, v3, 16, 1
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_add_u32_e32 v4, v4, v3
; GFX9-NEXT: s_and_b32 s4, s49, 0xffff0000
+; GFX9-NEXT: v_add_u32_e32 v4, v4, v3
+; GFX9-NEXT: v_or_b32_sdwa v15, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v3
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX9-NEXT: v_lshl_or_b32 v15, v0, 16, v1
-; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX9-NEXT: v_and_b32_sdwa v3, v32, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
-; GFX9-NEXT: v_lshl_or_b32 v14, v2, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v14, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_u32_e32 v1, 0x7fff, v1
; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
@@ -130783,11 +130779,11 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s48, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v13, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v13, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -130802,11 +130798,11 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s47, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v12, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v12, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -130821,11 +130817,11 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s46, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v11, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v11, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -130840,11 +130836,11 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s45, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v10, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v10, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -130859,11 +130855,11 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s44, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v9, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v9, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -130878,11 +130874,11 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s43, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v8, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v8, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -130897,11 +130893,11 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s42, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v7, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v7, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -130916,11 +130912,11 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s41, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v6, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v6, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -130935,11 +130931,11 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s40, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v5, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v5, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -130954,11 +130950,11 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s39, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v4, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v4, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -130973,11 +130969,11 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s38, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v3, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v3, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -130992,11 +130988,11 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v17, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v17, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s37, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v2, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v2, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -131011,11 +131007,11 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v17, 0x7fff, v17
; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v17, v18, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v17, v18, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s36, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v1, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v17, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v17, v17, v0
@@ -131030,11 +131026,11 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v18, 0x7fff, v18
; GFX9-NEXT: v_or_b32_e32 v19, 0x400000, v17
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
-; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v19, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v17, v32, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v19, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s67, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v17
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -131049,11 +131045,11 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s66, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v31, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v31, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -131068,11 +131064,11 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s65, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v30, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v30, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -131087,11 +131083,11 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s64, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v29, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v29, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -131106,11 +131102,11 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s63, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v28, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v28, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -131125,11 +131121,11 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s62, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v27, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v27, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -131144,11 +131140,11 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s61, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v26, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v26, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -131163,11 +131159,11 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s60, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v25, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v25, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -131182,11 +131178,11 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s59, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v24, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v24, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -131201,11 +131197,11 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s58, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v23, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v23, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -131220,11 +131216,11 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s57, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v22, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v22, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -131239,11 +131235,11 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s56, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v21, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v21, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -131258,11 +131254,11 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s55, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v20, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v20, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -131275,69 +131271,69 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_bfe_u32 v19, v18, 16, 1
; GFX9-NEXT: v_add_u32_e32 v19, v19, v18
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
-; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v18
+; GFX9-NEXT: v_or_b32_e32 v32, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v33, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v32, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s54, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v19, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v19, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
; GFX9-NEXT: v_add_u32_e32 v18, 0x7fff, v18
-; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v17
+; GFX9-NEXT: v_or_b32_e32 v32, 0x400000, v17
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
; GFX9-NEXT: s_lshl_b32 s4, s54, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v33, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v32, vcc
; GFX9-NEXT: v_add_f32_e32 v18, s4, v16
-; GFX9-NEXT: v_bfe_u32 v33, v18, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v33, v33, v18
-; GFX9-NEXT: v_add_u32_e32 v33, 0x7fff, v33
-; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v18
+; GFX9-NEXT: v_bfe_u32 v32, v18, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v32, v32, v18
+; GFX9-NEXT: v_add_u32_e32 v32, 0x7fff, v32
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v33, v34, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v32, v33, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s53, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v18, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v18, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
-; GFX9-NEXT: v_bfe_u32 v33, v17, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v33, v33, v17
-; GFX9-NEXT: v_add_u32_e32 v33, 0x7fff, v33
-; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v17
+; GFX9-NEXT: v_bfe_u32 v32, v17, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v32, v32, v17
+; GFX9-NEXT: v_add_u32_e32 v32, 0x7fff, v32
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v17
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
; GFX9-NEXT: s_lshl_b32 s4, s53, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v17, v33, v34, vcc
-; GFX9-NEXT: v_add_f32_e32 v33, s4, v16
-; GFX9-NEXT: v_bfe_u32 v34, v33, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v34, v34, v33
-; GFX9-NEXT: v_add_u32_e32 v34, 0x7fff, v34
-; GFX9-NEXT: v_or_b32_e32 v35, 0x400000, v33
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v33, v33
-; GFX9-NEXT: v_cndmask_b32_e32 v33, v34, v35, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v17, v32, v33, vcc
+; GFX9-NEXT: v_add_f32_e32 v32, s4, v16
+; GFX9-NEXT: v_bfe_u32 v33, v32, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v33, v33, v32
+; GFX9-NEXT: v_add_u32_e32 v33, 0x7fff, v33
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v32
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v32, v32
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v33, v32, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v32, v33, v34, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s52, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v17, v17, 16, v33
-; GFX9-NEXT: v_add_f32_e32 v33, s4, v16
-; GFX9-NEXT: v_bfe_u32 v34, v33, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v34, v34, v33
+; GFX9-NEXT: v_or_b32_sdwa v17, v17, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f32_e32 v32, s4, v16
+; GFX9-NEXT: v_bfe_u32 v33, v32, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v33, v33, v32
; GFX9-NEXT: s_lshl_b32 s4, s52, 16
-; GFX9-NEXT: v_add_u32_e32 v34, 0x7fff, v34
-; GFX9-NEXT: v_or_b32_e32 v35, 0x400000, v33
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v33, v33
+; GFX9-NEXT: v_add_u32_e32 v33, 0x7fff, v33
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v32
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v32, v32
; GFX9-NEXT: v_add_f32_e32 v16, s4, v16
-; GFX9-NEXT: v_cndmask_b32_e32 v33, v34, v35, vcc
-; GFX9-NEXT: v_bfe_u32 v34, v16, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v34, v34, v16
-; GFX9-NEXT: v_add_u32_e32 v34, 0x7fff, v34
-; GFX9-NEXT: v_or_b32_e32 v35, 0x400000, v16
+; GFX9-NEXT: v_cndmask_b32_e32 v32, v33, v34, vcc
+; GFX9-NEXT: v_bfe_u32 v33, v16, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v33, v33, v16
+; GFX9-NEXT: v_add_u32_e32 v33, 0x7fff, v33
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v16
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v16, v16
-; GFX9-NEXT: v_cndmask_b32_e32 v16, v34, v35, vcc
-; GFX9-NEXT: v_and_b32_sdwa v16, v32, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshrrev_b32_e32 v32, 16, v33
-; GFX9-NEXT: v_lshl_or_b32 v16, v32, 16, v16
+; GFX9-NEXT: v_lshrrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_cndmask_b32_e32 v16, v33, v34, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_or_b32_sdwa v16, v32, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_branch .LBB79_6
; GFX9-NEXT: .LBB79_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
@@ -131373,24 +131369,24 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
; GFX9-NEXT: .LBB79_6: ; %end
-; GFX9-NEXT: v_readlane_b32 s67, v36, 15
-; GFX9-NEXT: v_readlane_b32 s66, v36, 14
-; GFX9-NEXT: v_readlane_b32 s65, v36, 13
-; GFX9-NEXT: v_readlane_b32 s64, v36, 12
-; GFX9-NEXT: v_readlane_b32 s55, v36, 11
-; GFX9-NEXT: v_readlane_b32 s54, v36, 10
-; GFX9-NEXT: v_readlane_b32 s53, v36, 9
-; GFX9-NEXT: v_readlane_b32 s52, v36, 8
-; GFX9-NEXT: v_readlane_b32 s51, v36, 7
-; GFX9-NEXT: v_readlane_b32 s50, v36, 6
-; GFX9-NEXT: v_readlane_b32 s49, v36, 5
-; GFX9-NEXT: v_readlane_b32 s48, v36, 4
-; GFX9-NEXT: v_readlane_b32 s39, v36, 3
-; GFX9-NEXT: v_readlane_b32 s38, v36, 2
-; GFX9-NEXT: v_readlane_b32 s37, v36, 1
-; GFX9-NEXT: v_readlane_b32 s36, v36, 0
+; GFX9-NEXT: v_readlane_b32 s67, v35, 15
+; GFX9-NEXT: v_readlane_b32 s66, v35, 14
+; GFX9-NEXT: v_readlane_b32 s65, v35, 13
+; GFX9-NEXT: v_readlane_b32 s64, v35, 12
+; GFX9-NEXT: v_readlane_b32 s55, v35, 11
+; GFX9-NEXT: v_readlane_b32 s54, v35, 10
+; GFX9-NEXT: v_readlane_b32 s53, v35, 9
+; GFX9-NEXT: v_readlane_b32 s52, v35, 8
+; GFX9-NEXT: v_readlane_b32 s51, v35, 7
+; GFX9-NEXT: v_readlane_b32 s50, v35, 6
+; GFX9-NEXT: v_readlane_b32 s49, v35, 5
+; GFX9-NEXT: v_readlane_b32 s48, v35, 4
+; GFX9-NEXT: v_readlane_b32 s39, v35, 3
+; GFX9-NEXT: v_readlane_b32 s38, v35, 2
+; GFX9-NEXT: v_readlane_b32 s37, v35, 1
+; GFX9-NEXT: v_readlane_b32 s36, v35, 0
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
-; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v35, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-NEXT: s_mov_b64 exec, s[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -147910,25 +147906,25 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX9-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
+; GFX9-NEXT: v_readfirstlane_b32 s42, v13
; GFX9-NEXT: v_readfirstlane_b32 s46, v12
; GFX9-NEXT: v_readfirstlane_b32 s61, v11
-; GFX9-NEXT: v_readfirstlane_b32 s63, v10
+; GFX9-NEXT: v_readfirstlane_b32 s15, v3
; GFX9-NEXT: v_readfirstlane_b32 s7, v0
-; GFX9-NEXT: buffer_load_ushort v10, off, s[0:3], s32 offset:8
-; GFX9-NEXT: buffer_load_ushort v11, off, s[0:3], s32 offset:4
-; GFX9-NEXT: buffer_load_ushort v12, off, s[0:3], s32
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:328
-; GFX9-NEXT: v_readfirstlane_b32 s43, v2
-; GFX9-NEXT: v_readfirstlane_b32 s6, v1
+; GFX9-NEXT: buffer_load_ushort v11, off, s[0:3], s32 offset:8
+; GFX9-NEXT: buffer_load_ushort v12, off, s[0:3], s32 offset:4
+; GFX9-NEXT: buffer_load_ushort v13, off, s[0:3], s32
+; GFX9-NEXT: buffer_load_ushort v3, off, s[0:3], s32 offset:328
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:324
; GFX9-NEXT: v_readfirstlane_b32 s8, v5
; GFX9-NEXT: v_readfirstlane_b32 s10, v4
-; GFX9-NEXT: v_readfirstlane_b32 s15, v3
+; GFX9-NEXT: v_readfirstlane_b32 s43, v2
+; GFX9-NEXT: v_readfirstlane_b32 s6, v1
+; GFX9-NEXT: v_readfirstlane_b32 s63, v10
; GFX9-NEXT: v_readfirstlane_b32 s12, v9
; GFX9-NEXT: v_readfirstlane_b32 s14, v8
; GFX9-NEXT: v_readfirstlane_b32 s47, v7
-; GFX9-NEXT: v_readfirstlane_b32 s59, v6
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:332
-; GFX9-NEXT: v_readfirstlane_b32 s42, v13
; GFX9-NEXT: v_readfirstlane_b32 s74, v30
; GFX9-NEXT: v_readfirstlane_b32 s44, v29
; GFX9-NEXT: v_readfirstlane_b32 s56, v28
@@ -147946,401 +147942,403 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX9-NEXT: v_readfirstlane_b32 s58, v16
; GFX9-NEXT: v_readfirstlane_b32 s73, v15
; GFX9-NEXT: v_readfirstlane_b32 s75, v14
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:472 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:324
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:320
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:484 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:316
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:480 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v2, off, s[0:3], s32 offset:312
-; GFX9-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:308
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:304
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:496 ; 4-byte Folded Spill
+; GFX9-NEXT: v_readfirstlane_b32 s59, v6
+; GFX9-NEXT: buffer_load_ushort v39, off, s[0:3], s32 offset:296
+; GFX9-NEXT: buffer_load_ushort v50, off, s[0:3], s32 offset:292
+; GFX9-NEXT: buffer_load_ushort v62, off, s[0:3], s32 offset:288
+; GFX9-NEXT: buffer_load_ushort v6, off, s[0:3], s32 offset:284
+; GFX9-NEXT: s_waitcnt vmcnt(5)
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:320
+; GFX9-NEXT: buffer_load_ushort v2, off, s[0:3], s32 offset:316
+; GFX9-NEXT: buffer_load_ushort v4, off, s[0:3], s32 offset:312
+; GFX9-NEXT: buffer_load_ushort v5, off, s[0:3], s32 offset:308
+; GFX9-NEXT: buffer_load_ushort v56, off, s[0:3], s32 offset:304
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:300
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:492 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v4, off, s[0:3], s32 offset:296
-; GFX9-NEXT: buffer_load_ushort v5, off, s[0:3], s32 offset:292
-; GFX9-NEXT: buffer_load_ushort v3, off, s[0:3], s32 offset:288
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:284
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:488 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v8, off, s[0:3], s32 offset:280
-; GFX9-NEXT: buffer_load_ushort v7, off, s[0:3], s32 offset:276
-; GFX9-NEXT: buffer_load_ushort v6, off, s[0:3], s32 offset:272
-; GFX9-NEXT: buffer_load_ushort v63, off, s[0:3], s32 offset:268
-; GFX9-NEXT: buffer_load_ushort v48, off, s[0:3], s32 offset:264
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v63, off, s[0:3], s32 offset:280
+; GFX9-NEXT: buffer_load_ushort v38, off, s[0:3], s32 offset:276
+; GFX9-NEXT: buffer_load_ushort v52, off, s[0:3], s32 offset:272
+; GFX9-NEXT: buffer_load_ushort v7, off, s[0:3], s32 offset:268
+; GFX9-NEXT: buffer_load_ushort v8, off, s[0:3], s32 offset:264
; GFX9-NEXT: buffer_load_ushort v54, off, s[0:3], s32 offset:260
-; GFX9-NEXT: buffer_load_ushort v61, off, s[0:3], s32 offset:256
-; GFX9-NEXT: buffer_load_ushort v33, off, s[0:3], s32 offset:252
-; GFX9-NEXT: buffer_load_ushort v9, off, s[0:3], s32 offset:248
-; GFX9-NEXT: buffer_load_ushort v62, off, s[0:3], s32 offset:244
+; GFX9-NEXT: buffer_load_ushort v9, off, s[0:3], s32 offset:256
+; GFX9-NEXT: buffer_load_ushort v37, off, s[0:3], s32 offset:252
+; GFX9-NEXT: buffer_load_ushort v10, off, s[0:3], s32 offset:248
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:244
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:428 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:240
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:464 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v43, off, s[0:3], s32 offset:236
-; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:452 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:236
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:432 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:232
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:500 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:464 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:228
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:436 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:400 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:224
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:436 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:220
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:468 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:440 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:216
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:504 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:472 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:212
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:468 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:208
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:440 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:204
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:400 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:408 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v45, off, s[0:3], s32 offset:204
+; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:200
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:512 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:488 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:196
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:484 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:192
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v51, off, s[0:3], s32 offset:188
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:184
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:492 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:180
; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_ushort v61, off, s[0:3], s32 offset:140
+; GFX9-NEXT: buffer_load_ushort v36, off, s[0:3], s32 offset:136
+; GFX9-NEXT: buffer_load_ushort v60, off, s[0:3], s32 offset:132
+; GFX9-NEXT: buffer_load_ushort v43, off, s[0:3], s32 offset:128
+; GFX9-NEXT: buffer_load_ushort v58, off, s[0:3], s32 offset:124
+; GFX9-NEXT: buffer_load_ushort v34, off, s[0:3], s32 offset:120
+; GFX9-NEXT: buffer_load_ushort v55, off, s[0:3], s32 offset:116
+; GFX9-NEXT: buffer_load_ushort v42, off, s[0:3], s32 offset:112
+; GFX9-NEXT: buffer_load_ushort v47, off, s[0:3], s32 offset:108
+; GFX9-NEXT: buffer_load_ushort v28, off, s[0:3], s32 offset:104
+; GFX9-NEXT: buffer_load_ushort v35, off, s[0:3], s32 offset:100
+; GFX9-NEXT: buffer_load_ushort v48, off, s[0:3], s32 offset:96
+; GFX9-NEXT: buffer_load_ushort v44, off, s[0:3], s32 offset:92
+; GFX9-NEXT: buffer_load_ushort v24, off, s[0:3], s32 offset:88
+; GFX9-NEXT: buffer_load_ushort v29, off, s[0:3], s32 offset:84
+; GFX9-NEXT: buffer_load_ushort v53, off, s[0:3], s32 offset:80
+; GFX9-NEXT: buffer_load_ushort v49, off, s[0:3], s32 offset:76
+; GFX9-NEXT: buffer_load_ushort v20, off, s[0:3], s32 offset:72
+; GFX9-NEXT: buffer_load_ushort v25, off, s[0:3], s32 offset:68
+; GFX9-NEXT: buffer_load_ushort v30, off, s[0:3], s32 offset:64
+; GFX9-NEXT: buffer_load_ushort v33, off, s[0:3], s32 offset:60
+; GFX9-NEXT: buffer_load_ushort v17, off, s[0:3], s32 offset:56
+; GFX9-NEXT: buffer_load_ushort v21, off, s[0:3], s32 offset:52
+; GFX9-NEXT: buffer_load_ushort v26, off, s[0:3], s32 offset:48
+; GFX9-NEXT: buffer_load_ushort v31, off, s[0:3], s32 offset:44
+; GFX9-NEXT: buffer_load_ushort v15, off, s[0:3], s32 offset:40
+; GFX9-NEXT: buffer_load_ushort v18, off, s[0:3], s32 offset:36
+; GFX9-NEXT: buffer_load_ushort v22, off, s[0:3], s32 offset:32
+; GFX9-NEXT: buffer_load_ushort v27, off, s[0:3], s32 offset:28
+; GFX9-NEXT: buffer_load_ushort v14, off, s[0:3], s32 offset:24
+; GFX9-NEXT: buffer_load_ushort v16, off, s[0:3], s32 offset:20
+; GFX9-NEXT: buffer_load_ushort v19, off, s[0:3], s32 offset:16
+; GFX9-NEXT: buffer_load_ushort v23, off, s[0:3], s32 offset:12
; GFX9-NEXT: s_and_b64 s[4:5], vcc, exec
+; GFX9-NEXT: s_waitcnt vmcnt(33)
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:496 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v32, off, s[0:3], s32 offset:176
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:172
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:508 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:192
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:480 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v46, off, s[0:3], s32 offset:168
+; GFX9-NEXT: buffer_load_ushort v57, off, s[0:3], s32 offset:164
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_ushort v13, off, s[0:3], s32 offset:188
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:444 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v47, off, s[0:3], s32 offset:184
-; GFX9-NEXT: buffer_load_ushort v59, off, s[0:3], s32 offset:180
-; GFX9-NEXT: buffer_load_ushort v42, off, s[0:3], s32 offset:176
-; GFX9-NEXT: buffer_load_ushort v58, off, s[0:3], s32 offset:172
-; GFX9-NEXT: buffer_load_ushort v44, off, s[0:3], s32 offset:168
-; GFX9-NEXT: buffer_load_ushort v56, off, s[0:3], s32 offset:164
-; GFX9-NEXT: buffer_load_ushort v60, off, s[0:3], s32 offset:160
-; GFX9-NEXT: buffer_load_ushort v53, off, s[0:3], s32 offset:156
-; GFX9-NEXT: buffer_load_ushort v49, off, s[0:3], s32 offset:152
-; GFX9-NEXT: buffer_load_ushort v45, off, s[0:3], s32 offset:148
-; GFX9-NEXT: buffer_load_ushort v57, off, s[0:3], s32 offset:144
-; GFX9-NEXT: buffer_load_ushort v55, off, s[0:3], s32 offset:140
-; GFX9-NEXT: buffer_load_ushort v38, off, s[0:3], s32 offset:136
-; GFX9-NEXT: buffer_load_ushort v32, off, s[0:3], s32 offset:132
-; GFX9-NEXT: buffer_load_ushort v46, off, s[0:3], s32 offset:128
-; GFX9-NEXT: buffer_load_ushort v40, off, s[0:3], s32 offset:124
-; GFX9-NEXT: buffer_load_ushort v31, off, s[0:3], s32 offset:120
-; GFX9-NEXT: buffer_load_ushort v52, off, s[0:3], s32 offset:116
-; GFX9-NEXT: buffer_load_ushort v50, off, s[0:3], s32 offset:112
-; GFX9-NEXT: buffer_load_ushort v41, off, s[0:3], s32 offset:108
-; GFX9-NEXT: buffer_load_ushort v27, off, s[0:3], s32 offset:104
-; GFX9-NEXT: buffer_load_ushort v37, off, s[0:3], s32 offset:100
-; GFX9-NEXT: buffer_load_ushort v36, off, s[0:3], s32 offset:96
-; GFX9-NEXT: buffer_load_ushort v51, off, s[0:3], s32 offset:92
-; GFX9-NEXT: buffer_load_ushort v23, off, s[0:3], s32 offset:88
-; GFX9-NEXT: buffer_load_ushort v28, off, s[0:3], s32 offset:84
-; GFX9-NEXT: buffer_load_ushort v34, off, s[0:3], s32 offset:80
-; GFX9-NEXT: buffer_load_ushort v39, off, s[0:3], s32 offset:76
-; GFX9-NEXT: buffer_load_ushort v19, off, s[0:3], s32 offset:72
-; GFX9-NEXT: buffer_load_ushort v24, off, s[0:3], s32 offset:68
-; GFX9-NEXT: buffer_load_ushort v29, off, s[0:3], s32 offset:64
-; GFX9-NEXT: buffer_load_ushort v35, off, s[0:3], s32 offset:60
-; GFX9-NEXT: buffer_load_ushort v16, off, s[0:3], s32 offset:56
-; GFX9-NEXT: buffer_load_ushort v20, off, s[0:3], s32 offset:52
-; GFX9-NEXT: buffer_load_ushort v25, off, s[0:3], s32 offset:48
-; GFX9-NEXT: buffer_load_ushort v30, off, s[0:3], s32 offset:44
-; GFX9-NEXT: buffer_load_ushort v14, off, s[0:3], s32 offset:40
-; GFX9-NEXT: buffer_load_ushort v17, off, s[0:3], s32 offset:36
-; GFX9-NEXT: buffer_load_ushort v21, off, s[0:3], s32 offset:32
-; GFX9-NEXT: buffer_load_ushort v26, off, s[0:3], s32 offset:28
-; GFX9-NEXT: buffer_load_ushort v13, off, s[0:3], s32 offset:24
-; GFX9-NEXT: buffer_load_ushort v15, off, s[0:3], s32 offset:20
-; GFX9-NEXT: buffer_load_ushort v18, off, s[0:3], s32 offset:16
-; GFX9-NEXT: buffer_load_ushort v22, off, s[0:3], s32 offset:12
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:516 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:520 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(6)
-; GFX9-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:528 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:532 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(7)
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(9)
-; GFX9-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(12)
-; GFX9-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:160
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:500 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v40, off, s[0:3], s32 offset:156
+; GFX9-NEXT: buffer_load_ushort v59, off, s[0:3], s32 offset:152
+; GFX9-NEXT: buffer_load_ushort v41, off, s[0:3], s32 offset:148
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:144
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:512 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:516 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:520 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:528 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:532 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:408 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v49, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
+; GFX9-NEXT: s_waitcnt vmcnt(27)
+; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:412 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:420 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:424 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:428 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:432 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:420 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:424 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:628 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:636 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v52, off, s[0:3], s32 offset:640 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:644 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v38, off, s[0:3], s32 offset:648 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:444 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:448 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v48, off, s[0:3], s32 offset:628 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:452 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:636 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:640 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:644 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:648 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:656 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:660 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v48, off, s[0:3], s32 offset:664 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:668 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:672 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:508 ; 4-byte Folded Spill
+; GFX9-NEXT: s_waitcnt vmcnt(49)
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:504 ; 4-byte Folded Spill
; GFX9-NEXT: s_cbranch_scc0 .LBB89_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v8, 0xc0c0004
-; GFX9-NEXT: v_mov_b32_e32 v2, s21
-; GFX9-NEXT: v_perm_b32 v0, s16, v0, v8
+; GFX9-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:676 ; 4-byte Folded Spill
; GFX9-NEXT: v_mov_b32_e32 v1, s19
-; GFX9-NEXT: v_perm_b32 v2, s20, v2, v8
; GFX9-NEXT: v_mov_b32_e32 v3, s23
+; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_perm_b32 v1, s18, v1, v8
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:680 ; 4-byte Folded Spill
+; GFX9-NEXT: v_mov_b32_e32 v2, s21
; GFX9-NEXT: v_perm_b32 v3, s22, v3, v8
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v3, 16, v2
-; GFX9-NEXT: v_mov_b32_e32 v2, s25
-; GFX9-NEXT: v_perm_b32 v2, s24, v2, v8
+; GFX9-NEXT: v_perm_b32 v0, s16, v0, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_perm_b32 v2, s20, v2, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v3, s27
+; GFX9-NEXT: v_mov_b32_e32 v2, s25
; GFX9-NEXT: v_perm_b32 v3, s26, v3, v8
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_lshl_or_b32 v2, v3, 16, v2
-; GFX9-NEXT: v_mov_b32_e32 v3, s29
-; GFX9-NEXT: v_perm_b32 v3, s28, v3, v8
+; GFX9-NEXT: v_perm_b32 v2, s24, v2, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_mov_b32_e32 v4, s6
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v3, s29
; GFX9-NEXT: v_perm_b32 v4, s7, v4, v8
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_lshl_or_b32 v3, v4, 16, v3
-; GFX9-NEXT: v_mov_b32_e32 v4, s15
-; GFX9-NEXT: v_perm_b32 v4, s43, v4, v8
+; GFX9-NEXT: v_perm_b32 v3, s28, v3, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX9-NEXT: v_mov_b32_e32 v5, s8
+; GFX9-NEXT: v_or_b32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v4, s15
; GFX9-NEXT: v_perm_b32 v5, s10, v5, v8
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_lshl_or_b32 v4, v5, 16, v4
-; GFX9-NEXT: v_mov_b32_e32 v5, s47
-; GFX9-NEXT: v_perm_b32 v5, s59, v5, v8
+; GFX9-NEXT: v_perm_b32 v4, s43, v4, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX9-NEXT: v_mov_b32_e32 v6, s12
+; GFX9-NEXT: v_or_b32_sdwa v4, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v5, s47
; GFX9-NEXT: v_perm_b32 v6, s14, v6, v8
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_lshl_or_b32 v5, v6, 16, v5
-; GFX9-NEXT: v_mov_b32_e32 v6, s61
-; GFX9-NEXT: v_perm_b32 v6, s63, v6, v8
+; GFX9-NEXT: v_perm_b32 v5, s59, v5, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX9-NEXT: v_mov_b32_e32 v50, v54
+; GFX9-NEXT: v_mov_b32_e32 v54, v7
; GFX9-NEXT: v_mov_b32_e32 v7, s42
+; GFX9-NEXT: v_or_b32_sdwa v5, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v6, s61
; GFX9-NEXT: v_perm_b32 v7, s46, v7, v8
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_lshl_or_b32 v6, v7, 16, v6
-; GFX9-NEXT: v_mov_b32_e32 v7, s73
-; GFX9-NEXT: v_perm_b32 v7, s75, v7, v8
-; GFX9-NEXT: v_mov_b32_e32 v63, v33
-; GFX9-NEXT: v_mov_b32_e32 v33, v9
+; GFX9-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:684 ; 4-byte Folded Spill
+; GFX9-NEXT: v_perm_b32 v6, s63, v6, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX9-NEXT: v_mov_b32_e32 v39, v9
; GFX9-NEXT: v_mov_b32_e32 v9, s45
+; GFX9-NEXT: v_or_b32_sdwa v6, v7, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v7, s73
; GFX9-NEXT: v_perm_b32 v9, s58, v9, v8
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_lshl_or_b32 v7, v9, 16, v7
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v10
-; GFX9-NEXT: v_perm_b32 v10, s74, v12, v8
-; GFX9-NEXT: v_or_b32_sdwa v9, v11, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v9, 16, v10
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v18
-; GFX9-NEXT: v_or_b32_sdwa v9, v22, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v13
-; GFX9-NEXT: v_or_b32_sdwa v10, v15, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
+; GFX9-NEXT: v_perm_b32 v7, s75, v7, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX9-NEXT: s_and_b32 s4, s40, 0xff
+; GFX9-NEXT: s_lshl_b32 s5, s13, 8
+; GFX9-NEXT: v_or_b32_sdwa v7, v9, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v9, s57
+; GFX9-NEXT: s_or_b32 s4, s4, s5
+; GFX9-NEXT: v_mov_b32_e32 v56, v38
+; GFX9-NEXT: v_perm_b32 v9, s60, v9, v8
+; GFX9-NEXT: v_mov_b32_e32 v38, v37
+; GFX9-NEXT: v_mov_b32_e32 v37, v10
+; GFX9-NEXT: v_lshlrev_b32_e64 v10, 16, s4
+; GFX9-NEXT: v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v11
+; GFX9-NEXT: v_perm_b32 v8, s74, v13, v8
+; GFX9-NEXT: v_or_b32_sdwa v10, v12, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v11, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v19
+; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v14
+; GFX9-NEXT: v_or_b32_sdwa v8, v23, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v10, v16, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v12, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_mov_b32 s4, 0xc0c0004
-; GFX9-NEXT: v_lshl_or_b32 v12, v10, 16, v9
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v14
-; GFX9-NEXT: v_perm_b32 v10, v26, v21, s4
-; GFX9-NEXT: v_or_b32_sdwa v9, v17, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v13, v9, 16, v10
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v25
-; GFX9-NEXT: v_or_b32_sdwa v9, v30, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v16
-; GFX9-NEXT: v_or_b32_sdwa v10, v20, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_lshl_or_b32 v14, v10, 16, v9
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v19
-; GFX9-NEXT: v_perm_b32 v10, v35, v29, s4
-; GFX9-NEXT: v_or_b32_sdwa v9, v24, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v15, v9, 16, v10
-; GFX9-NEXT: v_perm_b32 v9, v39, v34, s4
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_perm_b32 v10, v51, v36, s4
-; GFX9-NEXT: v_perm_b32 v16, v28, v23, s4
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_perm_b32 v17, v37, v27, s4
-; GFX9-NEXT: v_lshl_or_b32 v16, v16, 16, v9
-; GFX9-NEXT: v_perm_b32 v9, v41, v50, s4
-; GFX9-NEXT: v_lshl_or_b32 v17, v17, 16, v10
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_perm_b32 v10, v52, v31, s4
-; GFX9-NEXT: v_lshl_or_b32 v18, v10, 16, v9
-; GFX9-NEXT: v_perm_b32 v9, v40, v46, s4
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_perm_b32 v10, v32, v38, s4
-; GFX9-NEXT: v_lshl_or_b32 v19, v10, 16, v9
-; GFX9-NEXT: v_perm_b32 v9, v55, v57, s4
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_perm_b32 v10, v45, v49, s4
-; GFX9-NEXT: v_lshl_or_b32 v20, v10, 16, v9
-; GFX9-NEXT: v_perm_b32 v9, v53, v60, s4
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_perm_b32 v10, v56, v44, s4
-; GFX9-NEXT: v_lshl_or_b32 v21, v10, 16, v9
-; GFX9-NEXT: v_perm_b32 v9, v58, v42, s4
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_perm_b32 v10, v59, v47, s4
-; GFX9-NEXT: v_lshl_or_b32 v22, v10, 16, v9
-; GFX9-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
-; GFX9-NEXT: v_mov_b32_e32 v50, v32
-; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
-; GFX9-NEXT: v_mov_b32_e32 v51, v49
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v15
+; GFX9-NEXT: v_or_b32_sdwa v8, v18, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_perm_b32 v10, v27, v22, s4
+; GFX9-NEXT: v_perm_b32 v16, v49, v53, s4
; GFX9-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:404 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
-; GFX9-NEXT: v_mov_b32_e32 v41, v46
-; GFX9-NEXT: v_mov_b32_e32 v39, v52
-; GFX9-NEXT: v_mov_b32_e32 v52, v38
-; GFX9-NEXT: v_mov_b32_e32 v34, v37
-; GFX9-NEXT: v_mov_b32_e32 v37, v31
-; GFX9-NEXT: s_lshl_b32 s5, s13, 8
+; GFX9-NEXT: v_or_b32_sdwa v13, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v17
+; GFX9-NEXT: v_perm_b32 v17, v44, v48, s4
+; GFX9-NEXT: v_mov_b32_e32 v44, v55
+; GFX9-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v26
+; GFX9-NEXT: v_or_b32_sdwa v8, v31, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v10, v21, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v14, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v20
+; GFX9-NEXT: v_or_b32_sdwa v8, v25, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_perm_b32 v10, v33, v30, s4
+; GFX9-NEXT: v_or_b32_sdwa v15, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_perm_b32 v8, v29, v24, s4
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX9-NEXT: v_perm_b32 v10, v35, v28, s4
+; GFX9-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX9-NEXT: v_or_b32_sdwa v16, v8, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_perm_b32 v8, v44, v34, s4
+; GFX9-NEXT: v_or_b32_sdwa v17, v10, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX9-NEXT: v_perm_b32 v10, v47, v42, s4
+; GFX9-NEXT: v_mov_b32_e32 v34, v36
+; GFX9-NEXT: v_mov_b32_e32 v35, v60
+; GFX9-NEXT: v_or_b32_sdwa v18, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_perm_b32 v8, v35, v34, s4
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX9-NEXT: v_perm_b32 v10, v58, v43, s4
+; GFX9-NEXT: v_mov_b32_e32 v53, v59
+; GFX9-NEXT: v_or_b32_sdwa v19, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_perm_b32 v8, v41, v53, s4
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX9-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:484 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
+; GFX9-NEXT: s_lshl_b32 s5, s41, 8
; GFX9-NEXT: s_lshl_b32 s77, s9, 8
; GFX9-NEXT: s_lshl_b32 s78, s44, 8
-; GFX9-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:452 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:428 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(9)
-; GFX9-NEXT: v_perm_b32 v9, v9, v10, s4
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: s_waitcnt vmcnt(7)
-; GFX9-NEXT: v_perm_b32 v10, v32, v58, s4
-; GFX9-NEXT: v_lshl_or_b32 v23, v10, 16, v9
-; GFX9-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:440 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:400 ; 4-byte Folded Reload
+; GFX9-NEXT: v_perm_b32 v10, v61, v49, s4
+; GFX9-NEXT: v_or_b32_sdwa v20, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_perm_b32 v8, v57, v46, s4
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX9-NEXT: s_waitcnt vmcnt(8)
+; GFX9-NEXT: v_perm_b32 v10, v40, v55, s4
+; GFX9-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
+; GFX9-NEXT: v_or_b32_sdwa v21, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: s_waitcnt vmcnt(9)
+; GFX9-NEXT: v_perm_b32 v10, v33, v32, s4
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_perm_b32 v8, v60, v40, s4
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX9-NEXT: v_or_b32_sdwa v22, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_perm_b32 v8, v42, v59, s4
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX9-NEXT: v_perm_b32 v10, v51, v36, s4
+; GFX9-NEXT: v_or_b32_sdwa v23, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v8
+; GFX9-NEXT: v_or_b32_sdwa v8, v45, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v9
+; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v51
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v49
-; GFX9-NEXT: v_or_b32_sdwa v10, v24, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_lshl_or_b32 v24, v10, 16, v9
-; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:436 ; 4-byte Folded Reload
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v35
+; GFX9-NEXT: v_or_b32_sdwa v10, v45, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v24, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:400 ; 4-byte Folded Reload
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v32
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
+; GFX9-NEXT: v_or_b32_sdwa v8, v10, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:436 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:440 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v10, v25, v10, s4
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v25, v9, 16, v10
-; GFX9-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
-; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v33
-; GFX9-NEXT: v_or_b32_sdwa v10, v62, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
-; GFX9-NEXT: v_mov_b32_e32 v33, v46
+; GFX9-NEXT: v_or_b32_sdwa v25, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:452 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:432 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v9
-; GFX9-NEXT: v_or_b32_sdwa v9, v43, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_lshl_or_b32 v26, v10, 16, v9
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v48
-; GFX9-NEXT: v_perm_b32 v10, v63, v61, s4
-; GFX9-NEXT: v_or_b32_sdwa v9, v54, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v27, v9, 16, v10
-; GFX9-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
-; GFX9-NEXT: v_mov_b32_e32 v54, v61
-; GFX9-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:484 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(4)
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v9
-; GFX9-NEXT: s_waitcnt vmcnt(3)
-; GFX9-NEXT: v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_mov_b32_e32 v48, v63
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v10
-; GFX9-NEXT: v_or_b32_sdwa v10, v28, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_lshl_or_b32 v28, v10, 16, v9
-; GFX9-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:428 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v8
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_or_b32_sdwa v8, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v37
+; GFX9-NEXT: v_or_b32_sdwa v10, v26, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v26, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:424 ; 4-byte Folded Reload
+; GFX9-NEXT: v_perm_b32 v10, v38, v39, s4
+; GFX9-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
+; GFX9-NEXT: v_mov_b32_e32 v38, v34
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v8
+; GFX9-NEXT: v_or_b32_sdwa v8, v50, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v27, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v52
+; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v63
+; GFX9-NEXT: v_or_b32_sdwa v8, v54, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v10, v56, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v28, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
+; GFX9-NEXT: v_mov_b32_e32 v54, v31
+; GFX9-NEXT: v_mov_b32_e32 v50, v53
+; GFX9-NEXT: v_mov_b32_e32 v56, v40
+; GFX9-NEXT: v_mov_b32_e32 v52, v32
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v9
+; GFX9-NEXT: v_mov_b32_e32 v63, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v8
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:432 ; 4-byte Folded Reload
+; GFX9-NEXT: v_or_b32_sdwa v8, v10, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v10, v36, v10, s4
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v29, v9, 16, v10
-; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:424 ; 4-byte Folded Reload
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v43
-; GFX9-NEXT: v_or_b32_sdwa v9, v62, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
+; GFX9-NEXT: v_perm_b32 v10, v10, v62, s4
+; GFX9-NEXT: v_or_b32_sdwa v29, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:404 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v8
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v10
-; GFX9-NEXT: v_or_b32_sdwa v10, v30, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_lshl_or_b32 v30, v10, 16, v9
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v38
-; GFX9-NEXT: v_perm_b32 v10, v63, v61, s4
-; GFX9-NEXT: v_or_b32_sdwa v9, v46, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v31, v9, 16, v10
-; GFX9-NEXT: v_mov_b32_e32 v9, s57
-; GFX9-NEXT: v_perm_b32 v8, s60, v9, v8
-; GFX9-NEXT: s_and_b32 s4, s40, 0xff
-; GFX9-NEXT: s_or_b32 s4, s4, s5
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, s4, 16, v8
+; GFX9-NEXT: v_or_b32_sdwa v8, v39, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v10, v30, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v30, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v8
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v62, v10
+; GFX9-NEXT: v_perm_b32 v10, v31, v10, s4
; GFX9-NEXT: s_and_b32 s4, s76, 0xff
-; GFX9-NEXT: s_lshl_b32 s5, s41, 8
; GFX9-NEXT: s_or_b32 s4, s4, s5
; GFX9-NEXT: s_and_b32 s5, s11, 0xff
; GFX9-NEXT: s_or_b32 s5, s5, s77
@@ -148350,60 +148348,54 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX9-NEXT: s_or_b32 s5, s5, s77
; GFX9-NEXT: s_and_b32 s77, s56, 0xff
; GFX9-NEXT: s_or_b32 s77, s77, s78
+; GFX9-NEXT: v_or_b32_sdwa v8, v48, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: s_pack_ll_b32_b16 s5, s5, s77
+; GFX9-NEXT: v_or_b32_sdwa v31, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v8, s4
; GFX9-NEXT: v_mov_b32_e32 v10, s5
; GFX9-NEXT: s_mov_b64 s[4:5], 0
-; GFX9-NEXT: v_mov_b32_e32 v63, v36
; GFX9-NEXT: s_branch .LBB89_3
; GFX9-NEXT: .LBB89_2:
-; GFX9-NEXT: v_mov_b32_e32 v54, v61
-; GFX9-NEXT: v_mov_b32_e32 v50, v32
-; GFX9-NEXT: v_mov_b32_e32 v51, v49
-; GFX9-NEXT: v_mov_b32_e32 v39, v52
-; GFX9-NEXT: v_mov_b32_e32 v52, v38
-; GFX9-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:484 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
+; GFX9-NEXT: v_mov_b32_e32 v63, v39
+; GFX9-NEXT: v_mov_b32_e32 v38, v36
+; GFX9-NEXT: v_mov_b32_e32 v44, v55
+; GFX9-NEXT: v_mov_b32_e32 v50, v59
+; GFX9-NEXT: v_mov_b32_e32 v35, v60
+; GFX9-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
-; GFX9-NEXT: v_mov_b32_e32 v41, v46
-; GFX9-NEXT: v_mov_b32_e32 v34, v37
-; GFX9-NEXT: v_mov_b32_e32 v37, v31
+; GFX9-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:484 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
+; GFX9-NEXT: v_mov_b32_e32 v54, v2
+; GFX9-NEXT: v_mov_b32_e32 v62, v1
; GFX9-NEXT: s_mov_b64 s[4:5], -1
; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
; GFX9-NEXT: ; implicit-def: $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
; GFX9-NEXT: .LBB89_3: ; %Flow
-; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:400 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
; GFX9-NEXT: s_cselect_b32 s4, 1, 0
; GFX9-NEXT: s_cmp_lg_u32 s4, 1
; GFX9-NEXT: s_cbranch_scc1 .LBB89_5
; GFX9-NEXT: ; %bb.4: ; %cmp.true
-; GFX9-NEXT: s_waitcnt vmcnt(5)
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v48
-; GFX9-NEXT: s_mov_b32 s4, 0xc0c0004
-; GFX9-NEXT: v_perm_b32 v0, v0, v61, s4
-; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:472 ; 4-byte Folded Spill
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v33
-; GFX9-NEXT: s_waitcnt vmcnt(4)
-; GFX9-NEXT: v_perm_b32 v0, v0, v38, s4
-; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v62
-; GFX9-NEXT: v_perm_b32 v0, v0, v43, s4
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v54
+; GFX9-NEXT: s_mov_b32 s5, 0xc0c0004
+; GFX9-NEXT: v_perm_b32 v0, v0, v62, s5
; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:480 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
+; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v48
+; GFX9-NEXT: s_movk_i32 s4, 0x300
; GFX9-NEXT: s_add_i32 s74, s74, 3
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:424 ; 4-byte Folded Reload
; GFX9-NEXT: s_add_i32 s72, s72, 3
; GFX9-NEXT: s_add_i32 s56, s56, 3
; GFX9-NEXT: s_add_i32 s60, s60, 3
@@ -148424,373 +148416,358 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX9-NEXT: s_add_i32 s26, s26, 3
; GFX9-NEXT: s_add_i32 s20, s20, 3
; GFX9-NEXT: s_add_i32 s22, s22, 3
+; GFX9-NEXT: v_mov_b32_e32 v32, s23
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: v_mov_b32_e32 v33, s17
; GFX9-NEXT: s_add_i32 s18, s18, 3
-; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
+; GFX9-NEXT: v_mov_b32_e32 v34, s19
+; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:432 ; 4-byte Folded Reload
-; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:408 ; 4-byte Folded Spill
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v63
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:404 ; 4-byte Folded Reload
+; GFX9-NEXT: v_add_u32_sdwa v0, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:444 ; 4-byte Folded Spill
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v39
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:424 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:428 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_sdwa v0, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:636 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:428 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:412 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT: v_perm_b32 v0, v0, v63, s5
+; GFX9-NEXT: v_add_u32_sdwa v0, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:448 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: v_add_u32_e32 v38, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:452 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_e32 v39, 0x300, v0
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: v_add_u32_e32 v62, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_sdwa v63, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:628 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT: v_perm_b32 v0, v0, v54, s4
-; GFX9-NEXT: v_add_u32_e32 v63, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:424 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_sdwa v48, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:432 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:452 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:412 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:632 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:420 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:428 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: v_add_u32_e32 v31, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_sdwa v31, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:440 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:436 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v30, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:436 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:440 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:400 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT: v_perm_b32 v0, v0, v35, s4
-; GFX9-NEXT: v_add_u32_e32 v29, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v36
+; GFX9-NEXT: v_perm_b32 v0, v0, v52, s5
+; GFX9-NEXT: v_add_u32_sdwa v29, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:632 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v28, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:404 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v45
+; GFX9-NEXT: v_perm_b32 v0, v0, v51, s5
+; GFX9-NEXT: v_add_u32_sdwa v27, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT: v_perm_b32 v0, v0, v49, s4
-; GFX9-NEXT: v_add_u32_e32 v27, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v36, s5
; GFX9-NEXT: v_add_u32_e32 v26, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v32
-; GFX9-NEXT: v_perm_b32 v0, v0, v58, s4
-; GFX9-NEXT: v_add_u32_e32 v25, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
-; GFX9-NEXT: v_mov_b32_e32 v32, s23
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v42
+; GFX9-NEXT: v_perm_b32 v0, v0, v59, s5
+; GFX9-NEXT: v_add_u32_sdwa v25, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v33
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_e32 v24, 0x300, v0
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v60
+; GFX9-NEXT: v_perm_b32 v0, v0, v56, s5
+; GFX9-NEXT: v_add_u32_sdwa v23, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
+; GFX9-NEXT: v_mov_b32_e32 v33, s17
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT: v_perm_b32 v0, v0, v42, s4
-; GFX9-NEXT: v_add_u32_e32 v24, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v59
-; GFX9-NEXT: v_perm_b32 v0, v0, v47, s4
-; GFX9-NEXT: v_add_u32_e32 v23, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v53
-; GFX9-NEXT: v_perm_b32 v0, v0, v60, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v55, s5
; GFX9-NEXT: v_add_u32_e32 v22, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v56
-; GFX9-NEXT: v_perm_b32 v0, v0, v44, s4
-; GFX9-NEXT: v_add_u32_e32 v21, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v55
-; GFX9-NEXT: v_perm_b32 v0, v0, v57, s4
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v57
+; GFX9-NEXT: v_perm_b32 v0, v0, v46, s5
+; GFX9-NEXT: v_add_u32_sdwa v21, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v61
+; GFX9-NEXT: v_perm_b32 v0, v0, v49, s5
; GFX9-NEXT: v_add_u32_e32 v20, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v45
-; GFX9-NEXT: v_perm_b32 v0, v0, v51, s4
-; GFX9-NEXT: v_add_u32_e32 v49, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v40
-; GFX9-NEXT: v_perm_b32 v0, v0, v41, s4
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v41
+; GFX9-NEXT: v_perm_b32 v0, v0, v50, s5
+; GFX9-NEXT: v_add_u32_sdwa v51, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v58
+; GFX9-NEXT: v_perm_b32 v0, v0, v43, s5
; GFX9-NEXT: v_add_u32_e32 v19, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v50
-; GFX9-NEXT: v_perm_b32 v0, v0, v52, s4
-; GFX9-NEXT: v_add_u32_e32 v52, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v22, 0xffff, v22
-; GFX9-NEXT: v_lshl_or_b32 v21, v21, 16, v22
-; GFX9-NEXT: v_and_b32_e32 v22, 0xffff, v24
-; GFX9-NEXT: v_lshl_or_b32 v22, v23, 16, v22
-; GFX9-NEXT: v_and_b32_e32 v23, 0xffff, v26
-; GFX9-NEXT: v_lshl_or_b32 v23, v25, 16, v23
-; GFX9-NEXT: v_and_b32_e32 v25, 0xffff, v30
-; GFX9-NEXT: v_lshl_or_b32 v25, v29, 16, v25
-; GFX9-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:424 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v24, 0xffff, v28
-; GFX9-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:428 ; 4-byte Folded Reload
-; GFX9-NEXT: v_lshl_or_b32 v24, v27, 16, v24
-; GFX9-NEXT: v_and_b32_e32 v27, 0xffff, v63
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_lshl_or_b32 v19, v52, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v49, 16, v20
-; GFX9-NEXT: s_waitcnt vmcnt(4)
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v35
+; GFX9-NEXT: v_perm_b32 v0, v0, v38, s5
+; GFX9-NEXT: v_add_u32_sdwa v38, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v47
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
; GFX9-NEXT: v_add_u32_e32 v18, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v39
-; GFX9-NEXT: v_perm_b32 v0, v0, v37, s4
-; GFX9-NEXT: v_add_u32_e32 v37, 0x300, v0
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v44
+; GFX9-NEXT: v_or_b32_sdwa v21, v21, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v22, v23, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v23, v25, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v25, v29, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
+; GFX9-NEXT: v_or_b32_sdwa v24, v27, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v38, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v20, v51, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v28, v63, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(4)
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_sdwa v53, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_lshl_or_b32 v18, v37, 16, v18
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
+; GFX9-NEXT: v_or_b32_sdwa v18, v53, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: s_waitcnt vmcnt(5)
+; GFX9-NEXT: v_or_b32_sdwa v26, v31, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: v_or_b32_sdwa v29, v30, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:404 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(4)
+; GFX9-NEXT: v_or_b32_sdwa v27, v48, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: s_waitcnt vmcnt(3)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
-; GFX9-NEXT: v_add_u32_e32 v54, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v34
-; GFX9-NEXT: v_mov_b32_e32 v34, s19
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v17, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: v_or_b32_sdwa v30, v31, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: v_add_u32_e32 v16, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:636 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_sdwa v54, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
+; GFX9-NEXT: v_or_b32_sdwa v17, v54, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT: v_perm_b32 v0, v0, v37, s5
+; GFX9-NEXT: v_add_u32_e32 v16, 0x300, v0
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: v_add_u32_e32 v55, 0x300, v0
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_sdwa v55, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
-; GFX9-NEXT: v_lshl_or_b32 v16, v55, 16, v16
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
+; GFX9-NEXT: v_or_b32_sdwa v16, v55, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v15, 0x300, v0
; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: v_add_u32_e32 v40, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_sdwa v40, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
-; GFX9-NEXT: v_lshl_or_b32 v15, v40, 16, v15
+; GFX9-NEXT: v_or_b32_sdwa v15, v40, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v14, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: v_add_u32_e32 v41, 0x300, v0
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_sdwa v41, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
-; GFX9-NEXT: v_lshl_or_b32 v14, v41, 16, v14
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
+; GFX9-NEXT: v_or_b32_sdwa v14, v41, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v13, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: v_add_u32_e32 v42, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
-; GFX9-NEXT: v_lshl_or_b32 v13, v42, 16, v13
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_sdwa v42, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
+; GFX9-NEXT: v_or_b32_sdwa v13, v42, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v12, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
-; GFX9-NEXT: v_add_u32_e32 v43, 0x300, v0
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
+; GFX9-NEXT: v_add_u32_sdwa v43, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX9-NEXT: v_perm_b32 v32, s22, v32, v0
-; GFX9-NEXT: v_add_u32_e32 v32, 0x300, v32
+; GFX9-NEXT: v_add_u32_sdwa v32, v32, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_perm_b32 v33, s16, v33, v0
; GFX9-NEXT: v_add_u32_e32 v33, 0x300, v33
-; GFX9-NEXT: v_and_b32_e32 v33, 0xffff, v33
-; GFX9-NEXT: v_lshl_or_b32 v12, v43, 16, v12
+; GFX9-NEXT: v_or_b32_sdwa v12, v43, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v1, s74, v1, v0
; GFX9-NEXT: v_add_u32_e32 v11, 0x300, v1
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_add_u32_e32 v1, 3, v1
-; GFX9-NEXT: v_perm_b32 v1, v1, v2, s4
-; GFX9-NEXT: v_add_u32_e32 v44, 0x300, v1
+; GFX9-NEXT: v_perm_b32 v1, v1, v2, s5
+; GFX9-NEXT: v_add_u32_sdwa v44, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v1, s62
; GFX9-NEXT: v_perm_b32 v1, s72, v1, v0
; GFX9-NEXT: v_add_u32_e32 v10, 0x300, v1
; GFX9-NEXT: v_mov_b32_e32 v1, s44
; GFX9-NEXT: v_perm_b32 v1, s56, v1, v0
-; GFX9-NEXT: v_add_u32_e32 v45, 0x300, v1
+; GFX9-NEXT: v_add_u32_sdwa v45, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v1, s57
; GFX9-NEXT: v_perm_b32 v1, s60, v1, v0
; GFX9-NEXT: v_add_u32_e32 v9, 0x300, v1
; GFX9-NEXT: v_mov_b32_e32 v1, s13
; GFX9-NEXT: v_perm_b32 v1, s40, v1, v0
-; GFX9-NEXT: v_add_u32_e32 v46, 0x300, v1
+; GFX9-NEXT: v_add_u32_sdwa v46, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v1, s41
; GFX9-NEXT: v_perm_b32 v1, s76, v1, v0
; GFX9-NEXT: v_add_u32_e32 v8, 0x300, v1
; GFX9-NEXT: v_mov_b32_e32 v1, s9
; GFX9-NEXT: v_perm_b32 v1, s11, v1, v0
-; GFX9-NEXT: v_add_u32_e32 v47, 0x300, v1
+; GFX9-NEXT: v_add_u32_sdwa v47, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v1, s73
; GFX9-NEXT: v_perm_b32 v1, s75, v1, v0
; GFX9-NEXT: v_add_u32_e32 v7, 0x300, v1
; GFX9-NEXT: v_mov_b32_e32 v1, s45
; GFX9-NEXT: v_perm_b32 v1, s58, v1, v0
-; GFX9-NEXT: v_add_u32_e32 v56, 0x300, v1
+; GFX9-NEXT: v_add_u32_sdwa v56, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v1, s61
; GFX9-NEXT: v_perm_b32 v1, s63, v1, v0
; GFX9-NEXT: v_add_u32_e32 v6, 0x300, v1
; GFX9-NEXT: v_mov_b32_e32 v1, s42
; GFX9-NEXT: v_perm_b32 v1, s46, v1, v0
-; GFX9-NEXT: v_add_u32_e32 v57, 0x300, v1
+; GFX9-NEXT: v_add_u32_sdwa v57, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v1, s47
; GFX9-NEXT: v_perm_b32 v1, s59, v1, v0
; GFX9-NEXT: v_add_u32_e32 v5, 0x300, v1
; GFX9-NEXT: v_mov_b32_e32 v1, s12
; GFX9-NEXT: v_perm_b32 v1, s14, v1, v0
-; GFX9-NEXT: v_add_u32_e32 v58, 0x300, v1
+; GFX9-NEXT: v_add_u32_sdwa v58, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v1, s15
; GFX9-NEXT: v_perm_b32 v1, s43, v1, v0
; GFX9-NEXT: v_add_u32_e32 v4, 0x300, v1
; GFX9-NEXT: v_mov_b32_e32 v1, s8
; GFX9-NEXT: v_perm_b32 v1, s10, v1, v0
-; GFX9-NEXT: v_add_u32_e32 v59, 0x300, v1
+; GFX9-NEXT: v_add_u32_sdwa v59, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v1, s29
; GFX9-NEXT: v_perm_b32 v1, s28, v1, v0
; GFX9-NEXT: v_add_u32_e32 v3, 0x300, v1
; GFX9-NEXT: v_mov_b32_e32 v1, s6
; GFX9-NEXT: v_perm_b32 v1, s7, v1, v0
-; GFX9-NEXT: v_add_u32_e32 v60, 0x300, v1
+; GFX9-NEXT: v_add_u32_sdwa v60, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v1, s25
; GFX9-NEXT: v_perm_b32 v1, s24, v1, v0
; GFX9-NEXT: v_add_u32_e32 v2, 0x300, v1
; GFX9-NEXT: v_mov_b32_e32 v1, s27
; GFX9-NEXT: v_perm_b32 v1, s26, v1, v0
-; GFX9-NEXT: v_add_u32_e32 v61, 0x300, v1
+; GFX9-NEXT: v_add_u32_sdwa v61, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v1, s21
; GFX9-NEXT: v_perm_b32 v1, s20, v1, v0
; GFX9-NEXT: v_add_u32_e32 v1, 0x300, v1
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_lshl_or_b32 v1, v32, 16, v1
-; GFX9-NEXT: v_and_b32_e32 v32, 0xffff, v54
-; GFX9-NEXT: v_lshl_or_b32 v17, v17, 16, v32
-; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v29, 0xffff, v29
-; GFX9-NEXT: v_and_b32_e32 v26, 0xffff, v26
-; GFX9-NEXT: v_lshl_or_b32 v29, v30, 16, v29
-; GFX9-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
-; GFX9-NEXT: v_lshl_or_b32 v26, v31, 16, v26
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
+; GFX9-NEXT: v_or_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
; GFX9-NEXT: v_perm_b32 v0, s18, v34, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v27, v28, 16, v27
-; GFX9-NEXT: v_and_b32_e32 v28, 0xffff, v38
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v33
-; GFX9-NEXT: v_lshl_or_b32 v2, v61, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v60, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v59, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v58, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v57, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v56, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v47, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v46, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v45, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v44, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v28, v62, 16, v28
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_and_b32_e32 v30, 0xffff, v30
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_lshl_or_b32 v30, v31, 16, v30
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_and_b32_e32 v31, 0xffff, v31
-; GFX9-NEXT: v_lshl_or_b32 v31, v32, 16, v31
+; GFX9-NEXT: v_add_u32_sdwa v0, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v61, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v60, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v4, v59, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v58, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v57, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v56, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v47, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v46, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v45, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v44, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_or_b32_sdwa v31, v32, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: .LBB89_5: ; %end
; GFX9-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
@@ -161331,11 +161308,11 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v4
; GFX9-NEXT: s_and_b32 s45, s44, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v2, v4, 16, v1
+; GFX9-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GFX9-NEXT: v_or_b32_sdwa v2, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v1, s45, v13
; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v1
@@ -161352,16 +161329,16 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: s_and_b32 s44, s43, 0xffff0000
; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
+; GFX9-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v5
+; GFX9-NEXT: s_and_b32 s44, s43, 0xffff0000
+; GFX9-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v3, s44, v13
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_bfe_u32 v4, v3, 16, 1
-; GFX9-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: v_add_u32_e32 v4, v4, v3
; GFX9-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshl_or_b32 v1, v5, 16, v1
; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v3
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
@@ -161375,11 +161352,11 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v3
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX9-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v6
; GFX9-NEXT: s_and_b32 s43, s42, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v4, v6, 16, v3
+; GFX9-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GFX9-NEXT: v_or_b32_sdwa v4, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v3, s43, v13
; GFX9-NEXT: v_bfe_u32 v5, v3, 16, 1
; GFX9-NEXT: v_add_u32_e32 v5, v5, v3
@@ -161396,16 +161373,16 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v3
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX9-NEXT: s_and_b32 s42, s41, 0xffff0000
; GFX9-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc
+; GFX9-NEXT: v_lshrrev_b32_e32 v5, 16, v3
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v7
+; GFX9-NEXT: s_and_b32 s42, s41, 0xffff0000
+; GFX9-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GFX9-NEXT: v_or_b32_sdwa v3, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v5, s42, v13
-; GFX9-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX9-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
; GFX9-NEXT: v_add_u32_e32 v6, v6, v5
; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshl_or_b32 v3, v7, 16, v3
; GFX9-NEXT: v_add_u32_e32 v6, 0x7fff, v6
; GFX9-NEXT: v_or_b32_e32 v7, 0x400000, v5
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
@@ -161419,11 +161396,11 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_or_b32_e32 v7, 0x400000, v5
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
; GFX9-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX9-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX9-NEXT: v_lshrrev_b32_e32 v6, 16, v5
+; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v8
; GFX9-NEXT: s_and_b32 s41, s40, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v6, v8, 16, v5
+; GFX9-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX9-NEXT: v_or_b32_sdwa v6, v5, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v5, s41, v13
; GFX9-NEXT: v_bfe_u32 v7, v5, 16, 1
; GFX9-NEXT: v_add_u32_e32 v7, v7, v5
@@ -161440,16 +161417,16 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v7, 0x7fff, v7
; GFX9-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX9-NEXT: s_and_b32 s40, s13, 0xffff0000
; GFX9-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc
+; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v5
+; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v9
+; GFX9-NEXT: s_and_b32 s40, s13, 0xffff0000
+; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GFX9-NEXT: v_or_b32_sdwa v5, v5, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v7, s40, v13
-; GFX9-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX9-NEXT: v_bfe_u32 v8, v7, 16, 1
-; GFX9-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
; GFX9-NEXT: v_add_u32_e32 v8, v8, v7
; GFX9-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshl_or_b32 v5, v9, 16, v5
; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v7
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
@@ -161463,12 +161440,12 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v7
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX9-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v10
; GFX9-NEXT: s_and_b32 s13, s12, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v11, v10, 16, v7
+; GFX9-NEXT: v_or_b32_sdwa v11, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v7, s13, v13
+; GFX9-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
; GFX9-NEXT: v_bfe_u32 v8, v7, 16, 1
; GFX9-NEXT: v_add_u32_e32 v8, v8, v7
; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
@@ -161485,13 +161462,13 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v7
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX9-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v10
; GFX9-NEXT: s_and_b32 s12, s15, 0xffff0000
; GFX9-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshl_or_b32 v10, v10, 16, v7
+; GFX9-NEXT: v_or_b32_sdwa v10, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v7, s12, v13
+; GFX9-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
; GFX9-NEXT: v_bfe_u32 v8, v7, 16, 1
; GFX9-NEXT: v_add_u32_e32 v8, v8, v7
; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
@@ -161507,12 +161484,12 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v7
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX9-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v12
; GFX9-NEXT: s_and_b32 s12, s14, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v17, v12, 16, v7
+; GFX9-NEXT: v_or_b32_sdwa v17, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v7, s12, v13
+; GFX9-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
; GFX9-NEXT: v_bfe_u32 v8, v7, 16, 1
; GFX9-NEXT: v_add_u32_e32 v8, v8, v7
; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
@@ -161529,12 +161506,12 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v7
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX9-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v12
; GFX9-NEXT: s_and_b32 s12, s11, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v16, v12, 16, v7
+; GFX9-NEXT: v_or_b32_sdwa v16, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v7, s12, v13
+; GFX9-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
; GFX9-NEXT: v_bfe_u32 v8, v7, 16, 1
; GFX9-NEXT: v_add_u32_e32 v8, v8, v7
; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
@@ -161551,12 +161528,12 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v7
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX9-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v12
; GFX9-NEXT: s_and_b32 s11, s10, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v19, v12, 16, v7
+; GFX9-NEXT: v_or_b32_sdwa v19, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v7, s11, v13
+; GFX9-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
; GFX9-NEXT: v_bfe_u32 v8, v7, 16, 1
; GFX9-NEXT: v_add_u32_e32 v8, v8, v7
; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
@@ -161573,12 +161550,12 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v7
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX9-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v12
; GFX9-NEXT: s_and_b32 s10, s9, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v18, v12, 16, v7
+; GFX9-NEXT: v_or_b32_sdwa v18, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v7, s10, v13
+; GFX9-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
; GFX9-NEXT: v_bfe_u32 v8, v7, 16, 1
; GFX9-NEXT: v_add_u32_e32 v8, v8, v7
; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
@@ -161595,12 +161572,12 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v7
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX9-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v12
; GFX9-NEXT: s_and_b32 s9, s8, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v23, v12, 16, v7
+; GFX9-NEXT: v_or_b32_sdwa v23, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v7, s9, v13
+; GFX9-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
; GFX9-NEXT: v_bfe_u32 v8, v7, 16, 1
; GFX9-NEXT: v_add_u32_e32 v8, v8, v7
; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
@@ -161617,12 +161594,12 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v7
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX9-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v12
; GFX9-NEXT: s_and_b32 s8, s7, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v22, v12, 16, v7
+; GFX9-NEXT: v_or_b32_sdwa v22, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v7, s8, v13
+; GFX9-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
; GFX9-NEXT: v_bfe_u32 v8, v7, 16, 1
; GFX9-NEXT: v_add_u32_e32 v8, v8, v7
; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
@@ -161639,12 +161616,12 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v7
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX9-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v12
; GFX9-NEXT: s_and_b32 s7, s6, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v25, v12, 16, v7
+; GFX9-NEXT: v_or_b32_sdwa v25, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v7, s7, v13
+; GFX9-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
; GFX9-NEXT: v_bfe_u32 v8, v7, 16, 1
; GFX9-NEXT: v_add_u32_e32 v8, v8, v7
; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
@@ -161661,12 +161638,12 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v7
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX9-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v12
; GFX9-NEXT: s_and_b32 s6, s17, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v24, v12, 16, v7
+; GFX9-NEXT: v_or_b32_sdwa v24, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v7, s6, v13
+; GFX9-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
; GFX9-NEXT: v_bfe_u32 v8, v7, 16, 1
; GFX9-NEXT: v_add_u32_e32 v8, v8, v7
; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
@@ -161683,12 +161660,12 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v7
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX9-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v12
; GFX9-NEXT: s_and_b32 s6, s16, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v27, v12, 16, v7
+; GFX9-NEXT: v_or_b32_sdwa v27, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v7, s6, v13
+; GFX9-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
; GFX9-NEXT: v_bfe_u32 v8, v7, 16, 1
; GFX9-NEXT: v_add_u32_e32 v8, v8, v7
; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
@@ -161705,12 +161682,12 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v7
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX9-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v12
; GFX9-NEXT: s_and_b32 s6, s19, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v26, v12, 16, v7
+; GFX9-NEXT: v_or_b32_sdwa v26, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v7, s6, v13
+; GFX9-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
; GFX9-NEXT: v_bfe_u32 v8, v7, 16, 1
; GFX9-NEXT: v_add_u32_e32 v8, v8, v7
; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
@@ -161727,12 +161704,12 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v7
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX9-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v12
; GFX9-NEXT: s_and_b32 s6, s18, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v29, v12, 16, v7
+; GFX9-NEXT: v_or_b32_sdwa v29, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v7, s6, v13
+; GFX9-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
; GFX9-NEXT: v_bfe_u32 v8, v7, 16, 1
; GFX9-NEXT: v_add_u32_e32 v8, v8, v7
; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
@@ -161749,12 +161726,12 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v7
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX9-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v12
; GFX9-NEXT: s_and_b32 s6, s21, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v28, v12, 16, v7
+; GFX9-NEXT: v_or_b32_sdwa v28, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v7, s6, v13
+; GFX9-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
; GFX9-NEXT: v_bfe_u32 v8, v7, 16, 1
; GFX9-NEXT: v_add_u32_e32 v8, v8, v7
; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
@@ -161771,12 +161748,12 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v7
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX9-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v12
; GFX9-NEXT: s_and_b32 s6, s20, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v53, v12, 16, v7
+; GFX9-NEXT: v_or_b32_sdwa v53, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v7, s6, v13
+; GFX9-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
; GFX9-NEXT: v_bfe_u32 v8, v7, 16, 1
; GFX9-NEXT: v_add_u32_e32 v8, v8, v7
; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
@@ -161793,12 +161770,12 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v7
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX9-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v12
; GFX9-NEXT: s_and_b32 s6, s23, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v52, v12, 16, v7
+; GFX9-NEXT: v_or_b32_sdwa v52, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v7, s6, v13
+; GFX9-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
; GFX9-NEXT: v_bfe_u32 v8, v7, 16, 1
; GFX9-NEXT: v_add_u32_e32 v8, v8, v7
; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
@@ -161815,12 +161792,12 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v7
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX9-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v12
; GFX9-NEXT: s_and_b32 s6, s22, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v55, v12, 16, v7
+; GFX9-NEXT: v_or_b32_sdwa v55, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v7, s6, v13
+; GFX9-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
; GFX9-NEXT: v_bfe_u32 v8, v7, 16, 1
; GFX9-NEXT: v_add_u32_e32 v8, v8, v7
; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
@@ -161837,12 +161814,12 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v7
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX9-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v12
; GFX9-NEXT: s_and_b32 s6, s25, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v54, v12, 16, v7
+; GFX9-NEXT: v_or_b32_sdwa v54, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v7, s6, v13
+; GFX9-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
; GFX9-NEXT: v_bfe_u32 v8, v7, 16, 1
; GFX9-NEXT: v_add_u32_e32 v8, v8, v7
; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
@@ -161859,12 +161836,12 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v7
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX9-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v12
; GFX9-NEXT: s_and_b32 s6, s24, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v41, v12, 16, v7
+; GFX9-NEXT: v_or_b32_sdwa v41, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v7, s6, v13
+; GFX9-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
; GFX9-NEXT: v_bfe_u32 v8, v7, 16, 1
; GFX9-NEXT: v_add_u32_e32 v8, v8, v7
; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
@@ -161881,12 +161858,12 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v7
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX9-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v12
; GFX9-NEXT: s_and_b32 s6, s27, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v40, v12, 16, v7
+; GFX9-NEXT: v_or_b32_sdwa v40, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v7, s6, v13
+; GFX9-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
; GFX9-NEXT: v_bfe_u32 v8, v7, 16, 1
; GFX9-NEXT: v_add_u32_e32 v8, v8, v7
; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
@@ -161902,12 +161879,12 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v7
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX9-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v14
; GFX9-NEXT: s_and_b32 s6, s26, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v43, v14, 16, v7
+; GFX9-NEXT: v_or_b32_sdwa v43, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v7, s6, v13
+; GFX9-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
; GFX9-NEXT: v_bfe_u32 v8, v7, 16, 1
; GFX9-NEXT: v_add_u32_e32 v8, v8, v7
; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
@@ -161923,12 +161900,12 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v7
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX9-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v57
; GFX9-NEXT: s_and_b32 s6, s29, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v42, v57, 16, v7
+; GFX9-NEXT: v_or_b32_sdwa v42, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v7, s6, v13
+; GFX9-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
; GFX9-NEXT: v_bfe_u32 v8, v7, 16, 1
; GFX9-NEXT: v_add_u32_e32 v8, v8, v7
; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
@@ -161944,12 +161921,12 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v7
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX9-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v56
; GFX9-NEXT: s_and_b32 s6, s28, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v45, v56, 16, v7
+; GFX9-NEXT: v_or_b32_sdwa v45, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v7, s6, v13
+; GFX9-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
; GFX9-NEXT: v_bfe_u32 v8, v7, 16, 1
; GFX9-NEXT: v_add_u32_e32 v8, v8, v7
; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
@@ -161966,12 +161943,12 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v7
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX9-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v14
; GFX9-NEXT: s_and_b32 s6, s5, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v44, v14, 16, v7
+; GFX9-NEXT: v_or_b32_sdwa v44, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v7, s6, v13
+; GFX9-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
; GFX9-NEXT: v_bfe_u32 v8, v7, 16, 1
; GFX9-NEXT: v_add_u32_e32 v8, v8, v7
; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
@@ -161988,9 +161965,9 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX9-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v61, 16, v7
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v61
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v58
; GFX9-NEXT: s_and_b32 s5, s4, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v47, v58, 16, v7
+; GFX9-NEXT: v_or_b32_sdwa v47, v7, v61 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v7, s5, v13
; GFX9-NEXT: v_bfe_u32 v8, v7, 16, 1
; GFX9-NEXT: v_add_u32_e32 v8, v8, v7
@@ -162008,8 +161985,8 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX9-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v13, 16, v7
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v13
-; GFX9-NEXT: v_lshl_or_b32 v46, v59, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v59
+; GFX9-NEXT: v_or_b32_sdwa v46, v7, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_lshrrev_b64 v[7:8], 24, v[24:25]
; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
@@ -174479,25 +174456,25 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX9-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
+; GFX9-NEXT: v_readfirstlane_b32 s42, v13
; GFX9-NEXT: v_readfirstlane_b32 s46, v12
; GFX9-NEXT: v_readfirstlane_b32 s61, v11
-; GFX9-NEXT: v_readfirstlane_b32 s63, v10
+; GFX9-NEXT: v_readfirstlane_b32 s15, v3
; GFX9-NEXT: v_readfirstlane_b32 s7, v0
-; GFX9-NEXT: buffer_load_ushort v10, off, s[0:3], s32 offset:8
-; GFX9-NEXT: buffer_load_ushort v11, off, s[0:3], s32 offset:4
-; GFX9-NEXT: buffer_load_ushort v12, off, s[0:3], s32
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:328
-; GFX9-NEXT: v_readfirstlane_b32 s43, v2
-; GFX9-NEXT: v_readfirstlane_b32 s6, v1
+; GFX9-NEXT: buffer_load_ushort v11, off, s[0:3], s32 offset:8
+; GFX9-NEXT: buffer_load_ushort v12, off, s[0:3], s32 offset:4
+; GFX9-NEXT: buffer_load_ushort v13, off, s[0:3], s32
+; GFX9-NEXT: buffer_load_ushort v3, off, s[0:3], s32 offset:328
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:324
; GFX9-NEXT: v_readfirstlane_b32 s8, v5
; GFX9-NEXT: v_readfirstlane_b32 s10, v4
-; GFX9-NEXT: v_readfirstlane_b32 s15, v3
+; GFX9-NEXT: v_readfirstlane_b32 s43, v2
+; GFX9-NEXT: v_readfirstlane_b32 s6, v1
+; GFX9-NEXT: v_readfirstlane_b32 s63, v10
; GFX9-NEXT: v_readfirstlane_b32 s12, v9
; GFX9-NEXT: v_readfirstlane_b32 s14, v8
; GFX9-NEXT: v_readfirstlane_b32 s47, v7
-; GFX9-NEXT: v_readfirstlane_b32 s59, v6
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:332
-; GFX9-NEXT: v_readfirstlane_b32 s42, v13
; GFX9-NEXT: v_readfirstlane_b32 s74, v30
; GFX9-NEXT: v_readfirstlane_b32 s44, v29
; GFX9-NEXT: v_readfirstlane_b32 s56, v28
@@ -174515,401 +174492,403 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX9-NEXT: v_readfirstlane_b32 s58, v16
; GFX9-NEXT: v_readfirstlane_b32 s73, v15
; GFX9-NEXT: v_readfirstlane_b32 s75, v14
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:472 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:324
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:320
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:484 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:316
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:480 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v2, off, s[0:3], s32 offset:312
-; GFX9-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:308
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:304
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:496 ; 4-byte Folded Spill
+; GFX9-NEXT: v_readfirstlane_b32 s59, v6
+; GFX9-NEXT: buffer_load_ushort v39, off, s[0:3], s32 offset:296
+; GFX9-NEXT: buffer_load_ushort v50, off, s[0:3], s32 offset:292
+; GFX9-NEXT: buffer_load_ushort v62, off, s[0:3], s32 offset:288
+; GFX9-NEXT: buffer_load_ushort v6, off, s[0:3], s32 offset:284
+; GFX9-NEXT: s_waitcnt vmcnt(5)
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:320
+; GFX9-NEXT: buffer_load_ushort v2, off, s[0:3], s32 offset:316
+; GFX9-NEXT: buffer_load_ushort v4, off, s[0:3], s32 offset:312
+; GFX9-NEXT: buffer_load_ushort v5, off, s[0:3], s32 offset:308
+; GFX9-NEXT: buffer_load_ushort v56, off, s[0:3], s32 offset:304
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:300
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:492 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v4, off, s[0:3], s32 offset:296
-; GFX9-NEXT: buffer_load_ushort v5, off, s[0:3], s32 offset:292
-; GFX9-NEXT: buffer_load_ushort v3, off, s[0:3], s32 offset:288
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:284
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:488 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v8, off, s[0:3], s32 offset:280
-; GFX9-NEXT: buffer_load_ushort v7, off, s[0:3], s32 offset:276
-; GFX9-NEXT: buffer_load_ushort v6, off, s[0:3], s32 offset:272
-; GFX9-NEXT: buffer_load_ushort v63, off, s[0:3], s32 offset:268
-; GFX9-NEXT: buffer_load_ushort v48, off, s[0:3], s32 offset:264
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v63, off, s[0:3], s32 offset:280
+; GFX9-NEXT: buffer_load_ushort v38, off, s[0:3], s32 offset:276
+; GFX9-NEXT: buffer_load_ushort v52, off, s[0:3], s32 offset:272
+; GFX9-NEXT: buffer_load_ushort v7, off, s[0:3], s32 offset:268
+; GFX9-NEXT: buffer_load_ushort v8, off, s[0:3], s32 offset:264
; GFX9-NEXT: buffer_load_ushort v54, off, s[0:3], s32 offset:260
-; GFX9-NEXT: buffer_load_ushort v61, off, s[0:3], s32 offset:256
-; GFX9-NEXT: buffer_load_ushort v33, off, s[0:3], s32 offset:252
-; GFX9-NEXT: buffer_load_ushort v9, off, s[0:3], s32 offset:248
-; GFX9-NEXT: buffer_load_ushort v62, off, s[0:3], s32 offset:244
+; GFX9-NEXT: buffer_load_ushort v9, off, s[0:3], s32 offset:256
+; GFX9-NEXT: buffer_load_ushort v37, off, s[0:3], s32 offset:252
+; GFX9-NEXT: buffer_load_ushort v10, off, s[0:3], s32 offset:248
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:244
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:428 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:240
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:464 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v43, off, s[0:3], s32 offset:236
-; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:452 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:236
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:432 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:232
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:500 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:464 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:228
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:436 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:400 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:224
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:436 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:220
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:468 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:440 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:216
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:504 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:472 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:212
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:468 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:208
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:440 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:204
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:400 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:408 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v45, off, s[0:3], s32 offset:204
+; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:200
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:512 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:488 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:196
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:484 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:192
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v51, off, s[0:3], s32 offset:188
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:184
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:492 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:180
; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_ushort v61, off, s[0:3], s32 offset:140
+; GFX9-NEXT: buffer_load_ushort v36, off, s[0:3], s32 offset:136
+; GFX9-NEXT: buffer_load_ushort v60, off, s[0:3], s32 offset:132
+; GFX9-NEXT: buffer_load_ushort v43, off, s[0:3], s32 offset:128
+; GFX9-NEXT: buffer_load_ushort v58, off, s[0:3], s32 offset:124
+; GFX9-NEXT: buffer_load_ushort v34, off, s[0:3], s32 offset:120
+; GFX9-NEXT: buffer_load_ushort v55, off, s[0:3], s32 offset:116
+; GFX9-NEXT: buffer_load_ushort v42, off, s[0:3], s32 offset:112
+; GFX9-NEXT: buffer_load_ushort v47, off, s[0:3], s32 offset:108
+; GFX9-NEXT: buffer_load_ushort v28, off, s[0:3], s32 offset:104
+; GFX9-NEXT: buffer_load_ushort v35, off, s[0:3], s32 offset:100
+; GFX9-NEXT: buffer_load_ushort v48, off, s[0:3], s32 offset:96
+; GFX9-NEXT: buffer_load_ushort v44, off, s[0:3], s32 offset:92
+; GFX9-NEXT: buffer_load_ushort v24, off, s[0:3], s32 offset:88
+; GFX9-NEXT: buffer_load_ushort v29, off, s[0:3], s32 offset:84
+; GFX9-NEXT: buffer_load_ushort v53, off, s[0:3], s32 offset:80
+; GFX9-NEXT: buffer_load_ushort v49, off, s[0:3], s32 offset:76
+; GFX9-NEXT: buffer_load_ushort v20, off, s[0:3], s32 offset:72
+; GFX9-NEXT: buffer_load_ushort v25, off, s[0:3], s32 offset:68
+; GFX9-NEXT: buffer_load_ushort v30, off, s[0:3], s32 offset:64
+; GFX9-NEXT: buffer_load_ushort v33, off, s[0:3], s32 offset:60
+; GFX9-NEXT: buffer_load_ushort v17, off, s[0:3], s32 offset:56
+; GFX9-NEXT: buffer_load_ushort v21, off, s[0:3], s32 offset:52
+; GFX9-NEXT: buffer_load_ushort v26, off, s[0:3], s32 offset:48
+; GFX9-NEXT: buffer_load_ushort v31, off, s[0:3], s32 offset:44
+; GFX9-NEXT: buffer_load_ushort v15, off, s[0:3], s32 offset:40
+; GFX9-NEXT: buffer_load_ushort v18, off, s[0:3], s32 offset:36
+; GFX9-NEXT: buffer_load_ushort v22, off, s[0:3], s32 offset:32
+; GFX9-NEXT: buffer_load_ushort v27, off, s[0:3], s32 offset:28
+; GFX9-NEXT: buffer_load_ushort v14, off, s[0:3], s32 offset:24
+; GFX9-NEXT: buffer_load_ushort v16, off, s[0:3], s32 offset:20
+; GFX9-NEXT: buffer_load_ushort v19, off, s[0:3], s32 offset:16
+; GFX9-NEXT: buffer_load_ushort v23, off, s[0:3], s32 offset:12
; GFX9-NEXT: s_and_b64 s[4:5], vcc, exec
+; GFX9-NEXT: s_waitcnt vmcnt(33)
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:496 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v32, off, s[0:3], s32 offset:176
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:172
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:508 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:192
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:480 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v46, off, s[0:3], s32 offset:168
+; GFX9-NEXT: buffer_load_ushort v57, off, s[0:3], s32 offset:164
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_ushort v13, off, s[0:3], s32 offset:188
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:444 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v47, off, s[0:3], s32 offset:184
-; GFX9-NEXT: buffer_load_ushort v59, off, s[0:3], s32 offset:180
-; GFX9-NEXT: buffer_load_ushort v42, off, s[0:3], s32 offset:176
-; GFX9-NEXT: buffer_load_ushort v58, off, s[0:3], s32 offset:172
-; GFX9-NEXT: buffer_load_ushort v44, off, s[0:3], s32 offset:168
-; GFX9-NEXT: buffer_load_ushort v56, off, s[0:3], s32 offset:164
-; GFX9-NEXT: buffer_load_ushort v60, off, s[0:3], s32 offset:160
-; GFX9-NEXT: buffer_load_ushort v53, off, s[0:3], s32 offset:156
-; GFX9-NEXT: buffer_load_ushort v49, off, s[0:3], s32 offset:152
-; GFX9-NEXT: buffer_load_ushort v45, off, s[0:3], s32 offset:148
-; GFX9-NEXT: buffer_load_ushort v57, off, s[0:3], s32 offset:144
-; GFX9-NEXT: buffer_load_ushort v55, off, s[0:3], s32 offset:140
-; GFX9-NEXT: buffer_load_ushort v38, off, s[0:3], s32 offset:136
-; GFX9-NEXT: buffer_load_ushort v32, off, s[0:3], s32 offset:132
-; GFX9-NEXT: buffer_load_ushort v46, off, s[0:3], s32 offset:128
-; GFX9-NEXT: buffer_load_ushort v40, off, s[0:3], s32 offset:124
-; GFX9-NEXT: buffer_load_ushort v31, off, s[0:3], s32 offset:120
-; GFX9-NEXT: buffer_load_ushort v52, off, s[0:3], s32 offset:116
-; GFX9-NEXT: buffer_load_ushort v50, off, s[0:3], s32 offset:112
-; GFX9-NEXT: buffer_load_ushort v41, off, s[0:3], s32 offset:108
-; GFX9-NEXT: buffer_load_ushort v27, off, s[0:3], s32 offset:104
-; GFX9-NEXT: buffer_load_ushort v37, off, s[0:3], s32 offset:100
-; GFX9-NEXT: buffer_load_ushort v36, off, s[0:3], s32 offset:96
-; GFX9-NEXT: buffer_load_ushort v51, off, s[0:3], s32 offset:92
-; GFX9-NEXT: buffer_load_ushort v23, off, s[0:3], s32 offset:88
-; GFX9-NEXT: buffer_load_ushort v28, off, s[0:3], s32 offset:84
-; GFX9-NEXT: buffer_load_ushort v34, off, s[0:3], s32 offset:80
-; GFX9-NEXT: buffer_load_ushort v39, off, s[0:3], s32 offset:76
-; GFX9-NEXT: buffer_load_ushort v19, off, s[0:3], s32 offset:72
-; GFX9-NEXT: buffer_load_ushort v24, off, s[0:3], s32 offset:68
-; GFX9-NEXT: buffer_load_ushort v29, off, s[0:3], s32 offset:64
-; GFX9-NEXT: buffer_load_ushort v35, off, s[0:3], s32 offset:60
-; GFX9-NEXT: buffer_load_ushort v16, off, s[0:3], s32 offset:56
-; GFX9-NEXT: buffer_load_ushort v20, off, s[0:3], s32 offset:52
-; GFX9-NEXT: buffer_load_ushort v25, off, s[0:3], s32 offset:48
-; GFX9-NEXT: buffer_load_ushort v30, off, s[0:3], s32 offset:44
-; GFX9-NEXT: buffer_load_ushort v14, off, s[0:3], s32 offset:40
-; GFX9-NEXT: buffer_load_ushort v17, off, s[0:3], s32 offset:36
-; GFX9-NEXT: buffer_load_ushort v21, off, s[0:3], s32 offset:32
-; GFX9-NEXT: buffer_load_ushort v26, off, s[0:3], s32 offset:28
-; GFX9-NEXT: buffer_load_ushort v13, off, s[0:3], s32 offset:24
-; GFX9-NEXT: buffer_load_ushort v15, off, s[0:3], s32 offset:20
-; GFX9-NEXT: buffer_load_ushort v18, off, s[0:3], s32 offset:16
-; GFX9-NEXT: buffer_load_ushort v22, off, s[0:3], s32 offset:12
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:516 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:520 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(6)
-; GFX9-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:528 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:532 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(7)
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(9)
-; GFX9-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(12)
-; GFX9-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:160
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:500 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v40, off, s[0:3], s32 offset:156
+; GFX9-NEXT: buffer_load_ushort v59, off, s[0:3], s32 offset:152
+; GFX9-NEXT: buffer_load_ushort v41, off, s[0:3], s32 offset:148
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:144
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:512 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:516 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:520 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:528 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:532 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:408 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v49, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
+; GFX9-NEXT: s_waitcnt vmcnt(27)
+; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:412 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:420 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:424 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:428 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:432 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:420 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:424 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:628 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:636 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v52, off, s[0:3], s32 offset:640 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:644 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v38, off, s[0:3], s32 offset:648 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:444 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:448 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v48, off, s[0:3], s32 offset:628 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:452 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:636 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:640 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:644 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:648 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:656 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:660 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v48, off, s[0:3], s32 offset:664 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:668 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:672 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:508 ; 4-byte Folded Spill
+; GFX9-NEXT: s_waitcnt vmcnt(49)
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:504 ; 4-byte Folded Spill
; GFX9-NEXT: s_cbranch_scc0 .LBB93_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v8, 0xc0c0004
-; GFX9-NEXT: v_mov_b32_e32 v2, s21
-; GFX9-NEXT: v_perm_b32 v0, s16, v0, v8
+; GFX9-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:676 ; 4-byte Folded Spill
; GFX9-NEXT: v_mov_b32_e32 v1, s19
-; GFX9-NEXT: v_perm_b32 v2, s20, v2, v8
; GFX9-NEXT: v_mov_b32_e32 v3, s23
+; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_perm_b32 v1, s18, v1, v8
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:680 ; 4-byte Folded Spill
+; GFX9-NEXT: v_mov_b32_e32 v2, s21
; GFX9-NEXT: v_perm_b32 v3, s22, v3, v8
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v3, 16, v2
-; GFX9-NEXT: v_mov_b32_e32 v2, s25
-; GFX9-NEXT: v_perm_b32 v2, s24, v2, v8
+; GFX9-NEXT: v_perm_b32 v0, s16, v0, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_perm_b32 v2, s20, v2, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v3, s27
+; GFX9-NEXT: v_mov_b32_e32 v2, s25
; GFX9-NEXT: v_perm_b32 v3, s26, v3, v8
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_lshl_or_b32 v2, v3, 16, v2
-; GFX9-NEXT: v_mov_b32_e32 v3, s29
-; GFX9-NEXT: v_perm_b32 v3, s28, v3, v8
+; GFX9-NEXT: v_perm_b32 v2, s24, v2, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_mov_b32_e32 v4, s6
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v3, s29
; GFX9-NEXT: v_perm_b32 v4, s7, v4, v8
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_lshl_or_b32 v3, v4, 16, v3
-; GFX9-NEXT: v_mov_b32_e32 v4, s15
-; GFX9-NEXT: v_perm_b32 v4, s43, v4, v8
+; GFX9-NEXT: v_perm_b32 v3, s28, v3, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX9-NEXT: v_mov_b32_e32 v5, s8
+; GFX9-NEXT: v_or_b32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v4, s15
; GFX9-NEXT: v_perm_b32 v5, s10, v5, v8
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_lshl_or_b32 v4, v5, 16, v4
-; GFX9-NEXT: v_mov_b32_e32 v5, s47
-; GFX9-NEXT: v_perm_b32 v5, s59, v5, v8
+; GFX9-NEXT: v_perm_b32 v4, s43, v4, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX9-NEXT: v_mov_b32_e32 v6, s12
+; GFX9-NEXT: v_or_b32_sdwa v4, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v5, s47
; GFX9-NEXT: v_perm_b32 v6, s14, v6, v8
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_lshl_or_b32 v5, v6, 16, v5
-; GFX9-NEXT: v_mov_b32_e32 v6, s61
-; GFX9-NEXT: v_perm_b32 v6, s63, v6, v8
+; GFX9-NEXT: v_perm_b32 v5, s59, v5, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX9-NEXT: v_mov_b32_e32 v50, v54
+; GFX9-NEXT: v_mov_b32_e32 v54, v7
; GFX9-NEXT: v_mov_b32_e32 v7, s42
+; GFX9-NEXT: v_or_b32_sdwa v5, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v6, s61
; GFX9-NEXT: v_perm_b32 v7, s46, v7, v8
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_lshl_or_b32 v6, v7, 16, v6
-; GFX9-NEXT: v_mov_b32_e32 v7, s73
-; GFX9-NEXT: v_perm_b32 v7, s75, v7, v8
-; GFX9-NEXT: v_mov_b32_e32 v63, v33
-; GFX9-NEXT: v_mov_b32_e32 v33, v9
+; GFX9-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:684 ; 4-byte Folded Spill
+; GFX9-NEXT: v_perm_b32 v6, s63, v6, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX9-NEXT: v_mov_b32_e32 v39, v9
; GFX9-NEXT: v_mov_b32_e32 v9, s45
+; GFX9-NEXT: v_or_b32_sdwa v6, v7, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v7, s73
; GFX9-NEXT: v_perm_b32 v9, s58, v9, v8
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_lshl_or_b32 v7, v9, 16, v7
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v10
-; GFX9-NEXT: v_perm_b32 v10, s74, v12, v8
-; GFX9-NEXT: v_or_b32_sdwa v9, v11, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v9, 16, v10
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v18
-; GFX9-NEXT: v_or_b32_sdwa v9, v22, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v13
-; GFX9-NEXT: v_or_b32_sdwa v10, v15, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
+; GFX9-NEXT: v_perm_b32 v7, s75, v7, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX9-NEXT: s_and_b32 s4, s40, 0xff
+; GFX9-NEXT: s_lshl_b32 s5, s13, 8
+; GFX9-NEXT: v_or_b32_sdwa v7, v9, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v9, s57
+; GFX9-NEXT: s_or_b32 s4, s4, s5
+; GFX9-NEXT: v_mov_b32_e32 v56, v38
+; GFX9-NEXT: v_perm_b32 v9, s60, v9, v8
+; GFX9-NEXT: v_mov_b32_e32 v38, v37
+; GFX9-NEXT: v_mov_b32_e32 v37, v10
+; GFX9-NEXT: v_lshlrev_b32_e64 v10, 16, s4
+; GFX9-NEXT: v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v11
+; GFX9-NEXT: v_perm_b32 v8, s74, v13, v8
+; GFX9-NEXT: v_or_b32_sdwa v10, v12, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v11, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v19
+; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v14
+; GFX9-NEXT: v_or_b32_sdwa v8, v23, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v10, v16, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v12, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_mov_b32 s4, 0xc0c0004
-; GFX9-NEXT: v_lshl_or_b32 v12, v10, 16, v9
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v14
-; GFX9-NEXT: v_perm_b32 v10, v26, v21, s4
-; GFX9-NEXT: v_or_b32_sdwa v9, v17, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v13, v9, 16, v10
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v25
-; GFX9-NEXT: v_or_b32_sdwa v9, v30, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v16
-; GFX9-NEXT: v_or_b32_sdwa v10, v20, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_lshl_or_b32 v14, v10, 16, v9
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v19
-; GFX9-NEXT: v_perm_b32 v10, v35, v29, s4
-; GFX9-NEXT: v_or_b32_sdwa v9, v24, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v15, v9, 16, v10
-; GFX9-NEXT: v_perm_b32 v9, v39, v34, s4
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_perm_b32 v10, v51, v36, s4
-; GFX9-NEXT: v_perm_b32 v16, v28, v23, s4
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_perm_b32 v17, v37, v27, s4
-; GFX9-NEXT: v_lshl_or_b32 v16, v16, 16, v9
-; GFX9-NEXT: v_perm_b32 v9, v41, v50, s4
-; GFX9-NEXT: v_lshl_or_b32 v17, v17, 16, v10
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_perm_b32 v10, v52, v31, s4
-; GFX9-NEXT: v_lshl_or_b32 v18, v10, 16, v9
-; GFX9-NEXT: v_perm_b32 v9, v40, v46, s4
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_perm_b32 v10, v32, v38, s4
-; GFX9-NEXT: v_lshl_or_b32 v19, v10, 16, v9
-; GFX9-NEXT: v_perm_b32 v9, v55, v57, s4
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_perm_b32 v10, v45, v49, s4
-; GFX9-NEXT: v_lshl_or_b32 v20, v10, 16, v9
-; GFX9-NEXT: v_perm_b32 v9, v53, v60, s4
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_perm_b32 v10, v56, v44, s4
-; GFX9-NEXT: v_lshl_or_b32 v21, v10, 16, v9
-; GFX9-NEXT: v_perm_b32 v9, v58, v42, s4
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_perm_b32 v10, v59, v47, s4
-; GFX9-NEXT: v_lshl_or_b32 v22, v10, 16, v9
-; GFX9-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
-; GFX9-NEXT: v_mov_b32_e32 v50, v32
-; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
-; GFX9-NEXT: v_mov_b32_e32 v51, v49
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v15
+; GFX9-NEXT: v_or_b32_sdwa v8, v18, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_perm_b32 v10, v27, v22, s4
+; GFX9-NEXT: v_perm_b32 v16, v49, v53, s4
; GFX9-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:404 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
-; GFX9-NEXT: v_mov_b32_e32 v41, v46
-; GFX9-NEXT: v_mov_b32_e32 v39, v52
-; GFX9-NEXT: v_mov_b32_e32 v52, v38
-; GFX9-NEXT: v_mov_b32_e32 v34, v37
-; GFX9-NEXT: v_mov_b32_e32 v37, v31
-; GFX9-NEXT: s_lshl_b32 s5, s13, 8
+; GFX9-NEXT: v_or_b32_sdwa v13, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v17
+; GFX9-NEXT: v_perm_b32 v17, v44, v48, s4
+; GFX9-NEXT: v_mov_b32_e32 v44, v55
+; GFX9-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v26
+; GFX9-NEXT: v_or_b32_sdwa v8, v31, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v10, v21, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v14, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v20
+; GFX9-NEXT: v_or_b32_sdwa v8, v25, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_perm_b32 v10, v33, v30, s4
+; GFX9-NEXT: v_or_b32_sdwa v15, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_perm_b32 v8, v29, v24, s4
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX9-NEXT: v_perm_b32 v10, v35, v28, s4
+; GFX9-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX9-NEXT: v_or_b32_sdwa v16, v8, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_perm_b32 v8, v44, v34, s4
+; GFX9-NEXT: v_or_b32_sdwa v17, v10, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX9-NEXT: v_perm_b32 v10, v47, v42, s4
+; GFX9-NEXT: v_mov_b32_e32 v34, v36
+; GFX9-NEXT: v_mov_b32_e32 v35, v60
+; GFX9-NEXT: v_or_b32_sdwa v18, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_perm_b32 v8, v35, v34, s4
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX9-NEXT: v_perm_b32 v10, v58, v43, s4
+; GFX9-NEXT: v_mov_b32_e32 v53, v59
+; GFX9-NEXT: v_or_b32_sdwa v19, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_perm_b32 v8, v41, v53, s4
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX9-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:484 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
+; GFX9-NEXT: s_lshl_b32 s5, s41, 8
; GFX9-NEXT: s_lshl_b32 s77, s9, 8
; GFX9-NEXT: s_lshl_b32 s78, s44, 8
-; GFX9-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:452 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:428 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(9)
-; GFX9-NEXT: v_perm_b32 v9, v9, v10, s4
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: s_waitcnt vmcnt(7)
-; GFX9-NEXT: v_perm_b32 v10, v32, v58, s4
-; GFX9-NEXT: v_lshl_or_b32 v23, v10, 16, v9
-; GFX9-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:440 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:400 ; 4-byte Folded Reload
+; GFX9-NEXT: v_perm_b32 v10, v61, v49, s4
+; GFX9-NEXT: v_or_b32_sdwa v20, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_perm_b32 v8, v57, v46, s4
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX9-NEXT: s_waitcnt vmcnt(8)
+; GFX9-NEXT: v_perm_b32 v10, v40, v55, s4
+; GFX9-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
+; GFX9-NEXT: v_or_b32_sdwa v21, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: s_waitcnt vmcnt(9)
+; GFX9-NEXT: v_perm_b32 v10, v33, v32, s4
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_perm_b32 v8, v60, v40, s4
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX9-NEXT: v_or_b32_sdwa v22, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_perm_b32 v8, v42, v59, s4
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX9-NEXT: v_perm_b32 v10, v51, v36, s4
+; GFX9-NEXT: v_or_b32_sdwa v23, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v8
+; GFX9-NEXT: v_or_b32_sdwa v8, v45, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v9
+; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v51
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v49
-; GFX9-NEXT: v_or_b32_sdwa v10, v24, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_lshl_or_b32 v24, v10, 16, v9
-; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:436 ; 4-byte Folded Reload
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v35
+; GFX9-NEXT: v_or_b32_sdwa v10, v45, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v24, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:400 ; 4-byte Folded Reload
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v32
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
+; GFX9-NEXT: v_or_b32_sdwa v8, v10, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:436 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:440 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v10, v25, v10, s4
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v25, v9, 16, v10
-; GFX9-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
-; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v33
-; GFX9-NEXT: v_or_b32_sdwa v10, v62, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
-; GFX9-NEXT: v_mov_b32_e32 v33, v46
+; GFX9-NEXT: v_or_b32_sdwa v25, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:452 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:432 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v9
-; GFX9-NEXT: v_or_b32_sdwa v9, v43, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_lshl_or_b32 v26, v10, 16, v9
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v48
-; GFX9-NEXT: v_perm_b32 v10, v63, v61, s4
-; GFX9-NEXT: v_or_b32_sdwa v9, v54, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v27, v9, 16, v10
-; GFX9-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
-; GFX9-NEXT: v_mov_b32_e32 v54, v61
-; GFX9-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:484 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(4)
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v9
-; GFX9-NEXT: s_waitcnt vmcnt(3)
-; GFX9-NEXT: v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_mov_b32_e32 v48, v63
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v10
-; GFX9-NEXT: v_or_b32_sdwa v10, v28, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_lshl_or_b32 v28, v10, 16, v9
-; GFX9-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:428 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v8
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_or_b32_sdwa v8, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v37
+; GFX9-NEXT: v_or_b32_sdwa v10, v26, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v26, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:424 ; 4-byte Folded Reload
+; GFX9-NEXT: v_perm_b32 v10, v38, v39, s4
+; GFX9-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
+; GFX9-NEXT: v_mov_b32_e32 v38, v34
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v8
+; GFX9-NEXT: v_or_b32_sdwa v8, v50, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v27, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v52
+; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v63
+; GFX9-NEXT: v_or_b32_sdwa v8, v54, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v10, v56, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v28, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
+; GFX9-NEXT: v_mov_b32_e32 v54, v31
+; GFX9-NEXT: v_mov_b32_e32 v50, v53
+; GFX9-NEXT: v_mov_b32_e32 v56, v40
+; GFX9-NEXT: v_mov_b32_e32 v52, v32
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v9
+; GFX9-NEXT: v_mov_b32_e32 v63, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v8
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:432 ; 4-byte Folded Reload
+; GFX9-NEXT: v_or_b32_sdwa v8, v10, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v10, v36, v10, s4
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v29, v9, 16, v10
-; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:424 ; 4-byte Folded Reload
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v43
-; GFX9-NEXT: v_or_b32_sdwa v9, v62, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
+; GFX9-NEXT: v_perm_b32 v10, v10, v62, s4
+; GFX9-NEXT: v_or_b32_sdwa v29, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:404 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v8
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v10
-; GFX9-NEXT: v_or_b32_sdwa v10, v30, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_lshl_or_b32 v30, v10, 16, v9
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v38
-; GFX9-NEXT: v_perm_b32 v10, v63, v61, s4
-; GFX9-NEXT: v_or_b32_sdwa v9, v46, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v31, v9, 16, v10
-; GFX9-NEXT: v_mov_b32_e32 v9, s57
-; GFX9-NEXT: v_perm_b32 v8, s60, v9, v8
-; GFX9-NEXT: s_and_b32 s4, s40, 0xff
-; GFX9-NEXT: s_or_b32 s4, s4, s5
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, s4, 16, v8
+; GFX9-NEXT: v_or_b32_sdwa v8, v39, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v10, v30, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v30, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v8
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v62, v10
+; GFX9-NEXT: v_perm_b32 v10, v31, v10, s4
; GFX9-NEXT: s_and_b32 s4, s76, 0xff
-; GFX9-NEXT: s_lshl_b32 s5, s41, 8
; GFX9-NEXT: s_or_b32 s4, s4, s5
; GFX9-NEXT: s_and_b32 s5, s11, 0xff
; GFX9-NEXT: s_or_b32 s5, s5, s77
@@ -174919,60 +174898,54 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX9-NEXT: s_or_b32 s5, s5, s77
; GFX9-NEXT: s_and_b32 s77, s56, 0xff
; GFX9-NEXT: s_or_b32 s77, s77, s78
+; GFX9-NEXT: v_or_b32_sdwa v8, v48, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: s_pack_ll_b32_b16 s5, s5, s77
+; GFX9-NEXT: v_or_b32_sdwa v31, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v8, s4
; GFX9-NEXT: v_mov_b32_e32 v10, s5
; GFX9-NEXT: s_mov_b64 s[4:5], 0
-; GFX9-NEXT: v_mov_b32_e32 v63, v36
; GFX9-NEXT: s_branch .LBB93_3
; GFX9-NEXT: .LBB93_2:
-; GFX9-NEXT: v_mov_b32_e32 v54, v61
-; GFX9-NEXT: v_mov_b32_e32 v50, v32
-; GFX9-NEXT: v_mov_b32_e32 v51, v49
-; GFX9-NEXT: v_mov_b32_e32 v39, v52
-; GFX9-NEXT: v_mov_b32_e32 v52, v38
-; GFX9-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:484 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
+; GFX9-NEXT: v_mov_b32_e32 v63, v39
+; GFX9-NEXT: v_mov_b32_e32 v38, v36
+; GFX9-NEXT: v_mov_b32_e32 v44, v55
+; GFX9-NEXT: v_mov_b32_e32 v50, v59
+; GFX9-NEXT: v_mov_b32_e32 v35, v60
+; GFX9-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
-; GFX9-NEXT: v_mov_b32_e32 v41, v46
-; GFX9-NEXT: v_mov_b32_e32 v34, v37
-; GFX9-NEXT: v_mov_b32_e32 v37, v31
+; GFX9-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:484 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
+; GFX9-NEXT: v_mov_b32_e32 v54, v2
+; GFX9-NEXT: v_mov_b32_e32 v62, v1
; GFX9-NEXT: s_mov_b64 s[4:5], -1
; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
; GFX9-NEXT: ; implicit-def: $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
; GFX9-NEXT: .LBB93_3: ; %Flow
-; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:400 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
; GFX9-NEXT: s_cselect_b32 s4, 1, 0
; GFX9-NEXT: s_cmp_lg_u32 s4, 1
; GFX9-NEXT: s_cbranch_scc1 .LBB93_5
; GFX9-NEXT: ; %bb.4: ; %cmp.true
-; GFX9-NEXT: s_waitcnt vmcnt(5)
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v48
-; GFX9-NEXT: s_mov_b32 s4, 0xc0c0004
-; GFX9-NEXT: v_perm_b32 v0, v0, v61, s4
-; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:472 ; 4-byte Folded Spill
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v33
-; GFX9-NEXT: s_waitcnt vmcnt(4)
-; GFX9-NEXT: v_perm_b32 v0, v0, v38, s4
-; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v62
-; GFX9-NEXT: v_perm_b32 v0, v0, v43, s4
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v54
+; GFX9-NEXT: s_mov_b32 s5, 0xc0c0004
+; GFX9-NEXT: v_perm_b32 v0, v0, v62, s5
; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:480 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
+; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v48
+; GFX9-NEXT: s_movk_i32 s4, 0x300
; GFX9-NEXT: s_add_i32 s74, s74, 3
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:424 ; 4-byte Folded Reload
; GFX9-NEXT: s_add_i32 s72, s72, 3
; GFX9-NEXT: s_add_i32 s56, s56, 3
; GFX9-NEXT: s_add_i32 s60, s60, 3
@@ -174993,373 +174966,358 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX9-NEXT: s_add_i32 s26, s26, 3
; GFX9-NEXT: s_add_i32 s20, s20, 3
; GFX9-NEXT: s_add_i32 s22, s22, 3
+; GFX9-NEXT: v_mov_b32_e32 v32, s23
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: v_mov_b32_e32 v33, s17
; GFX9-NEXT: s_add_i32 s18, s18, 3
-; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
+; GFX9-NEXT: v_mov_b32_e32 v34, s19
+; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:432 ; 4-byte Folded Reload
-; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:408 ; 4-byte Folded Spill
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v63
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:404 ; 4-byte Folded Reload
+; GFX9-NEXT: v_add_u32_sdwa v0, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:444 ; 4-byte Folded Spill
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v39
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:424 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:428 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_sdwa v0, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:636 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:428 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:412 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT: v_perm_b32 v0, v0, v63, s5
+; GFX9-NEXT: v_add_u32_sdwa v0, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:448 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: v_add_u32_e32 v38, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:452 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_e32 v39, 0x300, v0
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: v_add_u32_e32 v62, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_sdwa v63, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:628 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT: v_perm_b32 v0, v0, v54, s4
-; GFX9-NEXT: v_add_u32_e32 v63, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:424 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_sdwa v48, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:432 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:452 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:412 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:632 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:420 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:428 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: v_add_u32_e32 v31, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_sdwa v31, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:440 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:436 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v30, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:436 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:440 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:400 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT: v_perm_b32 v0, v0, v35, s4
-; GFX9-NEXT: v_add_u32_e32 v29, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v36
+; GFX9-NEXT: v_perm_b32 v0, v0, v52, s5
+; GFX9-NEXT: v_add_u32_sdwa v29, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:632 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v28, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:404 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v45
+; GFX9-NEXT: v_perm_b32 v0, v0, v51, s5
+; GFX9-NEXT: v_add_u32_sdwa v27, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT: v_perm_b32 v0, v0, v49, s4
-; GFX9-NEXT: v_add_u32_e32 v27, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v36, s5
; GFX9-NEXT: v_add_u32_e32 v26, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v32
-; GFX9-NEXT: v_perm_b32 v0, v0, v58, s4
-; GFX9-NEXT: v_add_u32_e32 v25, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
-; GFX9-NEXT: v_mov_b32_e32 v32, s23
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v42
+; GFX9-NEXT: v_perm_b32 v0, v0, v59, s5
+; GFX9-NEXT: v_add_u32_sdwa v25, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v33
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_e32 v24, 0x300, v0
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v60
+; GFX9-NEXT: v_perm_b32 v0, v0, v56, s5
+; GFX9-NEXT: v_add_u32_sdwa v23, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
+; GFX9-NEXT: v_mov_b32_e32 v33, s17
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT: v_perm_b32 v0, v0, v42, s4
-; GFX9-NEXT: v_add_u32_e32 v24, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v59
-; GFX9-NEXT: v_perm_b32 v0, v0, v47, s4
-; GFX9-NEXT: v_add_u32_e32 v23, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v53
-; GFX9-NEXT: v_perm_b32 v0, v0, v60, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v55, s5
; GFX9-NEXT: v_add_u32_e32 v22, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v56
-; GFX9-NEXT: v_perm_b32 v0, v0, v44, s4
-; GFX9-NEXT: v_add_u32_e32 v21, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v55
-; GFX9-NEXT: v_perm_b32 v0, v0, v57, s4
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v57
+; GFX9-NEXT: v_perm_b32 v0, v0, v46, s5
+; GFX9-NEXT: v_add_u32_sdwa v21, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v61
+; GFX9-NEXT: v_perm_b32 v0, v0, v49, s5
; GFX9-NEXT: v_add_u32_e32 v20, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v45
-; GFX9-NEXT: v_perm_b32 v0, v0, v51, s4
-; GFX9-NEXT: v_add_u32_e32 v49, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v40
-; GFX9-NEXT: v_perm_b32 v0, v0, v41, s4
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v41
+; GFX9-NEXT: v_perm_b32 v0, v0, v50, s5
+; GFX9-NEXT: v_add_u32_sdwa v51, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v58
+; GFX9-NEXT: v_perm_b32 v0, v0, v43, s5
; GFX9-NEXT: v_add_u32_e32 v19, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v50
-; GFX9-NEXT: v_perm_b32 v0, v0, v52, s4
-; GFX9-NEXT: v_add_u32_e32 v52, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v22, 0xffff, v22
-; GFX9-NEXT: v_lshl_or_b32 v21, v21, 16, v22
-; GFX9-NEXT: v_and_b32_e32 v22, 0xffff, v24
-; GFX9-NEXT: v_lshl_or_b32 v22, v23, 16, v22
-; GFX9-NEXT: v_and_b32_e32 v23, 0xffff, v26
-; GFX9-NEXT: v_lshl_or_b32 v23, v25, 16, v23
-; GFX9-NEXT: v_and_b32_e32 v25, 0xffff, v30
-; GFX9-NEXT: v_lshl_or_b32 v25, v29, 16, v25
-; GFX9-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:424 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v24, 0xffff, v28
-; GFX9-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:428 ; 4-byte Folded Reload
-; GFX9-NEXT: v_lshl_or_b32 v24, v27, 16, v24
-; GFX9-NEXT: v_and_b32_e32 v27, 0xffff, v63
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_lshl_or_b32 v19, v52, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v49, 16, v20
-; GFX9-NEXT: s_waitcnt vmcnt(4)
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v35
+; GFX9-NEXT: v_perm_b32 v0, v0, v38, s5
+; GFX9-NEXT: v_add_u32_sdwa v38, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v47
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
; GFX9-NEXT: v_add_u32_e32 v18, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v39
-; GFX9-NEXT: v_perm_b32 v0, v0, v37, s4
-; GFX9-NEXT: v_add_u32_e32 v37, 0x300, v0
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v44
+; GFX9-NEXT: v_or_b32_sdwa v21, v21, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v22, v23, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v23, v25, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v25, v29, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
+; GFX9-NEXT: v_or_b32_sdwa v24, v27, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v38, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v20, v51, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v28, v63, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(4)
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_sdwa v53, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_lshl_or_b32 v18, v37, 16, v18
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
+; GFX9-NEXT: v_or_b32_sdwa v18, v53, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: s_waitcnt vmcnt(5)
+; GFX9-NEXT: v_or_b32_sdwa v26, v31, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: v_or_b32_sdwa v29, v30, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:404 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(4)
+; GFX9-NEXT: v_or_b32_sdwa v27, v48, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: s_waitcnt vmcnt(3)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
-; GFX9-NEXT: v_add_u32_e32 v54, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v34
-; GFX9-NEXT: v_mov_b32_e32 v34, s19
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v17, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: v_or_b32_sdwa v30, v31, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: v_add_u32_e32 v16, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:636 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_sdwa v54, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
+; GFX9-NEXT: v_or_b32_sdwa v17, v54, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT: v_perm_b32 v0, v0, v37, s5
+; GFX9-NEXT: v_add_u32_e32 v16, 0x300, v0
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: v_add_u32_e32 v55, 0x300, v0
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_sdwa v55, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
-; GFX9-NEXT: v_lshl_or_b32 v16, v55, 16, v16
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
+; GFX9-NEXT: v_or_b32_sdwa v16, v55, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v15, 0x300, v0
; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: v_add_u32_e32 v40, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_sdwa v40, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
-; GFX9-NEXT: v_lshl_or_b32 v15, v40, 16, v15
+; GFX9-NEXT: v_or_b32_sdwa v15, v40, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v14, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: v_add_u32_e32 v41, 0x300, v0
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_sdwa v41, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
-; GFX9-NEXT: v_lshl_or_b32 v14, v41, 16, v14
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
+; GFX9-NEXT: v_or_b32_sdwa v14, v41, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v13, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: v_add_u32_e32 v42, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
-; GFX9-NEXT: v_lshl_or_b32 v13, v42, 16, v13
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_sdwa v42, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
+; GFX9-NEXT: v_or_b32_sdwa v13, v42, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v12, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
-; GFX9-NEXT: v_add_u32_e32 v43, 0x300, v0
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
+; GFX9-NEXT: v_add_u32_sdwa v43, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX9-NEXT: v_perm_b32 v32, s22, v32, v0
-; GFX9-NEXT: v_add_u32_e32 v32, 0x300, v32
+; GFX9-NEXT: v_add_u32_sdwa v32, v32, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_perm_b32 v33, s16, v33, v0
; GFX9-NEXT: v_add_u32_e32 v33, 0x300, v33
-; GFX9-NEXT: v_and_b32_e32 v33, 0xffff, v33
-; GFX9-NEXT: v_lshl_or_b32 v12, v43, 16, v12
+; GFX9-NEXT: v_or_b32_sdwa v12, v43, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v1, s74, v1, v0
; GFX9-NEXT: v_add_u32_e32 v11, 0x300, v1
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_add_u32_e32 v1, 3, v1
-; GFX9-NEXT: v_perm_b32 v1, v1, v2, s4
-; GFX9-NEXT: v_add_u32_e32 v44, 0x300, v1
+; GFX9-NEXT: v_perm_b32 v1, v1, v2, s5
+; GFX9-NEXT: v_add_u32_sdwa v44, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v1, s62
; GFX9-NEXT: v_perm_b32 v1, s72, v1, v0
; GFX9-NEXT: v_add_u32_e32 v10, 0x300, v1
; GFX9-NEXT: v_mov_b32_e32 v1, s44
; GFX9-NEXT: v_perm_b32 v1, s56, v1, v0
-; GFX9-NEXT: v_add_u32_e32 v45, 0x300, v1
+; GFX9-NEXT: v_add_u32_sdwa v45, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v1, s57
; GFX9-NEXT: v_perm_b32 v1, s60, v1, v0
; GFX9-NEXT: v_add_u32_e32 v9, 0x300, v1
; GFX9-NEXT: v_mov_b32_e32 v1, s13
; GFX9-NEXT: v_perm_b32 v1, s40, v1, v0
-; GFX9-NEXT: v_add_u32_e32 v46, 0x300, v1
+; GFX9-NEXT: v_add_u32_sdwa v46, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v1, s41
; GFX9-NEXT: v_perm_b32 v1, s76, v1, v0
; GFX9-NEXT: v_add_u32_e32 v8, 0x300, v1
; GFX9-NEXT: v_mov_b32_e32 v1, s9
; GFX9-NEXT: v_perm_b32 v1, s11, v1, v0
-; GFX9-NEXT: v_add_u32_e32 v47, 0x300, v1
+; GFX9-NEXT: v_add_u32_sdwa v47, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v1, s73
; GFX9-NEXT: v_perm_b32 v1, s75, v1, v0
; GFX9-NEXT: v_add_u32_e32 v7, 0x300, v1
; GFX9-NEXT: v_mov_b32_e32 v1, s45
; GFX9-NEXT: v_perm_b32 v1, s58, v1, v0
-; GFX9-NEXT: v_add_u32_e32 v56, 0x300, v1
+; GFX9-NEXT: v_add_u32_sdwa v56, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v1, s61
; GFX9-NEXT: v_perm_b32 v1, s63, v1, v0
; GFX9-NEXT: v_add_u32_e32 v6, 0x300, v1
; GFX9-NEXT: v_mov_b32_e32 v1, s42
; GFX9-NEXT: v_perm_b32 v1, s46, v1, v0
-; GFX9-NEXT: v_add_u32_e32 v57, 0x300, v1
+; GFX9-NEXT: v_add_u32_sdwa v57, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v1, s47
; GFX9-NEXT: v_perm_b32 v1, s59, v1, v0
; GFX9-NEXT: v_add_u32_e32 v5, 0x300, v1
; GFX9-NEXT: v_mov_b32_e32 v1, s12
; GFX9-NEXT: v_perm_b32 v1, s14, v1, v0
-; GFX9-NEXT: v_add_u32_e32 v58, 0x300, v1
+; GFX9-NEXT: v_add_u32_sdwa v58, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v1, s15
; GFX9-NEXT: v_perm_b32 v1, s43, v1, v0
; GFX9-NEXT: v_add_u32_e32 v4, 0x300, v1
; GFX9-NEXT: v_mov_b32_e32 v1, s8
; GFX9-NEXT: v_perm_b32 v1, s10, v1, v0
-; GFX9-NEXT: v_add_u32_e32 v59, 0x300, v1
+; GFX9-NEXT: v_add_u32_sdwa v59, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v1, s29
; GFX9-NEXT: v_perm_b32 v1, s28, v1, v0
; GFX9-NEXT: v_add_u32_e32 v3, 0x300, v1
; GFX9-NEXT: v_mov_b32_e32 v1, s6
; GFX9-NEXT: v_perm_b32 v1, s7, v1, v0
-; GFX9-NEXT: v_add_u32_e32 v60, 0x300, v1
+; GFX9-NEXT: v_add_u32_sdwa v60, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v1, s25
; GFX9-NEXT: v_perm_b32 v1, s24, v1, v0
; GFX9-NEXT: v_add_u32_e32 v2, 0x300, v1
; GFX9-NEXT: v_mov_b32_e32 v1, s27
; GFX9-NEXT: v_perm_b32 v1, s26, v1, v0
-; GFX9-NEXT: v_add_u32_e32 v61, 0x300, v1
+; GFX9-NEXT: v_add_u32_sdwa v61, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v1, s21
; GFX9-NEXT: v_perm_b32 v1, s20, v1, v0
; GFX9-NEXT: v_add_u32_e32 v1, 0x300, v1
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_lshl_or_b32 v1, v32, 16, v1
-; GFX9-NEXT: v_and_b32_e32 v32, 0xffff, v54
-; GFX9-NEXT: v_lshl_or_b32 v17, v17, 16, v32
-; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v29, 0xffff, v29
-; GFX9-NEXT: v_and_b32_e32 v26, 0xffff, v26
-; GFX9-NEXT: v_lshl_or_b32 v29, v30, 16, v29
-; GFX9-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
-; GFX9-NEXT: v_lshl_or_b32 v26, v31, 16, v26
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
+; GFX9-NEXT: v_or_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
; GFX9-NEXT: v_perm_b32 v0, s18, v34, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v27, v28, 16, v27
-; GFX9-NEXT: v_and_b32_e32 v28, 0xffff, v38
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v33
-; GFX9-NEXT: v_lshl_or_b32 v2, v61, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v60, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v59, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v58, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v57, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v56, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v47, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v46, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v45, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v44, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v28, v62, 16, v28
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_and_b32_e32 v30, 0xffff, v30
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_lshl_or_b32 v30, v31, 16, v30
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_and_b32_e32 v31, 0xffff, v31
-; GFX9-NEXT: v_lshl_or_b32 v31, v32, 16, v31
+; GFX9-NEXT: v_add_u32_sdwa v0, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v61, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v60, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v4, v59, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v58, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v57, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v56, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v47, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v46, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v45, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v44, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_or_b32_sdwa v31, v32, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: .LBB93_5: ; %end
; GFX9-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
@@ -195194,25 +195152,25 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX9-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
+; GFX9-NEXT: v_readfirstlane_b32 s42, v13
; GFX9-NEXT: v_readfirstlane_b32 s46, v12
; GFX9-NEXT: v_readfirstlane_b32 s61, v11
-; GFX9-NEXT: v_readfirstlane_b32 s63, v10
+; GFX9-NEXT: v_readfirstlane_b32 s15, v3
; GFX9-NEXT: v_readfirstlane_b32 s7, v0
-; GFX9-NEXT: buffer_load_ushort v10, off, s[0:3], s32 offset:8
-; GFX9-NEXT: buffer_load_ushort v11, off, s[0:3], s32 offset:4
-; GFX9-NEXT: buffer_load_ushort v12, off, s[0:3], s32
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:328
-; GFX9-NEXT: v_readfirstlane_b32 s43, v2
-; GFX9-NEXT: v_readfirstlane_b32 s6, v1
+; GFX9-NEXT: buffer_load_ushort v11, off, s[0:3], s32 offset:8
+; GFX9-NEXT: buffer_load_ushort v12, off, s[0:3], s32 offset:4
+; GFX9-NEXT: buffer_load_ushort v13, off, s[0:3], s32
+; GFX9-NEXT: buffer_load_ushort v3, off, s[0:3], s32 offset:328
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:324
; GFX9-NEXT: v_readfirstlane_b32 s8, v5
; GFX9-NEXT: v_readfirstlane_b32 s10, v4
-; GFX9-NEXT: v_readfirstlane_b32 s15, v3
+; GFX9-NEXT: v_readfirstlane_b32 s43, v2
+; GFX9-NEXT: v_readfirstlane_b32 s6, v1
+; GFX9-NEXT: v_readfirstlane_b32 s63, v10
; GFX9-NEXT: v_readfirstlane_b32 s12, v9
; GFX9-NEXT: v_readfirstlane_b32 s14, v8
; GFX9-NEXT: v_readfirstlane_b32 s47, v7
-; GFX9-NEXT: v_readfirstlane_b32 s59, v6
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:332
-; GFX9-NEXT: v_readfirstlane_b32 s42, v13
; GFX9-NEXT: v_readfirstlane_b32 s74, v30
; GFX9-NEXT: v_readfirstlane_b32 s44, v29
; GFX9-NEXT: v_readfirstlane_b32 s56, v28
@@ -195230,401 +195188,403 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX9-NEXT: v_readfirstlane_b32 s58, v16
; GFX9-NEXT: v_readfirstlane_b32 s73, v15
; GFX9-NEXT: v_readfirstlane_b32 s75, v14
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:472 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:324
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:320
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:484 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:316
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:480 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v2, off, s[0:3], s32 offset:312
-; GFX9-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:308
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:304
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:496 ; 4-byte Folded Spill
+; GFX9-NEXT: v_readfirstlane_b32 s59, v6
+; GFX9-NEXT: buffer_load_ushort v39, off, s[0:3], s32 offset:296
+; GFX9-NEXT: buffer_load_ushort v50, off, s[0:3], s32 offset:292
+; GFX9-NEXT: buffer_load_ushort v62, off, s[0:3], s32 offset:288
+; GFX9-NEXT: buffer_load_ushort v6, off, s[0:3], s32 offset:284
+; GFX9-NEXT: s_waitcnt vmcnt(5)
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:320
+; GFX9-NEXT: buffer_load_ushort v2, off, s[0:3], s32 offset:316
+; GFX9-NEXT: buffer_load_ushort v4, off, s[0:3], s32 offset:312
+; GFX9-NEXT: buffer_load_ushort v5, off, s[0:3], s32 offset:308
+; GFX9-NEXT: buffer_load_ushort v56, off, s[0:3], s32 offset:304
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:300
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:492 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v4, off, s[0:3], s32 offset:296
-; GFX9-NEXT: buffer_load_ushort v5, off, s[0:3], s32 offset:292
-; GFX9-NEXT: buffer_load_ushort v3, off, s[0:3], s32 offset:288
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:284
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:488 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v8, off, s[0:3], s32 offset:280
-; GFX9-NEXT: buffer_load_ushort v7, off, s[0:3], s32 offset:276
-; GFX9-NEXT: buffer_load_ushort v6, off, s[0:3], s32 offset:272
-; GFX9-NEXT: buffer_load_ushort v63, off, s[0:3], s32 offset:268
-; GFX9-NEXT: buffer_load_ushort v48, off, s[0:3], s32 offset:264
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v63, off, s[0:3], s32 offset:280
+; GFX9-NEXT: buffer_load_ushort v38, off, s[0:3], s32 offset:276
+; GFX9-NEXT: buffer_load_ushort v52, off, s[0:3], s32 offset:272
+; GFX9-NEXT: buffer_load_ushort v7, off, s[0:3], s32 offset:268
+; GFX9-NEXT: buffer_load_ushort v8, off, s[0:3], s32 offset:264
; GFX9-NEXT: buffer_load_ushort v54, off, s[0:3], s32 offset:260
-; GFX9-NEXT: buffer_load_ushort v61, off, s[0:3], s32 offset:256
-; GFX9-NEXT: buffer_load_ushort v33, off, s[0:3], s32 offset:252
-; GFX9-NEXT: buffer_load_ushort v9, off, s[0:3], s32 offset:248
-; GFX9-NEXT: buffer_load_ushort v62, off, s[0:3], s32 offset:244
+; GFX9-NEXT: buffer_load_ushort v9, off, s[0:3], s32 offset:256
+; GFX9-NEXT: buffer_load_ushort v37, off, s[0:3], s32 offset:252
+; GFX9-NEXT: buffer_load_ushort v10, off, s[0:3], s32 offset:248
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:244
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:428 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:240
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:464 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v43, off, s[0:3], s32 offset:236
-; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:452 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:236
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:432 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:232
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:500 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:464 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:228
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:436 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:400 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:224
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:436 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:220
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:468 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:440 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:216
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:504 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:472 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:212
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:468 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:208
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:440 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:204
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:400 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:408 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v45, off, s[0:3], s32 offset:204
+; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:200
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:512 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:488 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:196
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:484 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:192
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v51, off, s[0:3], s32 offset:188
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:184
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:492 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:180
; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_ushort v61, off, s[0:3], s32 offset:140
+; GFX9-NEXT: buffer_load_ushort v36, off, s[0:3], s32 offset:136
+; GFX9-NEXT: buffer_load_ushort v60, off, s[0:3], s32 offset:132
+; GFX9-NEXT: buffer_load_ushort v43, off, s[0:3], s32 offset:128
+; GFX9-NEXT: buffer_load_ushort v58, off, s[0:3], s32 offset:124
+; GFX9-NEXT: buffer_load_ushort v34, off, s[0:3], s32 offset:120
+; GFX9-NEXT: buffer_load_ushort v55, off, s[0:3], s32 offset:116
+; GFX9-NEXT: buffer_load_ushort v42, off, s[0:3], s32 offset:112
+; GFX9-NEXT: buffer_load_ushort v47, off, s[0:3], s32 offset:108
+; GFX9-NEXT: buffer_load_ushort v28, off, s[0:3], s32 offset:104
+; GFX9-NEXT: buffer_load_ushort v35, off, s[0:3], s32 offset:100
+; GFX9-NEXT: buffer_load_ushort v48, off, s[0:3], s32 offset:96
+; GFX9-NEXT: buffer_load_ushort v44, off, s[0:3], s32 offset:92
+; GFX9-NEXT: buffer_load_ushort v24, off, s[0:3], s32 offset:88
+; GFX9-NEXT: buffer_load_ushort v29, off, s[0:3], s32 offset:84
+; GFX9-NEXT: buffer_load_ushort v53, off, s[0:3], s32 offset:80
+; GFX9-NEXT: buffer_load_ushort v49, off, s[0:3], s32 offset:76
+; GFX9-NEXT: buffer_load_ushort v20, off, s[0:3], s32 offset:72
+; GFX9-NEXT: buffer_load_ushort v25, off, s[0:3], s32 offset:68
+; GFX9-NEXT: buffer_load_ushort v30, off, s[0:3], s32 offset:64
+; GFX9-NEXT: buffer_load_ushort v33, off, s[0:3], s32 offset:60
+; GFX9-NEXT: buffer_load_ushort v17, off, s[0:3], s32 offset:56
+; GFX9-NEXT: buffer_load_ushort v21, off, s[0:3], s32 offset:52
+; GFX9-NEXT: buffer_load_ushort v26, off, s[0:3], s32 offset:48
+; GFX9-NEXT: buffer_load_ushort v31, off, s[0:3], s32 offset:44
+; GFX9-NEXT: buffer_load_ushort v15, off, s[0:3], s32 offset:40
+; GFX9-NEXT: buffer_load_ushort v18, off, s[0:3], s32 offset:36
+; GFX9-NEXT: buffer_load_ushort v22, off, s[0:3], s32 offset:32
+; GFX9-NEXT: buffer_load_ushort v27, off, s[0:3], s32 offset:28
+; GFX9-NEXT: buffer_load_ushort v14, off, s[0:3], s32 offset:24
+; GFX9-NEXT: buffer_load_ushort v16, off, s[0:3], s32 offset:20
+; GFX9-NEXT: buffer_load_ushort v19, off, s[0:3], s32 offset:16
+; GFX9-NEXT: buffer_load_ushort v23, off, s[0:3], s32 offset:12
; GFX9-NEXT: s_and_b64 s[4:5], vcc, exec
+; GFX9-NEXT: s_waitcnt vmcnt(33)
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:496 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v32, off, s[0:3], s32 offset:176
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:172
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:508 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:192
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:480 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v46, off, s[0:3], s32 offset:168
+; GFX9-NEXT: buffer_load_ushort v57, off, s[0:3], s32 offset:164
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_ushort v13, off, s[0:3], s32 offset:188
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:444 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v47, off, s[0:3], s32 offset:184
-; GFX9-NEXT: buffer_load_ushort v59, off, s[0:3], s32 offset:180
-; GFX9-NEXT: buffer_load_ushort v42, off, s[0:3], s32 offset:176
-; GFX9-NEXT: buffer_load_ushort v58, off, s[0:3], s32 offset:172
-; GFX9-NEXT: buffer_load_ushort v44, off, s[0:3], s32 offset:168
-; GFX9-NEXT: buffer_load_ushort v56, off, s[0:3], s32 offset:164
-; GFX9-NEXT: buffer_load_ushort v60, off, s[0:3], s32 offset:160
-; GFX9-NEXT: buffer_load_ushort v53, off, s[0:3], s32 offset:156
-; GFX9-NEXT: buffer_load_ushort v49, off, s[0:3], s32 offset:152
-; GFX9-NEXT: buffer_load_ushort v45, off, s[0:3], s32 offset:148
-; GFX9-NEXT: buffer_load_ushort v57, off, s[0:3], s32 offset:144
-; GFX9-NEXT: buffer_load_ushort v55, off, s[0:3], s32 offset:140
-; GFX9-NEXT: buffer_load_ushort v38, off, s[0:3], s32 offset:136
-; GFX9-NEXT: buffer_load_ushort v32, off, s[0:3], s32 offset:132
-; GFX9-NEXT: buffer_load_ushort v46, off, s[0:3], s32 offset:128
-; GFX9-NEXT: buffer_load_ushort v40, off, s[0:3], s32 offset:124
-; GFX9-NEXT: buffer_load_ushort v31, off, s[0:3], s32 offset:120
-; GFX9-NEXT: buffer_load_ushort v52, off, s[0:3], s32 offset:116
-; GFX9-NEXT: buffer_load_ushort v50, off, s[0:3], s32 offset:112
-; GFX9-NEXT: buffer_load_ushort v41, off, s[0:3], s32 offset:108
-; GFX9-NEXT: buffer_load_ushort v27, off, s[0:3], s32 offset:104
-; GFX9-NEXT: buffer_load_ushort v37, off, s[0:3], s32 offset:100
-; GFX9-NEXT: buffer_load_ushort v36, off, s[0:3], s32 offset:96
-; GFX9-NEXT: buffer_load_ushort v51, off, s[0:3], s32 offset:92
-; GFX9-NEXT: buffer_load_ushort v23, off, s[0:3], s32 offset:88
-; GFX9-NEXT: buffer_load_ushort v28, off, s[0:3], s32 offset:84
-; GFX9-NEXT: buffer_load_ushort v34, off, s[0:3], s32 offset:80
-; GFX9-NEXT: buffer_load_ushort v39, off, s[0:3], s32 offset:76
-; GFX9-NEXT: buffer_load_ushort v19, off, s[0:3], s32 offset:72
-; GFX9-NEXT: buffer_load_ushort v24, off, s[0:3], s32 offset:68
-; GFX9-NEXT: buffer_load_ushort v29, off, s[0:3], s32 offset:64
-; GFX9-NEXT: buffer_load_ushort v35, off, s[0:3], s32 offset:60
-; GFX9-NEXT: buffer_load_ushort v16, off, s[0:3], s32 offset:56
-; GFX9-NEXT: buffer_load_ushort v20, off, s[0:3], s32 offset:52
-; GFX9-NEXT: buffer_load_ushort v25, off, s[0:3], s32 offset:48
-; GFX9-NEXT: buffer_load_ushort v30, off, s[0:3], s32 offset:44
-; GFX9-NEXT: buffer_load_ushort v14, off, s[0:3], s32 offset:40
-; GFX9-NEXT: buffer_load_ushort v17, off, s[0:3], s32 offset:36
-; GFX9-NEXT: buffer_load_ushort v21, off, s[0:3], s32 offset:32
-; GFX9-NEXT: buffer_load_ushort v26, off, s[0:3], s32 offset:28
-; GFX9-NEXT: buffer_load_ushort v13, off, s[0:3], s32 offset:24
-; GFX9-NEXT: buffer_load_ushort v15, off, s[0:3], s32 offset:20
-; GFX9-NEXT: buffer_load_ushort v18, off, s[0:3], s32 offset:16
-; GFX9-NEXT: buffer_load_ushort v22, off, s[0:3], s32 offset:12
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:516 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:520 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(6)
-; GFX9-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:528 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:532 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(7)
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(9)
-; GFX9-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(12)
-; GFX9-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:160
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:500 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v40, off, s[0:3], s32 offset:156
+; GFX9-NEXT: buffer_load_ushort v59, off, s[0:3], s32 offset:152
+; GFX9-NEXT: buffer_load_ushort v41, off, s[0:3], s32 offset:148
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:144
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:512 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:516 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:520 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:528 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:532 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:408 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v49, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
+; GFX9-NEXT: s_waitcnt vmcnt(27)
+; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:412 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:420 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:424 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:428 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:432 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:420 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:424 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:628 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:636 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v52, off, s[0:3], s32 offset:640 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:644 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v38, off, s[0:3], s32 offset:648 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:444 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:448 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v48, off, s[0:3], s32 offset:628 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:452 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:636 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:640 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:644 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:648 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:656 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:660 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v48, off, s[0:3], s32 offset:664 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:668 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:672 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:508 ; 4-byte Folded Spill
+; GFX9-NEXT: s_waitcnt vmcnt(49)
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:504 ; 4-byte Folded Spill
; GFX9-NEXT: s_cbranch_scc0 .LBB97_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v8, 0xc0c0004
-; GFX9-NEXT: v_mov_b32_e32 v2, s21
-; GFX9-NEXT: v_perm_b32 v0, s16, v0, v8
+; GFX9-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:676 ; 4-byte Folded Spill
; GFX9-NEXT: v_mov_b32_e32 v1, s19
-; GFX9-NEXT: v_perm_b32 v2, s20, v2, v8
; GFX9-NEXT: v_mov_b32_e32 v3, s23
+; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_perm_b32 v1, s18, v1, v8
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:680 ; 4-byte Folded Spill
+; GFX9-NEXT: v_mov_b32_e32 v2, s21
; GFX9-NEXT: v_perm_b32 v3, s22, v3, v8
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v3, 16, v2
-; GFX9-NEXT: v_mov_b32_e32 v2, s25
-; GFX9-NEXT: v_perm_b32 v2, s24, v2, v8
+; GFX9-NEXT: v_perm_b32 v0, s16, v0, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_perm_b32 v2, s20, v2, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v3, s27
+; GFX9-NEXT: v_mov_b32_e32 v2, s25
; GFX9-NEXT: v_perm_b32 v3, s26, v3, v8
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_lshl_or_b32 v2, v3, 16, v2
-; GFX9-NEXT: v_mov_b32_e32 v3, s29
-; GFX9-NEXT: v_perm_b32 v3, s28, v3, v8
+; GFX9-NEXT: v_perm_b32 v2, s24, v2, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_mov_b32_e32 v4, s6
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v3, s29
; GFX9-NEXT: v_perm_b32 v4, s7, v4, v8
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_lshl_or_b32 v3, v4, 16, v3
-; GFX9-NEXT: v_mov_b32_e32 v4, s15
-; GFX9-NEXT: v_perm_b32 v4, s43, v4, v8
+; GFX9-NEXT: v_perm_b32 v3, s28, v3, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX9-NEXT: v_mov_b32_e32 v5, s8
+; GFX9-NEXT: v_or_b32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v4, s15
; GFX9-NEXT: v_perm_b32 v5, s10, v5, v8
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_lshl_or_b32 v4, v5, 16, v4
-; GFX9-NEXT: v_mov_b32_e32 v5, s47
-; GFX9-NEXT: v_perm_b32 v5, s59, v5, v8
+; GFX9-NEXT: v_perm_b32 v4, s43, v4, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX9-NEXT: v_mov_b32_e32 v6, s12
+; GFX9-NEXT: v_or_b32_sdwa v4, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v5, s47
; GFX9-NEXT: v_perm_b32 v6, s14, v6, v8
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_lshl_or_b32 v5, v6, 16, v5
-; GFX9-NEXT: v_mov_b32_e32 v6, s61
-; GFX9-NEXT: v_perm_b32 v6, s63, v6, v8
+; GFX9-NEXT: v_perm_b32 v5, s59, v5, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX9-NEXT: v_mov_b32_e32 v50, v54
+; GFX9-NEXT: v_mov_b32_e32 v54, v7
; GFX9-NEXT: v_mov_b32_e32 v7, s42
+; GFX9-NEXT: v_or_b32_sdwa v5, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v6, s61
; GFX9-NEXT: v_perm_b32 v7, s46, v7, v8
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_lshl_or_b32 v6, v7, 16, v6
-; GFX9-NEXT: v_mov_b32_e32 v7, s73
-; GFX9-NEXT: v_perm_b32 v7, s75, v7, v8
-; GFX9-NEXT: v_mov_b32_e32 v63, v33
-; GFX9-NEXT: v_mov_b32_e32 v33, v9
+; GFX9-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:684 ; 4-byte Folded Spill
+; GFX9-NEXT: v_perm_b32 v6, s63, v6, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX9-NEXT: v_mov_b32_e32 v39, v9
; GFX9-NEXT: v_mov_b32_e32 v9, s45
+; GFX9-NEXT: v_or_b32_sdwa v6, v7, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v7, s73
; GFX9-NEXT: v_perm_b32 v9, s58, v9, v8
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_lshl_or_b32 v7, v9, 16, v7
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v10
-; GFX9-NEXT: v_perm_b32 v10, s74, v12, v8
-; GFX9-NEXT: v_or_b32_sdwa v9, v11, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v9, 16, v10
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v18
-; GFX9-NEXT: v_or_b32_sdwa v9, v22, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v13
-; GFX9-NEXT: v_or_b32_sdwa v10, v15, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
+; GFX9-NEXT: v_perm_b32 v7, s75, v7, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX9-NEXT: s_and_b32 s4, s40, 0xff
+; GFX9-NEXT: s_lshl_b32 s5, s13, 8
+; GFX9-NEXT: v_or_b32_sdwa v7, v9, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v9, s57
+; GFX9-NEXT: s_or_b32 s4, s4, s5
+; GFX9-NEXT: v_mov_b32_e32 v56, v38
+; GFX9-NEXT: v_perm_b32 v9, s60, v9, v8
+; GFX9-NEXT: v_mov_b32_e32 v38, v37
+; GFX9-NEXT: v_mov_b32_e32 v37, v10
+; GFX9-NEXT: v_lshlrev_b32_e64 v10, 16, s4
+; GFX9-NEXT: v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v11
+; GFX9-NEXT: v_perm_b32 v8, s74, v13, v8
+; GFX9-NEXT: v_or_b32_sdwa v10, v12, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v11, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v19
+; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v14
+; GFX9-NEXT: v_or_b32_sdwa v8, v23, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v10, v16, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v12, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_mov_b32 s4, 0xc0c0004
-; GFX9-NEXT: v_lshl_or_b32 v12, v10, 16, v9
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v14
-; GFX9-NEXT: v_perm_b32 v10, v26, v21, s4
-; GFX9-NEXT: v_or_b32_sdwa v9, v17, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v13, v9, 16, v10
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v25
-; GFX9-NEXT: v_or_b32_sdwa v9, v30, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v16
-; GFX9-NEXT: v_or_b32_sdwa v10, v20, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_lshl_or_b32 v14, v10, 16, v9
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v19
-; GFX9-NEXT: v_perm_b32 v10, v35, v29, s4
-; GFX9-NEXT: v_or_b32_sdwa v9, v24, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v15, v9, 16, v10
-; GFX9-NEXT: v_perm_b32 v9, v39, v34, s4
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_perm_b32 v10, v51, v36, s4
-; GFX9-NEXT: v_perm_b32 v16, v28, v23, s4
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_perm_b32 v17, v37, v27, s4
-; GFX9-NEXT: v_lshl_or_b32 v16, v16, 16, v9
-; GFX9-NEXT: v_perm_b32 v9, v41, v50, s4
-; GFX9-NEXT: v_lshl_or_b32 v17, v17, 16, v10
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_perm_b32 v10, v52, v31, s4
-; GFX9-NEXT: v_lshl_or_b32 v18, v10, 16, v9
-; GFX9-NEXT: v_perm_b32 v9, v40, v46, s4
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_perm_b32 v10, v32, v38, s4
-; GFX9-NEXT: v_lshl_or_b32 v19, v10, 16, v9
-; GFX9-NEXT: v_perm_b32 v9, v55, v57, s4
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_perm_b32 v10, v45, v49, s4
-; GFX9-NEXT: v_lshl_or_b32 v20, v10, 16, v9
-; GFX9-NEXT: v_perm_b32 v9, v53, v60, s4
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_perm_b32 v10, v56, v44, s4
-; GFX9-NEXT: v_lshl_or_b32 v21, v10, 16, v9
-; GFX9-NEXT: v_perm_b32 v9, v58, v42, s4
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_perm_b32 v10, v59, v47, s4
-; GFX9-NEXT: v_lshl_or_b32 v22, v10, 16, v9
-; GFX9-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
-; GFX9-NEXT: v_mov_b32_e32 v50, v32
-; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
-; GFX9-NEXT: v_mov_b32_e32 v51, v49
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v15
+; GFX9-NEXT: v_or_b32_sdwa v8, v18, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_perm_b32 v10, v27, v22, s4
+; GFX9-NEXT: v_perm_b32 v16, v49, v53, s4
; GFX9-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:404 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
-; GFX9-NEXT: v_mov_b32_e32 v41, v46
-; GFX9-NEXT: v_mov_b32_e32 v39, v52
-; GFX9-NEXT: v_mov_b32_e32 v52, v38
-; GFX9-NEXT: v_mov_b32_e32 v34, v37
-; GFX9-NEXT: v_mov_b32_e32 v37, v31
-; GFX9-NEXT: s_lshl_b32 s5, s13, 8
+; GFX9-NEXT: v_or_b32_sdwa v13, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v17
+; GFX9-NEXT: v_perm_b32 v17, v44, v48, s4
+; GFX9-NEXT: v_mov_b32_e32 v44, v55
+; GFX9-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v26
+; GFX9-NEXT: v_or_b32_sdwa v8, v31, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v10, v21, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v14, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v20
+; GFX9-NEXT: v_or_b32_sdwa v8, v25, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_perm_b32 v10, v33, v30, s4
+; GFX9-NEXT: v_or_b32_sdwa v15, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_perm_b32 v8, v29, v24, s4
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX9-NEXT: v_perm_b32 v10, v35, v28, s4
+; GFX9-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX9-NEXT: v_or_b32_sdwa v16, v8, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_perm_b32 v8, v44, v34, s4
+; GFX9-NEXT: v_or_b32_sdwa v17, v10, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX9-NEXT: v_perm_b32 v10, v47, v42, s4
+; GFX9-NEXT: v_mov_b32_e32 v34, v36
+; GFX9-NEXT: v_mov_b32_e32 v35, v60
+; GFX9-NEXT: v_or_b32_sdwa v18, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_perm_b32 v8, v35, v34, s4
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX9-NEXT: v_perm_b32 v10, v58, v43, s4
+; GFX9-NEXT: v_mov_b32_e32 v53, v59
+; GFX9-NEXT: v_or_b32_sdwa v19, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_perm_b32 v8, v41, v53, s4
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX9-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:484 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
+; GFX9-NEXT: s_lshl_b32 s5, s41, 8
; GFX9-NEXT: s_lshl_b32 s77, s9, 8
; GFX9-NEXT: s_lshl_b32 s78, s44, 8
-; GFX9-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:452 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:428 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(9)
-; GFX9-NEXT: v_perm_b32 v9, v9, v10, s4
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: s_waitcnt vmcnt(7)
-; GFX9-NEXT: v_perm_b32 v10, v32, v58, s4
-; GFX9-NEXT: v_lshl_or_b32 v23, v10, 16, v9
-; GFX9-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:440 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:400 ; 4-byte Folded Reload
+; GFX9-NEXT: v_perm_b32 v10, v61, v49, s4
+; GFX9-NEXT: v_or_b32_sdwa v20, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_perm_b32 v8, v57, v46, s4
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX9-NEXT: s_waitcnt vmcnt(8)
+; GFX9-NEXT: v_perm_b32 v10, v40, v55, s4
+; GFX9-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
+; GFX9-NEXT: v_or_b32_sdwa v21, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: s_waitcnt vmcnt(9)
+; GFX9-NEXT: v_perm_b32 v10, v33, v32, s4
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v9
+; GFX9-NEXT: v_perm_b32 v8, v60, v40, s4
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX9-NEXT: v_or_b32_sdwa v22, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_perm_b32 v8, v42, v59, s4
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX9-NEXT: v_perm_b32 v10, v51, v36, s4
+; GFX9-NEXT: v_or_b32_sdwa v23, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v8
+; GFX9-NEXT: v_or_b32_sdwa v8, v45, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v51
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v49
-; GFX9-NEXT: v_or_b32_sdwa v10, v24, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_lshl_or_b32 v24, v10, 16, v9
-; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:436 ; 4-byte Folded Reload
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v35
+; GFX9-NEXT: v_or_b32_sdwa v10, v45, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v24, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:400 ; 4-byte Folded Reload
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v32
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
+; GFX9-NEXT: v_or_b32_sdwa v8, v10, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:436 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:440 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v10, v25, v10, s4
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v25, v9, 16, v10
-; GFX9-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
-; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v33
-; GFX9-NEXT: v_or_b32_sdwa v10, v62, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
-; GFX9-NEXT: v_mov_b32_e32 v33, v46
+; GFX9-NEXT: v_or_b32_sdwa v25, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:452 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:432 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v9
-; GFX9-NEXT: v_or_b32_sdwa v9, v43, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_lshl_or_b32 v26, v10, 16, v9
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v48
-; GFX9-NEXT: v_perm_b32 v10, v63, v61, s4
-; GFX9-NEXT: v_or_b32_sdwa v9, v54, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v27, v9, 16, v10
-; GFX9-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
-; GFX9-NEXT: v_mov_b32_e32 v54, v61
-; GFX9-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:484 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(4)
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v9
-; GFX9-NEXT: s_waitcnt vmcnt(3)
-; GFX9-NEXT: v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_mov_b32_e32 v48, v63
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v10
-; GFX9-NEXT: v_or_b32_sdwa v10, v28, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_lshl_or_b32 v28, v10, 16, v9
-; GFX9-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:428 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v8
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_or_b32_sdwa v8, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v37
+; GFX9-NEXT: v_or_b32_sdwa v10, v26, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v26, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:424 ; 4-byte Folded Reload
+; GFX9-NEXT: v_perm_b32 v10, v38, v39, s4
+; GFX9-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
+; GFX9-NEXT: v_mov_b32_e32 v38, v34
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v8
+; GFX9-NEXT: v_or_b32_sdwa v8, v50, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v27, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v52
+; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v63
+; GFX9-NEXT: v_or_b32_sdwa v8, v54, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v10, v56, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v28, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
+; GFX9-NEXT: v_mov_b32_e32 v54, v31
+; GFX9-NEXT: v_mov_b32_e32 v50, v53
+; GFX9-NEXT: v_mov_b32_e32 v56, v40
+; GFX9-NEXT: v_mov_b32_e32 v52, v32
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v9
+; GFX9-NEXT: v_mov_b32_e32 v63, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v8
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:432 ; 4-byte Folded Reload
+; GFX9-NEXT: v_or_b32_sdwa v8, v10, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v10, v36, v10, s4
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v29, v9, 16, v10
-; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:424 ; 4-byte Folded Reload
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v43
-; GFX9-NEXT: v_or_b32_sdwa v9, v62, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
+; GFX9-NEXT: v_perm_b32 v10, v10, v62, s4
+; GFX9-NEXT: v_or_b32_sdwa v29, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:404 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v8
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v10
-; GFX9-NEXT: v_or_b32_sdwa v10, v30, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_lshl_or_b32 v30, v10, 16, v9
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v38
-; GFX9-NEXT: v_perm_b32 v10, v63, v61, s4
-; GFX9-NEXT: v_or_b32_sdwa v9, v46, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v31, v9, 16, v10
-; GFX9-NEXT: v_mov_b32_e32 v9, s57
-; GFX9-NEXT: v_perm_b32 v8, s60, v9, v8
-; GFX9-NEXT: s_and_b32 s4, s40, 0xff
-; GFX9-NEXT: s_or_b32 s4, s4, s5
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, s4, 16, v8
+; GFX9-NEXT: v_or_b32_sdwa v8, v39, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v10, v30, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v30, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v8
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v62, v10
+; GFX9-NEXT: v_perm_b32 v10, v31, v10, s4
; GFX9-NEXT: s_and_b32 s4, s76, 0xff
-; GFX9-NEXT: s_lshl_b32 s5, s41, 8
; GFX9-NEXT: s_or_b32 s4, s4, s5
; GFX9-NEXT: s_and_b32 s5, s11, 0xff
; GFX9-NEXT: s_or_b32 s5, s5, s77
@@ -195634,60 +195594,54 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX9-NEXT: s_or_b32 s5, s5, s77
; GFX9-NEXT: s_and_b32 s77, s56, 0xff
; GFX9-NEXT: s_or_b32 s77, s77, s78
+; GFX9-NEXT: v_or_b32_sdwa v8, v48, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: s_pack_ll_b32_b16 s5, s5, s77
+; GFX9-NEXT: v_or_b32_sdwa v31, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v8, s4
; GFX9-NEXT: v_mov_b32_e32 v10, s5
; GFX9-NEXT: s_mov_b64 s[4:5], 0
-; GFX9-NEXT: v_mov_b32_e32 v63, v36
; GFX9-NEXT: s_branch .LBB97_3
; GFX9-NEXT: .LBB97_2:
-; GFX9-NEXT: v_mov_b32_e32 v54, v61
-; GFX9-NEXT: v_mov_b32_e32 v50, v32
-; GFX9-NEXT: v_mov_b32_e32 v51, v49
-; GFX9-NEXT: v_mov_b32_e32 v39, v52
-; GFX9-NEXT: v_mov_b32_e32 v52, v38
-; GFX9-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:484 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
+; GFX9-NEXT: v_mov_b32_e32 v63, v39
+; GFX9-NEXT: v_mov_b32_e32 v38, v36
+; GFX9-NEXT: v_mov_b32_e32 v44, v55
+; GFX9-NEXT: v_mov_b32_e32 v50, v59
+; GFX9-NEXT: v_mov_b32_e32 v35, v60
+; GFX9-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
-; GFX9-NEXT: v_mov_b32_e32 v41, v46
-; GFX9-NEXT: v_mov_b32_e32 v34, v37
-; GFX9-NEXT: v_mov_b32_e32 v37, v31
+; GFX9-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:484 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
+; GFX9-NEXT: v_mov_b32_e32 v54, v2
+; GFX9-NEXT: v_mov_b32_e32 v62, v1
; GFX9-NEXT: s_mov_b64 s[4:5], -1
; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
; GFX9-NEXT: ; implicit-def: $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
; GFX9-NEXT: .LBB97_3: ; %Flow
-; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:400 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
; GFX9-NEXT: s_cselect_b32 s4, 1, 0
; GFX9-NEXT: s_cmp_lg_u32 s4, 1
; GFX9-NEXT: s_cbranch_scc1 .LBB97_5
; GFX9-NEXT: ; %bb.4: ; %cmp.true
-; GFX9-NEXT: s_waitcnt vmcnt(5)
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v48
-; GFX9-NEXT: s_mov_b32 s4, 0xc0c0004
-; GFX9-NEXT: v_perm_b32 v0, v0, v61, s4
-; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:472 ; 4-byte Folded Spill
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v33
-; GFX9-NEXT: s_waitcnt vmcnt(4)
-; GFX9-NEXT: v_perm_b32 v0, v0, v38, s4
-; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v62
-; GFX9-NEXT: v_perm_b32 v0, v0, v43, s4
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v54
+; GFX9-NEXT: s_mov_b32 s5, 0xc0c0004
+; GFX9-NEXT: v_perm_b32 v0, v0, v62, s5
; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:480 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
+; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v48
+; GFX9-NEXT: s_movk_i32 s4, 0x300
; GFX9-NEXT: s_add_i32 s74, s74, 3
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:424 ; 4-byte Folded Reload
; GFX9-NEXT: s_add_i32 s72, s72, 3
; GFX9-NEXT: s_add_i32 s56, s56, 3
; GFX9-NEXT: s_add_i32 s60, s60, 3
@@ -195708,373 +195662,358 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX9-NEXT: s_add_i32 s26, s26, 3
; GFX9-NEXT: s_add_i32 s20, s20, 3
; GFX9-NEXT: s_add_i32 s22, s22, 3
+; GFX9-NEXT: v_mov_b32_e32 v32, s23
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: v_mov_b32_e32 v33, s17
; GFX9-NEXT: s_add_i32 s18, s18, 3
-; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
+; GFX9-NEXT: v_mov_b32_e32 v34, s19
+; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:432 ; 4-byte Folded Reload
-; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:408 ; 4-byte Folded Spill
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v63
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:404 ; 4-byte Folded Reload
+; GFX9-NEXT: v_add_u32_sdwa v0, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:444 ; 4-byte Folded Spill
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v39
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:424 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:428 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_sdwa v0, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:636 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:428 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:412 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT: v_perm_b32 v0, v0, v63, s5
+; GFX9-NEXT: v_add_u32_sdwa v0, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:448 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: v_add_u32_e32 v38, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:452 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_e32 v39, 0x300, v0
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: v_add_u32_e32 v62, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_sdwa v63, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:628 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT: v_perm_b32 v0, v0, v54, s4
-; GFX9-NEXT: v_add_u32_e32 v63, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:424 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_sdwa v48, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:432 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:452 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:412 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:632 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:420 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:428 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: v_add_u32_e32 v31, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_sdwa v31, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:440 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:436 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v30, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:436 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:440 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:400 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT: v_perm_b32 v0, v0, v35, s4
-; GFX9-NEXT: v_add_u32_e32 v29, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v36
+; GFX9-NEXT: v_perm_b32 v0, v0, v52, s5
+; GFX9-NEXT: v_add_u32_sdwa v29, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:632 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v28, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:404 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v45
+; GFX9-NEXT: v_perm_b32 v0, v0, v51, s5
+; GFX9-NEXT: v_add_u32_sdwa v27, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT: v_perm_b32 v0, v0, v49, s4
-; GFX9-NEXT: v_add_u32_e32 v27, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v36, s5
; GFX9-NEXT: v_add_u32_e32 v26, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v32
-; GFX9-NEXT: v_perm_b32 v0, v0, v58, s4
-; GFX9-NEXT: v_add_u32_e32 v25, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
-; GFX9-NEXT: v_mov_b32_e32 v32, s23
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v42
+; GFX9-NEXT: v_perm_b32 v0, v0, v59, s5
+; GFX9-NEXT: v_add_u32_sdwa v25, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v33
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_e32 v24, 0x300, v0
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v60
+; GFX9-NEXT: v_perm_b32 v0, v0, v56, s5
+; GFX9-NEXT: v_add_u32_sdwa v23, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
+; GFX9-NEXT: v_mov_b32_e32 v33, s17
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT: v_perm_b32 v0, v0, v42, s4
-; GFX9-NEXT: v_add_u32_e32 v24, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v59
-; GFX9-NEXT: v_perm_b32 v0, v0, v47, s4
-; GFX9-NEXT: v_add_u32_e32 v23, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v53
-; GFX9-NEXT: v_perm_b32 v0, v0, v60, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v55, s5
; GFX9-NEXT: v_add_u32_e32 v22, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v56
-; GFX9-NEXT: v_perm_b32 v0, v0, v44, s4
-; GFX9-NEXT: v_add_u32_e32 v21, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v55
-; GFX9-NEXT: v_perm_b32 v0, v0, v57, s4
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v57
+; GFX9-NEXT: v_perm_b32 v0, v0, v46, s5
+; GFX9-NEXT: v_add_u32_sdwa v21, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v61
+; GFX9-NEXT: v_perm_b32 v0, v0, v49, s5
; GFX9-NEXT: v_add_u32_e32 v20, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v45
-; GFX9-NEXT: v_perm_b32 v0, v0, v51, s4
-; GFX9-NEXT: v_add_u32_e32 v49, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v40
-; GFX9-NEXT: v_perm_b32 v0, v0, v41, s4
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v41
+; GFX9-NEXT: v_perm_b32 v0, v0, v50, s5
+; GFX9-NEXT: v_add_u32_sdwa v51, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v58
+; GFX9-NEXT: v_perm_b32 v0, v0, v43, s5
; GFX9-NEXT: v_add_u32_e32 v19, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v50
-; GFX9-NEXT: v_perm_b32 v0, v0, v52, s4
-; GFX9-NEXT: v_add_u32_e32 v52, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v22, 0xffff, v22
-; GFX9-NEXT: v_lshl_or_b32 v21, v21, 16, v22
-; GFX9-NEXT: v_and_b32_e32 v22, 0xffff, v24
-; GFX9-NEXT: v_lshl_or_b32 v22, v23, 16, v22
-; GFX9-NEXT: v_and_b32_e32 v23, 0xffff, v26
-; GFX9-NEXT: v_lshl_or_b32 v23, v25, 16, v23
-; GFX9-NEXT: v_and_b32_e32 v25, 0xffff, v30
-; GFX9-NEXT: v_lshl_or_b32 v25, v29, 16, v25
-; GFX9-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:424 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v24, 0xffff, v28
-; GFX9-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:428 ; 4-byte Folded Reload
-; GFX9-NEXT: v_lshl_or_b32 v24, v27, 16, v24
-; GFX9-NEXT: v_and_b32_e32 v27, 0xffff, v63
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_lshl_or_b32 v19, v52, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v49, 16, v20
-; GFX9-NEXT: s_waitcnt vmcnt(4)
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v35
+; GFX9-NEXT: v_perm_b32 v0, v0, v38, s5
+; GFX9-NEXT: v_add_u32_sdwa v38, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v47
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
; GFX9-NEXT: v_add_u32_e32 v18, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v39
-; GFX9-NEXT: v_perm_b32 v0, v0, v37, s4
-; GFX9-NEXT: v_add_u32_e32 v37, 0x300, v0
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v44
+; GFX9-NEXT: v_or_b32_sdwa v21, v21, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v22, v23, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v23, v25, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v25, v29, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
+; GFX9-NEXT: v_or_b32_sdwa v24, v27, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v38, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v20, v51, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v28, v63, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(4)
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_sdwa v53, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_lshl_or_b32 v18, v37, 16, v18
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
+; GFX9-NEXT: v_or_b32_sdwa v18, v53, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: s_waitcnt vmcnt(5)
+; GFX9-NEXT: v_or_b32_sdwa v26, v31, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: v_or_b32_sdwa v29, v30, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:404 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(4)
+; GFX9-NEXT: v_or_b32_sdwa v27, v48, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: s_waitcnt vmcnt(3)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
-; GFX9-NEXT: v_add_u32_e32 v54, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v34
-; GFX9-NEXT: v_mov_b32_e32 v34, s19
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v17, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: v_or_b32_sdwa v30, v31, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: v_add_u32_e32 v16, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:636 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_sdwa v54, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
+; GFX9-NEXT: v_or_b32_sdwa v17, v54, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT: v_perm_b32 v0, v0, v37, s5
+; GFX9-NEXT: v_add_u32_e32 v16, 0x300, v0
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: v_add_u32_e32 v55, 0x300, v0
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_sdwa v55, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
-; GFX9-NEXT: v_lshl_or_b32 v16, v55, 16, v16
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
+; GFX9-NEXT: v_or_b32_sdwa v16, v55, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v15, 0x300, v0
; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: v_add_u32_e32 v40, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_sdwa v40, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
-; GFX9-NEXT: v_lshl_or_b32 v15, v40, 16, v15
+; GFX9-NEXT: v_or_b32_sdwa v15, v40, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v14, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: v_add_u32_e32 v41, 0x300, v0
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_sdwa v41, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
-; GFX9-NEXT: v_lshl_or_b32 v14, v41, 16, v14
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
+; GFX9-NEXT: v_or_b32_sdwa v14, v41, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v13, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: v_add_u32_e32 v42, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
-; GFX9-NEXT: v_lshl_or_b32 v13, v42, 16, v13
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_sdwa v42, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
+; GFX9-NEXT: v_or_b32_sdwa v13, v42, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v12, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
-; GFX9-NEXT: v_add_u32_e32 v43, 0x300, v0
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
+; GFX9-NEXT: v_add_u32_sdwa v43, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX9-NEXT: v_perm_b32 v32, s22, v32, v0
-; GFX9-NEXT: v_add_u32_e32 v32, 0x300, v32
+; GFX9-NEXT: v_add_u32_sdwa v32, v32, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_perm_b32 v33, s16, v33, v0
; GFX9-NEXT: v_add_u32_e32 v33, 0x300, v33
-; GFX9-NEXT: v_and_b32_e32 v33, 0xffff, v33
-; GFX9-NEXT: v_lshl_or_b32 v12, v43, 16, v12
+; GFX9-NEXT: v_or_b32_sdwa v12, v43, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v1, s74, v1, v0
; GFX9-NEXT: v_add_u32_e32 v11, 0x300, v1
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_add_u32_e32 v1, 3, v1
-; GFX9-NEXT: v_perm_b32 v1, v1, v2, s4
-; GFX9-NEXT: v_add_u32_e32 v44, 0x300, v1
+; GFX9-NEXT: v_perm_b32 v1, v1, v2, s5
+; GFX9-NEXT: v_add_u32_sdwa v44, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v1, s62
; GFX9-NEXT: v_perm_b32 v1, s72, v1, v0
; GFX9-NEXT: v_add_u32_e32 v10, 0x300, v1
; GFX9-NEXT: v_mov_b32_e32 v1, s44
; GFX9-NEXT: v_perm_b32 v1, s56, v1, v0
-; GFX9-NEXT: v_add_u32_e32 v45, 0x300, v1
+; GFX9-NEXT: v_add_u32_sdwa v45, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v1, s57
; GFX9-NEXT: v_perm_b32 v1, s60, v1, v0
; GFX9-NEXT: v_add_u32_e32 v9, 0x300, v1
; GFX9-NEXT: v_mov_b32_e32 v1, s13
; GFX9-NEXT: v_perm_b32 v1, s40, v1, v0
-; GFX9-NEXT: v_add_u32_e32 v46, 0x300, v1
+; GFX9-NEXT: v_add_u32_sdwa v46, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v1, s41
; GFX9-NEXT: v_perm_b32 v1, s76, v1, v0
; GFX9-NEXT: v_add_u32_e32 v8, 0x300, v1
; GFX9-NEXT: v_mov_b32_e32 v1, s9
; GFX9-NEXT: v_perm_b32 v1, s11, v1, v0
-; GFX9-NEXT: v_add_u32_e32 v47, 0x300, v1
+; GFX9-NEXT: v_add_u32_sdwa v47, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v1, s73
; GFX9-NEXT: v_perm_b32 v1, s75, v1, v0
; GFX9-NEXT: v_add_u32_e32 v7, 0x300, v1
; GFX9-NEXT: v_mov_b32_e32 v1, s45
; GFX9-NEXT: v_perm_b32 v1, s58, v1, v0
-; GFX9-NEXT: v_add_u32_e32 v56, 0x300, v1
+; GFX9-NEXT: v_add_u32_sdwa v56, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v1, s61
; GFX9-NEXT: v_perm_b32 v1, s63, v1, v0
; GFX9-NEXT: v_add_u32_e32 v6, 0x300, v1
; GFX9-NEXT: v_mov_b32_e32 v1, s42
; GFX9-NEXT: v_perm_b32 v1, s46, v1, v0
-; GFX9-NEXT: v_add_u32_e32 v57, 0x300, v1
+; GFX9-NEXT: v_add_u32_sdwa v57, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v1, s47
; GFX9-NEXT: v_perm_b32 v1, s59, v1, v0
; GFX9-NEXT: v_add_u32_e32 v5, 0x300, v1
; GFX9-NEXT: v_mov_b32_e32 v1, s12
; GFX9-NEXT: v_perm_b32 v1, s14, v1, v0
-; GFX9-NEXT: v_add_u32_e32 v58, 0x300, v1
+; GFX9-NEXT: v_add_u32_sdwa v58, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v1, s15
; GFX9-NEXT: v_perm_b32 v1, s43, v1, v0
; GFX9-NEXT: v_add_u32_e32 v4, 0x300, v1
; GFX9-NEXT: v_mov_b32_e32 v1, s8
; GFX9-NEXT: v_perm_b32 v1, s10, v1, v0
-; GFX9-NEXT: v_add_u32_e32 v59, 0x300, v1
+; GFX9-NEXT: v_add_u32_sdwa v59, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v1, s29
; GFX9-NEXT: v_perm_b32 v1, s28, v1, v0
; GFX9-NEXT: v_add_u32_e32 v3, 0x300, v1
; GFX9-NEXT: v_mov_b32_e32 v1, s6
; GFX9-NEXT: v_perm_b32 v1, s7, v1, v0
-; GFX9-NEXT: v_add_u32_e32 v60, 0x300, v1
+; GFX9-NEXT: v_add_u32_sdwa v60, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v1, s25
; GFX9-NEXT: v_perm_b32 v1, s24, v1, v0
; GFX9-NEXT: v_add_u32_e32 v2, 0x300, v1
; GFX9-NEXT: v_mov_b32_e32 v1, s27
; GFX9-NEXT: v_perm_b32 v1, s26, v1, v0
-; GFX9-NEXT: v_add_u32_e32 v61, 0x300, v1
+; GFX9-NEXT: v_add_u32_sdwa v61, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v1, s21
; GFX9-NEXT: v_perm_b32 v1, s20, v1, v0
; GFX9-NEXT: v_add_u32_e32 v1, 0x300, v1
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_lshl_or_b32 v1, v32, 16, v1
-; GFX9-NEXT: v_and_b32_e32 v32, 0xffff, v54
-; GFX9-NEXT: v_lshl_or_b32 v17, v17, 16, v32
-; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v29, 0xffff, v29
-; GFX9-NEXT: v_and_b32_e32 v26, 0xffff, v26
-; GFX9-NEXT: v_lshl_or_b32 v29, v30, 16, v29
-; GFX9-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
-; GFX9-NEXT: v_lshl_or_b32 v26, v31, 16, v26
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
+; GFX9-NEXT: v_or_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
; GFX9-NEXT: v_perm_b32 v0, s18, v34, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v27, v28, 16, v27
-; GFX9-NEXT: v_and_b32_e32 v28, 0xffff, v38
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v33
-; GFX9-NEXT: v_lshl_or_b32 v2, v61, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v60, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v59, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v58, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v57, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v56, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v47, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v46, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v45, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v44, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v28, v62, 16, v28
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_and_b32_e32 v30, 0xffff, v30
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_lshl_or_b32 v30, v31, 16, v30
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_and_b32_e32 v31, 0xffff, v31
-; GFX9-NEXT: v_lshl_or_b32 v31, v32, 16, v31
+; GFX9-NEXT: v_add_u32_sdwa v0, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v61, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v60, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v4, v59, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v58, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v57, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v56, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v47, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v46, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v45, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v44, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_or_b32_sdwa v31, v32, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: .LBB97_5: ; %end
; GFX9-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
@@ -212568,18 +212507,18 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
-; GFX9-NEXT: buffer_store_dword v36, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v35, off, s[0:3], s32 ; 4-byte Folded Spill
; GFX9-NEXT: s_mov_b64 exec, s[4:5]
-; GFX9-NEXT: v_writelane_b32 v36, s36, 0
-; GFX9-NEXT: v_writelane_b32 v36, s37, 1
-; GFX9-NEXT: v_writelane_b32 v36, s38, 2
-; GFX9-NEXT: v_writelane_b32 v36, s39, 3
-; GFX9-NEXT: v_writelane_b32 v36, s48, 4
-; GFX9-NEXT: v_writelane_b32 v36, s49, 5
-; GFX9-NEXT: v_writelane_b32 v36, s50, 6
-; GFX9-NEXT: v_writelane_b32 v36, s51, 7
-; GFX9-NEXT: v_writelane_b32 v36, s30, 8
-; GFX9-NEXT: v_writelane_b32 v36, s31, 9
+; GFX9-NEXT: v_writelane_b32 v35, s36, 0
+; GFX9-NEXT: v_writelane_b32 v35, s37, 1
+; GFX9-NEXT: v_writelane_b32 v35, s38, 2
+; GFX9-NEXT: v_writelane_b32 v35, s39, 3
+; GFX9-NEXT: v_writelane_b32 v35, s48, 4
+; GFX9-NEXT: v_writelane_b32 v35, s49, 5
+; GFX9-NEXT: v_writelane_b32 v35, s50, 6
+; GFX9-NEXT: v_writelane_b32 v35, s51, 7
+; GFX9-NEXT: v_writelane_b32 v35, s30, 8
+; GFX9-NEXT: v_writelane_b32 v35, s31, 9
; GFX9-NEXT: v_readfirstlane_b32 s4, v18
; GFX9-NEXT: v_readfirstlane_b32 s51, v17
; GFX9-NEXT: v_readfirstlane_b32 s50, v16
@@ -212617,6 +212556,14 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
+; GFX9-NEXT: s_lshl_b32 s4, s50, 16
+; GFX9-NEXT: v_add_u32_e32 v1, 0x7fff, v1
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v1, v2, vcc
+; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
; GFX9-NEXT: s_and_b32 s5, s30, 0xffff0000
; GFX9-NEXT: v_add_u32_e32 v1, 0x7fff, v1
; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
@@ -212645,25 +212592,24 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: s_lshl_b32 s5, s31, 16
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_add_f32_e32 v3, s5, v16
-; GFX9-NEXT: v_bfe_u32 v4, v3, 16, 1
-; GFX9-NEXT: v_mov_b32_e32 v32, 0xffff
; GFX9-NEXT: s_and_b32 s4, s29, 0xffff0000
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_bfe_u32 v4, v3, 16, 1
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_add_u32_e32 v4, v4, v3
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_or_b32_sdwa v14, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v3
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX9-NEXT: v_lshl_or_b32 v14, v0, 16, v1
-; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX9-NEXT: v_and_b32_sdwa v3, v32, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
-; GFX9-NEXT: v_lshl_or_b32 v15, v2, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v15, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_u32_e32 v1, 0x7fff, v1
; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
@@ -212675,11 +212621,11 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: s_and_b32 s4, s28, 0xffff0000
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v13, v0, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v13, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -212694,11 +212640,11 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: s_and_b32 s4, s27, 0xffff0000
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v12, v0, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v12, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -212713,11 +212659,11 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: s_and_b32 s4, s26, 0xffff0000
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v11, v0, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v11, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -212732,11 +212678,11 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: s_and_b32 s4, s25, 0xffff0000
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v10, v0, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v10, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -212751,11 +212697,11 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: s_and_b32 s4, s24, 0xffff0000
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v9, v0, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v9, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -212770,11 +212716,11 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: s_and_b32 s4, s23, 0xffff0000
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v8, v0, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v8, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -212789,11 +212735,11 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: s_and_b32 s4, s22, 0xffff0000
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v7, v0, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v7, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -212808,11 +212754,11 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: s_and_b32 s4, s21, 0xffff0000
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v6, v0, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v6, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -212827,11 +212773,11 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: s_and_b32 s4, s20, 0xffff0000
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v5, v0, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v5, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -212846,11 +212792,11 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: s_and_b32 s4, s19, 0xffff0000
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v4, v0, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v4, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -212865,11 +212811,11 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: s_and_b32 s4, s18, 0xffff0000
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v3, v0, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v3, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -212882,59 +212828,51 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
-; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v1
+; GFX9-NEXT: v_or_b32_e32 v19, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v18, vcc
-; GFX9-NEXT: s_and_b32 s4, s17, 0xffff0000
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v2, v0, 16, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v19, vcc
+; GFX9-NEXT: s_and_b32 s4, s17, 0xffff0000
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v2, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
; GFX9-NEXT: v_add_u32_e32 v1, 0x7fff, v1
-; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v0
+; GFX9-NEXT: v_or_b32_e32 v19, 0x400000, v0
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
; GFX9-NEXT: s_lshl_b32 s4, s17, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v18, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v19, vcc
; GFX9-NEXT: v_add_f32_e32 v1, s4, v16
-; GFX9-NEXT: v_bfe_u32 v18, v1, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v18, v18, v1
-; GFX9-NEXT: v_add_u32_e32 v18, 0x7fff, v18
-; GFX9-NEXT: v_or_b32_e32 v19, 0x400000, v1
+; GFX9-NEXT: v_bfe_u32 v19, v1, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v19, v19, v1
+; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
+; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v18, v19, vcc
-; GFX9-NEXT: s_and_b32 s4, s16, 0xffff0000
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v1, v0, 16, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v19, v20, vcc
+; GFX9-NEXT: s_and_b32 s4, s16, 0xffff0000
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
-; GFX9-NEXT: v_bfe_u32 v18, v0, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v18, v18, v0
-; GFX9-NEXT: v_add_u32_e32 v18, 0x7fff, v18
-; GFX9-NEXT: v_or_b32_e32 v19, 0x400000, v0
+; GFX9-NEXT: v_bfe_u32 v19, v0, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v19, v19, v0
+; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
+; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v0
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
; GFX9-NEXT: s_lshl_b32 s4, s16, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v18, v19, vcc
-; GFX9-NEXT: v_add_f32_e32 v18, s4, v16
-; GFX9-NEXT: v_bfe_u32 v19, v18, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v19, v19, v18
-; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
-; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
-; GFX9-NEXT: s_lshl_b32 s4, s50, 16
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v19, v20, vcc
+; GFX9-NEXT: v_add_f32_e32 v19, s4, v16
+; GFX9-NEXT: v_bfe_u32 v20, v19, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v20, v20, v19
+; GFX9-NEXT: v_add_u32_e32 v20, 0x7fff, v20
+; GFX9-NEXT: v_or_b32_e32 v21, 0x400000, v19
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v19, v19
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v18
-; GFX9-NEXT: v_add_f32_e32 v18, s4, v16
-; GFX9-NEXT: v_bfe_u32 v19, v18, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v19, v19, v18
-; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
-; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
+; GFX9-NEXT: v_cndmask_b32_e32 v19, v20, v21, vcc
; GFX9-NEXT: s_and_b32 s4, s51, 0xffff0000
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v19, s4, v16
; GFX9-NEXT: v_bfe_u32 v20, v19, 16, 1
; GFX9-NEXT: v_add_u32_e32 v20, v20, v19
@@ -212944,22 +212882,22 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; GFX9-NEXT: s_lshl_b32 s4, s51, 16
; GFX9-NEXT: v_cndmask_b32_e32 v19, v20, v21, vcc
; GFX9-NEXT: v_add_f32_e32 v20, s4, v16
+; GFX9-NEXT: v_lshrrev_b32_e32 v18, 16, v18
; GFX9-NEXT: v_bfe_u32 v21, v20, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v21, v21, v20
-; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v18, 16, v18
; GFX9-NEXT: s_and_b32 s4, s49, 0xffff0000
+; GFX9-NEXT: v_add_u32_e32 v21, v21, v20
+; GFX9-NEXT: v_or_b32_sdwa v30, v18, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_add_u32_e32 v21, 0x7fff, v21
; GFX9-NEXT: v_or_b32_e32 v22, 0x400000, v20
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v20, v20
-; GFX9-NEXT: v_lshl_or_b32 v30, v17, 16, v18
-; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
-; GFX9-NEXT: v_cndmask_b32_e32 v20, v21, v22, vcc
-; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_lshrrev_b32_e32 v19, 16, v19
-; GFX9-NEXT: v_and_b32_sdwa v20, v32, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
+; GFX9-NEXT: v_cndmask_b32_e32 v20, v21, v22, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v19, 16, v19
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
-; GFX9-NEXT: v_lshl_or_b32 v31, v19, 16, v20
+; GFX9-NEXT: v_or_b32_sdwa v31, v19, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_u32_e32 v18, 0x7fff, v18
; GFX9-NEXT: v_or_b32_e32 v19, 0x400000, v17
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
@@ -212971,11 +212909,11 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s48, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v29, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v29, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -212990,11 +212928,11 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s47, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v28, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v28, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -213009,11 +212947,11 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s46, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v27, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v27, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -213028,11 +212966,11 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s45, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v26, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v26, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -213047,11 +212985,11 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s44, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v25, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v25, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -213066,11 +213004,11 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s43, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v24, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v24, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -213085,11 +213023,11 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s42, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v23, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v23, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -213104,11 +213042,11 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s41, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v22, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v22, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -213123,11 +213061,11 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s40, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v21, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v21, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -213142,11 +213080,11 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s39, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v20, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v20, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -213159,69 +213097,69 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; GFX9-NEXT: v_bfe_u32 v19, v18, 16, 1
; GFX9-NEXT: v_add_u32_e32 v19, v19, v18
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
-; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v18
+; GFX9-NEXT: v_or_b32_e32 v32, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v33, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v32, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s38, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v19, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v19, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
; GFX9-NEXT: v_add_u32_e32 v18, 0x7fff, v18
-; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v17
+; GFX9-NEXT: v_or_b32_e32 v32, 0x400000, v17
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
; GFX9-NEXT: s_lshl_b32 s4, s38, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v33, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v32, vcc
; GFX9-NEXT: v_add_f32_e32 v18, s4, v16
-; GFX9-NEXT: v_bfe_u32 v33, v18, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v33, v33, v18
-; GFX9-NEXT: v_add_u32_e32 v33, 0x7fff, v33
-; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v18
+; GFX9-NEXT: v_bfe_u32 v32, v18, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v32, v32, v18
+; GFX9-NEXT: v_add_u32_e32 v32, 0x7fff, v32
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v33, v34, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v32, v33, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s37, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v18, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v18, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
-; GFX9-NEXT: v_bfe_u32 v33, v17, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v33, v33, v17
-; GFX9-NEXT: v_add_u32_e32 v33, 0x7fff, v33
-; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v17
+; GFX9-NEXT: v_bfe_u32 v32, v17, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v32, v32, v17
+; GFX9-NEXT: v_add_u32_e32 v32, 0x7fff, v32
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v17
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
; GFX9-NEXT: s_lshl_b32 s4, s37, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v17, v33, v34, vcc
-; GFX9-NEXT: v_add_f32_e32 v33, s4, v16
-; GFX9-NEXT: v_bfe_u32 v34, v33, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v34, v34, v33
-; GFX9-NEXT: v_add_u32_e32 v34, 0x7fff, v34
-; GFX9-NEXT: v_or_b32_e32 v35, 0x400000, v33
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v33, v33
-; GFX9-NEXT: v_cndmask_b32_e32 v33, v34, v35, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v17, v32, v33, vcc
+; GFX9-NEXT: v_add_f32_e32 v32, s4, v16
+; GFX9-NEXT: v_bfe_u32 v33, v32, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v33, v33, v32
+; GFX9-NEXT: v_add_u32_e32 v33, 0x7fff, v33
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v32
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v32, v32
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v33, v32, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v32, v33, v34, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s36, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v17, v17, 16, v33
-; GFX9-NEXT: v_add_f32_e32 v33, s4, v16
-; GFX9-NEXT: v_bfe_u32 v34, v33, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v34, v34, v33
+; GFX9-NEXT: v_or_b32_sdwa v17, v17, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f32_e32 v32, s4, v16
+; GFX9-NEXT: v_bfe_u32 v33, v32, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v33, v33, v32
; GFX9-NEXT: s_lshl_b32 s4, s36, 16
-; GFX9-NEXT: v_add_u32_e32 v34, 0x7fff, v34
-; GFX9-NEXT: v_or_b32_e32 v35, 0x400000, v33
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v33, v33
+; GFX9-NEXT: v_add_u32_e32 v33, 0x7fff, v33
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v32
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v32, v32
; GFX9-NEXT: v_add_f32_e32 v16, s4, v16
-; GFX9-NEXT: v_cndmask_b32_e32 v33, v34, v35, vcc
-; GFX9-NEXT: v_bfe_u32 v34, v16, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v34, v34, v16
-; GFX9-NEXT: v_add_u32_e32 v34, 0x7fff, v34
-; GFX9-NEXT: v_or_b32_e32 v35, 0x400000, v16
+; GFX9-NEXT: v_cndmask_b32_e32 v32, v33, v34, vcc
+; GFX9-NEXT: v_bfe_u32 v33, v16, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v33, v33, v16
+; GFX9-NEXT: v_add_u32_e32 v33, 0x7fff, v33
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v16
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v16, v16
-; GFX9-NEXT: v_cndmask_b32_e32 v16, v34, v35, vcc
-; GFX9-NEXT: v_and_b32_sdwa v16, v32, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshrrev_b32_e32 v32, 16, v33
-; GFX9-NEXT: v_lshl_or_b32 v16, v32, 16, v16
+; GFX9-NEXT: v_lshrrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_cndmask_b32_e32 v16, v33, v34, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_or_b32_sdwa v16, v32, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_branch .LBB101_6
; GFX9-NEXT: .LBB101_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
@@ -213257,18 +213195,18 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; GFX9-NEXT: v_mov_b32_e32 v30, s50
; GFX9-NEXT: v_mov_b32_e32 v31, s51
; GFX9-NEXT: .LBB101_6: ; %end
-; GFX9-NEXT: v_readlane_b32 s30, v36, 8
-; GFX9-NEXT: v_readlane_b32 s31, v36, 9
-; GFX9-NEXT: v_readlane_b32 s51, v36, 7
-; GFX9-NEXT: v_readlane_b32 s50, v36, 6
-; GFX9-NEXT: v_readlane_b32 s49, v36, 5
-; GFX9-NEXT: v_readlane_b32 s48, v36, 4
-; GFX9-NEXT: v_readlane_b32 s39, v36, 3
-; GFX9-NEXT: v_readlane_b32 s38, v36, 2
-; GFX9-NEXT: v_readlane_b32 s37, v36, 1
-; GFX9-NEXT: v_readlane_b32 s36, v36, 0
+; GFX9-NEXT: v_readlane_b32 s30, v35, 8
+; GFX9-NEXT: v_readlane_b32 s31, v35, 9
+; GFX9-NEXT: v_readlane_b32 s51, v35, 7
+; GFX9-NEXT: v_readlane_b32 s50, v35, 6
+; GFX9-NEXT: v_readlane_b32 s49, v35, 5
+; GFX9-NEXT: v_readlane_b32 s48, v35, 4
+; GFX9-NEXT: v_readlane_b32 s39, v35, 3
+; GFX9-NEXT: v_readlane_b32 s38, v35, 2
+; GFX9-NEXT: v_readlane_b32 s37, v35, 1
+; GFX9-NEXT: v_readlane_b32 s36, v35, 0
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
-; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v35, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-NEXT: s_mov_b64 exec, s[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll
index 72a4c2376fa3f..4477c73349266 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll
@@ -3310,71 +3310,70 @@ define inreg <4 x i32> @bitcast_v8bf16_to_v4i32_scalar(<8 x bfloat> inreg %a, i3
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX9-NEXT: v_mov_b32_e32 v4, 0xffff
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s18
-; GFX9-NEXT: v_lshl_or_b32 v3, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v3, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
-; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: s_lshl_b32 s4, s18, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v5, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
-; GFX9-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v5, v5, v2
-; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
-; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v2
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc
+; GFX9-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v4, v4, v2
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v2
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s17
-; GFX9-NEXT: v_lshl_or_b32 v2, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
-; GFX9-NEXT: v_bfe_u32 v5, v1, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v5, v5, v1
-; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
-; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v1
+; GFX9-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v4, v4, v1
+; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: s_lshl_b32 s4, s17, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v6, vcc
-; GFX9-NEXT: v_add_f32_e32 v5, s4, v0
-; GFX9-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v6, v6, v5
-; GFX9-NEXT: v_add_u32_e32 v6, 0x7fff, v6
-; GFX9-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX9-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v5, vcc
+; GFX9-NEXT: v_add_f32_e32 v4, s4, v0
+; GFX9-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v5, v5, v4
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v5, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
+; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
+; GFX9-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s16
-; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v5
-; GFX9-NEXT: v_add_f32_e32 v5, s4, v0
-; GFX9-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v6, v6, v5
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f32_e32 v4, s4, v0
+; GFX9-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v5, v5, v4
; GFX9-NEXT: s_lshl_b32 s4, s16, 16
-; GFX9-NEXT: v_add_u32_e32 v6, 0x7fff, v6
-; GFX9-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
+; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
; GFX9-NEXT: v_add_f32_e32 v0, s4, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc
-; GFX9-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v6, v6, v0
-; GFX9-NEXT: v_add_u32_e32 v6, 0x7fff, v6
-; GFX9-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc
+; GFX9-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v5, v5, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
+; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v0
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v7, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX9-NEXT: v_and_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v0, v5, 16, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v5, v6, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX9-NEXT: v_or_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_setpc_b64 s[30:31]
; GFX9-NEXT: .LBB23_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
@@ -8065,71 +8064,70 @@ define inreg <4 x float> @bitcast_v8bf16_to_v4f32_scalar(<8 x bfloat> inreg %a,
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX9-NEXT: v_mov_b32_e32 v4, 0xffff
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s18
-; GFX9-NEXT: v_lshl_or_b32 v3, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v3, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
-; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: s_lshl_b32 s4, s18, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v5, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
-; GFX9-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v5, v5, v2
-; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
-; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v2
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc
+; GFX9-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v4, v4, v2
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v2
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s17
-; GFX9-NEXT: v_lshl_or_b32 v2, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
-; GFX9-NEXT: v_bfe_u32 v5, v1, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v5, v5, v1
-; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
-; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v1
+; GFX9-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v4, v4, v1
+; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: s_lshl_b32 s4, s17, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v6, vcc
-; GFX9-NEXT: v_add_f32_e32 v5, s4, v0
-; GFX9-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v6, v6, v5
-; GFX9-NEXT: v_add_u32_e32 v6, 0x7fff, v6
-; GFX9-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX9-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v5, vcc
+; GFX9-NEXT: v_add_f32_e32 v4, s4, v0
+; GFX9-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v5, v5, v4
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v5, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
+; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
+; GFX9-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s16
-; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v5
-; GFX9-NEXT: v_add_f32_e32 v5, s4, v0
-; GFX9-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v6, v6, v5
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f32_e32 v4, s4, v0
+; GFX9-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v5, v5, v4
; GFX9-NEXT: s_lshl_b32 s4, s16, 16
-; GFX9-NEXT: v_add_u32_e32 v6, 0x7fff, v6
-; GFX9-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
+; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
; GFX9-NEXT: v_add_f32_e32 v0, s4, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc
-; GFX9-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v6, v6, v0
-; GFX9-NEXT: v_add_u32_e32 v6, 0x7fff, v6
-; GFX9-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc
+; GFX9-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v5, v5, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
+; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v0
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v7, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX9-NEXT: v_and_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v0, v5, 16, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v5, v6, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX9-NEXT: v_or_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_setpc_b64 s[30:31]
; GFX9-NEXT: .LBB47_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
@@ -12413,71 +12411,70 @@ define inreg <2 x i64> @bitcast_v8bf16_to_v2i64_scalar(<8 x bfloat> inreg %a, i3
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX9-NEXT: v_mov_b32_e32 v4, 0xffff
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s18
-; GFX9-NEXT: v_lshl_or_b32 v3, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v3, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
-; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: s_lshl_b32 s4, s18, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v5, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
-; GFX9-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v5, v5, v2
-; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
-; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v2
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc
+; GFX9-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v4, v4, v2
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v2
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s17
-; GFX9-NEXT: v_lshl_or_b32 v2, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
-; GFX9-NEXT: v_bfe_u32 v5, v1, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v5, v5, v1
-; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
-; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v1
+; GFX9-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v4, v4, v1
+; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: s_lshl_b32 s4, s17, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v6, vcc
-; GFX9-NEXT: v_add_f32_e32 v5, s4, v0
-; GFX9-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v6, v6, v5
-; GFX9-NEXT: v_add_u32_e32 v6, 0x7fff, v6
-; GFX9-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX9-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v5, vcc
+; GFX9-NEXT: v_add_f32_e32 v4, s4, v0
+; GFX9-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v5, v5, v4
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v5, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
+; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
+; GFX9-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s16
-; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v5
-; GFX9-NEXT: v_add_f32_e32 v5, s4, v0
-; GFX9-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v6, v6, v5
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f32_e32 v4, s4, v0
+; GFX9-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v5, v5, v4
; GFX9-NEXT: s_lshl_b32 s4, s16, 16
-; GFX9-NEXT: v_add_u32_e32 v6, 0x7fff, v6
-; GFX9-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
+; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
; GFX9-NEXT: v_add_f32_e32 v0, s4, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc
-; GFX9-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v6, v6, v0
-; GFX9-NEXT: v_add_u32_e32 v6, 0x7fff, v6
-; GFX9-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc
+; GFX9-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v5, v5, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
+; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v0
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v7, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX9-NEXT: v_and_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v0, v5, 16, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v5, v6, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX9-NEXT: v_or_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_setpc_b64 s[30:31]
; GFX9-NEXT: .LBB67_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
@@ -16320,71 +16317,70 @@ define inreg <2 x double> @bitcast_v8bf16_to_v2f64_scalar(<8 x bfloat> inreg %a,
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX9-NEXT: v_mov_b32_e32 v4, 0xffff
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s18
-; GFX9-NEXT: v_lshl_or_b32 v3, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v3, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
-; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: s_lshl_b32 s4, s18, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v5, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
-; GFX9-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v5, v5, v2
-; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
-; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v2
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc
+; GFX9-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v4, v4, v2
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v2
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s17
-; GFX9-NEXT: v_lshl_or_b32 v2, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
-; GFX9-NEXT: v_bfe_u32 v5, v1, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v5, v5, v1
-; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
-; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v1
+; GFX9-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v4, v4, v1
+; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: s_lshl_b32 s4, s17, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v6, vcc
-; GFX9-NEXT: v_add_f32_e32 v5, s4, v0
-; GFX9-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v6, v6, v5
-; GFX9-NEXT: v_add_u32_e32 v6, 0x7fff, v6
-; GFX9-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX9-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v5, vcc
+; GFX9-NEXT: v_add_f32_e32 v4, s4, v0
+; GFX9-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v5, v5, v4
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v5, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
+; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
+; GFX9-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s16
-; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v5
-; GFX9-NEXT: v_add_f32_e32 v5, s4, v0
-; GFX9-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v6, v6, v5
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f32_e32 v4, s4, v0
+; GFX9-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v5, v5, v4
; GFX9-NEXT: s_lshl_b32 s4, s16, 16
-; GFX9-NEXT: v_add_u32_e32 v6, 0x7fff, v6
-; GFX9-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
+; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
; GFX9-NEXT: v_add_f32_e32 v0, s4, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc
-; GFX9-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v6, v6, v0
-; GFX9-NEXT: v_add_u32_e32 v6, 0x7fff, v6
-; GFX9-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc
+; GFX9-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v5, v5, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
+; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v0
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v7, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX9-NEXT: v_and_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v0, v5, 16, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v5, v6, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX9-NEXT: v_or_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_setpc_b64 s[30:31]
; GFX9-NEXT: .LBB83_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
@@ -22944,22 +22940,21 @@ define inreg <8 x half> @bitcast_v8bf16_to_v8f16_scalar(<8 x bfloat> inreg %a, i
; GFX9-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc
; GFX9-NEXT: v_bfe_u32 v8, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v8, v8, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v0
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v8, v9, vcc
-; GFX9-NEXT: v_mov_b32_e32 v8, 0xffff
-; GFX9-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX9-NEXT: v_and_b32_sdwa v0, v8, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v3, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v0, v8, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v8, v9, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v2, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v0, v8, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v0, v8, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v0, v4, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v3, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v0, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v0, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v4
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_setpc_b64 s[30:31]
; GFX9-NEXT: .LBB103_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
@@ -25902,9 +25897,9 @@ define inreg <16 x i8> @bitcast_v8bf16_to_v16i8_scalar(<8 x bfloat> inreg %a, i3
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v0
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v17
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v6
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s16
-; GFX9-NEXT: v_lshl_or_b32 v19, v6, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v19, v0, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v0, s4, v1
; GFX9-NEXT: v_bfe_u32 v2, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v0
@@ -25922,9 +25917,9 @@ define inreg <16 x i8> @bitcast_v8bf16_to_v16i8_scalar(<8 x bfloat> inreg %a, i3
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s19
-; GFX9-NEXT: v_lshl_or_b32 v18, v2, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v18, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v3, s4, v1
; GFX9-NEXT: v_bfe_u32 v4, v3, 16, 1
; GFX9-NEXT: v_add_u32_e32 v4, v4, v3
@@ -25942,9 +25937,9 @@ define inreg <16 x i8> @bitcast_v8bf16_to_v16i8_scalar(<8 x bfloat> inreg %a, i3
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v16, 16, v3
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v16
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v14
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s18
-; GFX9-NEXT: v_lshl_or_b32 v21, v14, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v21, v3, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v3, s4, v1
; GFX9-NEXT: v_bfe_u32 v4, v3, 16, 1
; GFX9-NEXT: v_add_u32_e32 v4, v4, v3
@@ -25962,8 +25957,8 @@ define inreg <16 x i8> @bitcast_v8bf16_to_v16i8_scalar(<8 x bfloat> inreg %a, i3
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v1
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v8
-; GFX9-NEXT: v_lshl_or_b32 v20, v10, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v10
+; GFX9-NEXT: v_or_b32_sdwa v20, v1, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_lshrrev_b64 v[11:12], 24, v[20:21]
; GFX9-NEXT: v_lshrrev_b64 v[3:4], 24, v[18:19]
; GFX9-NEXT: v_lshrrev_b32_e32 v15, 24, v21
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll
index f2e9354a563d8..e902c19cfd022 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll
@@ -4667,15 +4667,14 @@ define inreg <8 x i32> @bitcast_v16bf16_to_v8i32_scalar(<16 x bfloat> inreg %a,
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX9-NEXT: v_mov_b32_e32 v8, 0xffff
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v8, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s22, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v7, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v7, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -4687,14 +4686,14 @@ define inreg <8 x i32> @bitcast_v16bf16_to_v8i32_scalar(<16 x bfloat> inreg %a,
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v8, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s21, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v6, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v6, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -4706,14 +4705,14 @@ define inreg <8 x i32> @bitcast_v16bf16_to_v8i32_scalar(<16 x bfloat> inreg %a,
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v8, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s20, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v5, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -4725,14 +4724,14 @@ define inreg <8 x i32> @bitcast_v16bf16_to_v8i32_scalar(<16 x bfloat> inreg %a,
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v8, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s19, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v4, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v4, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -4744,70 +4743,70 @@ define inreg <8 x i32> @bitcast_v16bf16_to_v8i32_scalar(<16 x bfloat> inreg %a,
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
-; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX9-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v9, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v8, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v8, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s18, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v3, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v3, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
-; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX9-NEXT: v_or_b32_e32 v8, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: s_lshl_b32 s4, s18, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v9, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v8, vcc
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
-; GFX9-NEXT: v_bfe_u32 v9, v2, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v9, v9, v2
-; GFX9-NEXT: v_add_u32_e32 v9, 0x7fff, v9
-; GFX9-NEXT: v_or_b32_e32 v10, 0x400000, v2
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v9, v10, vcc
+; GFX9-NEXT: v_bfe_u32 v8, v2, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v8, v8, v2
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v8, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
+; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v8, v9, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s17, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v2, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
-; GFX9-NEXT: v_bfe_u32 v9, v1, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v9, v9, v1
-; GFX9-NEXT: v_add_u32_e32 v9, 0x7fff, v9
-; GFX9-NEXT: v_or_b32_e32 v10, 0x400000, v1
+; GFX9-NEXT: v_bfe_u32 v8, v1, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v8, v8, v1
+; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
+; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: s_lshl_b32 s4, s17, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v9, v10, vcc
-; GFX9-NEXT: v_add_f32_e32 v9, s4, v0
-; GFX9-NEXT: v_bfe_u32 v10, v9, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v10, v10, v9
-; GFX9-NEXT: v_add_u32_e32 v10, 0x7fff, v10
-; GFX9-NEXT: v_or_b32_e32 v11, 0x400000, v9
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v9, v9
-; GFX9-NEXT: v_cndmask_b32_e32 v9, v10, v11, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v8, v9, vcc
+; GFX9-NEXT: v_add_f32_e32 v8, s4, v0
+; GFX9-NEXT: v_bfe_u32 v9, v8, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v9, v9, v8
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v9, v8, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_u32_e32 v9, 0x7fff, v9
+; GFX9-NEXT: v_or_b32_e32 v10, 0x400000, v8
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v8, v8
+; GFX9-NEXT: v_cndmask_b32_e32 v8, v9, v10, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s16, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v9
-; GFX9-NEXT: v_add_f32_e32 v9, s4, v0
-; GFX9-NEXT: v_bfe_u32 v10, v9, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v10, v10, v9
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f32_e32 v8, s4, v0
+; GFX9-NEXT: v_bfe_u32 v9, v8, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v9, v9, v8
; GFX9-NEXT: s_lshl_b32 s4, s16, 16
-; GFX9-NEXT: v_add_u32_e32 v10, 0x7fff, v10
-; GFX9-NEXT: v_or_b32_e32 v11, 0x400000, v9
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v9, v9
+; GFX9-NEXT: v_add_u32_e32 v9, 0x7fff, v9
+; GFX9-NEXT: v_or_b32_e32 v10, 0x400000, v8
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v8, v8
; GFX9-NEXT: v_add_f32_e32 v0, s4, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v9, v10, v11, vcc
-; GFX9-NEXT: v_bfe_u32 v10, v0, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v10, v10, v0
-; GFX9-NEXT: v_add_u32_e32 v10, 0x7fff, v10
-; GFX9-NEXT: v_or_b32_e32 v11, 0x400000, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v8, v9, v10, vcc
+; GFX9-NEXT: v_bfe_u32 v9, v0, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v9, v9, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; GFX9-NEXT: v_add_u32_e32 v9, 0x7fff, v9
+; GFX9-NEXT: v_or_b32_e32 v10, 0x400000, v0
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v10, v11, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v9, 16, v9
-; GFX9-NEXT: v_and_b32_sdwa v0, v8, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v0, v9, 16, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v9, v10, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX9-NEXT: v_or_b32_sdwa v0, v8, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_setpc_b64 s[30:31]
; GFX9-NEXT: .LBB23_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
@@ -12142,15 +12141,14 @@ define inreg <8 x float> @bitcast_v16bf16_to_v8f32_scalar(<16 x bfloat> inreg %a
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX9-NEXT: v_mov_b32_e32 v8, 0xffff
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v8, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s22, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v7, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v7, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -12162,14 +12160,14 @@ define inreg <8 x float> @bitcast_v16bf16_to_v8f32_scalar(<16 x bfloat> inreg %a
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v8, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s21, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v6, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v6, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -12181,14 +12179,14 @@ define inreg <8 x float> @bitcast_v16bf16_to_v8f32_scalar(<16 x bfloat> inreg %a
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v8, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s20, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v5, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -12200,14 +12198,14 @@ define inreg <8 x float> @bitcast_v16bf16_to_v8f32_scalar(<16 x bfloat> inreg %a
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v8, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s19, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v4, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v4, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -12219,70 +12217,70 @@ define inreg <8 x float> @bitcast_v16bf16_to_v8f32_scalar(<16 x bfloat> inreg %a
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
-; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX9-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v9, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v8, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v8, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s18, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v3, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v3, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
-; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX9-NEXT: v_or_b32_e32 v8, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: s_lshl_b32 s4, s18, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v9, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v8, vcc
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
-; GFX9-NEXT: v_bfe_u32 v9, v2, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v9, v9, v2
-; GFX9-NEXT: v_add_u32_e32 v9, 0x7fff, v9
-; GFX9-NEXT: v_or_b32_e32 v10, 0x400000, v2
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v9, v10, vcc
+; GFX9-NEXT: v_bfe_u32 v8, v2, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v8, v8, v2
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v8, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
+; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v8, v9, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s17, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v2, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
-; GFX9-NEXT: v_bfe_u32 v9, v1, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v9, v9, v1
-; GFX9-NEXT: v_add_u32_e32 v9, 0x7fff, v9
-; GFX9-NEXT: v_or_b32_e32 v10, 0x400000, v1
+; GFX9-NEXT: v_bfe_u32 v8, v1, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v8, v8, v1
+; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
+; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: s_lshl_b32 s4, s17, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v9, v10, vcc
-; GFX9-NEXT: v_add_f32_e32 v9, s4, v0
-; GFX9-NEXT: v_bfe_u32 v10, v9, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v10, v10, v9
-; GFX9-NEXT: v_add_u32_e32 v10, 0x7fff, v10
-; GFX9-NEXT: v_or_b32_e32 v11, 0x400000, v9
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v9, v9
-; GFX9-NEXT: v_cndmask_b32_e32 v9, v10, v11, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v8, v9, vcc
+; GFX9-NEXT: v_add_f32_e32 v8, s4, v0
+; GFX9-NEXT: v_bfe_u32 v9, v8, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v9, v9, v8
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v9, v8, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_u32_e32 v9, 0x7fff, v9
+; GFX9-NEXT: v_or_b32_e32 v10, 0x400000, v8
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v8, v8
+; GFX9-NEXT: v_cndmask_b32_e32 v8, v9, v10, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s16, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v9
-; GFX9-NEXT: v_add_f32_e32 v9, s4, v0
-; GFX9-NEXT: v_bfe_u32 v10, v9, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v10, v10, v9
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f32_e32 v8, s4, v0
+; GFX9-NEXT: v_bfe_u32 v9, v8, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v9, v9, v8
; GFX9-NEXT: s_lshl_b32 s4, s16, 16
-; GFX9-NEXT: v_add_u32_e32 v10, 0x7fff, v10
-; GFX9-NEXT: v_or_b32_e32 v11, 0x400000, v9
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v9, v9
+; GFX9-NEXT: v_add_u32_e32 v9, 0x7fff, v9
+; GFX9-NEXT: v_or_b32_e32 v10, 0x400000, v8
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v8, v8
; GFX9-NEXT: v_add_f32_e32 v0, s4, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v9, v10, v11, vcc
-; GFX9-NEXT: v_bfe_u32 v10, v0, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v10, v10, v0
-; GFX9-NEXT: v_add_u32_e32 v10, 0x7fff, v10
-; GFX9-NEXT: v_or_b32_e32 v11, 0x400000, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v8, v9, v10, vcc
+; GFX9-NEXT: v_bfe_u32 v9, v0, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v9, v9, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; GFX9-NEXT: v_add_u32_e32 v9, 0x7fff, v9
+; GFX9-NEXT: v_or_b32_e32 v10, 0x400000, v0
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v10, v11, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v9, 16, v9
-; GFX9-NEXT: v_and_b32_sdwa v0, v8, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v0, v9, 16, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v9, v10, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX9-NEXT: v_or_b32_sdwa v0, v8, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_setpc_b64 s[30:31]
; GFX9-NEXT: .LBB47_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
@@ -19116,15 +19114,14 @@ define inreg <4 x i64> @bitcast_v16bf16_to_v4i64_scalar(<16 x bfloat> inreg %a,
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX9-NEXT: v_mov_b32_e32 v8, 0xffff
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v8, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s22, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v7, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v7, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -19136,14 +19133,14 @@ define inreg <4 x i64> @bitcast_v16bf16_to_v4i64_scalar(<16 x bfloat> inreg %a,
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v8, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s21, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v6, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v6, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -19155,14 +19152,14 @@ define inreg <4 x i64> @bitcast_v16bf16_to_v4i64_scalar(<16 x bfloat> inreg %a,
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v8, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s20, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v5, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -19174,14 +19171,14 @@ define inreg <4 x i64> @bitcast_v16bf16_to_v4i64_scalar(<16 x bfloat> inreg %a,
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v8, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s19, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v4, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v4, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -19193,70 +19190,70 @@ define inreg <4 x i64> @bitcast_v16bf16_to_v4i64_scalar(<16 x bfloat> inreg %a,
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
-; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX9-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v9, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v8, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v8, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s18, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v3, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v3, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
-; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX9-NEXT: v_or_b32_e32 v8, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: s_lshl_b32 s4, s18, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v9, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v8, vcc
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
-; GFX9-NEXT: v_bfe_u32 v9, v2, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v9, v9, v2
-; GFX9-NEXT: v_add_u32_e32 v9, 0x7fff, v9
-; GFX9-NEXT: v_or_b32_e32 v10, 0x400000, v2
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v9, v10, vcc
+; GFX9-NEXT: v_bfe_u32 v8, v2, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v8, v8, v2
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v8, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
+; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v8, v9, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s17, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v2, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
-; GFX9-NEXT: v_bfe_u32 v9, v1, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v9, v9, v1
-; GFX9-NEXT: v_add_u32_e32 v9, 0x7fff, v9
-; GFX9-NEXT: v_or_b32_e32 v10, 0x400000, v1
+; GFX9-NEXT: v_bfe_u32 v8, v1, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v8, v8, v1
+; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
+; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: s_lshl_b32 s4, s17, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v9, v10, vcc
-; GFX9-NEXT: v_add_f32_e32 v9, s4, v0
-; GFX9-NEXT: v_bfe_u32 v10, v9, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v10, v10, v9
-; GFX9-NEXT: v_add_u32_e32 v10, 0x7fff, v10
-; GFX9-NEXT: v_or_b32_e32 v11, 0x400000, v9
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v9, v9
-; GFX9-NEXT: v_cndmask_b32_e32 v9, v10, v11, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v8, v9, vcc
+; GFX9-NEXT: v_add_f32_e32 v8, s4, v0
+; GFX9-NEXT: v_bfe_u32 v9, v8, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v9, v9, v8
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v9, v8, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_u32_e32 v9, 0x7fff, v9
+; GFX9-NEXT: v_or_b32_e32 v10, 0x400000, v8
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v8, v8
+; GFX9-NEXT: v_cndmask_b32_e32 v8, v9, v10, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s16, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v9
-; GFX9-NEXT: v_add_f32_e32 v9, s4, v0
-; GFX9-NEXT: v_bfe_u32 v10, v9, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v10, v10, v9
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f32_e32 v8, s4, v0
+; GFX9-NEXT: v_bfe_u32 v9, v8, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v9, v9, v8
; GFX9-NEXT: s_lshl_b32 s4, s16, 16
-; GFX9-NEXT: v_add_u32_e32 v10, 0x7fff, v10
-; GFX9-NEXT: v_or_b32_e32 v11, 0x400000, v9
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v9, v9
+; GFX9-NEXT: v_add_u32_e32 v9, 0x7fff, v9
+; GFX9-NEXT: v_or_b32_e32 v10, 0x400000, v8
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v8, v8
; GFX9-NEXT: v_add_f32_e32 v0, s4, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v9, v10, v11, vcc
-; GFX9-NEXT: v_bfe_u32 v10, v0, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v10, v10, v0
-; GFX9-NEXT: v_add_u32_e32 v10, 0x7fff, v10
-; GFX9-NEXT: v_or_b32_e32 v11, 0x400000, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v8, v9, v10, vcc
+; GFX9-NEXT: v_bfe_u32 v9, v0, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v9, v9, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; GFX9-NEXT: v_add_u32_e32 v9, 0x7fff, v9
+; GFX9-NEXT: v_or_b32_e32 v10, 0x400000, v0
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v10, v11, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v9, 16, v9
-; GFX9-NEXT: v_and_b32_sdwa v0, v8, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v0, v9, 16, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v9, v10, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX9-NEXT: v_or_b32_sdwa v0, v8, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_setpc_b64 s[30:31]
; GFX9-NEXT: .LBB67_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
@@ -25527,15 +25524,14 @@ define inreg <4 x double> @bitcast_v16bf16_to_v4f64_scalar(<16 x bfloat> inreg %
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX9-NEXT: v_mov_b32_e32 v8, 0xffff
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v8, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s22, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v7, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v7, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -25547,14 +25543,14 @@ define inreg <4 x double> @bitcast_v16bf16_to_v4f64_scalar(<16 x bfloat> inreg %
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v8, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s21, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v6, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v6, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -25566,14 +25562,14 @@ define inreg <4 x double> @bitcast_v16bf16_to_v4f64_scalar(<16 x bfloat> inreg %
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v8, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s20, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v5, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -25585,14 +25581,14 @@ define inreg <4 x double> @bitcast_v16bf16_to_v4f64_scalar(<16 x bfloat> inreg %
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v8, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s19, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v4, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v4, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -25604,70 +25600,70 @@ define inreg <4 x double> @bitcast_v16bf16_to_v4f64_scalar(<16 x bfloat> inreg %
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
-; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX9-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v9, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v8, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v8, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s18, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v3, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v3, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
-; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX9-NEXT: v_or_b32_e32 v8, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: s_lshl_b32 s4, s18, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v9, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v8, vcc
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
-; GFX9-NEXT: v_bfe_u32 v9, v2, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v9, v9, v2
-; GFX9-NEXT: v_add_u32_e32 v9, 0x7fff, v9
-; GFX9-NEXT: v_or_b32_e32 v10, 0x400000, v2
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v9, v10, vcc
+; GFX9-NEXT: v_bfe_u32 v8, v2, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v8, v8, v2
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v8, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
+; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v8, v9, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s17, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v2, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
-; GFX9-NEXT: v_bfe_u32 v9, v1, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v9, v9, v1
-; GFX9-NEXT: v_add_u32_e32 v9, 0x7fff, v9
-; GFX9-NEXT: v_or_b32_e32 v10, 0x400000, v1
+; GFX9-NEXT: v_bfe_u32 v8, v1, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v8, v8, v1
+; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
+; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: s_lshl_b32 s4, s17, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v9, v10, vcc
-; GFX9-NEXT: v_add_f32_e32 v9, s4, v0
-; GFX9-NEXT: v_bfe_u32 v10, v9, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v10, v10, v9
-; GFX9-NEXT: v_add_u32_e32 v10, 0x7fff, v10
-; GFX9-NEXT: v_or_b32_e32 v11, 0x400000, v9
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v9, v9
-; GFX9-NEXT: v_cndmask_b32_e32 v9, v10, v11, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v8, v9, vcc
+; GFX9-NEXT: v_add_f32_e32 v8, s4, v0
+; GFX9-NEXT: v_bfe_u32 v9, v8, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v9, v9, v8
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v9, v8, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_u32_e32 v9, 0x7fff, v9
+; GFX9-NEXT: v_or_b32_e32 v10, 0x400000, v8
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v8, v8
+; GFX9-NEXT: v_cndmask_b32_e32 v8, v9, v10, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s16, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v9
-; GFX9-NEXT: v_add_f32_e32 v9, s4, v0
-; GFX9-NEXT: v_bfe_u32 v10, v9, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v10, v10, v9
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f32_e32 v8, s4, v0
+; GFX9-NEXT: v_bfe_u32 v9, v8, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v9, v9, v8
; GFX9-NEXT: s_lshl_b32 s4, s16, 16
-; GFX9-NEXT: v_add_u32_e32 v10, 0x7fff, v10
-; GFX9-NEXT: v_or_b32_e32 v11, 0x400000, v9
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v9, v9
+; GFX9-NEXT: v_add_u32_e32 v9, 0x7fff, v9
+; GFX9-NEXT: v_or_b32_e32 v10, 0x400000, v8
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v8, v8
; GFX9-NEXT: v_add_f32_e32 v0, s4, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v9, v10, v11, vcc
-; GFX9-NEXT: v_bfe_u32 v10, v0, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v10, v10, v0
-; GFX9-NEXT: v_add_u32_e32 v10, 0x7fff, v10
-; GFX9-NEXT: v_or_b32_e32 v11, 0x400000, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v8, v9, v10, vcc
+; GFX9-NEXT: v_bfe_u32 v9, v0, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v9, v9, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; GFX9-NEXT: v_add_u32_e32 v9, 0x7fff, v9
+; GFX9-NEXT: v_or_b32_e32 v10, 0x400000, v0
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v10, v11, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v9, 16, v9
-; GFX9-NEXT: v_and_b32_sdwa v0, v8, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v0, v9, 16, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v9, v10, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX9-NEXT: v_or_b32_sdwa v0, v8, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_setpc_b64 s[30:31]
; GFX9-NEXT: .LBB83_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
@@ -34815,59 +34811,59 @@ define inreg <16 x i16> @bitcast_v32i8_to_v16i16_scalar(<32 x i8> inreg %a, i32
; GFX9-NEXT: v_readfirstlane_b32 s8, v0
; GFX9-NEXT: s_cbranch_scc0 .LBB99_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v4, 0xc0c0004
-; GFX9-NEXT: v_perm_b32 v0, s16, v0, v4
; GFX9-NEXT: v_mov_b32_e32 v1, s19
+; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_perm_b32 v1, s18, v1, v4
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX9-NEXT: v_mov_b32_e32 v1, s21
-; GFX9-NEXT: v_perm_b32 v1, s20, v1, v4
+; GFX9-NEXT: v_perm_b32 v0, s16, v0, v4
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_mov_b32_e32 v2, s23
-; GFX9-NEXT: v_perm_b32 v2, s22, v2, v4
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, s25
-; GFX9-NEXT: v_perm_b32 v2, s24, v2, v4
-; GFX9-NEXT: v_mov_b32_e32 v3, s27
-; GFX9-NEXT: v_perm_b32 v3, s26, v3, v4
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_lshl_or_b32 v2, v3, 16, v2
-; GFX9-NEXT: v_mov_b32_e32 v3, s29
-; GFX9-NEXT: v_perm_b32 v3, s28, v3, v4
-; GFX9-NEXT: v_mov_b32_e32 v5, s6
; GFX9-NEXT: s_and_b32 s4, s14, 0xff
; GFX9-NEXT: s_lshl_b32 s5, s12, 8
-; GFX9-NEXT: v_perm_b32 v5, s8, v5, v4
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v1, s21
+; GFX9-NEXT: v_perm_b32 v2, s22, v2, v4
; GFX9-NEXT: s_or_b32 s4, s4, s5
; GFX9-NEXT: s_and_b32 s5, s9, 0xff
; GFX9-NEXT: s_lshl_b32 s56, s7, 8
-; GFX9-NEXT: v_lshl_or_b32 v3, v5, 16, v3
+; GFX9-NEXT: v_perm_b32 v1, s20, v1, v4
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_mov_b32_e32 v3, s27
; GFX9-NEXT: s_or_b32 s5, s5, s56
-; GFX9-NEXT: v_mov_b32_e32 v5, s40
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v2, s25
+; GFX9-NEXT: v_perm_b32 v3, s26, v3, v4
; GFX9-NEXT: s_pack_ll_b32_b16 s4, s4, s5
-; GFX9-NEXT: v_perm_b32 v5, s42, v5, v4
; GFX9-NEXT: s_and_b32 s5, s11, 0xff
; GFX9-NEXT: s_lshl_b32 s56, s10, 8
+; GFX9-NEXT: v_perm_b32 v2, s24, v2, v4
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_mov_b32_e32 v5, s6
; GFX9-NEXT: s_or_b32 s5, s5, s56
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_lshl_or_b32 v5, s5, 16, v5
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v3, s29
+; GFX9-NEXT: v_perm_b32 v5, s8, v5, v4
+; GFX9-NEXT: v_lshlrev_b32_e64 v6, 16, s5
; GFX9-NEXT: s_and_b32 s5, s45, 0xff
; GFX9-NEXT: s_lshl_b32 s56, s44, 8
+; GFX9-NEXT: v_perm_b32 v3, s28, v3, v4
+; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX9-NEXT: s_or_b32 s5, s5, s56
; GFX9-NEXT: s_and_b32 s56, s15, 0xff
; GFX9-NEXT: s_lshl_b32 s57, s13, 8
+; GFX9-NEXT: v_or_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v5, s40
; GFX9-NEXT: s_or_b32 s56, s56, s57
-; GFX9-NEXT: v_mov_b32_e32 v6, s46
+; GFX9-NEXT: v_perm_b32 v5, s42, v5, v4
; GFX9-NEXT: s_pack_ll_b32_b16 s5, s5, s56
-; GFX9-NEXT: v_perm_b32 v4, s47, v6, v4
; GFX9-NEXT: s_and_b32 s56, s43, 0xff
; GFX9-NEXT: s_lshl_b32 s57, s41, 8
+; GFX9-NEXT: v_or_b32_sdwa v5, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v6, s46
; GFX9-NEXT: s_or_b32 s56, s56, s57
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_lshl_or_b32 v7, s56, 16, v4
+; GFX9-NEXT: v_perm_b32 v4, s47, v6, v4
+; GFX9-NEXT: v_lshlrev_b32_e64 v6, 16, s56
+; GFX9-NEXT: v_or_b32_sdwa v7, v6, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v4, s4
; GFX9-NEXT: v_mov_b32_e32 v6, s5
; GFX9-NEXT: s_mov_b64 s[4:5], 0
@@ -34915,53 +34911,46 @@ define inreg <16 x i16> @bitcast_v32i8_to_v16i16_scalar(<32 x i8> inreg %a, i32
; GFX9-NEXT: v_add_u32_e32 v1, 0x300, v0
; GFX9-NEXT: s_add_i32 s22, s22, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s23
+; GFX9-NEXT: s_movk_i32 s4, 0x300
; GFX9-NEXT: v_perm_b32 v0, s22, v0, v2
-; GFX9-NEXT: v_add_u32_e32 v12, 0x300, v0
+; GFX9-NEXT: v_add_u32_sdwa v12, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s16, s16, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s17
-; GFX9-NEXT: v_perm_b32 v0, s16, v0, v2
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: v_mov_b32_e32 v13, s19
-; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
+; GFX9-NEXT: v_perm_b32 v0, s16, v0, v2
; GFX9-NEXT: v_perm_b32 v13, s18, v13, v2
-; GFX9-NEXT: v_add_u32_e32 v13, 0x300, v13
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v0, v13, 16, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: s_add_i32 s24, s24, 3
-; GFX9-NEXT: v_mov_b32_e32 v13, s25
-; GFX9-NEXT: v_add_u32_e32 v16, 0x300, v5
-; GFX9-NEXT: v_add_u32_e32 v5, 0x300, v7
+; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
+; GFX9-NEXT: v_add_u32_sdwa v13, v13, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s8, s8, 3
-; GFX9-NEXT: v_lshl_or_b32 v1, v12, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v0, v13, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v12, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v12, s6
-; GFX9-NEXT: v_perm_b32 v13, s24, v13, v2
+; GFX9-NEXT: s_add_i32 s24, s24, 3
+; GFX9-NEXT: v_mov_b32_e32 v13, s25
; GFX9-NEXT: s_add_i32 s26, s26, 3
; GFX9-NEXT: v_mov_b32_e32 v14, s27
-; GFX9-NEXT: v_add_u32_e32 v7, 0x300, v8
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
; GFX9-NEXT: v_perm_b32 v12, s8, v12, v2
+; GFX9-NEXT: v_perm_b32 v13, s24, v13, v2
; GFX9-NEXT: v_perm_b32 v2, s26, v14, v2
; GFX9-NEXT: v_add_u32_e32 v14, 0x300, v3
-; GFX9-NEXT: v_add_u32_e32 v15, 0x300, v4
-; GFX9-NEXT: v_add_u32_e32 v6, 0x300, v6
+; GFX9-NEXT: v_add_u32_sdwa v15, v4, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_add_u32_e32 v16, 0x300, v5
+; GFX9-NEXT: v_add_u32_sdwa v6, v6, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_add_u32_e32 v5, 0x300, v7
+; GFX9-NEXT: v_add_u32_sdwa v7, v8, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_add_u32_e32 v4, 0x300, v9
-; GFX9-NEXT: v_add_u32_e32 v8, 0x300, v10
+; GFX9-NEXT: v_add_u32_sdwa v8, v10, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_add_u32_e32 v3, 0x300, v11
+; GFX9-NEXT: v_add_u32_sdwa v9, v12, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_add_u32_e32 v10, 0x300, v13
-; GFX9-NEXT: v_lshl_or_b32 v5, v7, 16, v5
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v16
-; GFX9-NEXT: v_add_u32_e32 v9, 0x300, v12
-; GFX9-NEXT: v_add_u32_e32 v2, 0x300, v2
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_lshl_or_b32 v6, v6, 16, v7
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v14
-; GFX9-NEXT: v_lshl_or_b32 v2, v2, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v3, v9, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v8, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v7, v15, 16, v7
+; GFX9-NEXT: v_add_u32_sdwa v2, v2, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v2, v2, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v9, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v4, v8, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v7, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v6, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v15, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: .LBB99_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -37072,107 +37061,106 @@ define inreg <16 x half> @bitcast_v16bf16_to_v16f16_scalar(<16 x bfloat> inreg %
; GFX9-NEXT: v_cndmask_b32_e32 v9, v6, v7, vcc
; GFX9-NEXT: v_bfe_u32 v6, v5, 16, 1
; GFX9-NEXT: v_add_u32_e32 v6, v6, v5
-; GFX9-NEXT: s_lshl_b32 s4, s19, 16
-; GFX9-NEXT: v_add_u32_e32 v6, 0x7fff, v6
-; GFX9-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX9-NEXT: v_add_f32_e32 v5, s4, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v10, v6, v7, vcc
-; GFX9-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v6, v6, v5
-; GFX9-NEXT: s_and_b32 s5, s22, 0xffff0000
; GFX9-NEXT: v_add_u32_e32 v6, 0x7fff, v6
; GFX9-NEXT: v_or_b32_e32 v7, 0x400000, v5
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX9-NEXT: v_add_f32_e32 v5, s5, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v11, v6, v7, vcc
-; GFX9-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v6, v6, v5
-; GFX9-NEXT: v_add_u32_e32 v6, 0x7fff, v6
-; GFX9-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX9-NEXT: s_lshl_b32 s5, s22, 16
+; GFX9-NEXT: s_lshl_b32 s4, s19, 16
; GFX9-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc
+; GFX9-NEXT: v_add_f32_e32 v6, s4, v1
+; GFX9-NEXT: v_bfe_u32 v7, v6, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v7, v7, v6
+; GFX9-NEXT: s_and_b32 s4, s20, 0xffff0000
+; GFX9-NEXT: v_add_u32_e32 v7, 0x7fff, v7
+; GFX9-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX9-NEXT: v_add_f32_e32 v6, s4, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v10, v7, v10, vcc
+; GFX9-NEXT: v_bfe_u32 v7, v6, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v7, v7, v6
+; GFX9-NEXT: s_and_b32 s5, s22, 0xffff0000
+; GFX9-NEXT: v_add_u32_e32 v7, 0x7fff, v7
+; GFX9-NEXT: v_or_b32_e32 v11, 0x400000, v6
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
; GFX9-NEXT: v_add_f32_e32 v6, s5, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v11, v7, v11, vcc
; GFX9-NEXT: v_bfe_u32 v7, v6, 16, 1
; GFX9-NEXT: v_add_u32_e32 v7, v7, v6
; GFX9-NEXT: v_add_u32_e32 v7, 0x7fff, v7
; GFX9-NEXT: v_or_b32_e32 v12, 0x400000, v6
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
-; GFX9-NEXT: s_and_b32 s5, s23, 0xffff0000
+; GFX9-NEXT: s_lshl_b32 s5, s22, 16
; GFX9-NEXT: v_cndmask_b32_e32 v6, v7, v12, vcc
; GFX9-NEXT: v_add_f32_e32 v7, s5, v1
; GFX9-NEXT: v_bfe_u32 v12, v7, 16, 1
; GFX9-NEXT: v_add_u32_e32 v12, v12, v7
+; GFX9-NEXT: s_and_b32 s5, s23, 0xffff0000
; GFX9-NEXT: v_add_u32_e32 v12, 0x7fff, v12
; GFX9-NEXT: v_or_b32_e32 v13, 0x400000, v7
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
+; GFX9-NEXT: v_add_f32_e32 v7, s5, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v12, v12, v13, vcc
+; GFX9-NEXT: v_bfe_u32 v13, v7, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v13, v13, v7
+; GFX9-NEXT: v_add_u32_e32 v13, 0x7fff, v13
+; GFX9-NEXT: v_or_b32_e32 v14, 0x400000, v7
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX9-NEXT: s_lshl_b32 s5, s23, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v7, v12, v13, vcc
-; GFX9-NEXT: v_add_f32_e32 v12, s5, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v7, v13, v14, vcc
+; GFX9-NEXT: v_add_f32_e32 v13, s5, v1
+; GFX9-NEXT: v_bfe_u32 v14, v13, 16, 1
+; GFX9-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX9-NEXT: v_add_u32_e32 v14, v14, v13
+; GFX9-NEXT: s_lshl_b32 s4, s20, 16
+; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v7
+; GFX9-NEXT: v_add_u32_e32 v14, 0x7fff, v14
+; GFX9-NEXT: v_or_b32_e32 v15, 0x400000, v13
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v13, v13
+; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX9-NEXT: v_cndmask_b32_e32 v13, v14, v15, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX9-NEXT: v_or_b32_sdwa v6, v6, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f32_e32 v12, s4, v1
+; GFX9-NEXT: v_or_b32_sdwa v7, v7, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_bfe_u32 v13, v12, 16, 1
; GFX9-NEXT: v_add_u32_e32 v13, v13, v12
; GFX9-NEXT: v_add_u32_e32 v13, 0x7fff, v13
; GFX9-NEXT: v_or_b32_e32 v14, 0x400000, v12
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v12, v12
-; GFX9-NEXT: v_cndmask_b32_e32 v12, v13, v14, vcc
-; GFX9-NEXT: v_mov_b32_e32 v13, 0xffff
; GFX9-NEXT: s_and_b32 s4, s21, 0xffff0000
-; GFX9-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX9-NEXT: v_and_b32_sdwa v6, v13, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX9-NEXT: v_and_b32_sdwa v12, v13, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v6, v5, 16, v6
-; GFX9-NEXT: v_add_f32_e32 v5, s4, v1
-; GFX9-NEXT: v_lshl_or_b32 v7, v7, 16, v12
-; GFX9-NEXT: v_bfe_u32 v12, v5, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v12, v12, v5
-; GFX9-NEXT: v_add_u32_e32 v12, 0x7fff, v12
-; GFX9-NEXT: v_or_b32_e32 v14, 0x400000, v5
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX9-NEXT: v_cndmask_b32_e32 v12, v13, v14, vcc
+; GFX9-NEXT: v_add_f32_e32 v13, s4, v1
+; GFX9-NEXT: v_bfe_u32 v14, v13, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v14, v14, v13
; GFX9-NEXT: s_lshl_b32 s4, s21, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v5, v12, v14, vcc
-; GFX9-NEXT: v_add_f32_e32 v12, s4, v1
-; GFX9-NEXT: v_bfe_u32 v14, v12, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v14, v14, v12
-; GFX9-NEXT: v_add_u32_e32 v14, 0x7fff, v14
-; GFX9-NEXT: v_or_b32_e32 v15, 0x400000, v12
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v12, v12
-; GFX9-NEXT: v_cndmask_b32_e32 v12, v14, v15, vcc
-; GFX9-NEXT: s_and_b32 s4, s20, 0xffff0000
-; GFX9-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX9-NEXT: v_and_b32_sdwa v12, v13, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v5, v5, 16, v12
-; GFX9-NEXT: v_add_f32_e32 v12, s4, v1
-; GFX9-NEXT: v_bfe_u32 v14, v12, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v14, v14, v12
-; GFX9-NEXT: s_lshl_b32 s4, s20, 16
; GFX9-NEXT: v_add_u32_e32 v14, 0x7fff, v14
-; GFX9-NEXT: v_or_b32_e32 v15, 0x400000, v12
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v12, v12
+; GFX9-NEXT: v_or_b32_e32 v15, 0x400000, v13
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v13, v13
; GFX9-NEXT: v_add_f32_e32 v1, s4, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v12, v14, v15, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v13, v14, v15, vcc
; GFX9-NEXT: v_bfe_u32 v14, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v14, v14, v1
; GFX9-NEXT: v_add_u32_e32 v14, 0x7fff, v14
; GFX9-NEXT: v_or_b32_e32 v15, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v3
-; GFX9-NEXT: v_lshrrev_b32_e32 v3, 16, v10
-; GFX9-NEXT: v_cndmask_b32_e32 v10, v14, v15, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v16, 16, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v14, v15, vcc
+; GFX9-NEXT: v_lshrrev_b32_e32 v15, 16, v3
+; GFX9-NEXT: v_lshrrev_b32_e32 v3, 16, v5
+; GFX9-NEXT: v_lshrrev_b32_e32 v5, 16, v13
+; GFX9-NEXT: v_lshrrev_b32_e32 v14, 16, v2
; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v4
-; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v12
-; GFX9-NEXT: v_and_b32_sdwa v10, v13, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v4, v4, 16, v10
-; GFX9-NEXT: v_and_b32_sdwa v10, v13, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_and_b32_sdwa v9, v13, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_and_b32_sdwa v8, v13, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_and_b32_sdwa v0, v13, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v3, v3, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v2, v2, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v0, v16, 16, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v11
+; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX9-NEXT: v_or_b32_sdwa v5, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX9-NEXT: v_or_b32_sdwa v4, v1, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v3, v1, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v1, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v15
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v14
+; GFX9-NEXT: v_or_b32_sdwa v0, v8, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_setpc_b64 s[30:31]
; GFX9-NEXT: .LBB103_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
@@ -40370,59 +40358,59 @@ define inreg <16 x half> @bitcast_v32i8_to_v16f16_scalar(<32 x i8> inreg %a, i32
; GFX9-NEXT: v_readfirstlane_b32 s8, v0
; GFX9-NEXT: s_cbranch_scc0 .LBB107_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v4, 0xc0c0004
-; GFX9-NEXT: v_perm_b32 v0, s16, v0, v4
; GFX9-NEXT: v_mov_b32_e32 v1, s19
+; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_perm_b32 v1, s18, v1, v4
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX9-NEXT: v_mov_b32_e32 v1, s21
-; GFX9-NEXT: v_perm_b32 v1, s20, v1, v4
+; GFX9-NEXT: v_perm_b32 v0, s16, v0, v4
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_mov_b32_e32 v2, s23
-; GFX9-NEXT: v_perm_b32 v2, s22, v2, v4
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, s25
-; GFX9-NEXT: v_perm_b32 v2, s24, v2, v4
-; GFX9-NEXT: v_mov_b32_e32 v3, s27
-; GFX9-NEXT: v_perm_b32 v3, s26, v3, v4
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_lshl_or_b32 v2, v3, 16, v2
-; GFX9-NEXT: v_mov_b32_e32 v3, s29
-; GFX9-NEXT: v_perm_b32 v3, s28, v3, v4
-; GFX9-NEXT: v_mov_b32_e32 v5, s6
; GFX9-NEXT: s_and_b32 s4, s14, 0xff
; GFX9-NEXT: s_lshl_b32 s5, s12, 8
-; GFX9-NEXT: v_perm_b32 v5, s8, v5, v4
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v1, s21
+; GFX9-NEXT: v_perm_b32 v2, s22, v2, v4
; GFX9-NEXT: s_or_b32 s4, s4, s5
; GFX9-NEXT: s_and_b32 s5, s9, 0xff
; GFX9-NEXT: s_lshl_b32 s56, s7, 8
-; GFX9-NEXT: v_lshl_or_b32 v3, v5, 16, v3
+; GFX9-NEXT: v_perm_b32 v1, s20, v1, v4
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_mov_b32_e32 v3, s27
; GFX9-NEXT: s_or_b32 s5, s5, s56
-; GFX9-NEXT: v_mov_b32_e32 v5, s40
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v2, s25
+; GFX9-NEXT: v_perm_b32 v3, s26, v3, v4
; GFX9-NEXT: s_pack_ll_b32_b16 s4, s4, s5
-; GFX9-NEXT: v_perm_b32 v5, s42, v5, v4
; GFX9-NEXT: s_and_b32 s5, s11, 0xff
; GFX9-NEXT: s_lshl_b32 s56, s10, 8
+; GFX9-NEXT: v_perm_b32 v2, s24, v2, v4
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_mov_b32_e32 v5, s6
; GFX9-NEXT: s_or_b32 s5, s5, s56
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_lshl_or_b32 v5, s5, 16, v5
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v3, s29
+; GFX9-NEXT: v_perm_b32 v5, s8, v5, v4
+; GFX9-NEXT: v_lshlrev_b32_e64 v6, 16, s5
; GFX9-NEXT: s_and_b32 s5, s45, 0xff
; GFX9-NEXT: s_lshl_b32 s56, s44, 8
+; GFX9-NEXT: v_perm_b32 v3, s28, v3, v4
+; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX9-NEXT: s_or_b32 s5, s5, s56
; GFX9-NEXT: s_and_b32 s56, s15, 0xff
; GFX9-NEXT: s_lshl_b32 s57, s13, 8
+; GFX9-NEXT: v_or_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v5, s40
; GFX9-NEXT: s_or_b32 s56, s56, s57
-; GFX9-NEXT: v_mov_b32_e32 v6, s46
+; GFX9-NEXT: v_perm_b32 v5, s42, v5, v4
; GFX9-NEXT: s_pack_ll_b32_b16 s5, s5, s56
-; GFX9-NEXT: v_perm_b32 v4, s47, v6, v4
; GFX9-NEXT: s_and_b32 s56, s43, 0xff
; GFX9-NEXT: s_lshl_b32 s57, s41, 8
+; GFX9-NEXT: v_or_b32_sdwa v5, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v6, s46
; GFX9-NEXT: s_or_b32 s56, s56, s57
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_lshl_or_b32 v7, s56, 16, v4
+; GFX9-NEXT: v_perm_b32 v4, s47, v6, v4
+; GFX9-NEXT: v_lshlrev_b32_e64 v6, 16, s56
+; GFX9-NEXT: v_or_b32_sdwa v7, v6, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v4, s4
; GFX9-NEXT: v_mov_b32_e32 v6, s5
; GFX9-NEXT: s_mov_b64 s[4:5], 0
@@ -40470,53 +40458,46 @@ define inreg <16 x half> @bitcast_v32i8_to_v16f16_scalar(<32 x i8> inreg %a, i32
; GFX9-NEXT: v_add_u32_e32 v1, 0x300, v0
; GFX9-NEXT: s_add_i32 s22, s22, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s23
+; GFX9-NEXT: s_movk_i32 s4, 0x300
; GFX9-NEXT: v_perm_b32 v0, s22, v0, v2
-; GFX9-NEXT: v_add_u32_e32 v12, 0x300, v0
+; GFX9-NEXT: v_add_u32_sdwa v12, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s16, s16, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s17
-; GFX9-NEXT: v_perm_b32 v0, s16, v0, v2
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: v_mov_b32_e32 v13, s19
-; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
+; GFX9-NEXT: v_perm_b32 v0, s16, v0, v2
; GFX9-NEXT: v_perm_b32 v13, s18, v13, v2
-; GFX9-NEXT: v_add_u32_e32 v13, 0x300, v13
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v0, v13, 16, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: s_add_i32 s24, s24, 3
-; GFX9-NEXT: v_mov_b32_e32 v13, s25
-; GFX9-NEXT: v_add_u32_e32 v16, 0x300, v5
-; GFX9-NEXT: v_add_u32_e32 v5, 0x300, v7
+; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
+; GFX9-NEXT: v_add_u32_sdwa v13, v13, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s8, s8, 3
-; GFX9-NEXT: v_lshl_or_b32 v1, v12, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v0, v13, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v12, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v12, s6
-; GFX9-NEXT: v_perm_b32 v13, s24, v13, v2
+; GFX9-NEXT: s_add_i32 s24, s24, 3
+; GFX9-NEXT: v_mov_b32_e32 v13, s25
; GFX9-NEXT: s_add_i32 s26, s26, 3
; GFX9-NEXT: v_mov_b32_e32 v14, s27
-; GFX9-NEXT: v_add_u32_e32 v7, 0x300, v8
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
; GFX9-NEXT: v_perm_b32 v12, s8, v12, v2
+; GFX9-NEXT: v_perm_b32 v13, s24, v13, v2
; GFX9-NEXT: v_perm_b32 v2, s26, v14, v2
; GFX9-NEXT: v_add_u32_e32 v14, 0x300, v3
-; GFX9-NEXT: v_add_u32_e32 v15, 0x300, v4
-; GFX9-NEXT: v_add_u32_e32 v6, 0x300, v6
+; GFX9-NEXT: v_add_u32_sdwa v15, v4, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_add_u32_e32 v16, 0x300, v5
+; GFX9-NEXT: v_add_u32_sdwa v6, v6, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_add_u32_e32 v5, 0x300, v7
+; GFX9-NEXT: v_add_u32_sdwa v7, v8, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_add_u32_e32 v4, 0x300, v9
-; GFX9-NEXT: v_add_u32_e32 v8, 0x300, v10
+; GFX9-NEXT: v_add_u32_sdwa v8, v10, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_add_u32_e32 v3, 0x300, v11
+; GFX9-NEXT: v_add_u32_sdwa v9, v12, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_add_u32_e32 v10, 0x300, v13
-; GFX9-NEXT: v_lshl_or_b32 v5, v7, 16, v5
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v16
-; GFX9-NEXT: v_add_u32_e32 v9, 0x300, v12
-; GFX9-NEXT: v_add_u32_e32 v2, 0x300, v2
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_lshl_or_b32 v6, v6, 16, v7
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v14
-; GFX9-NEXT: v_lshl_or_b32 v2, v2, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v3, v9, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v8, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v7, v15, 16, v7
+; GFX9-NEXT: v_add_u32_sdwa v2, v2, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v2, v2, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v9, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v4, v8, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v7, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v6, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v15, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: .LBB107_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -42583,9 +42564,9 @@ define inreg <32 x i8> @bitcast_v16bf16_to_v32i8_scalar(<16 x bfloat> inreg %a,
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v35, 16, v0
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v6
; GFX9-NEXT: s_and_b32 s4, s16, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v4, v6, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v4, v0, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v0, s4, v1
; GFX9-NEXT: v_bfe_u32 v2, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v0
@@ -42620,9 +42601,9 @@ define inreg <32 x i8> @bitcast_v16bf16_to_v32i8_scalar(<16 x bfloat> inreg %a,
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
; GFX9-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v34, 16, v5
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v14
; GFX9-NEXT: s_and_b32 s4, s18, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v12, v14, 16, v5
+; GFX9-NEXT: v_or_b32_sdwa v12, v5, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v5, s4, v1
; GFX9-NEXT: v_bfe_u32 v7, v5, 16, 1
; GFX9-NEXT: v_add_u32_e32 v7, v7, v5
@@ -42640,9 +42621,9 @@ define inreg <32 x i8> @bitcast_v16bf16_to_v32i8_scalar(<16 x bfloat> inreg %a,
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
; GFX9-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v5
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v10
; GFX9-NEXT: s_and_b32 s4, s21, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v11, v10, 16, v5
+; GFX9-NEXT: v_or_b32_sdwa v11, v5, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v5, s4, v1
; GFX9-NEXT: v_bfe_u32 v7, v5, 16, 1
; GFX9-NEXT: v_add_u32_e32 v7, v7, v5
@@ -42660,9 +42641,9 @@ define inreg <32 x i8> @bitcast_v16bf16_to_v32i8_scalar(<16 x bfloat> inreg %a,
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
; GFX9-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v33, 16, v5
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v22
; GFX9-NEXT: s_and_b32 s4, s20, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v20, v22, 16, v5
+; GFX9-NEXT: v_or_b32_sdwa v20, v5, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v5, s4, v1
; GFX9-NEXT: v_bfe_u32 v7, v5, 16, 1
; GFX9-NEXT: v_add_u32_e32 v7, v7, v5
@@ -42680,9 +42661,9 @@ define inreg <32 x i8> @bitcast_v16bf16_to_v32i8_scalar(<16 x bfloat> inreg %a,
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
; GFX9-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v16, 16, v5
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v16
+; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v18
; GFX9-NEXT: s_and_b32 s4, s23, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v19, v18, 16, v5
+; GFX9-NEXT: v_or_b32_sdwa v19, v5, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v5, s4, v1
; GFX9-NEXT: v_bfe_u32 v7, v5, 16, 1
; GFX9-NEXT: v_add_u32_e32 v7, v7, v5
@@ -42714,17 +42695,17 @@ define inreg <32 x i8> @bitcast_v16bf16_to_v32i8_scalar(<16 x bfloat> inreg %a,
; GFX9-NEXT: v_or_b32_e32 v15, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: v_cndmask_b32_e32 v1, v13, v15, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v32, 16, v7
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_lshrrev_b32_e32 v30, 16, v5
-; GFX9-NEXT: v_lshrrev_b32_e32 v24, 16, v1
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v32
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v32, 16, v7
; GFX9-NEXT: v_lshrrev_b32_e32 v26, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v28, v30, 16, v1
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v24
-; GFX9-NEXT: v_lshl_or_b32 v3, v2, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v27, v26, 16, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v24, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v30
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v28, v1, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v26
+; GFX9-NEXT: v_or_b32_sdwa v3, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v27, v1, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_lshrrev_b32_e32 v31, 24, v28
; GFX9-NEXT: v_lshrrev_b32_e32 v29, 8, v28
; GFX9-NEXT: v_lshrrev_b32_e32 v25, 8, v27
@@ -44806,59 +44787,59 @@ define inreg <16 x bfloat> @bitcast_v32i8_to_v16bf16_scalar(<32 x i8> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s8, v0
; GFX9-NEXT: s_cbranch_scc0 .LBB111_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v4, 0xc0c0004
-; GFX9-NEXT: v_perm_b32 v0, s16, v0, v4
; GFX9-NEXT: v_mov_b32_e32 v1, s19
+; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_perm_b32 v1, s18, v1, v4
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX9-NEXT: v_mov_b32_e32 v1, s21
-; GFX9-NEXT: v_perm_b32 v1, s20, v1, v4
+; GFX9-NEXT: v_perm_b32 v0, s16, v0, v4
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_mov_b32_e32 v2, s23
-; GFX9-NEXT: v_perm_b32 v2, s22, v2, v4
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, s25
-; GFX9-NEXT: v_perm_b32 v2, s24, v2, v4
-; GFX9-NEXT: v_mov_b32_e32 v3, s27
-; GFX9-NEXT: v_perm_b32 v3, s26, v3, v4
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_lshl_or_b32 v2, v3, 16, v2
-; GFX9-NEXT: v_mov_b32_e32 v3, s29
-; GFX9-NEXT: v_perm_b32 v3, s28, v3, v4
-; GFX9-NEXT: v_mov_b32_e32 v5, s6
; GFX9-NEXT: s_and_b32 s4, s14, 0xff
; GFX9-NEXT: s_lshl_b32 s5, s12, 8
-; GFX9-NEXT: v_perm_b32 v5, s8, v5, v4
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v1, s21
+; GFX9-NEXT: v_perm_b32 v2, s22, v2, v4
; GFX9-NEXT: s_or_b32 s4, s4, s5
; GFX9-NEXT: s_and_b32 s5, s9, 0xff
; GFX9-NEXT: s_lshl_b32 s56, s7, 8
-; GFX9-NEXT: v_lshl_or_b32 v3, v5, 16, v3
+; GFX9-NEXT: v_perm_b32 v1, s20, v1, v4
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_mov_b32_e32 v3, s27
; GFX9-NEXT: s_or_b32 s5, s5, s56
-; GFX9-NEXT: v_mov_b32_e32 v5, s40
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v2, s25
+; GFX9-NEXT: v_perm_b32 v3, s26, v3, v4
; GFX9-NEXT: s_pack_ll_b32_b16 s4, s4, s5
-; GFX9-NEXT: v_perm_b32 v5, s42, v5, v4
; GFX9-NEXT: s_and_b32 s5, s11, 0xff
; GFX9-NEXT: s_lshl_b32 s56, s10, 8
+; GFX9-NEXT: v_perm_b32 v2, s24, v2, v4
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_mov_b32_e32 v5, s6
; GFX9-NEXT: s_or_b32 s5, s5, s56
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_lshl_or_b32 v5, s5, 16, v5
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v3, s29
+; GFX9-NEXT: v_perm_b32 v5, s8, v5, v4
+; GFX9-NEXT: v_lshlrev_b32_e64 v6, 16, s5
; GFX9-NEXT: s_and_b32 s5, s45, 0xff
; GFX9-NEXT: s_lshl_b32 s56, s44, 8
+; GFX9-NEXT: v_perm_b32 v3, s28, v3, v4
+; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX9-NEXT: s_or_b32 s5, s5, s56
; GFX9-NEXT: s_and_b32 s56, s15, 0xff
; GFX9-NEXT: s_lshl_b32 s57, s13, 8
+; GFX9-NEXT: v_or_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v5, s40
; GFX9-NEXT: s_or_b32 s56, s56, s57
-; GFX9-NEXT: v_mov_b32_e32 v6, s46
+; GFX9-NEXT: v_perm_b32 v5, s42, v5, v4
; GFX9-NEXT: s_pack_ll_b32_b16 s5, s5, s56
-; GFX9-NEXT: v_perm_b32 v4, s47, v6, v4
; GFX9-NEXT: s_and_b32 s56, s43, 0xff
; GFX9-NEXT: s_lshl_b32 s57, s41, 8
+; GFX9-NEXT: v_or_b32_sdwa v5, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v6, s46
; GFX9-NEXT: s_or_b32 s56, s56, s57
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_lshl_or_b32 v7, s56, 16, v4
+; GFX9-NEXT: v_perm_b32 v4, s47, v6, v4
+; GFX9-NEXT: v_lshlrev_b32_e64 v6, 16, s56
+; GFX9-NEXT: v_or_b32_sdwa v7, v6, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v4, s4
; GFX9-NEXT: v_mov_b32_e32 v6, s5
; GFX9-NEXT: s_mov_b64 s[4:5], 0
@@ -44906,53 +44887,46 @@ define inreg <16 x bfloat> @bitcast_v32i8_to_v16bf16_scalar(<32 x i8> inreg %a,
; GFX9-NEXT: v_add_u32_e32 v1, 0x300, v0
; GFX9-NEXT: s_add_i32 s22, s22, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s23
+; GFX9-NEXT: s_movk_i32 s4, 0x300
; GFX9-NEXT: v_perm_b32 v0, s22, v0, v2
-; GFX9-NEXT: v_add_u32_e32 v12, 0x300, v0
+; GFX9-NEXT: v_add_u32_sdwa v12, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s16, s16, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s17
-; GFX9-NEXT: v_perm_b32 v0, s16, v0, v2
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: v_mov_b32_e32 v13, s19
-; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
+; GFX9-NEXT: v_perm_b32 v0, s16, v0, v2
; GFX9-NEXT: v_perm_b32 v13, s18, v13, v2
-; GFX9-NEXT: v_add_u32_e32 v13, 0x300, v13
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v0, v13, 16, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: s_add_i32 s24, s24, 3
-; GFX9-NEXT: v_mov_b32_e32 v13, s25
-; GFX9-NEXT: v_add_u32_e32 v16, 0x300, v5
-; GFX9-NEXT: v_add_u32_e32 v5, 0x300, v7
+; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
+; GFX9-NEXT: v_add_u32_sdwa v13, v13, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s8, s8, 3
-; GFX9-NEXT: v_lshl_or_b32 v1, v12, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v0, v13, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v12, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v12, s6
-; GFX9-NEXT: v_perm_b32 v13, s24, v13, v2
+; GFX9-NEXT: s_add_i32 s24, s24, 3
+; GFX9-NEXT: v_mov_b32_e32 v13, s25
; GFX9-NEXT: s_add_i32 s26, s26, 3
; GFX9-NEXT: v_mov_b32_e32 v14, s27
-; GFX9-NEXT: v_add_u32_e32 v7, 0x300, v8
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
; GFX9-NEXT: v_perm_b32 v12, s8, v12, v2
+; GFX9-NEXT: v_perm_b32 v13, s24, v13, v2
; GFX9-NEXT: v_perm_b32 v2, s26, v14, v2
; GFX9-NEXT: v_add_u32_e32 v14, 0x300, v3
-; GFX9-NEXT: v_add_u32_e32 v15, 0x300, v4
-; GFX9-NEXT: v_add_u32_e32 v6, 0x300, v6
+; GFX9-NEXT: v_add_u32_sdwa v15, v4, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_add_u32_e32 v16, 0x300, v5
+; GFX9-NEXT: v_add_u32_sdwa v6, v6, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_add_u32_e32 v5, 0x300, v7
+; GFX9-NEXT: v_add_u32_sdwa v7, v8, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_add_u32_e32 v4, 0x300, v9
-; GFX9-NEXT: v_add_u32_e32 v8, 0x300, v10
+; GFX9-NEXT: v_add_u32_sdwa v8, v10, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_add_u32_e32 v3, 0x300, v11
+; GFX9-NEXT: v_add_u32_sdwa v9, v12, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_add_u32_e32 v10, 0x300, v13
-; GFX9-NEXT: v_lshl_or_b32 v5, v7, 16, v5
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v16
-; GFX9-NEXT: v_add_u32_e32 v9, 0x300, v12
-; GFX9-NEXT: v_add_u32_e32 v2, 0x300, v2
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_lshl_or_b32 v6, v6, 16, v7
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v14
-; GFX9-NEXT: v_lshl_or_b32 v2, v2, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v3, v9, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v8, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v7, v15, 16, v7
+; GFX9-NEXT: v_add_u32_sdwa v2, v2, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v2, v2, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v9, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v4, v8, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v7, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v6, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v15, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: .LBB111_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.320bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.320bit.ll
index cad415ed988a3..a00f7b3e2a386 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.320bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.320bit.ll
@@ -19433,69 +19433,69 @@ define inreg <20 x i16> @bitcast_v40i8_to_v20i16_scalar(<40 x i8> inreg %a, i32
; GFX9-NEXT: v_readfirstlane_b32 s42, v0
; GFX9-NEXT: s_cbranch_scc0 .LBB51_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v8, 0xc0c0004
-; GFX9-NEXT: v_perm_b32 v0, s16, v0, v8
; GFX9-NEXT: v_mov_b32_e32 v1, s19
+; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_perm_b32 v1, s18, v1, v8
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX9-NEXT: v_mov_b32_e32 v1, s21
-; GFX9-NEXT: v_perm_b32 v1, s20, v1, v8
+; GFX9-NEXT: v_perm_b32 v0, s16, v0, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_mov_b32_e32 v2, s23
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v1, s21
; GFX9-NEXT: v_perm_b32 v2, s22, v2, v8
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, s25
-; GFX9-NEXT: v_perm_b32 v2, s24, v2, v8
+; GFX9-NEXT: v_perm_b32 v1, s20, v1, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX9-NEXT: v_mov_b32_e32 v3, s27
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v2, s25
; GFX9-NEXT: v_perm_b32 v3, s26, v3, v8
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_lshl_or_b32 v2, v3, 16, v2
-; GFX9-NEXT: v_mov_b32_e32 v3, s29
-; GFX9-NEXT: v_perm_b32 v3, s28, v3, v8
+; GFX9-NEXT: v_perm_b32 v2, s24, v2, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_mov_b32_e32 v4, s41
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v3, s29
; GFX9-NEXT: v_perm_b32 v4, s42, v4, v8
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_lshl_or_b32 v3, v4, 16, v3
-; GFX9-NEXT: v_mov_b32_e32 v4, s40
-; GFX9-NEXT: v_perm_b32 v4, s43, v4, v8
+; GFX9-NEXT: v_perm_b32 v3, s28, v3, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX9-NEXT: v_mov_b32_e32 v5, s14
+; GFX9-NEXT: v_or_b32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v4, s40
; GFX9-NEXT: v_perm_b32 v5, s15, v5, v8
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_lshl_or_b32 v4, v5, 16, v4
-; GFX9-NEXT: v_mov_b32_e32 v5, s13
-; GFX9-NEXT: v_perm_b32 v5, s44, v5, v8
+; GFX9-NEXT: v_perm_b32 v4, s43, v4, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX9-NEXT: v_mov_b32_e32 v6, s11
+; GFX9-NEXT: v_or_b32_sdwa v4, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v5, s13
; GFX9-NEXT: v_perm_b32 v6, s12, v6, v8
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_lshl_or_b32 v5, v6, 16, v5
-; GFX9-NEXT: v_mov_b32_e32 v6, s9
-; GFX9-NEXT: v_perm_b32 v6, s10, v6, v8
+; GFX9-NEXT: v_perm_b32 v5, s44, v5, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v6
; GFX9-NEXT: v_mov_b32_e32 v7, s6
+; GFX9-NEXT: v_or_b32_sdwa v5, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v6, s9
; GFX9-NEXT: v_perm_b32 v7, s8, v7, v8
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_lshl_or_b32 v6, v7, 16, v6
-; GFX9-NEXT: v_mov_b32_e32 v7, s56
-; GFX9-NEXT: v_perm_b32 v7, s58, v7, v8
-; GFX9-NEXT: v_mov_b32_e32 v9, s7
; GFX9-NEXT: s_and_b32 s4, s61, 0xff
; GFX9-NEXT: s_lshl_b32 s5, s60, 8
-; GFX9-NEXT: v_perm_b32 v9, s45, v9, v8
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX9-NEXT: v_perm_b32 v6, s10, v6, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX9-NEXT: v_mov_b32_e32 v9, s7
; GFX9-NEXT: s_or_b32 s4, s4, s5
; GFX9-NEXT: s_and_b32 s5, s47, 0xff
; GFX9-NEXT: s_lshl_b32 s72, s46, 8
-; GFX9-NEXT: v_lshl_or_b32 v7, v9, 16, v7
+; GFX9-NEXT: v_or_b32_sdwa v6, v7, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v7, s56
+; GFX9-NEXT: v_perm_b32 v9, s45, v9, v8
; GFX9-NEXT: s_or_b32 s5, s5, s72
-; GFX9-NEXT: v_mov_b32_e32 v9, s62
+; GFX9-NEXT: v_perm_b32 v7, s58, v7, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v9
; GFX9-NEXT: s_pack_ll_b32_b16 s4, s4, s5
-; GFX9-NEXT: v_perm_b32 v8, s63, v9, v8
; GFX9-NEXT: s_and_b32 s5, s59, 0xff
; GFX9-NEXT: s_lshl_b32 s72, s57, 8
+; GFX9-NEXT: v_or_b32_sdwa v7, v9, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v9, s62
; GFX9-NEXT: s_or_b32 s5, s5, s72
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, s5, 16, v8
+; GFX9-NEXT: v_perm_b32 v8, s63, v9, v8
+; GFX9-NEXT: v_lshlrev_b32_e64 v9, 16, s5
+; GFX9-NEXT: v_or_b32_sdwa v9, v9, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v8, s4
; GFX9-NEXT: s_mov_b64 s[4:5], 0
; GFX9-NEXT: s_branch .LBB51_3
@@ -19530,85 +19530,76 @@ define inreg <20 x i16> @bitcast_v40i8_to_v20i16_scalar(<40 x i8> inreg %a, i32
; GFX9-NEXT: v_add_u32_e32 v1, 0x300, v0
; GFX9-NEXT: s_add_i32 s22, s22, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s23
+; GFX9-NEXT: s_movk_i32 s4, 0x300
; GFX9-NEXT: v_perm_b32 v0, s22, v0, v6
-; GFX9-NEXT: v_add_u32_e32 v2, 0x300, v0
+; GFX9-NEXT: v_add_u32_sdwa v2, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s16, s16, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s17
-; GFX9-NEXT: v_perm_b32 v0, s16, v0, v6
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: v_mov_b32_e32 v3, s19
-; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
+; GFX9-NEXT: v_perm_b32 v0, s16, v0, v6
; GFX9-NEXT: v_perm_b32 v3, s18, v3, v6
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
+; GFX9-NEXT: v_add_u32_sdwa v3, v3, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s24, s24, 3
-; GFX9-NEXT: v_add_u32_e32 v3, 0x300, v3
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v2, s25
-; GFX9-NEXT: v_lshl_or_b32 v0, v3, 16, v0
-; GFX9-NEXT: v_perm_b32 v2, s24, v2, v6
; GFX9-NEXT: s_add_i32 s26, s26, 3
; GFX9-NEXT: v_mov_b32_e32 v3, s27
-; GFX9-NEXT: v_add_u32_e32 v2, 0x300, v2
+; GFX9-NEXT: v_perm_b32 v2, s24, v2, v6
; GFX9-NEXT: v_perm_b32 v3, s26, v3, v6
-; GFX9-NEXT: v_add_u32_e32 v3, 0x300, v3
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX9-NEXT: v_add_u32_e32 v2, 0x300, v2
+; GFX9-NEXT: v_add_u32_sdwa v3, v3, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s28, s28, 3
-; GFX9-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v3, s29
-; GFX9-NEXT: v_perm_b32 v3, s28, v3, v6
; GFX9-NEXT: s_add_i32 s42, s42, 3
; GFX9-NEXT: v_mov_b32_e32 v4, s41
-; GFX9-NEXT: v_add_u32_e32 v3, 0x300, v3
+; GFX9-NEXT: v_perm_b32 v3, s28, v3, v6
; GFX9-NEXT: v_perm_b32 v4, s42, v4, v6
-; GFX9-NEXT: v_add_u32_e32 v4, 0x300, v4
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX9-NEXT: v_add_u32_e32 v3, 0x300, v3
+; GFX9-NEXT: v_add_u32_sdwa v4, v4, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s43, s43, 3
-; GFX9-NEXT: v_lshl_or_b32 v3, v4, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v4, s40
-; GFX9-NEXT: v_perm_b32 v4, s43, v4, v6
; GFX9-NEXT: s_add_i32 s15, s15, 3
; GFX9-NEXT: v_mov_b32_e32 v5, s14
-; GFX9-NEXT: v_add_u32_e32 v4, 0x300, v4
+; GFX9-NEXT: v_perm_b32 v4, s43, v4, v6
; GFX9-NEXT: v_perm_b32 v5, s15, v5, v6
-; GFX9-NEXT: v_add_u32_e32 v5, 0x300, v5
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX9-NEXT: v_add_u32_e32 v4, 0x300, v4
+; GFX9-NEXT: v_add_u32_sdwa v5, v5, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s44, s44, 3
-; GFX9-NEXT: v_lshl_or_b32 v4, v5, 16, v4
+; GFX9-NEXT: v_or_b32_sdwa v4, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v5, s13
-; GFX9-NEXT: v_perm_b32 v5, s44, v5, v6
; GFX9-NEXT: s_add_i32 s12, s12, 3
; GFX9-NEXT: v_mov_b32_e32 v12, s11
-; GFX9-NEXT: v_add_u32_e32 v5, 0x300, v5
+; GFX9-NEXT: v_perm_b32 v5, s44, v5, v6
; GFX9-NEXT: v_perm_b32 v12, s12, v12, v6
-; GFX9-NEXT: v_add_u32_e32 v12, 0x300, v12
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: s_add_i32 s10, s10, 3
-; GFX9-NEXT: v_mov_b32_e32 v13, s9
+; GFX9-NEXT: v_add_u32_e32 v5, 0x300, v5
+; GFX9-NEXT: v_add_u32_sdwa v12, v12, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s45, s45, 3
-; GFX9-NEXT: v_lshl_or_b32 v5, v12, 16, v5
+; GFX9-NEXT: v_or_b32_sdwa v5, v12, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v12, s7
-; GFX9-NEXT: v_perm_b32 v13, s10, v13, v6
+; GFX9-NEXT: s_add_i32 s10, s10, 3
+; GFX9-NEXT: v_mov_b32_e32 v13, s9
; GFX9-NEXT: s_add_i32 s8, s8, 3
; GFX9-NEXT: v_mov_b32_e32 v14, s6
-; GFX9-NEXT: v_add_u32_e32 v15, 0x300, v8
-; GFX9-NEXT: v_add_u32_e32 v8, 0x300, v9
; GFX9-NEXT: v_perm_b32 v12, s45, v12, v6
+; GFX9-NEXT: v_perm_b32 v13, s10, v13, v6
; GFX9-NEXT: v_perm_b32 v6, s8, v14, v6
; GFX9-NEXT: v_add_u32_e32 v14, 0x300, v7
-; GFX9-NEXT: v_add_u32_e32 v9, 0x300, v10
+; GFX9-NEXT: v_add_u32_sdwa v15, v8, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_add_u32_e32 v8, 0x300, v9
+; GFX9-NEXT: v_add_u32_sdwa v9, v10, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_add_u32_e32 v7, 0x300, v11
+; GFX9-NEXT: v_add_u32_sdwa v10, v12, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_add_u32_e32 v11, 0x300, v13
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_add_u32_e32 v10, 0x300, v12
-; GFX9-NEXT: v_add_u32_e32 v6, 0x300, v6
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v9, 16, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v14
-; GFX9-NEXT: v_lshl_or_b32 v6, v6, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v7, v10, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v9, v15, 16, v9
+; GFX9-NEXT: v_add_u32_sdwa v6, v6, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v6, v6, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v10, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v9, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v15, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: .LBB51_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -25874,69 +25865,69 @@ define inreg <20 x half> @bitcast_v40i8_to_v20f16_scalar(<40 x i8> inreg %a, i32
; GFX9-NEXT: v_readfirstlane_b32 s42, v0
; GFX9-NEXT: s_cbranch_scc0 .LBB63_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v8, 0xc0c0004
-; GFX9-NEXT: v_perm_b32 v0, s16, v0, v8
; GFX9-NEXT: v_mov_b32_e32 v1, s19
+; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_perm_b32 v1, s18, v1, v8
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX9-NEXT: v_mov_b32_e32 v1, s21
-; GFX9-NEXT: v_perm_b32 v1, s20, v1, v8
+; GFX9-NEXT: v_perm_b32 v0, s16, v0, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_mov_b32_e32 v2, s23
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v1, s21
; GFX9-NEXT: v_perm_b32 v2, s22, v2, v8
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, s25
-; GFX9-NEXT: v_perm_b32 v2, s24, v2, v8
+; GFX9-NEXT: v_perm_b32 v1, s20, v1, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX9-NEXT: v_mov_b32_e32 v3, s27
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v2, s25
; GFX9-NEXT: v_perm_b32 v3, s26, v3, v8
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_lshl_or_b32 v2, v3, 16, v2
-; GFX9-NEXT: v_mov_b32_e32 v3, s29
-; GFX9-NEXT: v_perm_b32 v3, s28, v3, v8
+; GFX9-NEXT: v_perm_b32 v2, s24, v2, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_mov_b32_e32 v4, s41
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v3, s29
; GFX9-NEXT: v_perm_b32 v4, s42, v4, v8
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_lshl_or_b32 v3, v4, 16, v3
-; GFX9-NEXT: v_mov_b32_e32 v4, s40
-; GFX9-NEXT: v_perm_b32 v4, s43, v4, v8
+; GFX9-NEXT: v_perm_b32 v3, s28, v3, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX9-NEXT: v_mov_b32_e32 v5, s14
+; GFX9-NEXT: v_or_b32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v4, s40
; GFX9-NEXT: v_perm_b32 v5, s15, v5, v8
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_lshl_or_b32 v4, v5, 16, v4
-; GFX9-NEXT: v_mov_b32_e32 v5, s13
-; GFX9-NEXT: v_perm_b32 v5, s44, v5, v8
+; GFX9-NEXT: v_perm_b32 v4, s43, v4, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX9-NEXT: v_mov_b32_e32 v6, s11
+; GFX9-NEXT: v_or_b32_sdwa v4, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v5, s13
; GFX9-NEXT: v_perm_b32 v6, s12, v6, v8
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_lshl_or_b32 v5, v6, 16, v5
-; GFX9-NEXT: v_mov_b32_e32 v6, s9
-; GFX9-NEXT: v_perm_b32 v6, s10, v6, v8
+; GFX9-NEXT: v_perm_b32 v5, s44, v5, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v6
; GFX9-NEXT: v_mov_b32_e32 v7, s6
+; GFX9-NEXT: v_or_b32_sdwa v5, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v6, s9
; GFX9-NEXT: v_perm_b32 v7, s8, v7, v8
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_lshl_or_b32 v6, v7, 16, v6
-; GFX9-NEXT: v_mov_b32_e32 v7, s56
-; GFX9-NEXT: v_perm_b32 v7, s58, v7, v8
-; GFX9-NEXT: v_mov_b32_e32 v9, s7
; GFX9-NEXT: s_and_b32 s4, s61, 0xff
; GFX9-NEXT: s_lshl_b32 s5, s60, 8
-; GFX9-NEXT: v_perm_b32 v9, s45, v9, v8
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX9-NEXT: v_perm_b32 v6, s10, v6, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX9-NEXT: v_mov_b32_e32 v9, s7
; GFX9-NEXT: s_or_b32 s4, s4, s5
; GFX9-NEXT: s_and_b32 s5, s47, 0xff
; GFX9-NEXT: s_lshl_b32 s72, s46, 8
-; GFX9-NEXT: v_lshl_or_b32 v7, v9, 16, v7
+; GFX9-NEXT: v_or_b32_sdwa v6, v7, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v7, s56
+; GFX9-NEXT: v_perm_b32 v9, s45, v9, v8
; GFX9-NEXT: s_or_b32 s5, s5, s72
-; GFX9-NEXT: v_mov_b32_e32 v9, s62
+; GFX9-NEXT: v_perm_b32 v7, s58, v7, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v9
; GFX9-NEXT: s_pack_ll_b32_b16 s4, s4, s5
-; GFX9-NEXT: v_perm_b32 v8, s63, v9, v8
; GFX9-NEXT: s_and_b32 s5, s59, 0xff
; GFX9-NEXT: s_lshl_b32 s72, s57, 8
+; GFX9-NEXT: v_or_b32_sdwa v7, v9, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v9, s62
; GFX9-NEXT: s_or_b32 s5, s5, s72
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, s5, 16, v8
+; GFX9-NEXT: v_perm_b32 v8, s63, v9, v8
+; GFX9-NEXT: v_lshlrev_b32_e64 v9, 16, s5
+; GFX9-NEXT: v_or_b32_sdwa v9, v9, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v8, s4
; GFX9-NEXT: s_mov_b64 s[4:5], 0
; GFX9-NEXT: s_branch .LBB63_3
@@ -25971,85 +25962,76 @@ define inreg <20 x half> @bitcast_v40i8_to_v20f16_scalar(<40 x i8> inreg %a, i32
; GFX9-NEXT: v_add_u32_e32 v1, 0x300, v0
; GFX9-NEXT: s_add_i32 s22, s22, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s23
+; GFX9-NEXT: s_movk_i32 s4, 0x300
; GFX9-NEXT: v_perm_b32 v0, s22, v0, v6
-; GFX9-NEXT: v_add_u32_e32 v2, 0x300, v0
+; GFX9-NEXT: v_add_u32_sdwa v2, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s16, s16, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s17
-; GFX9-NEXT: v_perm_b32 v0, s16, v0, v6
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: v_mov_b32_e32 v3, s19
-; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
+; GFX9-NEXT: v_perm_b32 v0, s16, v0, v6
; GFX9-NEXT: v_perm_b32 v3, s18, v3, v6
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
+; GFX9-NEXT: v_add_u32_sdwa v3, v3, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s24, s24, 3
-; GFX9-NEXT: v_add_u32_e32 v3, 0x300, v3
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v2, s25
-; GFX9-NEXT: v_lshl_or_b32 v0, v3, 16, v0
-; GFX9-NEXT: v_perm_b32 v2, s24, v2, v6
; GFX9-NEXT: s_add_i32 s26, s26, 3
; GFX9-NEXT: v_mov_b32_e32 v3, s27
-; GFX9-NEXT: v_add_u32_e32 v2, 0x300, v2
+; GFX9-NEXT: v_perm_b32 v2, s24, v2, v6
; GFX9-NEXT: v_perm_b32 v3, s26, v3, v6
-; GFX9-NEXT: v_add_u32_e32 v3, 0x300, v3
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX9-NEXT: v_add_u32_e32 v2, 0x300, v2
+; GFX9-NEXT: v_add_u32_sdwa v3, v3, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s28, s28, 3
-; GFX9-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v3, s29
-; GFX9-NEXT: v_perm_b32 v3, s28, v3, v6
; GFX9-NEXT: s_add_i32 s42, s42, 3
; GFX9-NEXT: v_mov_b32_e32 v4, s41
-; GFX9-NEXT: v_add_u32_e32 v3, 0x300, v3
+; GFX9-NEXT: v_perm_b32 v3, s28, v3, v6
; GFX9-NEXT: v_perm_b32 v4, s42, v4, v6
-; GFX9-NEXT: v_add_u32_e32 v4, 0x300, v4
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX9-NEXT: v_add_u32_e32 v3, 0x300, v3
+; GFX9-NEXT: v_add_u32_sdwa v4, v4, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s43, s43, 3
-; GFX9-NEXT: v_lshl_or_b32 v3, v4, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v4, s40
-; GFX9-NEXT: v_perm_b32 v4, s43, v4, v6
; GFX9-NEXT: s_add_i32 s15, s15, 3
; GFX9-NEXT: v_mov_b32_e32 v5, s14
-; GFX9-NEXT: v_add_u32_e32 v4, 0x300, v4
+; GFX9-NEXT: v_perm_b32 v4, s43, v4, v6
; GFX9-NEXT: v_perm_b32 v5, s15, v5, v6
-; GFX9-NEXT: v_add_u32_e32 v5, 0x300, v5
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX9-NEXT: v_add_u32_e32 v4, 0x300, v4
+; GFX9-NEXT: v_add_u32_sdwa v5, v5, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s44, s44, 3
-; GFX9-NEXT: v_lshl_or_b32 v4, v5, 16, v4
+; GFX9-NEXT: v_or_b32_sdwa v4, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v5, s13
-; GFX9-NEXT: v_perm_b32 v5, s44, v5, v6
; GFX9-NEXT: s_add_i32 s12, s12, 3
; GFX9-NEXT: v_mov_b32_e32 v12, s11
-; GFX9-NEXT: v_add_u32_e32 v5, 0x300, v5
+; GFX9-NEXT: v_perm_b32 v5, s44, v5, v6
; GFX9-NEXT: v_perm_b32 v12, s12, v12, v6
-; GFX9-NEXT: v_add_u32_e32 v12, 0x300, v12
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: s_add_i32 s10, s10, 3
-; GFX9-NEXT: v_mov_b32_e32 v13, s9
+; GFX9-NEXT: v_add_u32_e32 v5, 0x300, v5
+; GFX9-NEXT: v_add_u32_sdwa v12, v12, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s45, s45, 3
-; GFX9-NEXT: v_lshl_or_b32 v5, v12, 16, v5
+; GFX9-NEXT: v_or_b32_sdwa v5, v12, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v12, s7
-; GFX9-NEXT: v_perm_b32 v13, s10, v13, v6
+; GFX9-NEXT: s_add_i32 s10, s10, 3
+; GFX9-NEXT: v_mov_b32_e32 v13, s9
; GFX9-NEXT: s_add_i32 s8, s8, 3
; GFX9-NEXT: v_mov_b32_e32 v14, s6
-; GFX9-NEXT: v_add_u32_e32 v15, 0x300, v8
-; GFX9-NEXT: v_add_u32_e32 v8, 0x300, v9
; GFX9-NEXT: v_perm_b32 v12, s45, v12, v6
+; GFX9-NEXT: v_perm_b32 v13, s10, v13, v6
; GFX9-NEXT: v_perm_b32 v6, s8, v14, v6
; GFX9-NEXT: v_add_u32_e32 v14, 0x300, v7
-; GFX9-NEXT: v_add_u32_e32 v9, 0x300, v10
+; GFX9-NEXT: v_add_u32_sdwa v15, v8, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_add_u32_e32 v8, 0x300, v9
+; GFX9-NEXT: v_add_u32_sdwa v9, v10, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_add_u32_e32 v7, 0x300, v11
+; GFX9-NEXT: v_add_u32_sdwa v10, v12, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_add_u32_e32 v11, 0x300, v13
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_add_u32_e32 v10, 0x300, v12
-; GFX9-NEXT: v_add_u32_e32 v6, 0x300, v6
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v9, 16, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v14
-; GFX9-NEXT: v_lshl_or_b32 v6, v6, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v7, v10, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v9, v15, 16, v9
+; GFX9-NEXT: v_add_u32_sdwa v6, v6, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v6, v6, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v10, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v9, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v15, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: .LBB63_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.32bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.32bit.ll
index 55f71864d7af3..48259e19a00c0 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.32bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.32bit.ll
@@ -1621,14 +1621,13 @@ define inreg i32 @bitcast_v2bf16_to_i32_scalar(<2 x bfloat> inreg %a, i32 inreg
; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_bfe_u32 v2, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v0
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
-; GFX9-NEXT: v_mov_b32_e32 v2, 0xffff
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_setpc_b64 s[30:31]
; GFX9-NEXT: .LBB15_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
@@ -4091,14 +4090,13 @@ define inreg float @bitcast_v2bf16_to_f32_scalar(<2 x bfloat> inreg %a, i32 inre
; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_bfe_u32 v2, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v0
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
-; GFX9-NEXT: v_mov_b32_e32 v2, 0xffff
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_setpc_b64 s[30:31]
; GFX9-NEXT: .LBB35_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
@@ -7949,14 +7947,13 @@ define inreg <2 x half> @bitcast_v2bf16_to_v2f16_scalar(<2 x bfloat> inreg %a, i
; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_bfe_u32 v2, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v0
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
-; GFX9-NEXT: v_mov_b32_e32 v2, 0xffff
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_setpc_b64 s[30:31]
; GFX9-NEXT: .LBB63_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
@@ -9513,14 +9510,13 @@ define inreg <1 x i32> @bitcast_v2bf16_to_v1i32_scalar(<2 x bfloat> inreg %a, i3
; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_bfe_u32 v2, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v0
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
-; GFX9-NEXT: v_mov_b32_e32 v2, 0xffff
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_setpc_b64 s[30:31]
; GFX9-NEXT: .LBB73_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
@@ -10208,8 +10204,8 @@ define inreg <4 x i8> @bitcast_v2bf16_to_v4i8_scalar(<2 x bfloat> inreg %a, i32
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_lshrrev_b32_e32 v3, 24, v1
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.48bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.48bit.ll
index cab6fe1ea0374..db1a89f868986 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.48bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.48bit.ll
@@ -378,17 +378,15 @@ define inreg <3 x half> @bitcast_v3bf16_to_v3f16_scalar(<3 x bfloat> inreg %a, i
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_bfe_u32 v3, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v0
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc
-; GFX9-NEXT: v_mov_b32_e32 v3, 0xffff
-; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX9-NEXT: v_and_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: s_movk_i32 s4, 0x7fc0
-; GFX9-NEXT: v_and_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v0, v2, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, s4, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_mov_b32_e32 v2, 0x7fc00000
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_setpc_b64 s[30:31]
; GFX9-NEXT: .LBB1_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
@@ -1144,10 +1142,8 @@ define inreg <3 x i16> @bitcast_v3bf16_to_v3i16_scalar(<3 x bfloat> inreg %a, i3
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_mov_b32_e32 v3, 0xffff0000
; GFX9-NEXT: v_and_or_b32 v0, v2, v3, v0
-; GFX9-NEXT: v_mov_b32_e32 v2, 0xffff
-; GFX9-NEXT: s_movk_i32 s4, 0x7fc0
-; GFX9-NEXT: v_and_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v1, s4, 16, v1
+; GFX9-NEXT: v_mov_b32_e32 v2, 0x7fc00000
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_setpc_b64 s[30:31]
; GFX9-NEXT: .LBB5_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll
index df420ccaa91c6..1b1ffe440ff4b 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll
@@ -7666,10 +7666,10 @@ define inreg <16 x i32> @bitcast_v32bf16_to_v16i32_scalar(<32 x bfloat> inreg %a
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
-; GFX9-NEXT: buffer_store_dword v20, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v19, off, s[0:3], s32 ; 4-byte Folded Spill
; GFX9-NEXT: s_mov_b64 exec, s[4:5]
-; GFX9-NEXT: v_writelane_b32 v20, s30, 0
-; GFX9-NEXT: v_writelane_b32 v20, s31, 1
+; GFX9-NEXT: v_writelane_b32 v19, s30, 0
+; GFX9-NEXT: v_writelane_b32 v19, s31, 1
; GFX9-NEXT: v_readfirstlane_b32 s4, v2
; GFX9-NEXT: v_readfirstlane_b32 s31, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
@@ -7713,20 +7713,19 @@ define inreg <16 x i32> @bitcast_v32bf16_to_v16i32_scalar(<32 x bfloat> inreg %a
; GFX9-NEXT: s_lshl_b32 s4, s30, 16
; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
; GFX9-NEXT: v_add_f32_e32 v4, s4, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v5, v5, v4
-; GFX9-NEXT: v_mov_b32_e32 v16, 0xffff
; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v4
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc
-; GFX9-NEXT: v_lshl_or_b32 v15, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v15, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v3
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s29, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v14, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v14, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -7741,11 +7740,11 @@ define inreg <16 x i32> @bitcast_v32bf16_to_v16i32_scalar(<32 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s28, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v13, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v13, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -7760,11 +7759,11 @@ define inreg <16 x i32> @bitcast_v32bf16_to_v16i32_scalar(<32 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s27, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v12, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v12, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -7779,11 +7778,11 @@ define inreg <16 x i32> @bitcast_v32bf16_to_v16i32_scalar(<32 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s26, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v11, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v11, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -7798,11 +7797,11 @@ define inreg <16 x i32> @bitcast_v32bf16_to_v16i32_scalar(<32 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s25, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v10, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v10, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -7817,11 +7816,11 @@ define inreg <16 x i32> @bitcast_v32bf16_to_v16i32_scalar(<32 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s24, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v9, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v9, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -7836,11 +7835,11 @@ define inreg <16 x i32> @bitcast_v32bf16_to_v16i32_scalar(<32 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s23, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v8, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v8, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -7855,11 +7854,11 @@ define inreg <16 x i32> @bitcast_v32bf16_to_v16i32_scalar(<32 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s22, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v7, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v7, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -7874,11 +7873,11 @@ define inreg <16 x i32> @bitcast_v32bf16_to_v16i32_scalar(<32 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s21, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v6, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v6, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -7893,11 +7892,11 @@ define inreg <16 x i32> @bitcast_v32bf16_to_v16i32_scalar(<32 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s20, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v5, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -7912,11 +7911,11 @@ define inreg <16 x i32> @bitcast_v32bf16_to_v16i32_scalar(<32 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s19, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v4, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v4, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -7929,69 +7928,69 @@ define inreg <16 x i32> @bitcast_v32bf16_to_v16i32_scalar(<32 x bfloat> inreg %a
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
-; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v2
+; GFX9-NEXT: v_or_b32_e32 v16, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v17, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v16, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s18, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v3, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v3, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
-; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v1
+; GFX9-NEXT: v_or_b32_e32 v16, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: s_lshl_b32 s4, s18, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v17, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v16, vcc
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
-; GFX9-NEXT: v_bfe_u32 v17, v2, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v17, v17, v2
-; GFX9-NEXT: v_add_u32_e32 v17, 0x7fff, v17
-; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v2
+; GFX9-NEXT: v_bfe_u32 v16, v2, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v16, v16, v2
+; GFX9-NEXT: v_add_u32_e32 v16, 0x7fff, v16
+; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v17, v18, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v16, v17, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s17, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v2, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
-; GFX9-NEXT: v_bfe_u32 v17, v1, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v17, v17, v1
-; GFX9-NEXT: v_add_u32_e32 v17, 0x7fff, v17
-; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v1
+; GFX9-NEXT: v_bfe_u32 v16, v1, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v16, v16, v1
+; GFX9-NEXT: v_add_u32_e32 v16, 0x7fff, v16
+; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: s_lshl_b32 s4, s17, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v17, v18, vcc
-; GFX9-NEXT: v_add_f32_e32 v17, s4, v0
-; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
-; GFX9-NEXT: v_add_u32_e32 v18, 0x7fff, v18
-; GFX9-NEXT: v_or_b32_e32 v19, 0x400000, v17
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
-; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v19, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v16, v17, vcc
+; GFX9-NEXT: v_add_f32_e32 v16, s4, v0
+; GFX9-NEXT: v_bfe_u32 v17, v16, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v17, v17, v16
+; GFX9-NEXT: v_add_u32_e32 v17, 0x7fff, v17
+; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v16
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v16, v16
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v17, v16, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v16, v17, v18, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s16, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v17
-; GFX9-NEXT: v_add_f32_e32 v17, s4, v0
-; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f32_e32 v16, s4, v0
+; GFX9-NEXT: v_bfe_u32 v17, v16, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v17, v17, v16
; GFX9-NEXT: s_lshl_b32 s4, s16, 16
-; GFX9-NEXT: v_add_u32_e32 v18, 0x7fff, v18
-; GFX9-NEXT: v_or_b32_e32 v19, 0x400000, v17
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
+; GFX9-NEXT: v_add_u32_e32 v17, 0x7fff, v17
+; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v16
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v16, v16
; GFX9-NEXT: v_add_f32_e32 v0, s4, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v19, vcc
-; GFX9-NEXT: v_bfe_u32 v18, v0, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v18, v18, v0
-; GFX9-NEXT: v_add_u32_e32 v18, 0x7fff, v18
-; GFX9-NEXT: v_or_b32_e32 v19, 0x400000, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v16, v17, v18, vcc
+; GFX9-NEXT: v_bfe_u32 v17, v0, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v17, v17, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v16, 16, v16
+; GFX9-NEXT: v_add_u32_e32 v17, 0x7fff, v17
+; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v0
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v17
-; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v19, vcc
-; GFX9-NEXT: v_and_b32_sdwa v16, v16, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v16
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v17, v18, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v16
+; GFX9-NEXT: v_or_b32_sdwa v0, v16, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_branch .LBB23_6
; GFX9-NEXT: .LBB23_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
@@ -8011,10 +8010,10 @@ define inreg <16 x i32> @bitcast_v32bf16_to_v16i32_scalar(<32 x bfloat> inreg %a
; GFX9-NEXT: v_mov_b32_e32 v14, s30
; GFX9-NEXT: v_mov_b32_e32 v15, s31
; GFX9-NEXT: .LBB23_6: ; %end
-; GFX9-NEXT: v_readlane_b32 s30, v20, 0
-; GFX9-NEXT: v_readlane_b32 s31, v20, 1
+; GFX9-NEXT: v_readlane_b32 s30, v19, 0
+; GFX9-NEXT: v_readlane_b32 s31, v19, 1
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
-; GFX9-NEXT: buffer_load_dword v20, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v19, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-NEXT: s_mov_b64 exec, s[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -22609,10 +22608,10 @@ define inreg <16 x float> @bitcast_v32bf16_to_v16f32_scalar(<32 x bfloat> inreg
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
-; GFX9-NEXT: buffer_store_dword v20, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v19, off, s[0:3], s32 ; 4-byte Folded Spill
; GFX9-NEXT: s_mov_b64 exec, s[4:5]
-; GFX9-NEXT: v_writelane_b32 v20, s30, 0
-; GFX9-NEXT: v_writelane_b32 v20, s31, 1
+; GFX9-NEXT: v_writelane_b32 v19, s30, 0
+; GFX9-NEXT: v_writelane_b32 v19, s31, 1
; GFX9-NEXT: v_readfirstlane_b32 s4, v2
; GFX9-NEXT: v_readfirstlane_b32 s31, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
@@ -22656,20 +22655,19 @@ define inreg <16 x float> @bitcast_v32bf16_to_v16f32_scalar(<32 x bfloat> inreg
; GFX9-NEXT: s_lshl_b32 s4, s30, 16
; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
; GFX9-NEXT: v_add_f32_e32 v4, s4, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v5, v5, v4
-; GFX9-NEXT: v_mov_b32_e32 v16, 0xffff
; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v4
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc
-; GFX9-NEXT: v_lshl_or_b32 v15, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v15, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v3
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s29, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v14, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v14, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -22684,11 +22682,11 @@ define inreg <16 x float> @bitcast_v32bf16_to_v16f32_scalar(<32 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s28, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v13, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v13, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -22703,11 +22701,11 @@ define inreg <16 x float> @bitcast_v32bf16_to_v16f32_scalar(<32 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s27, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v12, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v12, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -22722,11 +22720,11 @@ define inreg <16 x float> @bitcast_v32bf16_to_v16f32_scalar(<32 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s26, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v11, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v11, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -22741,11 +22739,11 @@ define inreg <16 x float> @bitcast_v32bf16_to_v16f32_scalar(<32 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s25, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v10, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v10, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -22760,11 +22758,11 @@ define inreg <16 x float> @bitcast_v32bf16_to_v16f32_scalar(<32 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s24, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v9, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v9, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -22779,11 +22777,11 @@ define inreg <16 x float> @bitcast_v32bf16_to_v16f32_scalar(<32 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s23, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v8, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v8, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -22798,11 +22796,11 @@ define inreg <16 x float> @bitcast_v32bf16_to_v16f32_scalar(<32 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s22, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v7, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v7, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -22817,11 +22815,11 @@ define inreg <16 x float> @bitcast_v32bf16_to_v16f32_scalar(<32 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s21, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v6, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v6, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -22836,11 +22834,11 @@ define inreg <16 x float> @bitcast_v32bf16_to_v16f32_scalar(<32 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s20, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v5, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -22855,11 +22853,11 @@ define inreg <16 x float> @bitcast_v32bf16_to_v16f32_scalar(<32 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s19, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v4, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v4, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -22872,69 +22870,69 @@ define inreg <16 x float> @bitcast_v32bf16_to_v16f32_scalar(<32 x bfloat> inreg
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
-; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v2
+; GFX9-NEXT: v_or_b32_e32 v16, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v17, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v16, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s18, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v3, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v3, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
-; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v1
+; GFX9-NEXT: v_or_b32_e32 v16, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: s_lshl_b32 s4, s18, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v17, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v16, vcc
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
-; GFX9-NEXT: v_bfe_u32 v17, v2, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v17, v17, v2
-; GFX9-NEXT: v_add_u32_e32 v17, 0x7fff, v17
-; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v2
+; GFX9-NEXT: v_bfe_u32 v16, v2, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v16, v16, v2
+; GFX9-NEXT: v_add_u32_e32 v16, 0x7fff, v16
+; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v17, v18, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v16, v17, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s17, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v2, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
-; GFX9-NEXT: v_bfe_u32 v17, v1, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v17, v17, v1
-; GFX9-NEXT: v_add_u32_e32 v17, 0x7fff, v17
-; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v1
+; GFX9-NEXT: v_bfe_u32 v16, v1, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v16, v16, v1
+; GFX9-NEXT: v_add_u32_e32 v16, 0x7fff, v16
+; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: s_lshl_b32 s4, s17, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v17, v18, vcc
-; GFX9-NEXT: v_add_f32_e32 v17, s4, v0
-; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
-; GFX9-NEXT: v_add_u32_e32 v18, 0x7fff, v18
-; GFX9-NEXT: v_or_b32_e32 v19, 0x400000, v17
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
-; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v19, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v16, v17, vcc
+; GFX9-NEXT: v_add_f32_e32 v16, s4, v0
+; GFX9-NEXT: v_bfe_u32 v17, v16, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v17, v17, v16
+; GFX9-NEXT: v_add_u32_e32 v17, 0x7fff, v17
+; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v16
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v16, v16
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v17, v16, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v16, v17, v18, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s16, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v17
-; GFX9-NEXT: v_add_f32_e32 v17, s4, v0
-; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f32_e32 v16, s4, v0
+; GFX9-NEXT: v_bfe_u32 v17, v16, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v17, v17, v16
; GFX9-NEXT: s_lshl_b32 s4, s16, 16
-; GFX9-NEXT: v_add_u32_e32 v18, 0x7fff, v18
-; GFX9-NEXT: v_or_b32_e32 v19, 0x400000, v17
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
+; GFX9-NEXT: v_add_u32_e32 v17, 0x7fff, v17
+; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v16
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v16, v16
; GFX9-NEXT: v_add_f32_e32 v0, s4, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v19, vcc
-; GFX9-NEXT: v_bfe_u32 v18, v0, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v18, v18, v0
-; GFX9-NEXT: v_add_u32_e32 v18, 0x7fff, v18
-; GFX9-NEXT: v_or_b32_e32 v19, 0x400000, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v16, v17, v18, vcc
+; GFX9-NEXT: v_bfe_u32 v17, v0, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v17, v17, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v16, 16, v16
+; GFX9-NEXT: v_add_u32_e32 v17, 0x7fff, v17
+; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v0
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v17
-; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v19, vcc
-; GFX9-NEXT: v_and_b32_sdwa v16, v16, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v16
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v17, v18, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v16
+; GFX9-NEXT: v_or_b32_sdwa v0, v16, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_branch .LBB47_6
; GFX9-NEXT: .LBB47_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
@@ -22954,10 +22952,10 @@ define inreg <16 x float> @bitcast_v32bf16_to_v16f32_scalar(<32 x bfloat> inreg
; GFX9-NEXT: v_mov_b32_e32 v14, s30
; GFX9-NEXT: v_mov_b32_e32 v15, s31
; GFX9-NEXT: .LBB47_6: ; %end
-; GFX9-NEXT: v_readlane_b32 s30, v20, 0
-; GFX9-NEXT: v_readlane_b32 s31, v20, 1
+; GFX9-NEXT: v_readlane_b32 s30, v19, 0
+; GFX9-NEXT: v_readlane_b32 s31, v19, 1
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
-; GFX9-NEXT: buffer_load_dword v20, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v19, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-NEXT: s_mov_b64 exec, s[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -36866,10 +36864,10 @@ define inreg <8 x i64> @bitcast_v32bf16_to_v8i64_scalar(<32 x bfloat> inreg %a,
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
-; GFX9-NEXT: buffer_store_dword v20, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v19, off, s[0:3], s32 ; 4-byte Folded Spill
; GFX9-NEXT: s_mov_b64 exec, s[4:5]
-; GFX9-NEXT: v_writelane_b32 v20, s30, 0
-; GFX9-NEXT: v_writelane_b32 v20, s31, 1
+; GFX9-NEXT: v_writelane_b32 v19, s30, 0
+; GFX9-NEXT: v_writelane_b32 v19, s31, 1
; GFX9-NEXT: v_readfirstlane_b32 s4, v2
; GFX9-NEXT: v_readfirstlane_b32 s31, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
@@ -36913,20 +36911,19 @@ define inreg <8 x i64> @bitcast_v32bf16_to_v8i64_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: s_lshl_b32 s4, s30, 16
; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
; GFX9-NEXT: v_add_f32_e32 v4, s4, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v5, v5, v4
-; GFX9-NEXT: v_mov_b32_e32 v16, 0xffff
; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v4
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc
-; GFX9-NEXT: v_lshl_or_b32 v15, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v15, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v3
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s29, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v14, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v14, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -36941,11 +36938,11 @@ define inreg <8 x i64> @bitcast_v32bf16_to_v8i64_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s28, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v13, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v13, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -36960,11 +36957,11 @@ define inreg <8 x i64> @bitcast_v32bf16_to_v8i64_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s27, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v12, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v12, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -36979,11 +36976,11 @@ define inreg <8 x i64> @bitcast_v32bf16_to_v8i64_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s26, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v11, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v11, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -36998,11 +36995,11 @@ define inreg <8 x i64> @bitcast_v32bf16_to_v8i64_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s25, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v10, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v10, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -37017,11 +37014,11 @@ define inreg <8 x i64> @bitcast_v32bf16_to_v8i64_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s24, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v9, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v9, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -37036,11 +37033,11 @@ define inreg <8 x i64> @bitcast_v32bf16_to_v8i64_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s23, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v8, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v8, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -37055,11 +37052,11 @@ define inreg <8 x i64> @bitcast_v32bf16_to_v8i64_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s22, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v7, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v7, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -37074,11 +37071,11 @@ define inreg <8 x i64> @bitcast_v32bf16_to_v8i64_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s21, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v6, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v6, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -37093,11 +37090,11 @@ define inreg <8 x i64> @bitcast_v32bf16_to_v8i64_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s20, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v5, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -37112,11 +37109,11 @@ define inreg <8 x i64> @bitcast_v32bf16_to_v8i64_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s19, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v4, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v4, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -37129,69 +37126,69 @@ define inreg <8 x i64> @bitcast_v32bf16_to_v8i64_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
-; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v2
+; GFX9-NEXT: v_or_b32_e32 v16, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v17, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v16, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s18, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v3, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v3, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
-; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v1
+; GFX9-NEXT: v_or_b32_e32 v16, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: s_lshl_b32 s4, s18, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v17, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v16, vcc
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
-; GFX9-NEXT: v_bfe_u32 v17, v2, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v17, v17, v2
-; GFX9-NEXT: v_add_u32_e32 v17, 0x7fff, v17
-; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v2
+; GFX9-NEXT: v_bfe_u32 v16, v2, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v16, v16, v2
+; GFX9-NEXT: v_add_u32_e32 v16, 0x7fff, v16
+; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v17, v18, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v16, v17, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s17, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v2, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
-; GFX9-NEXT: v_bfe_u32 v17, v1, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v17, v17, v1
-; GFX9-NEXT: v_add_u32_e32 v17, 0x7fff, v17
-; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v1
+; GFX9-NEXT: v_bfe_u32 v16, v1, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v16, v16, v1
+; GFX9-NEXT: v_add_u32_e32 v16, 0x7fff, v16
+; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: s_lshl_b32 s4, s17, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v17, v18, vcc
-; GFX9-NEXT: v_add_f32_e32 v17, s4, v0
-; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
-; GFX9-NEXT: v_add_u32_e32 v18, 0x7fff, v18
-; GFX9-NEXT: v_or_b32_e32 v19, 0x400000, v17
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
-; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v19, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v16, v17, vcc
+; GFX9-NEXT: v_add_f32_e32 v16, s4, v0
+; GFX9-NEXT: v_bfe_u32 v17, v16, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v17, v17, v16
+; GFX9-NEXT: v_add_u32_e32 v17, 0x7fff, v17
+; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v16
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v16, v16
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v17, v16, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v16, v17, v18, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s16, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v17
-; GFX9-NEXT: v_add_f32_e32 v17, s4, v0
-; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f32_e32 v16, s4, v0
+; GFX9-NEXT: v_bfe_u32 v17, v16, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v17, v17, v16
; GFX9-NEXT: s_lshl_b32 s4, s16, 16
-; GFX9-NEXT: v_add_u32_e32 v18, 0x7fff, v18
-; GFX9-NEXT: v_or_b32_e32 v19, 0x400000, v17
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
+; GFX9-NEXT: v_add_u32_e32 v17, 0x7fff, v17
+; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v16
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v16, v16
; GFX9-NEXT: v_add_f32_e32 v0, s4, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v19, vcc
-; GFX9-NEXT: v_bfe_u32 v18, v0, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v18, v18, v0
-; GFX9-NEXT: v_add_u32_e32 v18, 0x7fff, v18
-; GFX9-NEXT: v_or_b32_e32 v19, 0x400000, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v16, v17, v18, vcc
+; GFX9-NEXT: v_bfe_u32 v17, v0, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v17, v17, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v16, 16, v16
+; GFX9-NEXT: v_add_u32_e32 v17, 0x7fff, v17
+; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v0
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v17
-; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v19, vcc
-; GFX9-NEXT: v_and_b32_sdwa v16, v16, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v16
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v17, v18, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v16
+; GFX9-NEXT: v_or_b32_sdwa v0, v16, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_branch .LBB67_6
; GFX9-NEXT: .LBB67_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
@@ -37211,10 +37208,10 @@ define inreg <8 x i64> @bitcast_v32bf16_to_v8i64_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v14, s30
; GFX9-NEXT: v_mov_b32_e32 v15, s31
; GFX9-NEXT: .LBB67_6: ; %end
-; GFX9-NEXT: v_readlane_b32 s30, v20, 0
-; GFX9-NEXT: v_readlane_b32 s31, v20, 1
+; GFX9-NEXT: v_readlane_b32 s30, v19, 0
+; GFX9-NEXT: v_readlane_b32 s31, v19, 1
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
-; GFX9-NEXT: buffer_load_dword v20, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v19, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-NEXT: s_mov_b64 exec, s[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -50163,10 +50160,10 @@ define inreg <8 x double> @bitcast_v32bf16_to_v8f64_scalar(<32 x bfloat> inreg %
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
-; GFX9-NEXT: buffer_store_dword v20, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v19, off, s[0:3], s32 ; 4-byte Folded Spill
; GFX9-NEXT: s_mov_b64 exec, s[4:5]
-; GFX9-NEXT: v_writelane_b32 v20, s30, 0
-; GFX9-NEXT: v_writelane_b32 v20, s31, 1
+; GFX9-NEXT: v_writelane_b32 v19, s30, 0
+; GFX9-NEXT: v_writelane_b32 v19, s31, 1
; GFX9-NEXT: v_readfirstlane_b32 s4, v2
; GFX9-NEXT: v_readfirstlane_b32 s31, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
@@ -50210,20 +50207,19 @@ define inreg <8 x double> @bitcast_v32bf16_to_v8f64_scalar(<32 x bfloat> inreg %
; GFX9-NEXT: s_lshl_b32 s4, s30, 16
; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
; GFX9-NEXT: v_add_f32_e32 v4, s4, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v5, v5, v4
-; GFX9-NEXT: v_mov_b32_e32 v16, 0xffff
; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v4
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc
-; GFX9-NEXT: v_lshl_or_b32 v15, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v15, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v3
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s29, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v14, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v14, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -50238,11 +50234,11 @@ define inreg <8 x double> @bitcast_v32bf16_to_v8f64_scalar(<32 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s28, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v13, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v13, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -50257,11 +50253,11 @@ define inreg <8 x double> @bitcast_v32bf16_to_v8f64_scalar(<32 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s27, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v12, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v12, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -50276,11 +50272,11 @@ define inreg <8 x double> @bitcast_v32bf16_to_v8f64_scalar(<32 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s26, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v11, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v11, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -50295,11 +50291,11 @@ define inreg <8 x double> @bitcast_v32bf16_to_v8f64_scalar(<32 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s25, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v10, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v10, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -50314,11 +50310,11 @@ define inreg <8 x double> @bitcast_v32bf16_to_v8f64_scalar(<32 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s24, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v9, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v9, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -50333,11 +50329,11 @@ define inreg <8 x double> @bitcast_v32bf16_to_v8f64_scalar(<32 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s23, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v8, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v8, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -50352,11 +50348,11 @@ define inreg <8 x double> @bitcast_v32bf16_to_v8f64_scalar(<32 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s22, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v7, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v7, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -50371,11 +50367,11 @@ define inreg <8 x double> @bitcast_v32bf16_to_v8f64_scalar(<32 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s21, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v6, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v6, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -50390,11 +50386,11 @@ define inreg <8 x double> @bitcast_v32bf16_to_v8f64_scalar(<32 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s20, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v5, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -50409,11 +50405,11 @@ define inreg <8 x double> @bitcast_v32bf16_to_v8f64_scalar(<32 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s19, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v4, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v4, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -50426,69 +50422,69 @@ define inreg <8 x double> @bitcast_v32bf16_to_v8f64_scalar(<32 x bfloat> inreg %
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
-; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v2
+; GFX9-NEXT: v_or_b32_e32 v16, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v17, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v16, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s18, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v3, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v3, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
-; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v1
+; GFX9-NEXT: v_or_b32_e32 v16, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: s_lshl_b32 s4, s18, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v17, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v16, vcc
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
-; GFX9-NEXT: v_bfe_u32 v17, v2, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v17, v17, v2
-; GFX9-NEXT: v_add_u32_e32 v17, 0x7fff, v17
-; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v2
+; GFX9-NEXT: v_bfe_u32 v16, v2, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v16, v16, v2
+; GFX9-NEXT: v_add_u32_e32 v16, 0x7fff, v16
+; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v17, v18, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v16, v17, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s17, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v2, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
-; GFX9-NEXT: v_bfe_u32 v17, v1, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v17, v17, v1
-; GFX9-NEXT: v_add_u32_e32 v17, 0x7fff, v17
-; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v1
+; GFX9-NEXT: v_bfe_u32 v16, v1, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v16, v16, v1
+; GFX9-NEXT: v_add_u32_e32 v16, 0x7fff, v16
+; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: s_lshl_b32 s4, s17, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v17, v18, vcc
-; GFX9-NEXT: v_add_f32_e32 v17, s4, v0
-; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
-; GFX9-NEXT: v_add_u32_e32 v18, 0x7fff, v18
-; GFX9-NEXT: v_or_b32_e32 v19, 0x400000, v17
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
-; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v19, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v16, v17, vcc
+; GFX9-NEXT: v_add_f32_e32 v16, s4, v0
+; GFX9-NEXT: v_bfe_u32 v17, v16, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v17, v17, v16
+; GFX9-NEXT: v_add_u32_e32 v17, 0x7fff, v17
+; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v16
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v16, v16
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v17, v16, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v16, v17, v18, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s16, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v17
-; GFX9-NEXT: v_add_f32_e32 v17, s4, v0
-; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f32_e32 v16, s4, v0
+; GFX9-NEXT: v_bfe_u32 v17, v16, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v17, v17, v16
; GFX9-NEXT: s_lshl_b32 s4, s16, 16
-; GFX9-NEXT: v_add_u32_e32 v18, 0x7fff, v18
-; GFX9-NEXT: v_or_b32_e32 v19, 0x400000, v17
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
+; GFX9-NEXT: v_add_u32_e32 v17, 0x7fff, v17
+; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v16
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v16, v16
; GFX9-NEXT: v_add_f32_e32 v0, s4, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v19, vcc
-; GFX9-NEXT: v_bfe_u32 v18, v0, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v18, v18, v0
-; GFX9-NEXT: v_add_u32_e32 v18, 0x7fff, v18
-; GFX9-NEXT: v_or_b32_e32 v19, 0x400000, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v16, v17, v18, vcc
+; GFX9-NEXT: v_bfe_u32 v17, v0, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v17, v17, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v16, 16, v16
+; GFX9-NEXT: v_add_u32_e32 v17, 0x7fff, v17
+; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v0
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v17
-; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v19, vcc
-; GFX9-NEXT: v_and_b32_sdwa v16, v16, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v16
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v17, v18, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v16
+; GFX9-NEXT: v_or_b32_sdwa v0, v16, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_branch .LBB83_6
; GFX9-NEXT: .LBB83_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
@@ -50508,10 +50504,10 @@ define inreg <8 x double> @bitcast_v32bf16_to_v8f64_scalar(<32 x bfloat> inreg %
; GFX9-NEXT: v_mov_b32_e32 v14, s30
; GFX9-NEXT: v_mov_b32_e32 v15, s31
; GFX9-NEXT: .LBB83_6: ; %end
-; GFX9-NEXT: v_readlane_b32 s30, v20, 0
-; GFX9-NEXT: v_readlane_b32 s31, v20, 1
+; GFX9-NEXT: v_readlane_b32 s30, v19, 0
+; GFX9-NEXT: v_readlane_b32 s31, v19, 1
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
-; GFX9-NEXT: buffer_load_dword v20, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v19, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-NEXT: s_mov_b64 exec, s[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -71386,11 +71382,11 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
; GFX9-NEXT: v_readfirstlane_b32 s56, v29
; GFX9-NEXT: v_readfirstlane_b32 s57, v28
; GFX9-NEXT: v_readfirstlane_b32 s61, v27
-; GFX9-NEXT: v_readfirstlane_b32 s63, v26
+; GFX9-NEXT: v_readfirstlane_b32 s62, v26
; GFX9-NEXT: v_readfirstlane_b32 s7, v25
-; GFX9-NEXT: v_readfirstlane_b32 s47, v24
+; GFX9-NEXT: v_readfirstlane_b32 s46, v24
; GFX9-NEXT: v_readfirstlane_b32 s58, v23
-; GFX9-NEXT: v_readfirstlane_b32 s60, v22
+; GFX9-NEXT: v_readfirstlane_b32 s59, v22
; GFX9-NEXT: v_readfirstlane_b32 s6, v21
; GFX9-NEXT: v_readfirstlane_b32 s8, v20
; GFX9-NEXT: v_readfirstlane_b32 s9, v19
@@ -71404,12 +71400,12 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
; GFX9-NEXT: buffer_load_ushort v18, off, s[0:3], s32
; GFX9-NEXT: buffer_load_ushort v24, off, s[0:3], s32 offset:72
; GFX9-NEXT: buffer_load_ushort v27, off, s[0:3], s32 offset:68
-; GFX9-NEXT: buffer_load_ushort v32, off, s[0:3], s32 offset:64
+; GFX9-NEXT: buffer_load_ushort v33, off, s[0:3], s32 offset:64
; GFX9-NEXT: buffer_load_ushort v34, off, s[0:3], s32 offset:60
; GFX9-NEXT: buffer_load_ushort v22, off, s[0:3], s32 offset:56
; GFX9-NEXT: buffer_load_ushort v25, off, s[0:3], s32 offset:52
; GFX9-NEXT: buffer_load_ushort v30, off, s[0:3], s32 offset:48
-; GFX9-NEXT: buffer_load_ushort v33, off, s[0:3], s32 offset:44
+; GFX9-NEXT: buffer_load_ushort v32, off, s[0:3], s32 offset:44
; GFX9-NEXT: buffer_load_ushort v20, off, s[0:3], s32 offset:40
; GFX9-NEXT: buffer_load_ushort v23, off, s[0:3], s32 offset:36
; GFX9-NEXT: buffer_load_ushort v28, off, s[0:3], s32 offset:32
@@ -71429,103 +71425,95 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
; GFX9-NEXT: v_readfirstlane_b32 s75, v6
; GFX9-NEXT: v_readfirstlane_b32 s44, v5
; GFX9-NEXT: v_readfirstlane_b32 s45, v4
-; GFX9-NEXT: v_readfirstlane_b32 s46, v3
+; GFX9-NEXT: v_readfirstlane_b32 s47, v3
; GFX9-NEXT: v_readfirstlane_b32 s74, v2
-; GFX9-NEXT: v_readfirstlane_b32 s59, v1
-; GFX9-NEXT: v_readfirstlane_b32 s62, v0
+; GFX9-NEXT: v_readfirstlane_b32 s60, v1
+; GFX9-NEXT: v_readfirstlane_b32 s63, v0
; GFX9-NEXT: s_waitcnt vmcnt(19)
; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v12
; GFX9-NEXT: s_and_b64 s[4:5], vcc, exec
; GFX9-NEXT: s_cbranch_scc0 .LBB99_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v8, 0xc0c0004
-; GFX9-NEXT: v_mov_b32_e32 v2, s21
-; GFX9-NEXT: v_perm_b32 v0, s16, v0, v8
; GFX9-NEXT: v_mov_b32_e32 v1, s19
-; GFX9-NEXT: v_perm_b32 v2, s20, v2, v8
; GFX9-NEXT: v_mov_b32_e32 v3, s23
+; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_perm_b32 v1, s18, v1, v8
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: v_mov_b32_e32 v2, s21
; GFX9-NEXT: v_perm_b32 v3, s22, v3, v8
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v3, 16, v2
-; GFX9-NEXT: v_mov_b32_e32 v2, s25
-; GFX9-NEXT: v_perm_b32 v2, s24, v2, v8
+; GFX9-NEXT: v_perm_b32 v0, s16, v0, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_perm_b32 v2, s20, v2, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v3, s27
+; GFX9-NEXT: v_mov_b32_e32 v2, s25
; GFX9-NEXT: v_perm_b32 v3, s26, v3, v8
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX9-NEXT: v_perm_b32 v2, s24, v2, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_mov_b32_e32 v4, s60
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v3, s29
+; GFX9-NEXT: v_perm_b32 v4, s63, v4, v8
; GFX9-NEXT: v_perm_b32 v3, s28, v3, v8
-; GFX9-NEXT: v_mov_b32_e32 v4, s59
-; GFX9-NEXT: v_perm_b32 v4, s62, v4, v8
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_lshl_or_b32 v3, v4, 16, v3
-; GFX9-NEXT: v_mov_b32_e32 v4, s46
-; GFX9-NEXT: v_perm_b32 v4, s74, v4, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX9-NEXT: v_mov_b32_e32 v5, s44
+; GFX9-NEXT: v_or_b32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v4, s47
; GFX9-NEXT: v_perm_b32 v5, s45, v5, v8
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_lshl_or_b32 v4, v5, 16, v4
-; GFX9-NEXT: v_mov_b32_e32 v5, s43
-; GFX9-NEXT: v_perm_b32 v5, s75, v5, v8
+; GFX9-NEXT: v_perm_b32 v4, s74, v4, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX9-NEXT: v_mov_b32_e32 v6, s41
+; GFX9-NEXT: v_or_b32_sdwa v4, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v5, s43
; GFX9-NEXT: v_perm_b32 v6, s42, v6, v8
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_lshl_or_b32 v5, v6, 16, v5
-; GFX9-NEXT: v_mov_b32_e32 v6, s40
-; GFX9-NEXT: v_perm_b32 v6, s76, v6, v8
+; GFX9-NEXT: v_perm_b32 v5, s75, v5, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v6
; GFX9-NEXT: v_mov_b32_e32 v7, s14
+; GFX9-NEXT: v_or_b32_sdwa v5, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v6, s40
; GFX9-NEXT: v_perm_b32 v7, s15, v7, v8
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_lshl_or_b32 v6, v7, 16, v6
-; GFX9-NEXT: v_mov_b32_e32 v7, s13
-; GFX9-NEXT: v_perm_b32 v7, s73, v7, v8
+; GFX9-NEXT: v_perm_b32 v6, s76, v6, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v7
; GFX9-NEXT: v_mov_b32_e32 v9, s11
+; GFX9-NEXT: v_or_b32_sdwa v6, v7, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v7, s13
; GFX9-NEXT: v_perm_b32 v9, s12, v9, v8
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_lshl_or_b32 v7, v9, 16, v7
+; GFX9-NEXT: v_perm_b32 v7, s73, v7, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX9-NEXT: s_and_b32 s4, s46, 0xff
+; GFX9-NEXT: s_lshl_b32 s5, s7, 8
+; GFX9-NEXT: v_or_b32_sdwa v7, v9, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v9, s58
+; GFX9-NEXT: s_or_b32 s4, s4, s5
+; GFX9-NEXT: v_perm_b32 v9, s59, v9, v8
+; GFX9-NEXT: v_lshlrev_b32_e64 v10, 16, s4
+; GFX9-NEXT: v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(18)
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v16
+; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v16
; GFX9-NEXT: s_waitcnt vmcnt(16)
-; GFX9-NEXT: v_perm_b32 v10, s72, v18, v8
-; GFX9-NEXT: v_or_b32_sdwa v9, v17, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v9, 16, v10
+; GFX9-NEXT: v_perm_b32 v8, s72, v18, v8
+; GFX9-NEXT: v_or_b32_sdwa v10, v17, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v11, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v26
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_or_b32_sdwa v9, v29, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v26
; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v19
-; GFX9-NEXT: v_or_b32_sdwa v10, v21, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_or_b32_sdwa v8, v29, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v10, v21, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v12, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_mov_b32 s4, 0xc0c0004
-; GFX9-NEXT: v_lshl_or_b32 v12, v10, 16, v9
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v20
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v20
+; GFX9-NEXT: v_or_b32_sdwa v8, v23, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_perm_b32 v10, v31, v28, s4
-; GFX9-NEXT: v_or_b32_sdwa v9, v23, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v13, v9, 16, v10
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v30
-; GFX9-NEXT: v_or_b32_sdwa v9, v33, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v13, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v30
; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v22
-; GFX9-NEXT: v_or_b32_sdwa v10, v25, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_lshl_or_b32 v14, v10, 16, v9
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v24
-; GFX9-NEXT: v_perm_b32 v10, v34, v32, s4
-; GFX9-NEXT: v_or_b32_sdwa v9, v27, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v15, v9, 16, v10
-; GFX9-NEXT: v_mov_b32_e32 v9, s58
-; GFX9-NEXT: v_perm_b32 v8, s60, v9, v8
-; GFX9-NEXT: s_and_b32 s4, s47, 0xff
-; GFX9-NEXT: s_lshl_b32 s5, s7, 8
-; GFX9-NEXT: s_or_b32 s4, s4, s5
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, s4, 16, v8
+; GFX9-NEXT: v_or_b32_sdwa v8, v32, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v10, v25, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v14, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_perm_b32 v10, v34, v33, s4
; GFX9-NEXT: s_and_b32 s4, s10, 0xff
; GFX9-NEXT: s_lshl_b32 s5, s9, 8
; GFX9-NEXT: s_or_b32 s4, s4, s5
@@ -71533,13 +71521,16 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
; GFX9-NEXT: s_lshl_b32 s77, s6, 8
; GFX9-NEXT: s_or_b32 s5, s5, s77
; GFX9-NEXT: s_pack_ll_b32_b16 s4, s4, s5
-; GFX9-NEXT: s_and_b32 s5, s63, 0xff
+; GFX9-NEXT: s_and_b32 s5, s62, 0xff
; GFX9-NEXT: s_lshl_b32 s77, s61, 8
; GFX9-NEXT: s_or_b32 s5, s5, s77
; GFX9-NEXT: s_and_b32 s77, s57, 0xff
; GFX9-NEXT: s_lshl_b32 s78, s56, 8
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v24
; GFX9-NEXT: s_or_b32 s77, s77, s78
+; GFX9-NEXT: v_or_b32_sdwa v8, v27, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: s_pack_ll_b32_b16 s5, s5, s77
+; GFX9-NEXT: v_or_b32_sdwa v15, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v8, s4
; GFX9-NEXT: v_mov_b32_e32 v10, s5
; GFX9-NEXT: s_mov_b64 s[4:5], 0
@@ -71556,11 +71547,11 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
; GFX9-NEXT: s_waitcnt vmcnt(12)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v34
; GFX9-NEXT: s_mov_b32 s4, 0xc0c0004
-; GFX9-NEXT: v_perm_b32 v8, v0, v32, s4
+; GFX9-NEXT: v_perm_b32 v8, v0, v33, s4
; GFX9-NEXT: v_add_u32_e32 v0, 3, v27
; GFX9-NEXT: v_perm_b32 v9, v0, v24, s4
; GFX9-NEXT: s_waitcnt vmcnt(8)
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v33
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v32
; GFX9-NEXT: v_perm_b32 v10, v0, v30, s4
; GFX9-NEXT: v_add_u32_e32 v0, 3, v25
; GFX9-NEXT: v_perm_b32 v11, v0, v22, s4
@@ -71577,138 +71568,123 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
; GFX9-NEXT: v_add_u32_e32 v0, 3, v17
; GFX9-NEXT: v_mov_b32_e32 v19, 0xc0c0004
; GFX9-NEXT: v_perm_b32 v16, v0, v16, s4
-; GFX9-NEXT: s_add_i32 s63, s63, 3
+; GFX9-NEXT: s_add_i32 s62, s62, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s61
-; GFX9-NEXT: v_perm_b32 v17, s63, v0, v19
+; GFX9-NEXT: v_perm_b32 v17, s62, v0, v19
; GFX9-NEXT: s_add_i32 s57, s57, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s56
; GFX9-NEXT: v_perm_b32 v20, s57, v0, v19
-; GFX9-NEXT: s_add_i32 s60, s60, 3
+; GFX9-NEXT: s_add_i32 s59, s59, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s58
-; GFX9-NEXT: v_perm_b32 v21, s60, v0, v19
+; GFX9-NEXT: v_perm_b32 v21, s59, v0, v19
; GFX9-NEXT: s_add_i32 s20, s20, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s21
; GFX9-NEXT: v_perm_b32 v0, s20, v0, v19
; GFX9-NEXT: v_add_u32_e32 v1, 0x300, v0
; GFX9-NEXT: s_add_i32 s22, s22, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s23
+; GFX9-NEXT: s_movk_i32 s5, 0x300
; GFX9-NEXT: v_perm_b32 v0, s22, v0, v19
-; GFX9-NEXT: v_add_u32_e32 v2, 0x300, v0
+; GFX9-NEXT: v_add_u32_sdwa v2, v0, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s16, s16, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s17
-; GFX9-NEXT: v_perm_b32 v0, s16, v0, v19
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: v_mov_b32_e32 v3, s19
-; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
+; GFX9-NEXT: v_perm_b32 v0, s16, v0, v19
; GFX9-NEXT: v_perm_b32 v3, s18, v3, v19
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
+; GFX9-NEXT: v_add_u32_sdwa v3, v3, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s24, s24, 3
-; GFX9-NEXT: v_add_u32_e32 v3, 0x300, v3
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v2, s25
-; GFX9-NEXT: v_lshl_or_b32 v0, v3, 16, v0
-; GFX9-NEXT: v_perm_b32 v2, s24, v2, v19
; GFX9-NEXT: s_add_i32 s26, s26, 3
; GFX9-NEXT: v_mov_b32_e32 v3, s27
-; GFX9-NEXT: v_add_u32_e32 v2, 0x300, v2
+; GFX9-NEXT: v_perm_b32 v2, s24, v2, v19
; GFX9-NEXT: v_perm_b32 v3, s26, v3, v19
-; GFX9-NEXT: v_add_u32_e32 v3, 0x300, v3
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX9-NEXT: v_add_u32_e32 v2, 0x300, v2
+; GFX9-NEXT: v_add_u32_sdwa v3, v3, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s28, s28, 3
-; GFX9-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v3, s29
+; GFX9-NEXT: s_add_i32 s63, s63, 3
+; GFX9-NEXT: v_mov_b32_e32 v4, s60
; GFX9-NEXT: v_perm_b32 v3, s28, v3, v19
-; GFX9-NEXT: s_add_i32 s62, s62, 3
-; GFX9-NEXT: v_mov_b32_e32 v4, s59
+; GFX9-NEXT: v_perm_b32 v4, s63, v4, v19
; GFX9-NEXT: v_add_u32_e32 v3, 0x300, v3
-; GFX9-NEXT: v_perm_b32 v4, s62, v4, v19
-; GFX9-NEXT: v_add_u32_e32 v4, 0x300, v4
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX9-NEXT: v_add_u32_sdwa v4, v4, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s74, s74, 3
-; GFX9-NEXT: v_lshl_or_b32 v3, v4, 16, v3
-; GFX9-NEXT: v_mov_b32_e32 v4, s46
-; GFX9-NEXT: v_perm_b32 v4, s74, v4, v19
+; GFX9-NEXT: v_or_b32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v4, s47
; GFX9-NEXT: s_add_i32 s45, s45, 3
; GFX9-NEXT: v_mov_b32_e32 v5, s44
-; GFX9-NEXT: v_add_u32_e32 v4, 0x300, v4
+; GFX9-NEXT: v_perm_b32 v4, s74, v4, v19
; GFX9-NEXT: v_perm_b32 v5, s45, v5, v19
-; GFX9-NEXT: v_add_u32_e32 v5, 0x300, v5
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX9-NEXT: v_add_u32_e32 v4, 0x300, v4
+; GFX9-NEXT: v_add_u32_sdwa v5, v5, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s75, s75, 3
-; GFX9-NEXT: v_lshl_or_b32 v4, v5, 16, v4
+; GFX9-NEXT: v_or_b32_sdwa v4, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v5, s43
-; GFX9-NEXT: v_perm_b32 v5, s75, v5, v19
; GFX9-NEXT: s_add_i32 s42, s42, 3
; GFX9-NEXT: v_mov_b32_e32 v6, s41
-; GFX9-NEXT: v_add_u32_e32 v5, 0x300, v5
+; GFX9-NEXT: v_perm_b32 v5, s75, v5, v19
; GFX9-NEXT: v_perm_b32 v6, s42, v6, v19
-; GFX9-NEXT: v_add_u32_e32 v6, 0x300, v6
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX9-NEXT: v_add_u32_e32 v5, 0x300, v5
+; GFX9-NEXT: v_add_u32_sdwa v6, v6, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s76, s76, 3
-; GFX9-NEXT: v_lshl_or_b32 v5, v6, 16, v5
+; GFX9-NEXT: v_or_b32_sdwa v5, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v6, s40
-; GFX9-NEXT: v_perm_b32 v6, s76, v6, v19
; GFX9-NEXT: s_add_i32 s15, s15, 3
; GFX9-NEXT: v_mov_b32_e32 v7, s14
-; GFX9-NEXT: v_add_u32_e32 v6, 0x300, v6
+; GFX9-NEXT: v_perm_b32 v6, s76, v6, v19
; GFX9-NEXT: v_perm_b32 v7, s15, v7, v19
-; GFX9-NEXT: v_add_u32_e32 v7, 0x300, v7
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; GFX9-NEXT: v_add_u32_e32 v6, 0x300, v6
+; GFX9-NEXT: v_add_u32_sdwa v7, v7, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s73, s73, 3
-; GFX9-NEXT: v_lshl_or_b32 v6, v7, 16, v6
+; GFX9-NEXT: v_or_b32_sdwa v6, v7, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v7, s13
-; GFX9-NEXT: v_perm_b32 v7, s73, v7, v19
; GFX9-NEXT: s_add_i32 s12, s12, 3
; GFX9-NEXT: v_mov_b32_e32 v22, s11
-; GFX9-NEXT: s_add_i32 s72, s72, 3
-; GFX9-NEXT: v_add_u32_e32 v7, 0x300, v7
+; GFX9-NEXT: v_perm_b32 v7, s73, v7, v19
; GFX9-NEXT: v_perm_b32 v22, s12, v22, v19
-; GFX9-NEXT: v_perm_b32 v18, s72, v18, v19
-; GFX9-NEXT: v_add_u32_e32 v22, 0x300, v22
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX9-NEXT: v_add_u32_e32 v7, 0x300, v7
+; GFX9-NEXT: v_add_u32_sdwa v22, v22, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: s_add_i32 s72, s72, 3
+; GFX9-NEXT: s_add_i32 s46, s46, 3
+; GFX9-NEXT: v_or_b32_sdwa v7, v22, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v22, s7
; GFX9-NEXT: s_add_i32 s10, s10, 3
; GFX9-NEXT: v_mov_b32_e32 v23, s9
-; GFX9-NEXT: v_add_u32_e32 v28, 0x300, v12
-; GFX9-NEXT: v_add_u32_e32 v12, 0x300, v14
-; GFX9-NEXT: s_add_i32 s47, s47, 3
-; GFX9-NEXT: v_lshl_or_b32 v7, v22, 16, v7
-; GFX9-NEXT: v_mov_b32_e32 v22, s7
-; GFX9-NEXT: v_perm_b32 v23, s10, v23, v19
; GFX9-NEXT: s_add_i32 s8, s8, 3
; GFX9-NEXT: v_mov_b32_e32 v24, s6
-; GFX9-NEXT: v_add_u32_e32 v27, 0x300, v11
-; GFX9-NEXT: v_add_u32_e32 v14, 0x300, v15
-; GFX9-NEXT: v_add_u32_e32 v11, 0x300, v18
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_perm_b32 v22, s47, v22, v19
+; GFX9-NEXT: v_perm_b32 v18, s72, v18, v19
+; GFX9-NEXT: v_perm_b32 v22, s46, v22, v19
+; GFX9-NEXT: v_perm_b32 v23, s10, v23, v19
; GFX9-NEXT: v_perm_b32 v19, s8, v24, v19
; GFX9-NEXT: v_add_u32_e32 v24, 0x300, v8
-; GFX9-NEXT: v_add_u32_e32 v25, 0x300, v9
+; GFX9-NEXT: v_add_u32_sdwa v25, v9, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_add_u32_e32 v26, 0x300, v10
-; GFX9-NEXT: v_add_u32_e32 v13, 0x300, v13
-; GFX9-NEXT: v_add_u32_e32 v15, 0x300, v16
+; GFX9-NEXT: v_add_u32_sdwa v27, v11, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_add_u32_e32 v28, 0x300, v12
+; GFX9-NEXT: v_add_u32_sdwa v13, v13, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_add_u32_e32 v12, 0x300, v14
+; GFX9-NEXT: v_add_u32_sdwa v14, v15, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_add_u32_e32 v11, 0x300, v18
+; GFX9-NEXT: v_add_u32_sdwa v15, v16, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_add_u32_e32 v10, 0x300, v17
+; GFX9-NEXT: v_add_u32_sdwa v16, v20, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_add_u32_e32 v9, 0x300, v21
+; GFX9-NEXT: v_add_u32_sdwa v17, v22, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_add_u32_e32 v8, 0x300, v23
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v14, 16, v12
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v28
-; GFX9-NEXT: v_add_u32_e32 v16, 0x300, v20
-; GFX9-NEXT: v_add_u32_e32 v17, 0x300, v22
-; GFX9-NEXT: v_add_u32_e32 v18, 0x300, v19
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v15, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v13, v13, 16, v14
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v26
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v24
-; GFX9-NEXT: v_lshl_or_b32 v8, v18, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v17, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v16, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v14, v27, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v25, 16, v15
+; GFX9-NEXT: v_add_u32_sdwa v18, v19, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v8, v18, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v17, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v16, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v15, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v12, v14, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v13, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v14, v27, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v25, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: .LBB99_5: ; %end
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -75848,15 +75824,15 @@ define inreg <32 x half> @bitcast_v32bf16_to_v32f16_scalar(<32 x bfloat> inreg %
; GFX9-NEXT: s_cmp_lg_u32 s4, 1
; GFX9-NEXT: s_cbranch_scc1 .LBB103_5
; GFX9-NEXT: ; %bb.4: ; %cmp.true
+; GFX9-NEXT: s_and_b32 s4, s16, 0xffff0000
; GFX9-NEXT: v_mov_b32_e32 v0, 0x40c00000
-; GFX9-NEXT: s_and_b32 s5, s30, 0xffff0000
-; GFX9-NEXT: v_add_f32_e32 v1, s5, v0
+; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: s_lshl_b32 s5, s30, 16
+; GFX9-NEXT: s_and_b32 s5, s30, 0xffff0000
; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_add_f32_e32 v2, s5, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
@@ -75864,7 +75840,7 @@ define inreg <32 x half> @bitcast_v32bf16_to_v32f16_scalar(<32 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: s_and_b32 s5, s31, 0xffff0000
+; GFX9-NEXT: s_lshl_b32 s5, s30, 16
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_add_f32_e32 v3, s5, v0
; GFX9-NEXT: v_bfe_u32 v4, v3, 16, 1
@@ -75872,274 +75848,270 @@ define inreg <32 x half> @bitcast_v32bf16_to_v32f16_scalar(<32 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v3
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX9-NEXT: s_lshl_b32 s5, s31, 16
+; GFX9-NEXT: s_and_b32 s5, s31, 0xffff0000
; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
; GFX9-NEXT: v_add_f32_e32 v4, s5, v0
; GFX9-NEXT: v_bfe_u32 v5, v4, 16, 1
-; GFX9-NEXT: v_mov_b32_e32 v16, 0xffff
-; GFX9-NEXT: s_and_b32 s4, s29, 0xffff0000
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v5, v5, v4
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v4
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX9-NEXT: v_lshl_or_b32 v14, v1, 16, v2
-; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
+; GFX9-NEXT: s_lshl_b32 s5, s31, 16
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX9-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc
-; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
-; GFX9-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX9-NEXT: v_and_b32_sdwa v4, v16, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
-; GFX9-NEXT: v_lshl_or_b32 v15, v3, 16, v4
-; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
-; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: s_lshl_b32 s4, s29, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_add_f32_e32 v5, s5, v0
+; GFX9-NEXT: s_and_b32 s4, s29, 0xffff0000
+; GFX9-NEXT: v_bfe_u32 v6, v5, 16, 1
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_add_u32_e32 v6, v6, v5
+; GFX9-NEXT: v_or_b32_sdwa v14, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX9-NEXT: v_add_u32_e32 v6, 0x7fff, v6
+; GFX9-NEXT: v_or_b32_e32 v7, 0x400000, v5
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
+; GFX9-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_or_b32_sdwa v15, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT: s_lshl_b32 s4, s29, 16
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_add_f32_e32 v3, s4, v0
+; GFX9-NEXT: v_bfe_u32 v4, v3, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v4, v4, v3
+; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
; GFX9-NEXT: s_and_b32 s4, s28, 0xffff0000
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v13, v1, 16, v2
-; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
-; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
-; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
-; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: s_lshl_b32 s4, s28, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v13, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT: s_lshl_b32 s4, s28, 16
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_add_f32_e32 v3, s4, v0
+; GFX9-NEXT: v_bfe_u32 v4, v3, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v4, v4, v3
+; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
; GFX9-NEXT: s_and_b32 s4, s27, 0xffff0000
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v12, v1, 16, v2
-; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
-; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
-; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
-; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: s_lshl_b32 s4, s27, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v12, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT: s_lshl_b32 s4, s27, 16
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_add_f32_e32 v3, s4, v0
+; GFX9-NEXT: v_bfe_u32 v4, v3, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v4, v4, v3
+; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
; GFX9-NEXT: s_and_b32 s4, s26, 0xffff0000
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v11, v1, 16, v2
-; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
-; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
-; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
-; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: s_lshl_b32 s4, s26, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v11, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT: s_lshl_b32 s4, s26, 16
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_add_f32_e32 v3, s4, v0
+; GFX9-NEXT: v_bfe_u32 v4, v3, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v4, v4, v3
+; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
; GFX9-NEXT: s_and_b32 s4, s25, 0xffff0000
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v10, v1, 16, v2
-; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
-; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
-; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
-; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: s_lshl_b32 s4, s25, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v10, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT: s_lshl_b32 s4, s25, 16
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_add_f32_e32 v3, s4, v0
+; GFX9-NEXT: v_bfe_u32 v4, v3, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v4, v4, v3
+; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
; GFX9-NEXT: s_and_b32 s4, s24, 0xffff0000
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v9, v1, 16, v2
-; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
-; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
-; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
-; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: s_lshl_b32 s4, s24, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v9, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT: s_lshl_b32 s4, s24, 16
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_add_f32_e32 v3, s4, v0
+; GFX9-NEXT: v_bfe_u32 v4, v3, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v4, v4, v3
+; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
; GFX9-NEXT: s_and_b32 s4, s23, 0xffff0000
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v8, v1, 16, v2
-; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
-; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
-; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
-; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: s_lshl_b32 s4, s23, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v8, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT: s_lshl_b32 s4, s23, 16
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_add_f32_e32 v3, s4, v0
+; GFX9-NEXT: v_bfe_u32 v4, v3, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v4, v4, v3
+; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
; GFX9-NEXT: s_and_b32 s4, s22, 0xffff0000
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v7, v1, 16, v2
-; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
-; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
-; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
-; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: s_lshl_b32 s4, s22, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v7, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT: s_lshl_b32 s4, s22, 16
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_add_f32_e32 v3, s4, v0
+; GFX9-NEXT: v_bfe_u32 v4, v3, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v4, v4, v3
+; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
; GFX9-NEXT: s_and_b32 s4, s21, 0xffff0000
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v6, v1, 16, v2
-; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
-; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
-; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
-; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: s_lshl_b32 s4, s21, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v6, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT: s_lshl_b32 s4, s21, 16
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_add_f32_e32 v3, s4, v0
+; GFX9-NEXT: v_bfe_u32 v4, v3, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v4, v4, v3
+; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
; GFX9-NEXT: s_and_b32 s4, s20, 0xffff0000
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v5, v1, 16, v2
-; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
-; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
-; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
-; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: s_lshl_b32 s4, s20, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v5, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT: s_lshl_b32 s4, s20, 16
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_add_f32_e32 v3, s4, v0
+; GFX9-NEXT: v_bfe_u32 v4, v3, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v4, v4, v3
+; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
+; GFX9-NEXT: v_or_b32_e32 v16, 0x400000, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v16, vcc
; GFX9-NEXT: s_and_b32 s4, s19, 0xffff0000
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v4, v1, 16, v2
-; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
-; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
-; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
-; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: s_lshl_b32 s4, s19, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v4, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
-; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v2
+; GFX9-NEXT: v_or_b32_e32 v16, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v17, vcc
+; GFX9-NEXT: s_lshl_b32 s4, s19, 16
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v16, vcc
+; GFX9-NEXT: v_add_f32_e32 v3, s4, v0
+; GFX9-NEXT: v_bfe_u32 v16, v3, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v16, v16, v3
+; GFX9-NEXT: v_add_u32_e32 v16, 0x7fff, v16
+; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v16, v17, vcc
; GFX9-NEXT: s_and_b32 s4, s18, 0xffff0000
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v3, v1, 16, v2
-; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
-; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
-; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
-; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v1
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: s_lshl_b32 s4, s18, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v17, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v3, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
-; GFX9-NEXT: v_bfe_u32 v17, v2, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v17, v17, v2
-; GFX9-NEXT: v_add_u32_e32 v17, 0x7fff, v17
-; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v2
+; GFX9-NEXT: v_bfe_u32 v16, v2, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v16, v16, v2
+; GFX9-NEXT: v_add_u32_e32 v16, 0x7fff, v16
+; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v17, v18, vcc
-; GFX9-NEXT: s_and_b32 s4, s17, 0xffff0000
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v2, v1, 16, v2
-; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
-; GFX9-NEXT: v_bfe_u32 v17, v1, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v17, v17, v1
+; GFX9-NEXT: s_lshl_b32 s4, s18, 16
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v16, v17, vcc
+; GFX9-NEXT: v_add_f32_e32 v16, s4, v0
+; GFX9-NEXT: v_bfe_u32 v17, v16, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v17, v17, v16
; GFX9-NEXT: v_add_u32_e32 v17, 0x7fff, v17
-; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v1
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: s_lshl_b32 s4, s17, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v17, v18, vcc
-; GFX9-NEXT: v_add_f32_e32 v17, s4, v0
-; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
-; GFX9-NEXT: v_add_u32_e32 v18, 0x7fff, v18
-; GFX9-NEXT: v_or_b32_e32 v19, 0x400000, v17
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
-; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v19, vcc
-; GFX9-NEXT: s_and_b32 s4, s16, 0xffff0000
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v17, v16, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v17
+; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v16
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v16, v16
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v16, v17, v18, vcc
+; GFX9-NEXT: s_lshl_b32 s4, s16, 16
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v2, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f32_e32 v16, s4, v0
+; GFX9-NEXT: v_bfe_u32 v17, v16, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v17, v17, v16
+; GFX9-NEXT: v_add_u32_e32 v17, 0x7fff, v17
+; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v16
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v16, v16
+; GFX9-NEXT: s_and_b32 s4, s17, 0xffff0000
+; GFX9-NEXT: v_cndmask_b32_e32 v16, v17, v18, vcc
; GFX9-NEXT: v_add_f32_e32 v17, s4, v0
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
-; GFX9-NEXT: s_lshl_b32 s4, s16, 16
+; GFX9-NEXT: s_lshl_b32 s4, s17, 16
; GFX9-NEXT: v_add_u32_e32 v18, 0x7fff, v18
; GFX9-NEXT: v_or_b32_e32 v19, 0x400000, v17
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
@@ -76150,10 +76122,13 @@ define inreg <32 x half> @bitcast_v32bf16_to_v32f16_scalar(<32 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v18, 0x7fff, v18
; GFX9-NEXT: v_or_b32_e32 v19, 0x400000, v0
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v17
-; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v19, vcc
-; GFX9-NEXT: v_and_b32_sdwa v16, v16, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v16
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v18, v19, vcc
+; GFX9-NEXT: v_lshrrev_b32_e32 v18, 16, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v17
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_branch .LBB103_6
; GFX9-NEXT: .LBB103_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
@@ -84161,11 +84136,11 @@ define inreg <32 x half> @bitcast_v64i8_to_v32f16_scalar(<64 x i8> inreg %a, i32
; GFX9-NEXT: v_readfirstlane_b32 s56, v29
; GFX9-NEXT: v_readfirstlane_b32 s57, v28
; GFX9-NEXT: v_readfirstlane_b32 s61, v27
-; GFX9-NEXT: v_readfirstlane_b32 s63, v26
+; GFX9-NEXT: v_readfirstlane_b32 s62, v26
; GFX9-NEXT: v_readfirstlane_b32 s7, v25
-; GFX9-NEXT: v_readfirstlane_b32 s47, v24
+; GFX9-NEXT: v_readfirstlane_b32 s46, v24
; GFX9-NEXT: v_readfirstlane_b32 s58, v23
-; GFX9-NEXT: v_readfirstlane_b32 s60, v22
+; GFX9-NEXT: v_readfirstlane_b32 s59, v22
; GFX9-NEXT: v_readfirstlane_b32 s6, v21
; GFX9-NEXT: v_readfirstlane_b32 s8, v20
; GFX9-NEXT: v_readfirstlane_b32 s9, v19
@@ -84179,12 +84154,12 @@ define inreg <32 x half> @bitcast_v64i8_to_v32f16_scalar(<64 x i8> inreg %a, i32
; GFX9-NEXT: buffer_load_ushort v18, off, s[0:3], s32
; GFX9-NEXT: buffer_load_ushort v24, off, s[0:3], s32 offset:72
; GFX9-NEXT: buffer_load_ushort v27, off, s[0:3], s32 offset:68
-; GFX9-NEXT: buffer_load_ushort v32, off, s[0:3], s32 offset:64
+; GFX9-NEXT: buffer_load_ushort v33, off, s[0:3], s32 offset:64
; GFX9-NEXT: buffer_load_ushort v34, off, s[0:3], s32 offset:60
; GFX9-NEXT: buffer_load_ushort v22, off, s[0:3], s32 offset:56
; GFX9-NEXT: buffer_load_ushort v25, off, s[0:3], s32 offset:52
; GFX9-NEXT: buffer_load_ushort v30, off, s[0:3], s32 offset:48
-; GFX9-NEXT: buffer_load_ushort v33, off, s[0:3], s32 offset:44
+; GFX9-NEXT: buffer_load_ushort v32, off, s[0:3], s32 offset:44
; GFX9-NEXT: buffer_load_ushort v20, off, s[0:3], s32 offset:40
; GFX9-NEXT: buffer_load_ushort v23, off, s[0:3], s32 offset:36
; GFX9-NEXT: buffer_load_ushort v28, off, s[0:3], s32 offset:32
@@ -84204,103 +84179,95 @@ define inreg <32 x half> @bitcast_v64i8_to_v32f16_scalar(<64 x i8> inreg %a, i32
; GFX9-NEXT: v_readfirstlane_b32 s75, v6
; GFX9-NEXT: v_readfirstlane_b32 s44, v5
; GFX9-NEXT: v_readfirstlane_b32 s45, v4
-; GFX9-NEXT: v_readfirstlane_b32 s46, v3
+; GFX9-NEXT: v_readfirstlane_b32 s47, v3
; GFX9-NEXT: v_readfirstlane_b32 s74, v2
-; GFX9-NEXT: v_readfirstlane_b32 s59, v1
-; GFX9-NEXT: v_readfirstlane_b32 s62, v0
+; GFX9-NEXT: v_readfirstlane_b32 s60, v1
+; GFX9-NEXT: v_readfirstlane_b32 s63, v0
; GFX9-NEXT: s_waitcnt vmcnt(19)
; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v12
; GFX9-NEXT: s_and_b64 s[4:5], vcc, exec
; GFX9-NEXT: s_cbranch_scc0 .LBB107_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v8, 0xc0c0004
-; GFX9-NEXT: v_mov_b32_e32 v2, s21
-; GFX9-NEXT: v_perm_b32 v0, s16, v0, v8
; GFX9-NEXT: v_mov_b32_e32 v1, s19
-; GFX9-NEXT: v_perm_b32 v2, s20, v2, v8
; GFX9-NEXT: v_mov_b32_e32 v3, s23
+; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_perm_b32 v1, s18, v1, v8
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: v_mov_b32_e32 v2, s21
; GFX9-NEXT: v_perm_b32 v3, s22, v3, v8
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v3, 16, v2
-; GFX9-NEXT: v_mov_b32_e32 v2, s25
-; GFX9-NEXT: v_perm_b32 v2, s24, v2, v8
+; GFX9-NEXT: v_perm_b32 v0, s16, v0, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_perm_b32 v2, s20, v2, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v3, s27
+; GFX9-NEXT: v_mov_b32_e32 v2, s25
; GFX9-NEXT: v_perm_b32 v3, s26, v3, v8
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX9-NEXT: v_perm_b32 v2, s24, v2, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_mov_b32_e32 v4, s60
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v3, s29
+; GFX9-NEXT: v_perm_b32 v4, s63, v4, v8
; GFX9-NEXT: v_perm_b32 v3, s28, v3, v8
-; GFX9-NEXT: v_mov_b32_e32 v4, s59
-; GFX9-NEXT: v_perm_b32 v4, s62, v4, v8
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_lshl_or_b32 v3, v4, 16, v3
-; GFX9-NEXT: v_mov_b32_e32 v4, s46
-; GFX9-NEXT: v_perm_b32 v4, s74, v4, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX9-NEXT: v_mov_b32_e32 v5, s44
+; GFX9-NEXT: v_or_b32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v4, s47
; GFX9-NEXT: v_perm_b32 v5, s45, v5, v8
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_lshl_or_b32 v4, v5, 16, v4
-; GFX9-NEXT: v_mov_b32_e32 v5, s43
-; GFX9-NEXT: v_perm_b32 v5, s75, v5, v8
+; GFX9-NEXT: v_perm_b32 v4, s74, v4, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX9-NEXT: v_mov_b32_e32 v6, s41
+; GFX9-NEXT: v_or_b32_sdwa v4, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v5, s43
; GFX9-NEXT: v_perm_b32 v6, s42, v6, v8
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_lshl_or_b32 v5, v6, 16, v5
-; GFX9-NEXT: v_mov_b32_e32 v6, s40
-; GFX9-NEXT: v_perm_b32 v6, s76, v6, v8
+; GFX9-NEXT: v_perm_b32 v5, s75, v5, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v6
; GFX9-NEXT: v_mov_b32_e32 v7, s14
+; GFX9-NEXT: v_or_b32_sdwa v5, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v6, s40
; GFX9-NEXT: v_perm_b32 v7, s15, v7, v8
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_lshl_or_b32 v6, v7, 16, v6
-; GFX9-NEXT: v_mov_b32_e32 v7, s13
-; GFX9-NEXT: v_perm_b32 v7, s73, v7, v8
+; GFX9-NEXT: v_perm_b32 v6, s76, v6, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v7
; GFX9-NEXT: v_mov_b32_e32 v9, s11
+; GFX9-NEXT: v_or_b32_sdwa v6, v7, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v7, s13
; GFX9-NEXT: v_perm_b32 v9, s12, v9, v8
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_lshl_or_b32 v7, v9, 16, v7
+; GFX9-NEXT: v_perm_b32 v7, s73, v7, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX9-NEXT: s_and_b32 s4, s46, 0xff
+; GFX9-NEXT: s_lshl_b32 s5, s7, 8
+; GFX9-NEXT: v_or_b32_sdwa v7, v9, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v9, s58
+; GFX9-NEXT: s_or_b32 s4, s4, s5
+; GFX9-NEXT: v_perm_b32 v9, s59, v9, v8
+; GFX9-NEXT: v_lshlrev_b32_e64 v10, 16, s4
+; GFX9-NEXT: v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(18)
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v16
+; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v16
; GFX9-NEXT: s_waitcnt vmcnt(16)
-; GFX9-NEXT: v_perm_b32 v10, s72, v18, v8
-; GFX9-NEXT: v_or_b32_sdwa v9, v17, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v9, 16, v10
+; GFX9-NEXT: v_perm_b32 v8, s72, v18, v8
+; GFX9-NEXT: v_or_b32_sdwa v10, v17, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v11, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v26
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_or_b32_sdwa v9, v29, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v26
; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v19
-; GFX9-NEXT: v_or_b32_sdwa v10, v21, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_or_b32_sdwa v8, v29, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v10, v21, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v12, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_mov_b32 s4, 0xc0c0004
-; GFX9-NEXT: v_lshl_or_b32 v12, v10, 16, v9
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v20
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v20
+; GFX9-NEXT: v_or_b32_sdwa v8, v23, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_perm_b32 v10, v31, v28, s4
-; GFX9-NEXT: v_or_b32_sdwa v9, v23, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v13, v9, 16, v10
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v30
-; GFX9-NEXT: v_or_b32_sdwa v9, v33, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v13, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v30
; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v22
-; GFX9-NEXT: v_or_b32_sdwa v10, v25, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_lshl_or_b32 v14, v10, 16, v9
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v24
-; GFX9-NEXT: v_perm_b32 v10, v34, v32, s4
-; GFX9-NEXT: v_or_b32_sdwa v9, v27, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v15, v9, 16, v10
-; GFX9-NEXT: v_mov_b32_e32 v9, s58
-; GFX9-NEXT: v_perm_b32 v8, s60, v9, v8
-; GFX9-NEXT: s_and_b32 s4, s47, 0xff
-; GFX9-NEXT: s_lshl_b32 s5, s7, 8
-; GFX9-NEXT: s_or_b32 s4, s4, s5
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, s4, 16, v8
+; GFX9-NEXT: v_or_b32_sdwa v8, v32, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v10, v25, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v14, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_perm_b32 v10, v34, v33, s4
; GFX9-NEXT: s_and_b32 s4, s10, 0xff
; GFX9-NEXT: s_lshl_b32 s5, s9, 8
; GFX9-NEXT: s_or_b32 s4, s4, s5
@@ -84308,13 +84275,16 @@ define inreg <32 x half> @bitcast_v64i8_to_v32f16_scalar(<64 x i8> inreg %a, i32
; GFX9-NEXT: s_lshl_b32 s77, s6, 8
; GFX9-NEXT: s_or_b32 s5, s5, s77
; GFX9-NEXT: s_pack_ll_b32_b16 s4, s4, s5
-; GFX9-NEXT: s_and_b32 s5, s63, 0xff
+; GFX9-NEXT: s_and_b32 s5, s62, 0xff
; GFX9-NEXT: s_lshl_b32 s77, s61, 8
; GFX9-NEXT: s_or_b32 s5, s5, s77
; GFX9-NEXT: s_and_b32 s77, s57, 0xff
; GFX9-NEXT: s_lshl_b32 s78, s56, 8
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v24
; GFX9-NEXT: s_or_b32 s77, s77, s78
+; GFX9-NEXT: v_or_b32_sdwa v8, v27, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: s_pack_ll_b32_b16 s5, s5, s77
+; GFX9-NEXT: v_or_b32_sdwa v15, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v8, s4
; GFX9-NEXT: v_mov_b32_e32 v10, s5
; GFX9-NEXT: s_mov_b64 s[4:5], 0
@@ -84331,11 +84301,11 @@ define inreg <32 x half> @bitcast_v64i8_to_v32f16_scalar(<64 x i8> inreg %a, i32
; GFX9-NEXT: s_waitcnt vmcnt(12)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v34
; GFX9-NEXT: s_mov_b32 s4, 0xc0c0004
-; GFX9-NEXT: v_perm_b32 v8, v0, v32, s4
+; GFX9-NEXT: v_perm_b32 v8, v0, v33, s4
; GFX9-NEXT: v_add_u32_e32 v0, 3, v27
; GFX9-NEXT: v_perm_b32 v9, v0, v24, s4
; GFX9-NEXT: s_waitcnt vmcnt(8)
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v33
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v32
; GFX9-NEXT: v_perm_b32 v10, v0, v30, s4
; GFX9-NEXT: v_add_u32_e32 v0, 3, v25
; GFX9-NEXT: v_perm_b32 v11, v0, v22, s4
@@ -84352,138 +84322,123 @@ define inreg <32 x half> @bitcast_v64i8_to_v32f16_scalar(<64 x i8> inreg %a, i32
; GFX9-NEXT: v_add_u32_e32 v0, 3, v17
; GFX9-NEXT: v_mov_b32_e32 v19, 0xc0c0004
; GFX9-NEXT: v_perm_b32 v16, v0, v16, s4
-; GFX9-NEXT: s_add_i32 s63, s63, 3
+; GFX9-NEXT: s_add_i32 s62, s62, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s61
-; GFX9-NEXT: v_perm_b32 v17, s63, v0, v19
+; GFX9-NEXT: v_perm_b32 v17, s62, v0, v19
; GFX9-NEXT: s_add_i32 s57, s57, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s56
; GFX9-NEXT: v_perm_b32 v20, s57, v0, v19
-; GFX9-NEXT: s_add_i32 s60, s60, 3
+; GFX9-NEXT: s_add_i32 s59, s59, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s58
-; GFX9-NEXT: v_perm_b32 v21, s60, v0, v19
+; GFX9-NEXT: v_perm_b32 v21, s59, v0, v19
; GFX9-NEXT: s_add_i32 s20, s20, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s21
; GFX9-NEXT: v_perm_b32 v0, s20, v0, v19
; GFX9-NEXT: v_add_u32_e32 v1, 0x300, v0
; GFX9-NEXT: s_add_i32 s22, s22, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s23
+; GFX9-NEXT: s_movk_i32 s5, 0x300
; GFX9-NEXT: v_perm_b32 v0, s22, v0, v19
-; GFX9-NEXT: v_add_u32_e32 v2, 0x300, v0
+; GFX9-NEXT: v_add_u32_sdwa v2, v0, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s16, s16, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s17
-; GFX9-NEXT: v_perm_b32 v0, s16, v0, v19
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: v_mov_b32_e32 v3, s19
-; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
+; GFX9-NEXT: v_perm_b32 v0, s16, v0, v19
; GFX9-NEXT: v_perm_b32 v3, s18, v3, v19
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
+; GFX9-NEXT: v_add_u32_sdwa v3, v3, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s24, s24, 3
-; GFX9-NEXT: v_add_u32_e32 v3, 0x300, v3
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v2, s25
-; GFX9-NEXT: v_lshl_or_b32 v0, v3, 16, v0
-; GFX9-NEXT: v_perm_b32 v2, s24, v2, v19
; GFX9-NEXT: s_add_i32 s26, s26, 3
; GFX9-NEXT: v_mov_b32_e32 v3, s27
-; GFX9-NEXT: v_add_u32_e32 v2, 0x300, v2
+; GFX9-NEXT: v_perm_b32 v2, s24, v2, v19
; GFX9-NEXT: v_perm_b32 v3, s26, v3, v19
-; GFX9-NEXT: v_add_u32_e32 v3, 0x300, v3
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX9-NEXT: v_add_u32_e32 v2, 0x300, v2
+; GFX9-NEXT: v_add_u32_sdwa v3, v3, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s28, s28, 3
-; GFX9-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v3, s29
+; GFX9-NEXT: s_add_i32 s63, s63, 3
+; GFX9-NEXT: v_mov_b32_e32 v4, s60
; GFX9-NEXT: v_perm_b32 v3, s28, v3, v19
-; GFX9-NEXT: s_add_i32 s62, s62, 3
-; GFX9-NEXT: v_mov_b32_e32 v4, s59
+; GFX9-NEXT: v_perm_b32 v4, s63, v4, v19
; GFX9-NEXT: v_add_u32_e32 v3, 0x300, v3
-; GFX9-NEXT: v_perm_b32 v4, s62, v4, v19
-; GFX9-NEXT: v_add_u32_e32 v4, 0x300, v4
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX9-NEXT: v_add_u32_sdwa v4, v4, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s74, s74, 3
-; GFX9-NEXT: v_lshl_or_b32 v3, v4, 16, v3
-; GFX9-NEXT: v_mov_b32_e32 v4, s46
-; GFX9-NEXT: v_perm_b32 v4, s74, v4, v19
+; GFX9-NEXT: v_or_b32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v4, s47
; GFX9-NEXT: s_add_i32 s45, s45, 3
; GFX9-NEXT: v_mov_b32_e32 v5, s44
-; GFX9-NEXT: v_add_u32_e32 v4, 0x300, v4
+; GFX9-NEXT: v_perm_b32 v4, s74, v4, v19
; GFX9-NEXT: v_perm_b32 v5, s45, v5, v19
-; GFX9-NEXT: v_add_u32_e32 v5, 0x300, v5
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX9-NEXT: v_add_u32_e32 v4, 0x300, v4
+; GFX9-NEXT: v_add_u32_sdwa v5, v5, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s75, s75, 3
-; GFX9-NEXT: v_lshl_or_b32 v4, v5, 16, v4
+; GFX9-NEXT: v_or_b32_sdwa v4, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v5, s43
-; GFX9-NEXT: v_perm_b32 v5, s75, v5, v19
; GFX9-NEXT: s_add_i32 s42, s42, 3
; GFX9-NEXT: v_mov_b32_e32 v6, s41
-; GFX9-NEXT: v_add_u32_e32 v5, 0x300, v5
+; GFX9-NEXT: v_perm_b32 v5, s75, v5, v19
; GFX9-NEXT: v_perm_b32 v6, s42, v6, v19
-; GFX9-NEXT: v_add_u32_e32 v6, 0x300, v6
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX9-NEXT: v_add_u32_e32 v5, 0x300, v5
+; GFX9-NEXT: v_add_u32_sdwa v6, v6, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s76, s76, 3
-; GFX9-NEXT: v_lshl_or_b32 v5, v6, 16, v5
+; GFX9-NEXT: v_or_b32_sdwa v5, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v6, s40
-; GFX9-NEXT: v_perm_b32 v6, s76, v6, v19
; GFX9-NEXT: s_add_i32 s15, s15, 3
; GFX9-NEXT: v_mov_b32_e32 v7, s14
-; GFX9-NEXT: v_add_u32_e32 v6, 0x300, v6
+; GFX9-NEXT: v_perm_b32 v6, s76, v6, v19
; GFX9-NEXT: v_perm_b32 v7, s15, v7, v19
-; GFX9-NEXT: v_add_u32_e32 v7, 0x300, v7
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; GFX9-NEXT: v_add_u32_e32 v6, 0x300, v6
+; GFX9-NEXT: v_add_u32_sdwa v7, v7, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s73, s73, 3
-; GFX9-NEXT: v_lshl_or_b32 v6, v7, 16, v6
+; GFX9-NEXT: v_or_b32_sdwa v6, v7, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v7, s13
-; GFX9-NEXT: v_perm_b32 v7, s73, v7, v19
; GFX9-NEXT: s_add_i32 s12, s12, 3
; GFX9-NEXT: v_mov_b32_e32 v22, s11
-; GFX9-NEXT: s_add_i32 s72, s72, 3
-; GFX9-NEXT: v_add_u32_e32 v7, 0x300, v7
+; GFX9-NEXT: v_perm_b32 v7, s73, v7, v19
; GFX9-NEXT: v_perm_b32 v22, s12, v22, v19
-; GFX9-NEXT: v_perm_b32 v18, s72, v18, v19
-; GFX9-NEXT: v_add_u32_e32 v22, 0x300, v22
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX9-NEXT: v_add_u32_e32 v7, 0x300, v7
+; GFX9-NEXT: v_add_u32_sdwa v22, v22, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: s_add_i32 s72, s72, 3
+; GFX9-NEXT: s_add_i32 s46, s46, 3
+; GFX9-NEXT: v_or_b32_sdwa v7, v22, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v22, s7
; GFX9-NEXT: s_add_i32 s10, s10, 3
; GFX9-NEXT: v_mov_b32_e32 v23, s9
-; GFX9-NEXT: v_add_u32_e32 v28, 0x300, v12
-; GFX9-NEXT: v_add_u32_e32 v12, 0x300, v14
-; GFX9-NEXT: s_add_i32 s47, s47, 3
-; GFX9-NEXT: v_lshl_or_b32 v7, v22, 16, v7
-; GFX9-NEXT: v_mov_b32_e32 v22, s7
-; GFX9-NEXT: v_perm_b32 v23, s10, v23, v19
; GFX9-NEXT: s_add_i32 s8, s8, 3
; GFX9-NEXT: v_mov_b32_e32 v24, s6
-; GFX9-NEXT: v_add_u32_e32 v27, 0x300, v11
-; GFX9-NEXT: v_add_u32_e32 v14, 0x300, v15
-; GFX9-NEXT: v_add_u32_e32 v11, 0x300, v18
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_perm_b32 v22, s47, v22, v19
+; GFX9-NEXT: v_perm_b32 v18, s72, v18, v19
+; GFX9-NEXT: v_perm_b32 v22, s46, v22, v19
+; GFX9-NEXT: v_perm_b32 v23, s10, v23, v19
; GFX9-NEXT: v_perm_b32 v19, s8, v24, v19
; GFX9-NEXT: v_add_u32_e32 v24, 0x300, v8
-; GFX9-NEXT: v_add_u32_e32 v25, 0x300, v9
+; GFX9-NEXT: v_add_u32_sdwa v25, v9, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_add_u32_e32 v26, 0x300, v10
-; GFX9-NEXT: v_add_u32_e32 v13, 0x300, v13
-; GFX9-NEXT: v_add_u32_e32 v15, 0x300, v16
+; GFX9-NEXT: v_add_u32_sdwa v27, v11, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_add_u32_e32 v28, 0x300, v12
+; GFX9-NEXT: v_add_u32_sdwa v13, v13, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_add_u32_e32 v12, 0x300, v14
+; GFX9-NEXT: v_add_u32_sdwa v14, v15, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_add_u32_e32 v11, 0x300, v18
+; GFX9-NEXT: v_add_u32_sdwa v15, v16, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_add_u32_e32 v10, 0x300, v17
+; GFX9-NEXT: v_add_u32_sdwa v16, v20, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_add_u32_e32 v9, 0x300, v21
+; GFX9-NEXT: v_add_u32_sdwa v17, v22, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_add_u32_e32 v8, 0x300, v23
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v14, 16, v12
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v28
-; GFX9-NEXT: v_add_u32_e32 v16, 0x300, v20
-; GFX9-NEXT: v_add_u32_e32 v17, 0x300, v22
-; GFX9-NEXT: v_add_u32_e32 v18, 0x300, v19
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v15, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v13, v13, 16, v14
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v26
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v24
-; GFX9-NEXT: v_lshl_or_b32 v8, v18, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v17, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v16, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v14, v27, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v25, 16, v15
+; GFX9-NEXT: v_add_u32_sdwa v18, v19, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v8, v18, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v17, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v16, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v15, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v12, v14, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v13, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v14, v27, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v25, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: .LBB107_5: ; %end
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -89578,12 +89533,12 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX9-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v5
; GFX9-NEXT: s_and_b32 s6, s16, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v4, v5, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v4, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v2, s6, v1
+; GFX9-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
; GFX9-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
@@ -89601,9 +89556,9 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v5, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v30, 16, v2
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v30
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v31
; GFX9-NEXT: s_and_b32 s6, s19, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v3, v31, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v3, v2, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v2, s6, v1
; GFX9-NEXT: v_bfe_u32 v5, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v5, v5, v2
@@ -89621,9 +89576,9 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v32, 16, v2
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v29
; GFX9-NEXT: s_and_b32 s6, s18, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v11, v29, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v11, v2, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v2, s6, v1
; GFX9-NEXT: v_bfe_u32 v5, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v5, v5, v2
@@ -89641,9 +89596,9 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v35, 16, v2
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v37
; GFX9-NEXT: s_and_b32 s6, s21, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v10, v37, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v10, v2, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v2, s6, v1
; GFX9-NEXT: v_bfe_u32 v5, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v5, v5, v2
@@ -89661,9 +89616,9 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v28, 16, v2
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v28
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v27
; GFX9-NEXT: s_and_b32 s6, s20, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v15, v27, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v15, v2, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v2, s6, v1
; GFX9-NEXT: v_bfe_u32 v5, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v5, v5, v2
@@ -89681,9 +89636,9 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v50, 16, v2
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v50
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v36
; GFX9-NEXT: s_and_b32 s6, s23, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v14, v36, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v14, v2, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v2, s6, v1
; GFX9-NEXT: v_bfe_u32 v5, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v5, v5, v2
@@ -89701,9 +89656,9 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v34, 16, v2
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v33
; GFX9-NEXT: s_and_b32 s6, s22, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v20, v33, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v20, v2, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v2, s6, v1
; GFX9-NEXT: v_bfe_u32 v5, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v5, v5, v2
@@ -89721,9 +89676,9 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v48, 16, v2
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v48
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v38
; GFX9-NEXT: s_and_b32 s6, s25, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v19, v38, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v19, v2, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v2, s6, v1
; GFX9-NEXT: v_bfe_u32 v5, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v5, v5, v2
@@ -89741,9 +89696,9 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v49, 16, v2
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v49
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v39
; GFX9-NEXT: s_and_b32 s6, s24, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v22, v39, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v22, v2, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v2, s6, v1
; GFX9-NEXT: v_bfe_u32 v5, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v5, v5, v2
@@ -89761,9 +89716,9 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v53, 16, v2
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v53
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v51
; GFX9-NEXT: s_and_b32 s6, s27, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v21, v51, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v21, v2, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v2, s6, v1
; GFX9-NEXT: v_bfe_u32 v5, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v5, v5, v2
@@ -89781,9 +89736,9 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v54, 16, v2
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v54
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v52
; GFX9-NEXT: s_and_b32 s6, s26, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v24, v52, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v24, v2, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v2, s6, v1
; GFX9-NEXT: v_bfe_u32 v5, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v5, v5, v2
@@ -89801,9 +89756,9 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v41, 16, v2
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v41
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v55
; GFX9-NEXT: s_and_b32 s6, s29, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v23, v55, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v23, v2, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v2, s6, v1
; GFX9-NEXT: v_bfe_u32 v5, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v5, v5, v2
@@ -89821,9 +89776,9 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v42, 16, v2
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v42
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v40
; GFX9-NEXT: s_and_b32 s6, s28, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v26, v40, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v26, v2, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v2, s6, v1
; GFX9-NEXT: v_bfe_u32 v5, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v5, v5, v2
@@ -89841,9 +89796,9 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v45, 16, v2
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v45
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v43
; GFX9-NEXT: s_and_b32 s6, s5, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v25, v43, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v25, v2, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v2, s6, v1
; GFX9-NEXT: v_bfe_u32 v5, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v5, v5, v2
@@ -89861,9 +89816,9 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v46, 16, v2
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v46
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v44
; GFX9-NEXT: s_and_b32 s5, s4, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v58, v44, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v58, v2, v46 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v2, s5, v1
; GFX9-NEXT: v_bfe_u32 v5, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v5, v5, v2
@@ -89880,11 +89835,11 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v5, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v56, 16, v1
; GFX9-NEXT: v_lshrrev_b64 v[5:6], 24, v[25:26]
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v56
+; GFX9-NEXT: v_lshrrev_b32_e32 v56, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v47
; GFX9-NEXT: v_lshrrev_b64 v[6:7], 24, v[23:24]
-; GFX9-NEXT: v_lshl_or_b32 v57, v47, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v57, v1, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_lshrrev_b64 v[7:8], 24, v[21:22]
; GFX9-NEXT: v_lshrrev_b64 v[16:17], 24, v[10:11]
; GFX9-NEXT: v_lshrrev_b64 v[1:2], 24, v[57:58]
@@ -94888,11 +94843,11 @@ define inreg <32 x bfloat> @bitcast_v64i8_to_v32bf16_scalar(<64 x i8> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s56, v29
; GFX9-NEXT: v_readfirstlane_b32 s57, v28
; GFX9-NEXT: v_readfirstlane_b32 s61, v27
-; GFX9-NEXT: v_readfirstlane_b32 s63, v26
+; GFX9-NEXT: v_readfirstlane_b32 s62, v26
; GFX9-NEXT: v_readfirstlane_b32 s7, v25
-; GFX9-NEXT: v_readfirstlane_b32 s47, v24
+; GFX9-NEXT: v_readfirstlane_b32 s46, v24
; GFX9-NEXT: v_readfirstlane_b32 s58, v23
-; GFX9-NEXT: v_readfirstlane_b32 s60, v22
+; GFX9-NEXT: v_readfirstlane_b32 s59, v22
; GFX9-NEXT: v_readfirstlane_b32 s6, v21
; GFX9-NEXT: v_readfirstlane_b32 s8, v20
; GFX9-NEXT: v_readfirstlane_b32 s9, v19
@@ -94906,12 +94861,12 @@ define inreg <32 x bfloat> @bitcast_v64i8_to_v32bf16_scalar(<64 x i8> inreg %a,
; GFX9-NEXT: buffer_load_ushort v18, off, s[0:3], s32
; GFX9-NEXT: buffer_load_ushort v24, off, s[0:3], s32 offset:72
; GFX9-NEXT: buffer_load_ushort v27, off, s[0:3], s32 offset:68
-; GFX9-NEXT: buffer_load_ushort v32, off, s[0:3], s32 offset:64
+; GFX9-NEXT: buffer_load_ushort v33, off, s[0:3], s32 offset:64
; GFX9-NEXT: buffer_load_ushort v34, off, s[0:3], s32 offset:60
; GFX9-NEXT: buffer_load_ushort v22, off, s[0:3], s32 offset:56
; GFX9-NEXT: buffer_load_ushort v25, off, s[0:3], s32 offset:52
; GFX9-NEXT: buffer_load_ushort v30, off, s[0:3], s32 offset:48
-; GFX9-NEXT: buffer_load_ushort v33, off, s[0:3], s32 offset:44
+; GFX9-NEXT: buffer_load_ushort v32, off, s[0:3], s32 offset:44
; GFX9-NEXT: buffer_load_ushort v20, off, s[0:3], s32 offset:40
; GFX9-NEXT: buffer_load_ushort v23, off, s[0:3], s32 offset:36
; GFX9-NEXT: buffer_load_ushort v28, off, s[0:3], s32 offset:32
@@ -94931,103 +94886,95 @@ define inreg <32 x bfloat> @bitcast_v64i8_to_v32bf16_scalar(<64 x i8> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s75, v6
; GFX9-NEXT: v_readfirstlane_b32 s44, v5
; GFX9-NEXT: v_readfirstlane_b32 s45, v4
-; GFX9-NEXT: v_readfirstlane_b32 s46, v3
+; GFX9-NEXT: v_readfirstlane_b32 s47, v3
; GFX9-NEXT: v_readfirstlane_b32 s74, v2
-; GFX9-NEXT: v_readfirstlane_b32 s59, v1
-; GFX9-NEXT: v_readfirstlane_b32 s62, v0
+; GFX9-NEXT: v_readfirstlane_b32 s60, v1
+; GFX9-NEXT: v_readfirstlane_b32 s63, v0
; GFX9-NEXT: s_waitcnt vmcnt(19)
; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v12
; GFX9-NEXT: s_and_b64 s[4:5], vcc, exec
; GFX9-NEXT: s_cbranch_scc0 .LBB111_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v8, 0xc0c0004
-; GFX9-NEXT: v_mov_b32_e32 v2, s21
-; GFX9-NEXT: v_perm_b32 v0, s16, v0, v8
; GFX9-NEXT: v_mov_b32_e32 v1, s19
-; GFX9-NEXT: v_perm_b32 v2, s20, v2, v8
; GFX9-NEXT: v_mov_b32_e32 v3, s23
+; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_perm_b32 v1, s18, v1, v8
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: v_mov_b32_e32 v2, s21
; GFX9-NEXT: v_perm_b32 v3, s22, v3, v8
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v3, 16, v2
-; GFX9-NEXT: v_mov_b32_e32 v2, s25
-; GFX9-NEXT: v_perm_b32 v2, s24, v2, v8
+; GFX9-NEXT: v_perm_b32 v0, s16, v0, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_perm_b32 v2, s20, v2, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v3, s27
+; GFX9-NEXT: v_mov_b32_e32 v2, s25
; GFX9-NEXT: v_perm_b32 v3, s26, v3, v8
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX9-NEXT: v_perm_b32 v2, s24, v2, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_mov_b32_e32 v4, s60
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v3, s29
+; GFX9-NEXT: v_perm_b32 v4, s63, v4, v8
; GFX9-NEXT: v_perm_b32 v3, s28, v3, v8
-; GFX9-NEXT: v_mov_b32_e32 v4, s59
-; GFX9-NEXT: v_perm_b32 v4, s62, v4, v8
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_lshl_or_b32 v3, v4, 16, v3
-; GFX9-NEXT: v_mov_b32_e32 v4, s46
-; GFX9-NEXT: v_perm_b32 v4, s74, v4, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX9-NEXT: v_mov_b32_e32 v5, s44
+; GFX9-NEXT: v_or_b32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v4, s47
; GFX9-NEXT: v_perm_b32 v5, s45, v5, v8
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_lshl_or_b32 v4, v5, 16, v4
-; GFX9-NEXT: v_mov_b32_e32 v5, s43
-; GFX9-NEXT: v_perm_b32 v5, s75, v5, v8
+; GFX9-NEXT: v_perm_b32 v4, s74, v4, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX9-NEXT: v_mov_b32_e32 v6, s41
+; GFX9-NEXT: v_or_b32_sdwa v4, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v5, s43
; GFX9-NEXT: v_perm_b32 v6, s42, v6, v8
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_lshl_or_b32 v5, v6, 16, v5
-; GFX9-NEXT: v_mov_b32_e32 v6, s40
-; GFX9-NEXT: v_perm_b32 v6, s76, v6, v8
+; GFX9-NEXT: v_perm_b32 v5, s75, v5, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v6
; GFX9-NEXT: v_mov_b32_e32 v7, s14
+; GFX9-NEXT: v_or_b32_sdwa v5, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v6, s40
; GFX9-NEXT: v_perm_b32 v7, s15, v7, v8
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_lshl_or_b32 v6, v7, 16, v6
-; GFX9-NEXT: v_mov_b32_e32 v7, s13
-; GFX9-NEXT: v_perm_b32 v7, s73, v7, v8
+; GFX9-NEXT: v_perm_b32 v6, s76, v6, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v7
; GFX9-NEXT: v_mov_b32_e32 v9, s11
+; GFX9-NEXT: v_or_b32_sdwa v6, v7, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v7, s13
; GFX9-NEXT: v_perm_b32 v9, s12, v9, v8
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_lshl_or_b32 v7, v9, 16, v7
+; GFX9-NEXT: v_perm_b32 v7, s73, v7, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX9-NEXT: s_and_b32 s4, s46, 0xff
+; GFX9-NEXT: s_lshl_b32 s5, s7, 8
+; GFX9-NEXT: v_or_b32_sdwa v7, v9, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v9, s58
+; GFX9-NEXT: s_or_b32 s4, s4, s5
+; GFX9-NEXT: v_perm_b32 v9, s59, v9, v8
+; GFX9-NEXT: v_lshlrev_b32_e64 v10, 16, s4
+; GFX9-NEXT: v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(18)
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v16
+; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v16
; GFX9-NEXT: s_waitcnt vmcnt(16)
-; GFX9-NEXT: v_perm_b32 v10, s72, v18, v8
-; GFX9-NEXT: v_or_b32_sdwa v9, v17, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v9, 16, v10
+; GFX9-NEXT: v_perm_b32 v8, s72, v18, v8
+; GFX9-NEXT: v_or_b32_sdwa v10, v17, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v11, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v26
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_or_b32_sdwa v9, v29, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v26
; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v19
-; GFX9-NEXT: v_or_b32_sdwa v10, v21, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_or_b32_sdwa v8, v29, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v10, v21, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v12, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_mov_b32 s4, 0xc0c0004
-; GFX9-NEXT: v_lshl_or_b32 v12, v10, 16, v9
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v20
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v20
+; GFX9-NEXT: v_or_b32_sdwa v8, v23, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_perm_b32 v10, v31, v28, s4
-; GFX9-NEXT: v_or_b32_sdwa v9, v23, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v13, v9, 16, v10
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v30
-; GFX9-NEXT: v_or_b32_sdwa v9, v33, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v13, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v30
; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v22
-; GFX9-NEXT: v_or_b32_sdwa v10, v25, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_lshl_or_b32 v14, v10, 16, v9
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v24
-; GFX9-NEXT: v_perm_b32 v10, v34, v32, s4
-; GFX9-NEXT: v_or_b32_sdwa v9, v27, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v15, v9, 16, v10
-; GFX9-NEXT: v_mov_b32_e32 v9, s58
-; GFX9-NEXT: v_perm_b32 v8, s60, v9, v8
-; GFX9-NEXT: s_and_b32 s4, s47, 0xff
-; GFX9-NEXT: s_lshl_b32 s5, s7, 8
-; GFX9-NEXT: s_or_b32 s4, s4, s5
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, s4, 16, v8
+; GFX9-NEXT: v_or_b32_sdwa v8, v32, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v10, v25, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v14, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_perm_b32 v10, v34, v33, s4
; GFX9-NEXT: s_and_b32 s4, s10, 0xff
; GFX9-NEXT: s_lshl_b32 s5, s9, 8
; GFX9-NEXT: s_or_b32 s4, s4, s5
@@ -95035,13 +94982,16 @@ define inreg <32 x bfloat> @bitcast_v64i8_to_v32bf16_scalar(<64 x i8> inreg %a,
; GFX9-NEXT: s_lshl_b32 s77, s6, 8
; GFX9-NEXT: s_or_b32 s5, s5, s77
; GFX9-NEXT: s_pack_ll_b32_b16 s4, s4, s5
-; GFX9-NEXT: s_and_b32 s5, s63, 0xff
+; GFX9-NEXT: s_and_b32 s5, s62, 0xff
; GFX9-NEXT: s_lshl_b32 s77, s61, 8
; GFX9-NEXT: s_or_b32 s5, s5, s77
; GFX9-NEXT: s_and_b32 s77, s57, 0xff
; GFX9-NEXT: s_lshl_b32 s78, s56, 8
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v24
; GFX9-NEXT: s_or_b32 s77, s77, s78
+; GFX9-NEXT: v_or_b32_sdwa v8, v27, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: s_pack_ll_b32_b16 s5, s5, s77
+; GFX9-NEXT: v_or_b32_sdwa v15, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v8, s4
; GFX9-NEXT: v_mov_b32_e32 v10, s5
; GFX9-NEXT: s_mov_b64 s[4:5], 0
@@ -95058,11 +95008,11 @@ define inreg <32 x bfloat> @bitcast_v64i8_to_v32bf16_scalar(<64 x i8> inreg %a,
; GFX9-NEXT: s_waitcnt vmcnt(12)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v34
; GFX9-NEXT: s_mov_b32 s4, 0xc0c0004
-; GFX9-NEXT: v_perm_b32 v8, v0, v32, s4
+; GFX9-NEXT: v_perm_b32 v8, v0, v33, s4
; GFX9-NEXT: v_add_u32_e32 v0, 3, v27
; GFX9-NEXT: v_perm_b32 v9, v0, v24, s4
; GFX9-NEXT: s_waitcnt vmcnt(8)
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v33
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v32
; GFX9-NEXT: v_perm_b32 v10, v0, v30, s4
; GFX9-NEXT: v_add_u32_e32 v0, 3, v25
; GFX9-NEXT: v_perm_b32 v11, v0, v22, s4
@@ -95079,138 +95029,123 @@ define inreg <32 x bfloat> @bitcast_v64i8_to_v32bf16_scalar(<64 x i8> inreg %a,
; GFX9-NEXT: v_add_u32_e32 v0, 3, v17
; GFX9-NEXT: v_mov_b32_e32 v19, 0xc0c0004
; GFX9-NEXT: v_perm_b32 v16, v0, v16, s4
-; GFX9-NEXT: s_add_i32 s63, s63, 3
+; GFX9-NEXT: s_add_i32 s62, s62, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s61
-; GFX9-NEXT: v_perm_b32 v17, s63, v0, v19
+; GFX9-NEXT: v_perm_b32 v17, s62, v0, v19
; GFX9-NEXT: s_add_i32 s57, s57, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s56
; GFX9-NEXT: v_perm_b32 v20, s57, v0, v19
-; GFX9-NEXT: s_add_i32 s60, s60, 3
+; GFX9-NEXT: s_add_i32 s59, s59, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s58
-; GFX9-NEXT: v_perm_b32 v21, s60, v0, v19
+; GFX9-NEXT: v_perm_b32 v21, s59, v0, v19
; GFX9-NEXT: s_add_i32 s20, s20, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s21
; GFX9-NEXT: v_perm_b32 v0, s20, v0, v19
; GFX9-NEXT: v_add_u32_e32 v1, 0x300, v0
; GFX9-NEXT: s_add_i32 s22, s22, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s23
+; GFX9-NEXT: s_movk_i32 s5, 0x300
; GFX9-NEXT: v_perm_b32 v0, s22, v0, v19
-; GFX9-NEXT: v_add_u32_e32 v2, 0x300, v0
+; GFX9-NEXT: v_add_u32_sdwa v2, v0, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s16, s16, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s17
-; GFX9-NEXT: v_perm_b32 v0, s16, v0, v19
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: v_mov_b32_e32 v3, s19
-; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
+; GFX9-NEXT: v_perm_b32 v0, s16, v0, v19
; GFX9-NEXT: v_perm_b32 v3, s18, v3, v19
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
+; GFX9-NEXT: v_add_u32_sdwa v3, v3, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s24, s24, 3
-; GFX9-NEXT: v_add_u32_e32 v3, 0x300, v3
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v2, s25
-; GFX9-NEXT: v_lshl_or_b32 v0, v3, 16, v0
-; GFX9-NEXT: v_perm_b32 v2, s24, v2, v19
; GFX9-NEXT: s_add_i32 s26, s26, 3
; GFX9-NEXT: v_mov_b32_e32 v3, s27
-; GFX9-NEXT: v_add_u32_e32 v2, 0x300, v2
+; GFX9-NEXT: v_perm_b32 v2, s24, v2, v19
; GFX9-NEXT: v_perm_b32 v3, s26, v3, v19
-; GFX9-NEXT: v_add_u32_e32 v3, 0x300, v3
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX9-NEXT: v_add_u32_e32 v2, 0x300, v2
+; GFX9-NEXT: v_add_u32_sdwa v3, v3, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s28, s28, 3
-; GFX9-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v3, s29
+; GFX9-NEXT: s_add_i32 s63, s63, 3
+; GFX9-NEXT: v_mov_b32_e32 v4, s60
; GFX9-NEXT: v_perm_b32 v3, s28, v3, v19
-; GFX9-NEXT: s_add_i32 s62, s62, 3
-; GFX9-NEXT: v_mov_b32_e32 v4, s59
+; GFX9-NEXT: v_perm_b32 v4, s63, v4, v19
; GFX9-NEXT: v_add_u32_e32 v3, 0x300, v3
-; GFX9-NEXT: v_perm_b32 v4, s62, v4, v19
-; GFX9-NEXT: v_add_u32_e32 v4, 0x300, v4
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX9-NEXT: v_add_u32_sdwa v4, v4, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s74, s74, 3
-; GFX9-NEXT: v_lshl_or_b32 v3, v4, 16, v3
-; GFX9-NEXT: v_mov_b32_e32 v4, s46
-; GFX9-NEXT: v_perm_b32 v4, s74, v4, v19
+; GFX9-NEXT: v_or_b32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v4, s47
; GFX9-NEXT: s_add_i32 s45, s45, 3
; GFX9-NEXT: v_mov_b32_e32 v5, s44
-; GFX9-NEXT: v_add_u32_e32 v4, 0x300, v4
+; GFX9-NEXT: v_perm_b32 v4, s74, v4, v19
; GFX9-NEXT: v_perm_b32 v5, s45, v5, v19
-; GFX9-NEXT: v_add_u32_e32 v5, 0x300, v5
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX9-NEXT: v_add_u32_e32 v4, 0x300, v4
+; GFX9-NEXT: v_add_u32_sdwa v5, v5, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s75, s75, 3
-; GFX9-NEXT: v_lshl_or_b32 v4, v5, 16, v4
+; GFX9-NEXT: v_or_b32_sdwa v4, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v5, s43
-; GFX9-NEXT: v_perm_b32 v5, s75, v5, v19
; GFX9-NEXT: s_add_i32 s42, s42, 3
; GFX9-NEXT: v_mov_b32_e32 v6, s41
-; GFX9-NEXT: v_add_u32_e32 v5, 0x300, v5
+; GFX9-NEXT: v_perm_b32 v5, s75, v5, v19
; GFX9-NEXT: v_perm_b32 v6, s42, v6, v19
-; GFX9-NEXT: v_add_u32_e32 v6, 0x300, v6
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX9-NEXT: v_add_u32_e32 v5, 0x300, v5
+; GFX9-NEXT: v_add_u32_sdwa v6, v6, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s76, s76, 3
-; GFX9-NEXT: v_lshl_or_b32 v5, v6, 16, v5
+; GFX9-NEXT: v_or_b32_sdwa v5, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v6, s40
-; GFX9-NEXT: v_perm_b32 v6, s76, v6, v19
; GFX9-NEXT: s_add_i32 s15, s15, 3
; GFX9-NEXT: v_mov_b32_e32 v7, s14
-; GFX9-NEXT: v_add_u32_e32 v6, 0x300, v6
+; GFX9-NEXT: v_perm_b32 v6, s76, v6, v19
; GFX9-NEXT: v_perm_b32 v7, s15, v7, v19
-; GFX9-NEXT: v_add_u32_e32 v7, 0x300, v7
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; GFX9-NEXT: v_add_u32_e32 v6, 0x300, v6
+; GFX9-NEXT: v_add_u32_sdwa v7, v7, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s73, s73, 3
-; GFX9-NEXT: v_lshl_or_b32 v6, v7, 16, v6
+; GFX9-NEXT: v_or_b32_sdwa v6, v7, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v7, s13
-; GFX9-NEXT: v_perm_b32 v7, s73, v7, v19
; GFX9-NEXT: s_add_i32 s12, s12, 3
; GFX9-NEXT: v_mov_b32_e32 v22, s11
-; GFX9-NEXT: s_add_i32 s72, s72, 3
-; GFX9-NEXT: v_add_u32_e32 v7, 0x300, v7
+; GFX9-NEXT: v_perm_b32 v7, s73, v7, v19
; GFX9-NEXT: v_perm_b32 v22, s12, v22, v19
-; GFX9-NEXT: v_perm_b32 v18, s72, v18, v19
-; GFX9-NEXT: v_add_u32_e32 v22, 0x300, v22
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX9-NEXT: v_add_u32_e32 v7, 0x300, v7
+; GFX9-NEXT: v_add_u32_sdwa v22, v22, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: s_add_i32 s72, s72, 3
+; GFX9-NEXT: s_add_i32 s46, s46, 3
+; GFX9-NEXT: v_or_b32_sdwa v7, v22, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v22, s7
; GFX9-NEXT: s_add_i32 s10, s10, 3
; GFX9-NEXT: v_mov_b32_e32 v23, s9
-; GFX9-NEXT: v_add_u32_e32 v28, 0x300, v12
-; GFX9-NEXT: v_add_u32_e32 v12, 0x300, v14
-; GFX9-NEXT: s_add_i32 s47, s47, 3
-; GFX9-NEXT: v_lshl_or_b32 v7, v22, 16, v7
-; GFX9-NEXT: v_mov_b32_e32 v22, s7
-; GFX9-NEXT: v_perm_b32 v23, s10, v23, v19
; GFX9-NEXT: s_add_i32 s8, s8, 3
; GFX9-NEXT: v_mov_b32_e32 v24, s6
-; GFX9-NEXT: v_add_u32_e32 v27, 0x300, v11
-; GFX9-NEXT: v_add_u32_e32 v14, 0x300, v15
-; GFX9-NEXT: v_add_u32_e32 v11, 0x300, v18
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_perm_b32 v22, s47, v22, v19
+; GFX9-NEXT: v_perm_b32 v18, s72, v18, v19
+; GFX9-NEXT: v_perm_b32 v22, s46, v22, v19
+; GFX9-NEXT: v_perm_b32 v23, s10, v23, v19
; GFX9-NEXT: v_perm_b32 v19, s8, v24, v19
; GFX9-NEXT: v_add_u32_e32 v24, 0x300, v8
-; GFX9-NEXT: v_add_u32_e32 v25, 0x300, v9
+; GFX9-NEXT: v_add_u32_sdwa v25, v9, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_add_u32_e32 v26, 0x300, v10
-; GFX9-NEXT: v_add_u32_e32 v13, 0x300, v13
-; GFX9-NEXT: v_add_u32_e32 v15, 0x300, v16
+; GFX9-NEXT: v_add_u32_sdwa v27, v11, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_add_u32_e32 v28, 0x300, v12
+; GFX9-NEXT: v_add_u32_sdwa v13, v13, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_add_u32_e32 v12, 0x300, v14
+; GFX9-NEXT: v_add_u32_sdwa v14, v15, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_add_u32_e32 v11, 0x300, v18
+; GFX9-NEXT: v_add_u32_sdwa v15, v16, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_add_u32_e32 v10, 0x300, v17
+; GFX9-NEXT: v_add_u32_sdwa v16, v20, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_add_u32_e32 v9, 0x300, v21
+; GFX9-NEXT: v_add_u32_sdwa v17, v22, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_add_u32_e32 v8, 0x300, v23
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v14, 16, v12
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v28
-; GFX9-NEXT: v_add_u32_e32 v16, 0x300, v20
-; GFX9-NEXT: v_add_u32_e32 v17, 0x300, v22
-; GFX9-NEXT: v_add_u32_e32 v18, 0x300, v19
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v15, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v13, v13, 16, v14
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v26
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v24
-; GFX9-NEXT: v_lshl_or_b32 v8, v18, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v17, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v16, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v14, v27, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v25, 16, v15
+; GFX9-NEXT: v_add_u32_sdwa v18, v19, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v8, v18, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v17, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v16, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v15, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v12, v14, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v13, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v14, v27, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v25, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: .LBB111_5: ; %end
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.576bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.576bit.ll
index 5075c732b6a17..9ce295efedfc9 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.576bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.576bit.ll
@@ -11201,42 +11201,42 @@ define inreg <36 x i16> @bitcast_v18f32_to_v36i16_scalar(<18 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v19, s11
; GFX9-NEXT: v_mov_b32_e32 v18, s10
; GFX9-NEXT: .LBB29_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_lshl_or_b32 v0, v35, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v34, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v33, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v32, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v31, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v30, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v29, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v28, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v27, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v26, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v25, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v24, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v23, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v22, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v21, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v20, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v19, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v18, 16, v17
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; GFX9-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; GFX9-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; GFX9-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX9-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; GFX9-NEXT: v_lshlrev_b32_e32 v20, 16, v20
+; GFX9-NEXT: v_lshlrev_b32_e32 v19, 16, v19
+; GFX9-NEXT: v_lshlrev_b32_e32 v18, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v0, v35, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v34, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v33, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v32, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v4, v31, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v30, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v29, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v28, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v27, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v26, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v25, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v24, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v12, v23, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v22, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v14, v21, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v20, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v16, v19, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v18, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: bitcast_v18f32_to_v36i16_scalar:
@@ -14015,42 +14015,42 @@ define inreg <36 x half> @bitcast_v18f32_to_v36f16_scalar(<18 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v19, s11
; GFX9-NEXT: v_mov_b32_e32 v18, s10
; GFX9-NEXT: .LBB33_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_lshl_or_b32 v0, v35, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v34, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v33, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v32, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v31, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v30, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v29, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v28, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v27, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v26, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v25, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v24, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v23, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v22, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v21, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v20, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v19, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v18, 16, v17
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; GFX9-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; GFX9-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; GFX9-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX9-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; GFX9-NEXT: v_lshlrev_b32_e32 v20, 16, v20
+; GFX9-NEXT: v_lshlrev_b32_e32 v19, 16, v19
+; GFX9-NEXT: v_lshlrev_b32_e32 v18, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v0, v35, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v34, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v33, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v32, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v4, v31, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v30, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v29, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v28, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v27, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v26, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v25, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v24, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v12, v23, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v22, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v14, v21, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v20, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v16, v19, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v18, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: bitcast_v18f32_to_v36f16_scalar:
@@ -23236,42 +23236,42 @@ define inreg <36 x i16> @bitcast_v9f64_to_v36i16_scalar(<9 x double> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v22, s11
; GFX9-NEXT: v_mov_b32_e32 v20, s10
; GFX9-NEXT: .LBB49_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_lshl_or_b32 v0, v33, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v35, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v31, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v34, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v29, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v32, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v28, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v30, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v27, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v26, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v23, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v25, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v21, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v24, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v19, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v22, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v18, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v20, 16, v17
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; GFX9-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; GFX9-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; GFX9-NEXT: v_lshlrev_b32_e32 v19, 16, v19
+; GFX9-NEXT: v_lshlrev_b32_e32 v18, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v0, v33, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v35
+; GFX9-NEXT: v_or_b32_sdwa v2, v31, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v34
+; GFX9-NEXT: v_or_b32_sdwa v4, v29, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v32
+; GFX9-NEXT: v_or_b32_sdwa v6, v28, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v30
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX9-NEXT: v_or_b32_sdwa v10, v23, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v23, 16, v25
+; GFX9-NEXT: v_or_b32_sdwa v12, v21, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v21, 16, v24
+; GFX9-NEXT: v_or_b32_sdwa v14, v19, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v19, 16, v22
+; GFX9-NEXT: v_or_b32_sdwa v16, v18, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v18, 16, v20
+; GFX9-NEXT: v_or_b32_sdwa v1, v33, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v31, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v29, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v28, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v27, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v26, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v23, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v21, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v19, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v18, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: bitcast_v9f64_to_v36i16_scalar:
@@ -25978,42 +25978,42 @@ define inreg <36 x half> @bitcast_v9f64_to_v36f16_scalar(<9 x double> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v22, s11
; GFX9-NEXT: v_mov_b32_e32 v20, s10
; GFX9-NEXT: .LBB53_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_lshl_or_b32 v0, v33, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v35, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v31, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v34, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v29, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v32, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v28, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v30, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v27, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v26, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v23, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v25, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v21, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v24, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v19, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v22, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v18, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v20, 16, v17
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; GFX9-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; GFX9-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; GFX9-NEXT: v_lshlrev_b32_e32 v19, 16, v19
+; GFX9-NEXT: v_lshlrev_b32_e32 v18, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v0, v33, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v35
+; GFX9-NEXT: v_or_b32_sdwa v2, v31, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v34
+; GFX9-NEXT: v_or_b32_sdwa v4, v29, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v32
+; GFX9-NEXT: v_or_b32_sdwa v6, v28, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v30
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX9-NEXT: v_or_b32_sdwa v10, v23, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v23, 16, v25
+; GFX9-NEXT: v_or_b32_sdwa v12, v21, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v21, 16, v24
+; GFX9-NEXT: v_or_b32_sdwa v14, v19, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v19, 16, v22
+; GFX9-NEXT: v_or_b32_sdwa v16, v18, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v18, 16, v20
+; GFX9-NEXT: v_or_b32_sdwa v1, v33, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v31, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v29, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v28, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v27, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v26, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v23, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v21, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v19, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v18, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: bitcast_v9f64_to_v36f16_scalar:
@@ -29300,42 +29300,42 @@ define inreg <36 x half> @bitcast_v36i16_to_v36f16_scalar(<36 x i16> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v34, s7
; GFX9-NEXT: v_mov_b32_e32 v35, s6
; GFX9-NEXT: .LBB57_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_lshl_or_b32 v0, v35, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v34, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v33, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v32, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v31, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v30, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v29, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v28, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v27, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v26, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v25, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v24, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v23, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v22, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v21, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v20, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v19, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v18, 16, v17
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; GFX9-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; GFX9-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; GFX9-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX9-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; GFX9-NEXT: v_lshlrev_b32_e32 v20, 16, v20
+; GFX9-NEXT: v_lshlrev_b32_e32 v19, 16, v19
+; GFX9-NEXT: v_lshlrev_b32_e32 v18, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v0, v35, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v34, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v33, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v32, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v4, v31, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v30, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v29, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v28, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v27, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v26, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v25, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v24, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v12, v23, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v22, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v14, v21, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v20, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v16, v19, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v18, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: bitcast_v36i16_to_v36f16_scalar:
@@ -30824,42 +30824,42 @@ define inreg <36 x i16> @bitcast_v36f16_to_v36i16_scalar(<36 x half> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v34, s7
; GFX9-NEXT: v_mov_b32_e32 v35, s6
; GFX9-NEXT: .LBB59_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_lshl_or_b32 v0, v35, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v34, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v33, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v32, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v31, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v30, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v29, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v28, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v27, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v26, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v25, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v24, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v23, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v22, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v21, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v20, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v19, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v18, 16, v17
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; GFX9-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; GFX9-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; GFX9-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX9-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; GFX9-NEXT: v_lshlrev_b32_e32 v20, 16, v20
+; GFX9-NEXT: v_lshlrev_b32_e32 v19, 16, v19
+; GFX9-NEXT: v_lshlrev_b32_e32 v18, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v0, v35, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v34, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v33, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v32, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v4, v31, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v30, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v29, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v28, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v27, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v26, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v25, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v24, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v12, v23, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v22, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v14, v21, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v20, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v16, v19, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v18, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: bitcast_v36f16_to_v36i16_scalar:
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.640bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.640bit.ll
index 0a6f2c6597a2f..765b7eb6c2a01 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.640bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.640bit.ll
@@ -12218,46 +12218,46 @@ define inreg <40 x i16> @bitcast_v20f32_to_v40i16_scalar(<20 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v21, s13
; GFX9-NEXT: v_mov_b32_e32 v20, s12
; GFX9-NEXT: .LBB29_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_lshl_or_b32 v0, v39, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v38, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v37, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v36, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v35, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v34, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v33, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v32, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v31, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v30, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v29, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v28, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v27, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v26, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v25, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v24, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v23, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v22, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v21, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v20, 16, v19
+; GFX9-NEXT: v_lshlrev_b32_e32 v39, 16, v39
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; GFX9-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; GFX9-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; GFX9-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; GFX9-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX9-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; GFX9-NEXT: v_lshlrev_b32_e32 v20, 16, v20
+; GFX9-NEXT: v_or_b32_sdwa v0, v39, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v38, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v37, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v36, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v4, v35, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v34, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v33, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v32, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v31, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v30, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v29, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v28, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v12, v27, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v26, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v14, v25, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v24, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v16, v23, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v22, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v18, v21, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v20, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: bitcast_v20f32_to_v40i16_scalar:
@@ -15339,46 +15339,46 @@ define inreg <40 x half> @bitcast_v20f32_to_v40f16_scalar(<20 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v21, s13
; GFX9-NEXT: v_mov_b32_e32 v20, s12
; GFX9-NEXT: .LBB33_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_lshl_or_b32 v0, v39, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v38, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v37, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v36, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v35, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v34, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v33, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v32, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v31, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v30, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v29, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v28, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v27, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v26, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v25, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v24, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v23, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v22, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v21, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v20, 16, v19
+; GFX9-NEXT: v_lshlrev_b32_e32 v39, 16, v39
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; GFX9-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; GFX9-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; GFX9-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; GFX9-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX9-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; GFX9-NEXT: v_lshlrev_b32_e32 v20, 16, v20
+; GFX9-NEXT: v_or_b32_sdwa v0, v39, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v38, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v37, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v36, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v4, v35, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v34, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v33, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v32, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v31, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v30, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v29, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v28, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v12, v27, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v26, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v14, v25, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v24, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v16, v23, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v22, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v18, v21, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v20, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: bitcast_v20f32_to_v40f16_scalar:
@@ -25518,46 +25518,46 @@ define inreg <40 x i16> @bitcast_v10f64_to_v40i16_scalar(<10 x double> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v24, s13
; GFX9-NEXT: v_mov_b32_e32 v22, s12
; GFX9-NEXT: .LBB49_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_lshl_or_b32 v0, v37, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v39, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v35, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v38, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v33, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v36, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v32, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v34, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v31, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v30, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v27, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v29, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v25, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v28, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v23, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v26, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v21, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v24, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v20, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v22, 16, v19
+; GFX9-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; GFX9-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; GFX9-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; GFX9-NEXT: v_lshlrev_b32_e32 v20, 16, v20
+; GFX9-NEXT: v_or_b32_sdwa v0, v37, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v37, 16, v39
+; GFX9-NEXT: v_or_b32_sdwa v2, v35, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v38
+; GFX9-NEXT: v_or_b32_sdwa v4, v33, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v36
+; GFX9-NEXT: v_or_b32_sdwa v6, v32, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; GFX9-NEXT: v_or_b32_sdwa v10, v27, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v29
+; GFX9-NEXT: v_or_b32_sdwa v12, v25, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v28
+; GFX9-NEXT: v_or_b32_sdwa v14, v23, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v23, 16, v26
+; GFX9-NEXT: v_or_b32_sdwa v16, v21, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v21, 16, v24
+; GFX9-NEXT: v_or_b32_sdwa v18, v20, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v20, 16, v22
+; GFX9-NEXT: v_or_b32_sdwa v1, v37, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v35, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v33, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v32, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v31, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v30, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v27, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v25, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v23, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v21, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v20, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: bitcast_v10f64_to_v40i16_scalar:
@@ -28559,46 +28559,46 @@ define inreg <40 x half> @bitcast_v10f64_to_v40f16_scalar(<10 x double> inreg %a
; GFX9-NEXT: v_mov_b32_e32 v24, s13
; GFX9-NEXT: v_mov_b32_e32 v22, s12
; GFX9-NEXT: .LBB53_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_lshl_or_b32 v0, v37, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v39, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v35, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v38, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v33, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v36, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v32, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v34, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v31, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v30, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v27, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v29, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v25, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v28, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v23, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v26, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v21, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v24, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v20, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v22, 16, v19
+; GFX9-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; GFX9-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; GFX9-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; GFX9-NEXT: v_lshlrev_b32_e32 v20, 16, v20
+; GFX9-NEXT: v_or_b32_sdwa v0, v37, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v37, 16, v39
+; GFX9-NEXT: v_or_b32_sdwa v2, v35, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v38
+; GFX9-NEXT: v_or_b32_sdwa v4, v33, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v36
+; GFX9-NEXT: v_or_b32_sdwa v6, v32, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; GFX9-NEXT: v_or_b32_sdwa v10, v27, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v29
+; GFX9-NEXT: v_or_b32_sdwa v12, v25, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v28
+; GFX9-NEXT: v_or_b32_sdwa v14, v23, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v23, 16, v26
+; GFX9-NEXT: v_or_b32_sdwa v16, v21, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v21, 16, v24
+; GFX9-NEXT: v_or_b32_sdwa v18, v20, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v20, 16, v22
+; GFX9-NEXT: v_or_b32_sdwa v1, v37, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v35, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v33, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v32, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v31, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v30, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v27, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v25, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v23, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v21, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v20, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: bitcast_v10f64_to_v40f16_scalar:
@@ -32293,46 +32293,46 @@ define inreg <40 x half> @bitcast_v40i16_to_v40f16_scalar(<40 x i16> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v38, s7
; GFX9-NEXT: v_mov_b32_e32 v39, s6
; GFX9-NEXT: .LBB57_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_lshl_or_b32 v0, v39, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v38, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v37, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v36, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v35, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v34, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v33, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v32, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v31, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v30, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v29, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v28, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v27, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v26, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v25, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v24, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v23, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v22, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v21, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v20, 16, v19
+; GFX9-NEXT: v_lshlrev_b32_e32 v39, 16, v39
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; GFX9-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; GFX9-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; GFX9-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; GFX9-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX9-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; GFX9-NEXT: v_lshlrev_b32_e32 v20, 16, v20
+; GFX9-NEXT: v_or_b32_sdwa v0, v39, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v38, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v37, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v36, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v4, v35, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v34, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v33, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v32, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v31, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v30, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v29, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v28, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v12, v27, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v26, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v14, v25, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v24, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v16, v23, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v22, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v18, v21, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v20, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: bitcast_v40i16_to_v40f16_scalar:
@@ -33981,46 +33981,46 @@ define inreg <40 x i16> @bitcast_v40f16_to_v40i16_scalar(<40 x half> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v38, s7
; GFX9-NEXT: v_mov_b32_e32 v39, s6
; GFX9-NEXT: .LBB59_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_lshl_or_b32 v0, v39, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v38, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v37, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v36, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v35, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v34, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v33, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v32, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v31, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v30, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v29, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v28, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v27, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v26, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v25, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v24, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v23, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v22, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v21, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v20, 16, v19
+; GFX9-NEXT: v_lshlrev_b32_e32 v39, 16, v39
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; GFX9-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; GFX9-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; GFX9-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; GFX9-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX9-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; GFX9-NEXT: v_lshlrev_b32_e32 v20, 16, v20
+; GFX9-NEXT: v_or_b32_sdwa v0, v39, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v38, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v37, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v36, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v4, v35, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v34, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v33, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v32, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v31, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v30, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v29, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v28, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v12, v27, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v26, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v14, v25, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v24, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v16, v23, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v22, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v18, v21, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v20, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: bitcast_v40f16_to_v40i16_scalar:
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll
index 0da1fde3f0549..39709428fecdb 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll
@@ -2644,33 +2644,32 @@ define inreg i64 @bitcast_v4bf16_to_i64_scalar(<4 x bfloat> inreg %a, i32 inreg
; GFX9-NEXT: v_add_f32_e32 v2, s4, v1
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX9-NEXT: v_mov_b32_e32 v3, 0xffff
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s17
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v2, s4, v1
-; GFX9-NEXT: v_bfe_u32 v4, v2, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v4, v4, v2
+; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
; GFX9-NEXT: s_lshl_b32 s4, s17, 16
-; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
-; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v2
+; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_add_f32_e32 v1, s4, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc
-; GFX9-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v4, v4, v1
-; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
-; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v5, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v3, v3, v1
; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX9-NEXT: v_and_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v1
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_setpc_b64 s[30:31]
; GFX9-NEXT: .LBB23_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
@@ -6070,33 +6069,32 @@ define inreg double @bitcast_v4bf16_to_f64_scalar(<4 x bfloat> inreg %a, i32 inr
; GFX9-NEXT: v_add_f32_e32 v2, s4, v1
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX9-NEXT: v_mov_b32_e32 v3, 0xffff
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s17
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v2, s4, v1
-; GFX9-NEXT: v_bfe_u32 v4, v2, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v4, v4, v2
+; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
; GFX9-NEXT: s_lshl_b32 s4, s17, 16
-; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
-; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v2
+; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_add_f32_e32 v1, s4, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc
-; GFX9-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v4, v4, v1
-; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
-; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v5, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v3, v3, v1
; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX9-NEXT: v_and_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v1
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_setpc_b64 s[30:31]
; GFX9-NEXT: .LBB47_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
@@ -9149,33 +9147,32 @@ define inreg <2 x i32> @bitcast_v4bf16_to_v2i32_scalar(<4 x bfloat> inreg %a, i3
; GFX9-NEXT: v_add_f32_e32 v2, s4, v1
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX9-NEXT: v_mov_b32_e32 v3, 0xffff
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s17
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v2, s4, v1
-; GFX9-NEXT: v_bfe_u32 v4, v2, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v4, v4, v2
+; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
; GFX9-NEXT: s_lshl_b32 s4, s17, 16
-; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
-; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v2
+; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_add_f32_e32 v1, s4, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc
-; GFX9-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v4, v4, v1
-; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
-; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v5, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v3, v3, v1
; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX9-NEXT: v_and_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v1
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_setpc_b64 s[30:31]
; GFX9-NEXT: .LBB67_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
@@ -11870,33 +11867,32 @@ define inreg <2 x float> @bitcast_v4bf16_to_v2f32_scalar(<4 x bfloat> inreg %a,
; GFX9-NEXT: v_add_f32_e32 v2, s4, v1
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX9-NEXT: v_mov_b32_e32 v3, 0xffff
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s17
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v2, s4, v1
-; GFX9-NEXT: v_bfe_u32 v4, v2, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v4, v4, v2
+; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
; GFX9-NEXT: s_lshl_b32 s4, s17, 16
-; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
-; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v2
+; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_add_f32_e32 v1, s4, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc
-; GFX9-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v4, v4, v1
-; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
-; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v5, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v3, v3, v1
; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX9-NEXT: v_and_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v1
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_setpc_b64 s[30:31]
; GFX9-NEXT: .LBB83_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
@@ -16165,17 +16161,16 @@ define inreg <4 x half> @bitcast_v4bf16_to_v4f16_scalar(<4 x bfloat> inreg %a, i
; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
; GFX9-NEXT: v_bfe_u32 v4, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v4, v4, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v0
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc
-; GFX9-NEXT: v_mov_b32_e32 v4, 0xffff
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX9-NEXT: v_and_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_and_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v0, v3, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_setpc_b64 s[30:31]
; GFX9-NEXT: .LBB103_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
@@ -17989,9 +17984,9 @@ define inreg <8 x i8> @bitcast_v4bf16_to_v8i8_scalar(<4 x bfloat> inreg %a, i32
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s17
-; GFX9-NEXT: v_lshl_or_b32 v9, v2, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v9, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v3, s4, v1
; GFX9-NEXT: v_bfe_u32 v4, v3, 16, 1
; GFX9-NEXT: v_add_u32_e32 v4, v4, v3
@@ -18009,8 +18004,8 @@ define inreg <8 x i8> @bitcast_v4bf16_to_v8i8_scalar(<4 x bfloat> inreg %a, i32
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v1
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v8
-; GFX9-NEXT: v_lshl_or_b32 v10, v6, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v6
+; GFX9-NEXT: v_or_b32_sdwa v10, v1, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_lshrrev_b64 v[3:4], 24, v[9:10]
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 8, v9
; GFX9-NEXT: v_lshrrev_b32_e32 v7, 24, v10
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.704bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.704bit.ll
index bbf979d84112e..7d99803b4c22e 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.704bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.704bit.ll
@@ -13154,50 +13154,50 @@ define inreg <44 x i16> @bitcast_v22f32_to_v44i16_scalar(<22 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v23, s15
; GFX9-NEXT: v_mov_b32_e32 v22, s14
; GFX9-NEXT: .LBB29_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_and_b32_e32 v21, 0xffff, v21
-; GFX9-NEXT: v_lshl_or_b32 v0, v51, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v50, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v49, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v48, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v39, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v38, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v37, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v36, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v35, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v34, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v33, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v32, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v31, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v30, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v29, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v28, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v27, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v26, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v25, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v24, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v23, 16, v20
-; GFX9-NEXT: v_lshl_or_b32 v21, v22, 16, v21
+; GFX9-NEXT: v_lshlrev_b32_e32 v51, 16, v51
+; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v50
+; GFX9-NEXT: v_lshlrev_b32_e32 v49, 16, v49
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; GFX9-NEXT: v_lshlrev_b32_e32 v39, 16, v39
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; GFX9-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; GFX9-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; GFX9-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; GFX9-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX9-NEXT: v_or_b32_sdwa v0, v51, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v50, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v49, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v48, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v4, v39, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v38, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v37, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v36, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v35, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v34, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v33, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v32, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v12, v31, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v30, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v14, v29, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v28, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v16, v27, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v26, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v18, v25, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v24, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v20, v23, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v21, v22, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: bitcast_v22f32_to_v44i16_scalar:
@@ -16579,50 +16579,50 @@ define inreg <44 x half> @bitcast_v22f32_to_v44f16_scalar(<22 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v23, s15
; GFX9-NEXT: v_mov_b32_e32 v22, s14
; GFX9-NEXT: .LBB33_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_and_b32_e32 v21, 0xffff, v21
-; GFX9-NEXT: v_lshl_or_b32 v0, v51, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v50, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v49, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v48, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v39, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v38, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v37, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v36, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v35, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v34, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v33, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v32, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v31, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v30, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v29, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v28, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v27, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v26, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v25, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v24, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v23, 16, v20
-; GFX9-NEXT: v_lshl_or_b32 v21, v22, 16, v21
+; GFX9-NEXT: v_lshlrev_b32_e32 v51, 16, v51
+; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v50
+; GFX9-NEXT: v_lshlrev_b32_e32 v49, 16, v49
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; GFX9-NEXT: v_lshlrev_b32_e32 v39, 16, v39
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; GFX9-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; GFX9-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; GFX9-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; GFX9-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX9-NEXT: v_or_b32_sdwa v0, v51, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v50, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v49, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v48, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v4, v39, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v38, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v37, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v36, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v35, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v34, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v33, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v32, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v12, v31, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v30, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v14, v29, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v28, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v16, v27, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v26, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v18, v25, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v24, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v20, v23, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v21, v22, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: bitcast_v22f32_to_v44f16_scalar:
@@ -27702,50 +27702,50 @@ define inreg <44 x i16> @bitcast_v11f64_to_v44i16_scalar(<11 x double> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v26, s15
; GFX9-NEXT: v_mov_b32_e32 v24, s14
; GFX9-NEXT: .LBB49_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_and_b32_e32 v21, 0xffff, v21
-; GFX9-NEXT: v_lshl_or_b32 v0, v49, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v51, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v39, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v50, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v37, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v48, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v35, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v38, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v33, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v36, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v31, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v34, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v29, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v32, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v27, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v30, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v25, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v28, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v23, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v26, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v22, 16, v20
-; GFX9-NEXT: v_lshl_or_b32 v21, v24, 16, v21
+; GFX9-NEXT: v_lshlrev_b32_e32 v49, 16, v49
+; GFX9-NEXT: v_lshlrev_b32_e32 v39, 16, v39
+; GFX9-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; GFX9-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; GFX9-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX9-NEXT: v_or_b32_sdwa v0, v49, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v49, 16, v51
+; GFX9-NEXT: v_or_b32_sdwa v2, v39, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v39, 16, v50
+; GFX9-NEXT: v_or_b32_sdwa v4, v37, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v37, 16, v48
+; GFX9-NEXT: v_or_b32_sdwa v6, v35, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v38
+; GFX9-NEXT: v_or_b32_sdwa v8, v33, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v36
+; GFX9-NEXT: v_or_b32_sdwa v10, v31, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v34
+; GFX9-NEXT: v_or_b32_sdwa v12, v29, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v32
+; GFX9-NEXT: v_or_b32_sdwa v14, v27, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v30
+; GFX9-NEXT: v_or_b32_sdwa v16, v25, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v28
+; GFX9-NEXT: v_or_b32_sdwa v18, v23, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v23, 16, v26
+; GFX9-NEXT: v_or_b32_sdwa v20, v22, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v22, 16, v24
+; GFX9-NEXT: v_or_b32_sdwa v1, v49, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v39, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v37, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v35, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v33, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v31, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v29, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v27, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v25, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v23, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v21, v22, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: bitcast_v11f64_to_v44i16_scalar:
@@ -31039,50 +31039,50 @@ define inreg <44 x half> @bitcast_v11f64_to_v44f16_scalar(<11 x double> inreg %a
; GFX9-NEXT: v_mov_b32_e32 v26, s15
; GFX9-NEXT: v_mov_b32_e32 v24, s14
; GFX9-NEXT: .LBB53_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_and_b32_e32 v21, 0xffff, v21
-; GFX9-NEXT: v_lshl_or_b32 v0, v49, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v51, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v39, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v50, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v37, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v48, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v35, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v38, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v33, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v36, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v31, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v34, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v29, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v32, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v27, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v30, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v25, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v28, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v23, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v26, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v22, 16, v20
-; GFX9-NEXT: v_lshl_or_b32 v21, v24, 16, v21
+; GFX9-NEXT: v_lshlrev_b32_e32 v49, 16, v49
+; GFX9-NEXT: v_lshlrev_b32_e32 v39, 16, v39
+; GFX9-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; GFX9-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; GFX9-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX9-NEXT: v_or_b32_sdwa v0, v49, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v49, 16, v51
+; GFX9-NEXT: v_or_b32_sdwa v2, v39, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v39, 16, v50
+; GFX9-NEXT: v_or_b32_sdwa v4, v37, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v37, 16, v48
+; GFX9-NEXT: v_or_b32_sdwa v6, v35, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v38
+; GFX9-NEXT: v_or_b32_sdwa v8, v33, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v36
+; GFX9-NEXT: v_or_b32_sdwa v10, v31, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v34
+; GFX9-NEXT: v_or_b32_sdwa v12, v29, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v32
+; GFX9-NEXT: v_or_b32_sdwa v14, v27, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v30
+; GFX9-NEXT: v_or_b32_sdwa v16, v25, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v28
+; GFX9-NEXT: v_or_b32_sdwa v18, v23, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v23, 16, v26
+; GFX9-NEXT: v_or_b32_sdwa v20, v22, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v22, 16, v24
+; GFX9-NEXT: v_or_b32_sdwa v1, v49, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v39, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v37, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v35, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v33, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v31, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v29, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v27, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v25, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v23, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v21, v22, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: bitcast_v11f64_to_v44f16_scalar:
@@ -35188,50 +35188,50 @@ define inreg <44 x half> @bitcast_v44i16_to_v44f16_scalar(<44 x i16> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v50, s7
; GFX9-NEXT: v_mov_b32_e32 v51, s6
; GFX9-NEXT: .LBB57_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_and_b32_e32 v21, 0xffff, v21
-; GFX9-NEXT: v_lshl_or_b32 v0, v51, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v50, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v49, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v48, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v39, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v38, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v37, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v36, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v35, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v34, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v33, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v32, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v31, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v30, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v29, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v28, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v27, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v26, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v25, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v24, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v23, 16, v20
-; GFX9-NEXT: v_lshl_or_b32 v21, v22, 16, v21
+; GFX9-NEXT: v_lshlrev_b32_e32 v51, 16, v51
+; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v50
+; GFX9-NEXT: v_lshlrev_b32_e32 v49, 16, v49
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; GFX9-NEXT: v_lshlrev_b32_e32 v39, 16, v39
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; GFX9-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; GFX9-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; GFX9-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; GFX9-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX9-NEXT: v_or_b32_sdwa v0, v51, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v50, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v49, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v48, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v4, v39, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v38, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v37, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v36, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v35, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v34, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v33, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v32, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v12, v31, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v30, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v14, v29, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v28, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v16, v27, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v26, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v18, v25, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v24, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v20, v23, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v21, v22, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: bitcast_v44i16_to_v44f16_scalar:
@@ -37044,50 +37044,50 @@ define inreg <44 x i16> @bitcast_v44f16_to_v44i16_scalar(<44 x half> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v50, s7
; GFX9-NEXT: v_mov_b32_e32 v51, s6
; GFX9-NEXT: .LBB59_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_and_b32_e32 v21, 0xffff, v21
-; GFX9-NEXT: v_lshl_or_b32 v0, v51, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v50, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v49, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v48, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v39, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v38, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v37, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v36, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v35, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v34, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v33, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v32, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v31, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v30, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v29, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v28, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v27, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v26, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v25, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v24, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v23, 16, v20
-; GFX9-NEXT: v_lshl_or_b32 v21, v22, 16, v21
+; GFX9-NEXT: v_lshlrev_b32_e32 v51, 16, v51
+; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v50
+; GFX9-NEXT: v_lshlrev_b32_e32 v49, 16, v49
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; GFX9-NEXT: v_lshlrev_b32_e32 v39, 16, v39
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; GFX9-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; GFX9-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; GFX9-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; GFX9-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX9-NEXT: v_or_b32_sdwa v0, v51, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v50, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v49, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v48, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v4, v39, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v38, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v37, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v36, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v35, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v34, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v33, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v32, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v12, v31, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v30, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v14, v29, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v28, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v16, v27, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v26, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v18, v25, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v24, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v20, v23, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v21, v22, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: bitcast_v44f16_to_v44i16_scalar:
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.768bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.768bit.ll
index 1cd0817e859f1..d046cb754ce49 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.768bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.768bit.ll
@@ -14129,54 +14129,54 @@ define inreg <48 x i16> @bitcast_v24f32_to_v48i16_scalar(<24 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v25, s41
; GFX9-NEXT: v_mov_b32_e32 v24, s40
; GFX9-NEXT: .LBB29_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_and_b32_e32 v21, 0xffff, v21
-; GFX9-NEXT: v_and_b32_e32 v22, 0xffff, v22
-; GFX9-NEXT: v_and_b32_e32 v23, 0xffff, v23
-; GFX9-NEXT: v_lshl_or_b32 v0, v55, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v54, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v53, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v52, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v51, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v50, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v49, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v48, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v39, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v38, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v37, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v36, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v35, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v34, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v33, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v32, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v31, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v30, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v29, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v28, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v27, 16, v20
-; GFX9-NEXT: v_lshl_or_b32 v21, v26, 16, v21
-; GFX9-NEXT: v_lshl_or_b32 v22, v25, 16, v22
-; GFX9-NEXT: v_lshl_or_b32 v23, v24, 16, v23
+; GFX9-NEXT: v_lshlrev_b32_e32 v55, 16, v55
+; GFX9-NEXT: v_lshlrev_b32_e32 v54, 16, v54
+; GFX9-NEXT: v_lshlrev_b32_e32 v53, 16, v53
+; GFX9-NEXT: v_lshlrev_b32_e32 v52, 16, v52
+; GFX9-NEXT: v_lshlrev_b32_e32 v51, 16, v51
+; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v50
+; GFX9-NEXT: v_lshlrev_b32_e32 v49, 16, v49
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; GFX9-NEXT: v_lshlrev_b32_e32 v39, 16, v39
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; GFX9-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; GFX9-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; GFX9-NEXT: v_or_b32_sdwa v0, v55, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v54, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v53, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v52, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v4, v51, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v50, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v49, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v48, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v39, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v38, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v37, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v36, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v12, v35, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v34, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v14, v33, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v32, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v16, v31, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v30, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v18, v29, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v28, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v20, v27, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v21, v26, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v22, v25, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v23, v24, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: bitcast_v24f32_to_v48i16_scalar:
@@ -17873,54 +17873,54 @@ define inreg <48 x half> @bitcast_v24f32_to_v48f16_scalar(<24 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v25, s41
; GFX9-NEXT: v_mov_b32_e32 v24, s40
; GFX9-NEXT: .LBB33_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_and_b32_e32 v21, 0xffff, v21
-; GFX9-NEXT: v_and_b32_e32 v22, 0xffff, v22
-; GFX9-NEXT: v_and_b32_e32 v23, 0xffff, v23
-; GFX9-NEXT: v_lshl_or_b32 v0, v55, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v54, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v53, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v52, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v51, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v50, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v49, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v48, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v39, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v38, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v37, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v36, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v35, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v34, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v33, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v32, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v31, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v30, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v29, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v28, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v27, 16, v20
-; GFX9-NEXT: v_lshl_or_b32 v21, v26, 16, v21
-; GFX9-NEXT: v_lshl_or_b32 v22, v25, 16, v22
-; GFX9-NEXT: v_lshl_or_b32 v23, v24, 16, v23
+; GFX9-NEXT: v_lshlrev_b32_e32 v55, 16, v55
+; GFX9-NEXT: v_lshlrev_b32_e32 v54, 16, v54
+; GFX9-NEXT: v_lshlrev_b32_e32 v53, 16, v53
+; GFX9-NEXT: v_lshlrev_b32_e32 v52, 16, v52
+; GFX9-NEXT: v_lshlrev_b32_e32 v51, 16, v51
+; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v50
+; GFX9-NEXT: v_lshlrev_b32_e32 v49, 16, v49
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; GFX9-NEXT: v_lshlrev_b32_e32 v39, 16, v39
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; GFX9-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; GFX9-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; GFX9-NEXT: v_or_b32_sdwa v0, v55, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v54, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v53, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v52, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v4, v51, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v50, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v49, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v48, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v39, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v38, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v37, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v36, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v12, v35, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v34, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v14, v33, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v32, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v16, v31, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v30, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v18, v29, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v28, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v20, v27, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v21, v26, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v22, v25, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v23, v24, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: bitcast_v24f32_to_v48f16_scalar:
@@ -29978,54 +29978,54 @@ define inreg <48 x i16> @bitcast_v12f64_to_v48i16_scalar(<12 x double> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v28, s41
; GFX9-NEXT: v_mov_b32_e32 v26, s40
; GFX9-NEXT: .LBB49_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_and_b32_e32 v21, 0xffff, v21
-; GFX9-NEXT: v_and_b32_e32 v22, 0xffff, v22
-; GFX9-NEXT: v_and_b32_e32 v23, 0xffff, v23
-; GFX9-NEXT: v_lshl_or_b32 v0, v53, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v55, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v51, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v54, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v49, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v52, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v48, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v50, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v38, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v39, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v35, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v37, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v33, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v36, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v31, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v34, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v29, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v32, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v27, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v30, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v25, 16, v20
-; GFX9-NEXT: v_lshl_or_b32 v21, v28, 16, v21
-; GFX9-NEXT: v_lshl_or_b32 v22, v24, 16, v22
-; GFX9-NEXT: v_lshl_or_b32 v23, v26, 16, v23
+; GFX9-NEXT: v_lshlrev_b32_e32 v53, 16, v53
+; GFX9-NEXT: v_lshlrev_b32_e32 v51, 16, v51
+; GFX9-NEXT: v_lshlrev_b32_e32 v49, 16, v49
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; GFX9-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; GFX9-NEXT: v_or_b32_sdwa v0, v53, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v53, 16, v55
+; GFX9-NEXT: v_or_b32_sdwa v2, v51, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v51, 16, v54
+; GFX9-NEXT: v_or_b32_sdwa v4, v49, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v49, 16, v52
+; GFX9-NEXT: v_or_b32_sdwa v6, v48, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v50
+; GFX9-NEXT: v_or_b32_sdwa v8, v38, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v39
+; GFX9-NEXT: v_or_b32_sdwa v10, v35, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v37
+; GFX9-NEXT: v_or_b32_sdwa v12, v33, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v36
+; GFX9-NEXT: v_or_b32_sdwa v14, v31, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v34
+; GFX9-NEXT: v_or_b32_sdwa v16, v29, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v32
+; GFX9-NEXT: v_or_b32_sdwa v18, v27, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v30
+; GFX9-NEXT: v_or_b32_sdwa v20, v25, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v28
+; GFX9-NEXT: v_or_b32_sdwa v22, v24, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v24, 16, v26
+; GFX9-NEXT: v_or_b32_sdwa v1, v53, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v51, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v49, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v48, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v38, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v35, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v33, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v31, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v29, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v27, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v21, v25, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v23, v24, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: bitcast_v12f64_to_v48i16_scalar:
@@ -33626,54 +33626,54 @@ define inreg <48 x half> @bitcast_v12f64_to_v48f16_scalar(<12 x double> inreg %a
; GFX9-NEXT: v_mov_b32_e32 v28, s41
; GFX9-NEXT: v_mov_b32_e32 v26, s40
; GFX9-NEXT: .LBB53_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_and_b32_e32 v21, 0xffff, v21
-; GFX9-NEXT: v_and_b32_e32 v22, 0xffff, v22
-; GFX9-NEXT: v_and_b32_e32 v23, 0xffff, v23
-; GFX9-NEXT: v_lshl_or_b32 v0, v53, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v55, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v51, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v54, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v49, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v52, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v48, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v50, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v38, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v39, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v35, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v37, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v33, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v36, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v31, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v34, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v29, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v32, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v27, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v30, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v25, 16, v20
-; GFX9-NEXT: v_lshl_or_b32 v21, v28, 16, v21
-; GFX9-NEXT: v_lshl_or_b32 v22, v24, 16, v22
-; GFX9-NEXT: v_lshl_or_b32 v23, v26, 16, v23
+; GFX9-NEXT: v_lshlrev_b32_e32 v53, 16, v53
+; GFX9-NEXT: v_lshlrev_b32_e32 v51, 16, v51
+; GFX9-NEXT: v_lshlrev_b32_e32 v49, 16, v49
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; GFX9-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; GFX9-NEXT: v_or_b32_sdwa v0, v53, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v53, 16, v55
+; GFX9-NEXT: v_or_b32_sdwa v2, v51, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v51, 16, v54
+; GFX9-NEXT: v_or_b32_sdwa v4, v49, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v49, 16, v52
+; GFX9-NEXT: v_or_b32_sdwa v6, v48, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v50
+; GFX9-NEXT: v_or_b32_sdwa v8, v38, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v39
+; GFX9-NEXT: v_or_b32_sdwa v10, v35, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v37
+; GFX9-NEXT: v_or_b32_sdwa v12, v33, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v36
+; GFX9-NEXT: v_or_b32_sdwa v14, v31, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v34
+; GFX9-NEXT: v_or_b32_sdwa v16, v29, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v32
+; GFX9-NEXT: v_or_b32_sdwa v18, v27, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v30
+; GFX9-NEXT: v_or_b32_sdwa v20, v25, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v28
+; GFX9-NEXT: v_or_b32_sdwa v22, v24, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v24, 16, v26
+; GFX9-NEXT: v_or_b32_sdwa v1, v53, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v51, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v49, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v48, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v38, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v35, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v33, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v31, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v29, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v27, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v21, v25, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v23, v24, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: bitcast_v12f64_to_v48f16_scalar:
@@ -38182,54 +38182,54 @@ define inreg <48 x half> @bitcast_v48i16_to_v48f16_scalar(<48 x i16> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v54, s7
; GFX9-NEXT: v_mov_b32_e32 v55, s6
; GFX9-NEXT: .LBB57_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_and_b32_e32 v21, 0xffff, v21
-; GFX9-NEXT: v_and_b32_e32 v22, 0xffff, v22
-; GFX9-NEXT: v_and_b32_e32 v23, 0xffff, v23
-; GFX9-NEXT: v_lshl_or_b32 v0, v55, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v54, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v53, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v52, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v51, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v50, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v49, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v48, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v39, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v38, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v37, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v36, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v35, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v34, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v33, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v32, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v31, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v30, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v29, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v28, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v27, 16, v20
-; GFX9-NEXT: v_lshl_or_b32 v21, v26, 16, v21
-; GFX9-NEXT: v_lshl_or_b32 v22, v25, 16, v22
-; GFX9-NEXT: v_lshl_or_b32 v23, v24, 16, v23
+; GFX9-NEXT: v_lshlrev_b32_e32 v55, 16, v55
+; GFX9-NEXT: v_lshlrev_b32_e32 v54, 16, v54
+; GFX9-NEXT: v_lshlrev_b32_e32 v53, 16, v53
+; GFX9-NEXT: v_lshlrev_b32_e32 v52, 16, v52
+; GFX9-NEXT: v_lshlrev_b32_e32 v51, 16, v51
+; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v50
+; GFX9-NEXT: v_lshlrev_b32_e32 v49, 16, v49
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; GFX9-NEXT: v_lshlrev_b32_e32 v39, 16, v39
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; GFX9-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; GFX9-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; GFX9-NEXT: v_or_b32_sdwa v0, v55, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v54, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v53, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v52, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v4, v51, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v50, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v49, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v48, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v39, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v38, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v37, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v36, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v12, v35, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v34, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v14, v33, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v32, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v16, v31, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v30, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v18, v29, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v28, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v20, v27, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v21, v26, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v22, v25, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v23, v24, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: bitcast_v48i16_to_v48f16_scalar:
@@ -40209,54 +40209,54 @@ define inreg <48 x i16> @bitcast_v48f16_to_v48i16_scalar(<48 x half> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v54, s7
; GFX9-NEXT: v_mov_b32_e32 v55, s6
; GFX9-NEXT: .LBB59_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_and_b32_e32 v21, 0xffff, v21
-; GFX9-NEXT: v_and_b32_e32 v22, 0xffff, v22
-; GFX9-NEXT: v_and_b32_e32 v23, 0xffff, v23
-; GFX9-NEXT: v_lshl_or_b32 v0, v55, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v54, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v53, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v52, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v51, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v50, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v49, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v48, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v39, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v38, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v37, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v36, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v35, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v34, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v33, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v32, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v31, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v30, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v29, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v28, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v27, 16, v20
-; GFX9-NEXT: v_lshl_or_b32 v21, v26, 16, v21
-; GFX9-NEXT: v_lshl_or_b32 v22, v25, 16, v22
-; GFX9-NEXT: v_lshl_or_b32 v23, v24, 16, v23
+; GFX9-NEXT: v_lshlrev_b32_e32 v55, 16, v55
+; GFX9-NEXT: v_lshlrev_b32_e32 v54, 16, v54
+; GFX9-NEXT: v_lshlrev_b32_e32 v53, 16, v53
+; GFX9-NEXT: v_lshlrev_b32_e32 v52, 16, v52
+; GFX9-NEXT: v_lshlrev_b32_e32 v51, 16, v51
+; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v50
+; GFX9-NEXT: v_lshlrev_b32_e32 v49, 16, v49
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; GFX9-NEXT: v_lshlrev_b32_e32 v39, 16, v39
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; GFX9-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; GFX9-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; GFX9-NEXT: v_or_b32_sdwa v0, v55, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v54, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v53, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v52, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v4, v51, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v50, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v49, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v48, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v39, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v38, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v37, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v36, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v12, v35, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v34, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v14, v33, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v32, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v16, v31, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v30, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v18, v29, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v28, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v20, v27, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v21, v26, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v22, v25, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v23, v24, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: bitcast_v48f16_to_v48i16_scalar:
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.832bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.832bit.ll
index a211d0f6ecfcd..047cddc9c2d8f 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.832bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.832bit.ll
@@ -15239,62 +15239,62 @@ define inreg <52 x i16> @bitcast_v26f32_to_v52i16_scalar(<26 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v27, s43
; GFX9-NEXT: v_mov_b32_e32 v26, s42
; GFX9-NEXT: .LBB29_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_lshl_or_b32 v0, v43, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v42, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v41, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v40, 16, v3
+; GFX9-NEXT: v_lshlrev_b32_e32 v43, 16, v43
+; GFX9-NEXT: v_lshlrev_b32_e32 v42, 16, v42
+; GFX9-NEXT: v_lshlrev_b32_e32 v41, 16, v41
+; GFX9-NEXT: v_lshlrev_b32_e32 v40, 16, v40
+; GFX9-NEXT: v_or_b32_sdwa v0, v43, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v42, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v41, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v40, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: buffer_load_dword v43, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_and_b32_e32 v21, 0xffff, v21
-; GFX9-NEXT: v_and_b32_e32 v22, 0xffff, v22
-; GFX9-NEXT: v_and_b32_e32 v23, 0xffff, v23
-; GFX9-NEXT: v_and_b32_e32 v24, 0xffff, v24
-; GFX9-NEXT: v_and_b32_e32 v25, 0xffff, v25
-; GFX9-NEXT: v_lshl_or_b32 v4, v55, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v54, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v53, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v52, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v51, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v50, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v49, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v48, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v39, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v38, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v37, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v36, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v35, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v34, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v33, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v32, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v31, 16, v20
-; GFX9-NEXT: v_lshl_or_b32 v21, v30, 16, v21
-; GFX9-NEXT: v_lshl_or_b32 v22, v29, 16, v22
-; GFX9-NEXT: v_lshl_or_b32 v23, v28, 16, v23
-; GFX9-NEXT: v_lshl_or_b32 v24, v27, 16, v24
-; GFX9-NEXT: v_lshl_or_b32 v25, v26, 16, v25
+; GFX9-NEXT: v_lshlrev_b32_e32 v55, 16, v55
+; GFX9-NEXT: v_lshlrev_b32_e32 v54, 16, v54
+; GFX9-NEXT: v_lshlrev_b32_e32 v53, 16, v53
+; GFX9-NEXT: v_lshlrev_b32_e32 v52, 16, v52
+; GFX9-NEXT: v_lshlrev_b32_e32 v51, 16, v51
+; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v50
+; GFX9-NEXT: v_lshlrev_b32_e32 v49, 16, v49
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; GFX9-NEXT: v_lshlrev_b32_e32 v39, 16, v39
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; GFX9-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX9-NEXT: v_or_b32_sdwa v4, v55, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v54, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v53, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v52, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v51, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v50, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v49, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v48, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v12, v39, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v38, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v14, v37, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v36, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v16, v35, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v34, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v18, v33, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v32, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v20, v31, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v21, v30, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v22, v29, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v23, v28, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v24, v27, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v25, v26, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -19373,62 +19373,62 @@ define inreg <52 x half> @bitcast_v26f32_to_v52f16_scalar(<26 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v27, s43
; GFX9-NEXT: v_mov_b32_e32 v26, s42
; GFX9-NEXT: .LBB33_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_lshl_or_b32 v0, v43, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v42, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v41, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v40, 16, v3
+; GFX9-NEXT: v_lshlrev_b32_e32 v43, 16, v43
+; GFX9-NEXT: v_lshlrev_b32_e32 v42, 16, v42
+; GFX9-NEXT: v_lshlrev_b32_e32 v41, 16, v41
+; GFX9-NEXT: v_lshlrev_b32_e32 v40, 16, v40
+; GFX9-NEXT: v_or_b32_sdwa v0, v43, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v42, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v41, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v40, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: buffer_load_dword v43, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_and_b32_e32 v21, 0xffff, v21
-; GFX9-NEXT: v_and_b32_e32 v22, 0xffff, v22
-; GFX9-NEXT: v_and_b32_e32 v23, 0xffff, v23
-; GFX9-NEXT: v_and_b32_e32 v24, 0xffff, v24
-; GFX9-NEXT: v_and_b32_e32 v25, 0xffff, v25
-; GFX9-NEXT: v_lshl_or_b32 v4, v55, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v54, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v53, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v52, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v51, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v50, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v49, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v48, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v39, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v38, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v37, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v36, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v35, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v34, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v33, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v32, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v31, 16, v20
-; GFX9-NEXT: v_lshl_or_b32 v21, v30, 16, v21
-; GFX9-NEXT: v_lshl_or_b32 v22, v29, 16, v22
-; GFX9-NEXT: v_lshl_or_b32 v23, v28, 16, v23
-; GFX9-NEXT: v_lshl_or_b32 v24, v27, 16, v24
-; GFX9-NEXT: v_lshl_or_b32 v25, v26, 16, v25
+; GFX9-NEXT: v_lshlrev_b32_e32 v55, 16, v55
+; GFX9-NEXT: v_lshlrev_b32_e32 v54, 16, v54
+; GFX9-NEXT: v_lshlrev_b32_e32 v53, 16, v53
+; GFX9-NEXT: v_lshlrev_b32_e32 v52, 16, v52
+; GFX9-NEXT: v_lshlrev_b32_e32 v51, 16, v51
+; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v50
+; GFX9-NEXT: v_lshlrev_b32_e32 v49, 16, v49
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; GFX9-NEXT: v_lshlrev_b32_e32 v39, 16, v39
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; GFX9-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX9-NEXT: v_or_b32_sdwa v4, v55, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v54, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v53, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v52, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v51, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v50, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v49, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v48, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v12, v39, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v38, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v14, v37, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v36, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v16, v35, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v34, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v18, v33, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v32, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v20, v31, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v21, v30, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v22, v29, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v23, v28, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v24, v27, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v25, v26, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -32606,62 +32606,62 @@ define inreg <52 x i16> @bitcast_v13f64_to_v52i16_scalar(<13 x double> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v31, s43
; GFX9-NEXT: v_mov_b32_e32 v28, s42
; GFX9-NEXT: .LBB49_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_lshl_or_b32 v0, v40, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v43, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v3, v42, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v5, v41, 16, v5
+; GFX9-NEXT: v_lshlrev_b32_e32 v40, 16, v40
+; GFX9-NEXT: v_or_b32_sdwa v0, v40, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v40, 16, v43
+; GFX9-NEXT: v_lshlrev_b32_e32 v54, 16, v54
+; GFX9-NEXT: v_lshlrev_b32_e32 v52, 16, v52
+; GFX9-NEXT: v_or_b32_sdwa v1, v40, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v54, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v54, 16, v42
+; GFX9-NEXT: v_or_b32_sdwa v4, v52, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v52, 16, v41
; GFX9-NEXT: buffer_load_dword v43, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_and_b32_e32 v21, 0xffff, v21
-; GFX9-NEXT: v_and_b32_e32 v22, 0xffff, v22
-; GFX9-NEXT: v_and_b32_e32 v23, 0xffff, v23
-; GFX9-NEXT: v_and_b32_e32 v24, 0xffff, v24
-; GFX9-NEXT: v_and_b32_e32 v25, 0xffff, v25
-; GFX9-NEXT: v_lshl_or_b32 v2, v54, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v4, v52, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v6, v50, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v55, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v48, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v53, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v38, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v51, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v36, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v49, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v34, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v39, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v32, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v37, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v30, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v35, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v29, 16, v20
-; GFX9-NEXT: v_lshl_or_b32 v21, v33, 16, v21
-; GFX9-NEXT: v_lshl_or_b32 v22, v27, 16, v22
-; GFX9-NEXT: v_lshl_or_b32 v23, v31, 16, v23
-; GFX9-NEXT: v_lshl_or_b32 v24, v26, 16, v24
-; GFX9-NEXT: v_lshl_or_b32 v25, v28, 16, v25
+; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v50
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX9-NEXT: v_or_b32_sdwa v6, v50, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v55
+; GFX9-NEXT: v_or_b32_sdwa v8, v48, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v53
+; GFX9-NEXT: v_or_b32_sdwa v10, v38, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v51
+; GFX9-NEXT: v_or_b32_sdwa v12, v36, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v49
+; GFX9-NEXT: v_or_b32_sdwa v14, v34, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v39
+; GFX9-NEXT: v_or_b32_sdwa v16, v32, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v37
+; GFX9-NEXT: v_or_b32_sdwa v18, v30, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v35
+; GFX9-NEXT: v_or_b32_sdwa v20, v29, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v33
+; GFX9-NEXT: v_or_b32_sdwa v22, v27, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v31
+; GFX9-NEXT: v_or_b32_sdwa v24, v26, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v26, 16, v28
+; GFX9-NEXT: v_or_b32_sdwa v3, v54, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v52, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v50, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v48, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v38, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v36, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v34, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v32, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v30, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v21, v29, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v23, v27, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v25, v26, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -36640,62 +36640,62 @@ define inreg <52 x half> @bitcast_v13f64_to_v52f16_scalar(<13 x double> inreg %a
; GFX9-NEXT: v_mov_b32_e32 v31, s43
; GFX9-NEXT: v_mov_b32_e32 v28, s42
; GFX9-NEXT: .LBB53_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_lshl_or_b32 v0, v40, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v43, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v3, v42, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v5, v41, 16, v5
+; GFX9-NEXT: v_lshlrev_b32_e32 v40, 16, v40
+; GFX9-NEXT: v_or_b32_sdwa v0, v40, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v40, 16, v43
+; GFX9-NEXT: v_lshlrev_b32_e32 v54, 16, v54
+; GFX9-NEXT: v_lshlrev_b32_e32 v52, 16, v52
+; GFX9-NEXT: v_or_b32_sdwa v1, v40, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v54, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v54, 16, v42
+; GFX9-NEXT: v_or_b32_sdwa v4, v52, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v52, 16, v41
; GFX9-NEXT: buffer_load_dword v43, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_and_b32_e32 v21, 0xffff, v21
-; GFX9-NEXT: v_and_b32_e32 v22, 0xffff, v22
-; GFX9-NEXT: v_and_b32_e32 v23, 0xffff, v23
-; GFX9-NEXT: v_and_b32_e32 v24, 0xffff, v24
-; GFX9-NEXT: v_and_b32_e32 v25, 0xffff, v25
-; GFX9-NEXT: v_lshl_or_b32 v2, v54, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v4, v52, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v6, v50, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v55, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v48, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v53, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v38, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v51, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v36, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v49, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v34, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v39, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v32, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v37, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v30, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v35, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v29, 16, v20
-; GFX9-NEXT: v_lshl_or_b32 v21, v33, 16, v21
-; GFX9-NEXT: v_lshl_or_b32 v22, v27, 16, v22
-; GFX9-NEXT: v_lshl_or_b32 v23, v31, 16, v23
-; GFX9-NEXT: v_lshl_or_b32 v24, v26, 16, v24
-; GFX9-NEXT: v_lshl_or_b32 v25, v28, 16, v25
+; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v50
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX9-NEXT: v_or_b32_sdwa v6, v50, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v55
+; GFX9-NEXT: v_or_b32_sdwa v8, v48, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v53
+; GFX9-NEXT: v_or_b32_sdwa v10, v38, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v51
+; GFX9-NEXT: v_or_b32_sdwa v12, v36, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v49
+; GFX9-NEXT: v_or_b32_sdwa v14, v34, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v39
+; GFX9-NEXT: v_or_b32_sdwa v16, v32, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v37
+; GFX9-NEXT: v_or_b32_sdwa v18, v30, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v35
+; GFX9-NEXT: v_or_b32_sdwa v20, v29, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v33
+; GFX9-NEXT: v_or_b32_sdwa v22, v27, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v31
+; GFX9-NEXT: v_or_b32_sdwa v24, v26, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v26, 16, v28
+; GFX9-NEXT: v_or_b32_sdwa v3, v54, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v52, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v50, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v48, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v38, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v36, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v34, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v32, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v30, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v21, v29, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v23, v27, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v25, v26, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -41638,62 +41638,62 @@ define inreg <52 x half> @bitcast_v52i16_to_v52f16_scalar(<52 x i16> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v42, s7
; GFX9-NEXT: v_mov_b32_e32 v43, s6
; GFX9-NEXT: .LBB57_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_lshl_or_b32 v0, v43, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v42, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v41, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v40, 16, v3
+; GFX9-NEXT: v_lshlrev_b32_e32 v43, 16, v43
+; GFX9-NEXT: v_lshlrev_b32_e32 v42, 16, v42
+; GFX9-NEXT: v_lshlrev_b32_e32 v41, 16, v41
+; GFX9-NEXT: v_lshlrev_b32_e32 v40, 16, v40
+; GFX9-NEXT: v_or_b32_sdwa v0, v43, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v42, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v41, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v40, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: buffer_load_dword v43, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_and_b32_e32 v21, 0xffff, v21
-; GFX9-NEXT: v_and_b32_e32 v22, 0xffff, v22
-; GFX9-NEXT: v_and_b32_e32 v23, 0xffff, v23
-; GFX9-NEXT: v_and_b32_e32 v24, 0xffff, v24
-; GFX9-NEXT: v_and_b32_e32 v25, 0xffff, v25
-; GFX9-NEXT: v_lshl_or_b32 v4, v55, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v54, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v53, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v52, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v51, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v50, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v49, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v48, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v39, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v38, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v37, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v36, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v35, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v34, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v33, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v32, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v31, 16, v20
-; GFX9-NEXT: v_lshl_or_b32 v21, v30, 16, v21
-; GFX9-NEXT: v_lshl_or_b32 v22, v29, 16, v22
-; GFX9-NEXT: v_lshl_or_b32 v23, v28, 16, v23
-; GFX9-NEXT: v_lshl_or_b32 v24, v27, 16, v24
-; GFX9-NEXT: v_lshl_or_b32 v25, v26, 16, v25
+; GFX9-NEXT: v_lshlrev_b32_e32 v55, 16, v55
+; GFX9-NEXT: v_lshlrev_b32_e32 v54, 16, v54
+; GFX9-NEXT: v_lshlrev_b32_e32 v53, 16, v53
+; GFX9-NEXT: v_lshlrev_b32_e32 v52, 16, v52
+; GFX9-NEXT: v_lshlrev_b32_e32 v51, 16, v51
+; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v50
+; GFX9-NEXT: v_lshlrev_b32_e32 v49, 16, v49
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; GFX9-NEXT: v_lshlrev_b32_e32 v39, 16, v39
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; GFX9-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX9-NEXT: v_or_b32_sdwa v4, v55, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v54, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v53, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v52, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v51, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v50, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v49, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v48, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v12, v39, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v38, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v14, v37, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v36, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v16, v35, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v34, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v18, v33, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v32, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v20, v31, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v21, v30, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v22, v29, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v23, v28, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v24, v27, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v25, v26, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -43873,62 +43873,62 @@ define inreg <52 x i16> @bitcast_v52f16_to_v52i16_scalar(<52 x half> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v42, s7
; GFX9-NEXT: v_mov_b32_e32 v43, s6
; GFX9-NEXT: .LBB59_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_lshl_or_b32 v0, v43, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v42, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v41, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v40, 16, v3
+; GFX9-NEXT: v_lshlrev_b32_e32 v43, 16, v43
+; GFX9-NEXT: v_lshlrev_b32_e32 v42, 16, v42
+; GFX9-NEXT: v_lshlrev_b32_e32 v41, 16, v41
+; GFX9-NEXT: v_lshlrev_b32_e32 v40, 16, v40
+; GFX9-NEXT: v_or_b32_sdwa v0, v43, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v42, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v41, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v40, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: buffer_load_dword v43, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_and_b32_e32 v21, 0xffff, v21
-; GFX9-NEXT: v_and_b32_e32 v22, 0xffff, v22
-; GFX9-NEXT: v_and_b32_e32 v23, 0xffff, v23
-; GFX9-NEXT: v_and_b32_e32 v24, 0xffff, v24
-; GFX9-NEXT: v_and_b32_e32 v25, 0xffff, v25
-; GFX9-NEXT: v_lshl_or_b32 v4, v55, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v54, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v53, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v52, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v51, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v50, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v49, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v48, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v39, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v38, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v37, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v36, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v35, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v34, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v33, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v32, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v31, 16, v20
-; GFX9-NEXT: v_lshl_or_b32 v21, v30, 16, v21
-; GFX9-NEXT: v_lshl_or_b32 v22, v29, 16, v22
-; GFX9-NEXT: v_lshl_or_b32 v23, v28, 16, v23
-; GFX9-NEXT: v_lshl_or_b32 v24, v27, 16, v24
-; GFX9-NEXT: v_lshl_or_b32 v25, v26, 16, v25
+; GFX9-NEXT: v_lshlrev_b32_e32 v55, 16, v55
+; GFX9-NEXT: v_lshlrev_b32_e32 v54, 16, v54
+; GFX9-NEXT: v_lshlrev_b32_e32 v53, 16, v53
+; GFX9-NEXT: v_lshlrev_b32_e32 v52, 16, v52
+; GFX9-NEXT: v_lshlrev_b32_e32 v51, 16, v51
+; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v50
+; GFX9-NEXT: v_lshlrev_b32_e32 v49, 16, v49
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; GFX9-NEXT: v_lshlrev_b32_e32 v39, 16, v39
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; GFX9-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX9-NEXT: v_or_b32_sdwa v4, v55, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v54, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v53, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v52, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v51, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v50, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v49, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v48, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v12, v39, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v38, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v14, v37, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v36, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v16, v35, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v34, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v18, v33, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v32, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v20, v31, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v21, v30, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v22, v29, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v23, v28, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v24, v27, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v25, v26, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.896bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.896bit.ll
index 8eca8e84d6d5d..7ff802e2b2121 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.896bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.896bit.ll
@@ -16446,22 +16446,22 @@ define inreg <56 x i16> @bitcast_v28f32_to_v56i16_scalar(<28 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v29, s45
; GFX9-NEXT: v_mov_b32_e32 v28, s44
; GFX9-NEXT: .LBB29_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_lshl_or_b32 v0, v47, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v46, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v45, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v44, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v43, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v42, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v41, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v40, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v47, 16, v47
+; GFX9-NEXT: v_lshlrev_b32_e32 v46, 16, v46
+; GFX9-NEXT: v_lshlrev_b32_e32 v45, 16, v45
+; GFX9-NEXT: v_lshlrev_b32_e32 v44, 16, v44
+; GFX9-NEXT: v_lshlrev_b32_e32 v43, 16, v43
+; GFX9-NEXT: v_lshlrev_b32_e32 v42, 16, v42
+; GFX9-NEXT: v_lshlrev_b32_e32 v41, 16, v41
+; GFX9-NEXT: v_lshlrev_b32_e32 v40, 16, v40
+; GFX9-NEXT: v_or_b32_sdwa v0, v47, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v46, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v45, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v44, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v4, v43, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v42, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v41, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v40, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: buffer_load_dword v47, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
@@ -16470,46 +16470,46 @@ define inreg <56 x i16> @bitcast_v28f32_to_v56i16_scalar(<28 x float> inreg %a,
; GFX9-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_and_b32_e32 v21, 0xffff, v21
-; GFX9-NEXT: v_and_b32_e32 v22, 0xffff, v22
-; GFX9-NEXT: v_and_b32_e32 v23, 0xffff, v23
-; GFX9-NEXT: v_and_b32_e32 v24, 0xffff, v24
-; GFX9-NEXT: v_and_b32_e32 v25, 0xffff, v25
-; GFX9-NEXT: v_and_b32_e32 v26, 0xffff, v26
-; GFX9-NEXT: v_and_b32_e32 v27, 0xffff, v27
-; GFX9-NEXT: v_lshl_or_b32 v8, v55, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v54, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v53, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v52, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v51, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v50, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v49, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v48, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v39, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v38, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v37, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v36, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v35, 16, v20
-; GFX9-NEXT: v_lshl_or_b32 v21, v34, 16, v21
-; GFX9-NEXT: v_lshl_or_b32 v22, v33, 16, v22
-; GFX9-NEXT: v_lshl_or_b32 v23, v32, 16, v23
-; GFX9-NEXT: v_lshl_or_b32 v24, v31, 16, v24
-; GFX9-NEXT: v_lshl_or_b32 v25, v30, 16, v25
-; GFX9-NEXT: v_lshl_or_b32 v26, v29, 16, v26
-; GFX9-NEXT: v_lshl_or_b32 v27, v28, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v55, 16, v55
+; GFX9-NEXT: v_lshlrev_b32_e32 v54, 16, v54
+; GFX9-NEXT: v_lshlrev_b32_e32 v53, 16, v53
+; GFX9-NEXT: v_lshlrev_b32_e32 v52, 16, v52
+; GFX9-NEXT: v_lshlrev_b32_e32 v51, 16, v51
+; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v50
+; GFX9-NEXT: v_lshlrev_b32_e32 v49, 16, v49
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; GFX9-NEXT: v_lshlrev_b32_e32 v39, 16, v39
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; GFX9-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; GFX9-NEXT: v_or_b32_sdwa v8, v55, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v54, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v53, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v52, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v12, v51, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v50, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v14, v49, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v48, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v16, v39, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v38, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v18, v37, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v36, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v20, v35, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v21, v34, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v22, v33, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v23, v32, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v24, v31, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v25, v30, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v26, v29, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v27, v28, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -21038,22 +21038,22 @@ define inreg <56 x half> @bitcast_v28f32_to_v56f16_scalar(<28 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v29, s45
; GFX9-NEXT: v_mov_b32_e32 v28, s44
; GFX9-NEXT: .LBB33_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_lshl_or_b32 v0, v47, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v46, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v45, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v44, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v43, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v42, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v41, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v40, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v47, 16, v47
+; GFX9-NEXT: v_lshlrev_b32_e32 v46, 16, v46
+; GFX9-NEXT: v_lshlrev_b32_e32 v45, 16, v45
+; GFX9-NEXT: v_lshlrev_b32_e32 v44, 16, v44
+; GFX9-NEXT: v_lshlrev_b32_e32 v43, 16, v43
+; GFX9-NEXT: v_lshlrev_b32_e32 v42, 16, v42
+; GFX9-NEXT: v_lshlrev_b32_e32 v41, 16, v41
+; GFX9-NEXT: v_lshlrev_b32_e32 v40, 16, v40
+; GFX9-NEXT: v_or_b32_sdwa v0, v47, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v46, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v45, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v44, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v4, v43, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v42, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v41, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v40, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: buffer_load_dword v47, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
@@ -21062,46 +21062,46 @@ define inreg <56 x half> @bitcast_v28f32_to_v56f16_scalar(<28 x float> inreg %a,
; GFX9-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_and_b32_e32 v21, 0xffff, v21
-; GFX9-NEXT: v_and_b32_e32 v22, 0xffff, v22
-; GFX9-NEXT: v_and_b32_e32 v23, 0xffff, v23
-; GFX9-NEXT: v_and_b32_e32 v24, 0xffff, v24
-; GFX9-NEXT: v_and_b32_e32 v25, 0xffff, v25
-; GFX9-NEXT: v_and_b32_e32 v26, 0xffff, v26
-; GFX9-NEXT: v_and_b32_e32 v27, 0xffff, v27
-; GFX9-NEXT: v_lshl_or_b32 v8, v55, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v54, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v53, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v52, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v51, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v50, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v49, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v48, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v39, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v38, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v37, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v36, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v35, 16, v20
-; GFX9-NEXT: v_lshl_or_b32 v21, v34, 16, v21
-; GFX9-NEXT: v_lshl_or_b32 v22, v33, 16, v22
-; GFX9-NEXT: v_lshl_or_b32 v23, v32, 16, v23
-; GFX9-NEXT: v_lshl_or_b32 v24, v31, 16, v24
-; GFX9-NEXT: v_lshl_or_b32 v25, v30, 16, v25
-; GFX9-NEXT: v_lshl_or_b32 v26, v29, 16, v26
-; GFX9-NEXT: v_lshl_or_b32 v27, v28, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v55, 16, v55
+; GFX9-NEXT: v_lshlrev_b32_e32 v54, 16, v54
+; GFX9-NEXT: v_lshlrev_b32_e32 v53, 16, v53
+; GFX9-NEXT: v_lshlrev_b32_e32 v52, 16, v52
+; GFX9-NEXT: v_lshlrev_b32_e32 v51, 16, v51
+; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v50
+; GFX9-NEXT: v_lshlrev_b32_e32 v49, 16, v49
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; GFX9-NEXT: v_lshlrev_b32_e32 v39, 16, v39
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; GFX9-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; GFX9-NEXT: v_or_b32_sdwa v8, v55, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v54, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v53, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v52, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v12, v51, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v50, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v14, v49, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v48, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v16, v39, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v38, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v18, v37, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v36, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v20, v35, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v21, v34, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v22, v33, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v23, v32, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v24, v31, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v25, v30, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v26, v29, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v27, v28, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -35510,22 +35510,24 @@ define inreg <56 x i16> @bitcast_v14f64_to_v56i16_scalar(<14 x double> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v33, s45
; GFX9-NEXT: v_mov_b32_e32 v31, s44
; GFX9-NEXT: .LBB49_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_lshl_or_b32 v0, v44, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v47, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v42, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v46, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v40, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v45, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v7, v43, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v9, v41, 16, v9
+; GFX9-NEXT: v_lshlrev_b32_e32 v44, 16, v44
+; GFX9-NEXT: v_lshlrev_b32_e32 v42, 16, v42
+; GFX9-NEXT: v_lshlrev_b32_e32 v40, 16, v40
+; GFX9-NEXT: v_or_b32_sdwa v0, v44, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v44, 16, v47
+; GFX9-NEXT: v_or_b32_sdwa v2, v42, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v42, 16, v46
+; GFX9-NEXT: v_or_b32_sdwa v4, v40, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v40, 16, v45
+; GFX9-NEXT: v_lshlrev_b32_e32 v54, 16, v54
+; GFX9-NEXT: v_lshlrev_b32_e32 v52, 16, v52
+; GFX9-NEXT: v_or_b32_sdwa v1, v44, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v42, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v40, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v54, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v54, 16, v43
+; GFX9-NEXT: v_or_b32_sdwa v8, v52, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v52, 16, v41
; GFX9-NEXT: buffer_load_dword v47, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
@@ -35534,46 +35536,44 @@ define inreg <56 x i16> @bitcast_v14f64_to_v56i16_scalar(<14 x double> inreg %a,
; GFX9-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_and_b32_e32 v21, 0xffff, v21
-; GFX9-NEXT: v_and_b32_e32 v22, 0xffff, v22
-; GFX9-NEXT: v_and_b32_e32 v23, 0xffff, v23
-; GFX9-NEXT: v_and_b32_e32 v24, 0xffff, v24
-; GFX9-NEXT: v_and_b32_e32 v25, 0xffff, v25
-; GFX9-NEXT: v_and_b32_e32 v26, 0xffff, v26
-; GFX9-NEXT: v_and_b32_e32 v27, 0xffff, v27
-; GFX9-NEXT: v_lshl_or_b32 v6, v54, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v8, v52, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v10, v50, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v55, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v48, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v53, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v38, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v51, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v36, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v49, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v34, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v39, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v32, 16, v20
-; GFX9-NEXT: v_lshl_or_b32 v21, v37, 16, v21
-; GFX9-NEXT: v_lshl_or_b32 v22, v30, 16, v22
-; GFX9-NEXT: v_lshl_or_b32 v23, v35, 16, v23
-; GFX9-NEXT: v_lshl_or_b32 v24, v29, 16, v24
-; GFX9-NEXT: v_lshl_or_b32 v25, v33, 16, v25
-; GFX9-NEXT: v_lshl_or_b32 v26, v28, 16, v26
-; GFX9-NEXT: v_lshl_or_b32 v27, v31, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v50
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; GFX9-NEXT: v_or_b32_sdwa v10, v50, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v55
+; GFX9-NEXT: v_or_b32_sdwa v12, v48, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v53
+; GFX9-NEXT: v_or_b32_sdwa v14, v38, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v51
+; GFX9-NEXT: v_or_b32_sdwa v16, v36, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v49
+; GFX9-NEXT: v_or_b32_sdwa v18, v34, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v39
+; GFX9-NEXT: v_or_b32_sdwa v20, v32, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v37
+; GFX9-NEXT: v_or_b32_sdwa v22, v30, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v35
+; GFX9-NEXT: v_or_b32_sdwa v24, v29, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v33
+; GFX9-NEXT: v_or_b32_sdwa v26, v28, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v31
+; GFX9-NEXT: v_or_b32_sdwa v7, v54, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v52, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v50, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v48, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v38, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v36, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v34, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v21, v32, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v23, v30, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v25, v29, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v27, v28, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -39997,22 +39997,24 @@ define inreg <56 x half> @bitcast_v14f64_to_v56f16_scalar(<14 x double> inreg %a
; GFX9-NEXT: v_mov_b32_e32 v33, s45
; GFX9-NEXT: v_mov_b32_e32 v31, s44
; GFX9-NEXT: .LBB53_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_lshl_or_b32 v0, v44, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v47, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v42, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v46, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v40, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v45, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v7, v43, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v9, v41, 16, v9
+; GFX9-NEXT: v_lshlrev_b32_e32 v44, 16, v44
+; GFX9-NEXT: v_lshlrev_b32_e32 v42, 16, v42
+; GFX9-NEXT: v_lshlrev_b32_e32 v40, 16, v40
+; GFX9-NEXT: v_or_b32_sdwa v0, v44, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v44, 16, v47
+; GFX9-NEXT: v_or_b32_sdwa v2, v42, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v42, 16, v46
+; GFX9-NEXT: v_or_b32_sdwa v4, v40, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v40, 16, v45
+; GFX9-NEXT: v_lshlrev_b32_e32 v54, 16, v54
+; GFX9-NEXT: v_lshlrev_b32_e32 v52, 16, v52
+; GFX9-NEXT: v_or_b32_sdwa v1, v44, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v42, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v40, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v54, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v54, 16, v43
+; GFX9-NEXT: v_or_b32_sdwa v8, v52, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v52, 16, v41
; GFX9-NEXT: buffer_load_dword v47, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
@@ -40021,46 +40023,44 @@ define inreg <56 x half> @bitcast_v14f64_to_v56f16_scalar(<14 x double> inreg %a
; GFX9-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_and_b32_e32 v21, 0xffff, v21
-; GFX9-NEXT: v_and_b32_e32 v22, 0xffff, v22
-; GFX9-NEXT: v_and_b32_e32 v23, 0xffff, v23
-; GFX9-NEXT: v_and_b32_e32 v24, 0xffff, v24
-; GFX9-NEXT: v_and_b32_e32 v25, 0xffff, v25
-; GFX9-NEXT: v_and_b32_e32 v26, 0xffff, v26
-; GFX9-NEXT: v_and_b32_e32 v27, 0xffff, v27
-; GFX9-NEXT: v_lshl_or_b32 v6, v54, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v8, v52, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v10, v50, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v55, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v48, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v53, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v38, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v51, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v36, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v49, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v34, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v39, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v32, 16, v20
-; GFX9-NEXT: v_lshl_or_b32 v21, v37, 16, v21
-; GFX9-NEXT: v_lshl_or_b32 v22, v30, 16, v22
-; GFX9-NEXT: v_lshl_or_b32 v23, v35, 16, v23
-; GFX9-NEXT: v_lshl_or_b32 v24, v29, 16, v24
-; GFX9-NEXT: v_lshl_or_b32 v25, v33, 16, v25
-; GFX9-NEXT: v_lshl_or_b32 v26, v28, 16, v26
-; GFX9-NEXT: v_lshl_or_b32 v27, v31, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v50
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; GFX9-NEXT: v_or_b32_sdwa v10, v50, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v55
+; GFX9-NEXT: v_or_b32_sdwa v12, v48, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v53
+; GFX9-NEXT: v_or_b32_sdwa v14, v38, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v51
+; GFX9-NEXT: v_or_b32_sdwa v16, v36, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v49
+; GFX9-NEXT: v_or_b32_sdwa v18, v34, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v39
+; GFX9-NEXT: v_or_b32_sdwa v20, v32, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v37
+; GFX9-NEXT: v_or_b32_sdwa v22, v30, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v35
+; GFX9-NEXT: v_or_b32_sdwa v24, v29, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v33
+; GFX9-NEXT: v_or_b32_sdwa v26, v28, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v31
+; GFX9-NEXT: v_or_b32_sdwa v7, v54, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v52, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v50, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v48, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v38, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v36, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v34, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v21, v32, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v23, v30, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v25, v29, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v27, v28, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -45462,22 +45462,22 @@ define inreg <56 x half> @bitcast_v56i16_to_v56f16_scalar(<56 x i16> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v46, s7
; GFX9-NEXT: v_mov_b32_e32 v47, s6
; GFX9-NEXT: .LBB57_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_lshl_or_b32 v0, v47, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v46, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v45, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v44, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v43, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v42, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v41, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v40, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v47, 16, v47
+; GFX9-NEXT: v_lshlrev_b32_e32 v46, 16, v46
+; GFX9-NEXT: v_lshlrev_b32_e32 v45, 16, v45
+; GFX9-NEXT: v_lshlrev_b32_e32 v44, 16, v44
+; GFX9-NEXT: v_lshlrev_b32_e32 v43, 16, v43
+; GFX9-NEXT: v_lshlrev_b32_e32 v42, 16, v42
+; GFX9-NEXT: v_lshlrev_b32_e32 v41, 16, v41
+; GFX9-NEXT: v_lshlrev_b32_e32 v40, 16, v40
+; GFX9-NEXT: v_or_b32_sdwa v0, v47, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v46, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v45, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v44, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v4, v43, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v42, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v41, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v40, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: buffer_load_dword v47, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
@@ -45486,46 +45486,46 @@ define inreg <56 x half> @bitcast_v56i16_to_v56f16_scalar(<56 x i16> inreg %a, i
; GFX9-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_and_b32_e32 v21, 0xffff, v21
-; GFX9-NEXT: v_and_b32_e32 v22, 0xffff, v22
-; GFX9-NEXT: v_and_b32_e32 v23, 0xffff, v23
-; GFX9-NEXT: v_and_b32_e32 v24, 0xffff, v24
-; GFX9-NEXT: v_and_b32_e32 v25, 0xffff, v25
-; GFX9-NEXT: v_and_b32_e32 v26, 0xffff, v26
-; GFX9-NEXT: v_and_b32_e32 v27, 0xffff, v27
-; GFX9-NEXT: v_lshl_or_b32 v8, v55, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v54, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v53, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v52, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v51, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v50, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v49, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v48, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v39, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v38, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v37, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v36, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v35, 16, v20
-; GFX9-NEXT: v_lshl_or_b32 v21, v34, 16, v21
-; GFX9-NEXT: v_lshl_or_b32 v22, v33, 16, v22
-; GFX9-NEXT: v_lshl_or_b32 v23, v32, 16, v23
-; GFX9-NEXT: v_lshl_or_b32 v24, v31, 16, v24
-; GFX9-NEXT: v_lshl_or_b32 v25, v30, 16, v25
-; GFX9-NEXT: v_lshl_or_b32 v26, v29, 16, v26
-; GFX9-NEXT: v_lshl_or_b32 v27, v28, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v55, 16, v55
+; GFX9-NEXT: v_lshlrev_b32_e32 v54, 16, v54
+; GFX9-NEXT: v_lshlrev_b32_e32 v53, 16, v53
+; GFX9-NEXT: v_lshlrev_b32_e32 v52, 16, v52
+; GFX9-NEXT: v_lshlrev_b32_e32 v51, 16, v51
+; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v50
+; GFX9-NEXT: v_lshlrev_b32_e32 v49, 16, v49
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; GFX9-NEXT: v_lshlrev_b32_e32 v39, 16, v39
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; GFX9-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; GFX9-NEXT: v_or_b32_sdwa v8, v55, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v54, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v53, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v52, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v12, v51, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v50, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v14, v49, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v48, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v16, v39, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v38, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v18, v37, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v36, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v20, v35, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v21, v34, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v22, v33, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v23, v32, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v24, v31, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v25, v30, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v26, v29, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v27, v28, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -47906,22 +47906,22 @@ define inreg <56 x i16> @bitcast_v56f16_to_v56i16_scalar(<56 x half> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v46, s7
; GFX9-NEXT: v_mov_b32_e32 v47, s6
; GFX9-NEXT: .LBB59_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_lshl_or_b32 v0, v47, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v46, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v45, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v44, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v43, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v42, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v41, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v40, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v47, 16, v47
+; GFX9-NEXT: v_lshlrev_b32_e32 v46, 16, v46
+; GFX9-NEXT: v_lshlrev_b32_e32 v45, 16, v45
+; GFX9-NEXT: v_lshlrev_b32_e32 v44, 16, v44
+; GFX9-NEXT: v_lshlrev_b32_e32 v43, 16, v43
+; GFX9-NEXT: v_lshlrev_b32_e32 v42, 16, v42
+; GFX9-NEXT: v_lshlrev_b32_e32 v41, 16, v41
+; GFX9-NEXT: v_lshlrev_b32_e32 v40, 16, v40
+; GFX9-NEXT: v_or_b32_sdwa v0, v47, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v46, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v45, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v44, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v4, v43, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v42, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v41, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v40, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: buffer_load_dword v47, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
@@ -47930,46 +47930,46 @@ define inreg <56 x i16> @bitcast_v56f16_to_v56i16_scalar(<56 x half> inreg %a, i
; GFX9-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_and_b32_e32 v21, 0xffff, v21
-; GFX9-NEXT: v_and_b32_e32 v22, 0xffff, v22
-; GFX9-NEXT: v_and_b32_e32 v23, 0xffff, v23
-; GFX9-NEXT: v_and_b32_e32 v24, 0xffff, v24
-; GFX9-NEXT: v_and_b32_e32 v25, 0xffff, v25
-; GFX9-NEXT: v_and_b32_e32 v26, 0xffff, v26
-; GFX9-NEXT: v_and_b32_e32 v27, 0xffff, v27
-; GFX9-NEXT: v_lshl_or_b32 v8, v55, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v54, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v53, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v52, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v51, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v50, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v49, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v48, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v39, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v38, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v37, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v36, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v35, 16, v20
-; GFX9-NEXT: v_lshl_or_b32 v21, v34, 16, v21
-; GFX9-NEXT: v_lshl_or_b32 v22, v33, 16, v22
-; GFX9-NEXT: v_lshl_or_b32 v23, v32, 16, v23
-; GFX9-NEXT: v_lshl_or_b32 v24, v31, 16, v24
-; GFX9-NEXT: v_lshl_or_b32 v25, v30, 16, v25
-; GFX9-NEXT: v_lshl_or_b32 v26, v29, 16, v26
-; GFX9-NEXT: v_lshl_or_b32 v27, v28, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v55, 16, v55
+; GFX9-NEXT: v_lshlrev_b32_e32 v54, 16, v54
+; GFX9-NEXT: v_lshlrev_b32_e32 v53, 16, v53
+; GFX9-NEXT: v_lshlrev_b32_e32 v52, 16, v52
+; GFX9-NEXT: v_lshlrev_b32_e32 v51, 16, v51
+; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v50
+; GFX9-NEXT: v_lshlrev_b32_e32 v49, 16, v49
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; GFX9-NEXT: v_lshlrev_b32_e32 v39, 16, v39
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; GFX9-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; GFX9-NEXT: v_or_b32_sdwa v8, v55, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v54, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v53, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v52, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v12, v51, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v50, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v14, v49, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v48, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v16, v39, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v38, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v18, v37, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v36, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v20, v35, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v21, v34, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v22, v33, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v23, v32, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v24, v31, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v25, v30, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v26, v29, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v27, v28, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.960bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.960bit.ll
index 821ff4360b2b9..4d6c87f4a6619 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.960bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.960bit.ll
@@ -17644,30 +17644,30 @@ define inreg <60 x i16> @bitcast_v30f32_to_v60i16_scalar(<30 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v31, s47
; GFX9-NEXT: v_mov_b32_e32 v30, s46
; GFX9-NEXT: .LBB29_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_lshl_or_b32 v0, v59, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v58, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v57, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v56, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v47, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v46, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v45, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v44, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v43, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v42, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v41, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v40, 16, v11
+; GFX9-NEXT: v_lshlrev_b32_e32 v59, 16, v59
+; GFX9-NEXT: v_lshlrev_b32_e32 v58, 16, v58
+; GFX9-NEXT: v_lshlrev_b32_e32 v57, 16, v57
+; GFX9-NEXT: v_lshlrev_b32_e32 v56, 16, v56
+; GFX9-NEXT: v_lshlrev_b32_e32 v47, 16, v47
+; GFX9-NEXT: v_lshlrev_b32_e32 v46, 16, v46
+; GFX9-NEXT: v_lshlrev_b32_e32 v45, 16, v45
+; GFX9-NEXT: v_lshlrev_b32_e32 v44, 16, v44
+; GFX9-NEXT: v_lshlrev_b32_e32 v43, 16, v43
+; GFX9-NEXT: v_lshlrev_b32_e32 v42, 16, v42
+; GFX9-NEXT: v_lshlrev_b32_e32 v41, 16, v41
+; GFX9-NEXT: v_lshlrev_b32_e32 v40, 16, v40
+; GFX9-NEXT: v_or_b32_sdwa v0, v59, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v58, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v57, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v56, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v4, v47, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v46, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v45, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v44, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v43, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v42, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v41, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v40, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: buffer_load_dword v59, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
@@ -17680,43 +17680,43 @@ define inreg <60 x i16> @bitcast_v30f32_to_v60i16_scalar(<30 x float> inreg %a,
; GFX9-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_and_b32_e32 v21, 0xffff, v21
-; GFX9-NEXT: v_and_b32_e32 v22, 0xffff, v22
-; GFX9-NEXT: v_and_b32_e32 v23, 0xffff, v23
-; GFX9-NEXT: v_and_b32_e32 v24, 0xffff, v24
-; GFX9-NEXT: v_and_b32_e32 v25, 0xffff, v25
-; GFX9-NEXT: v_and_b32_e32 v26, 0xffff, v26
-; GFX9-NEXT: v_and_b32_e32 v27, 0xffff, v27
-; GFX9-NEXT: v_and_b32_e32 v28, 0xffff, v28
-; GFX9-NEXT: v_and_b32_e32 v29, 0xffff, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v55, 16, v55
+; GFX9-NEXT: v_lshlrev_b32_e32 v54, 16, v54
+; GFX9-NEXT: v_lshlrev_b32_e32 v53, 16, v53
+; GFX9-NEXT: v_lshlrev_b32_e32 v52, 16, v52
+; GFX9-NEXT: v_lshlrev_b32_e32 v51, 16, v51
+; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v50
+; GFX9-NEXT: v_lshlrev_b32_e32 v49, 16, v49
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; GFX9-NEXT: v_lshlrev_b32_e32 v39, 16, v39
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; GFX9-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
; GFX9-NEXT: v_readlane_b32 s30, v60, 0
-; GFX9-NEXT: v_lshl_or_b32 v12, v55, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v54, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v53, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v52, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v51, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v50, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v49, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v48, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v39, 16, v20
-; GFX9-NEXT: v_lshl_or_b32 v21, v38, 16, v21
-; GFX9-NEXT: v_lshl_or_b32 v22, v37, 16, v22
-; GFX9-NEXT: v_lshl_or_b32 v23, v36, 16, v23
-; GFX9-NEXT: v_lshl_or_b32 v24, v35, 16, v24
-; GFX9-NEXT: v_lshl_or_b32 v25, v34, 16, v25
-; GFX9-NEXT: v_lshl_or_b32 v26, v33, 16, v26
-; GFX9-NEXT: v_lshl_or_b32 v27, v32, 16, v27
-; GFX9-NEXT: v_lshl_or_b32 v28, v31, 16, v28
-; GFX9-NEXT: v_lshl_or_b32 v29, v30, 16, v29
+; GFX9-NEXT: v_or_b32_sdwa v12, v55, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v54, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v14, v53, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v52, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v16, v51, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v50, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v18, v49, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v48, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v20, v39, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v21, v38, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v22, v37, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v23, v36, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v24, v35, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v25, v34, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v26, v33, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v27, v32, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v28, v31, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v29, v30, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_readlane_b32 s31, v60, 1
; GFX9-NEXT: s_or_saveexec_b64 s[4:5], -1
; GFX9-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
@@ -22628,30 +22628,30 @@ define inreg <60 x half> @bitcast_v30f32_to_v60f16_scalar(<30 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v31, s47
; GFX9-NEXT: v_mov_b32_e32 v30, s46
; GFX9-NEXT: .LBB33_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_lshl_or_b32 v0, v59, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v58, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v57, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v56, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v47, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v46, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v45, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v44, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v43, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v42, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v41, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v40, 16, v11
+; GFX9-NEXT: v_lshlrev_b32_e32 v59, 16, v59
+; GFX9-NEXT: v_lshlrev_b32_e32 v58, 16, v58
+; GFX9-NEXT: v_lshlrev_b32_e32 v57, 16, v57
+; GFX9-NEXT: v_lshlrev_b32_e32 v56, 16, v56
+; GFX9-NEXT: v_lshlrev_b32_e32 v47, 16, v47
+; GFX9-NEXT: v_lshlrev_b32_e32 v46, 16, v46
+; GFX9-NEXT: v_lshlrev_b32_e32 v45, 16, v45
+; GFX9-NEXT: v_lshlrev_b32_e32 v44, 16, v44
+; GFX9-NEXT: v_lshlrev_b32_e32 v43, 16, v43
+; GFX9-NEXT: v_lshlrev_b32_e32 v42, 16, v42
+; GFX9-NEXT: v_lshlrev_b32_e32 v41, 16, v41
+; GFX9-NEXT: v_lshlrev_b32_e32 v40, 16, v40
+; GFX9-NEXT: v_or_b32_sdwa v0, v59, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v58, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v57, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v56, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v4, v47, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v46, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v45, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v44, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v43, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v42, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v41, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v40, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: buffer_load_dword v59, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
@@ -22664,43 +22664,43 @@ define inreg <60 x half> @bitcast_v30f32_to_v60f16_scalar(<30 x float> inreg %a,
; GFX9-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_and_b32_e32 v21, 0xffff, v21
-; GFX9-NEXT: v_and_b32_e32 v22, 0xffff, v22
-; GFX9-NEXT: v_and_b32_e32 v23, 0xffff, v23
-; GFX9-NEXT: v_and_b32_e32 v24, 0xffff, v24
-; GFX9-NEXT: v_and_b32_e32 v25, 0xffff, v25
-; GFX9-NEXT: v_and_b32_e32 v26, 0xffff, v26
-; GFX9-NEXT: v_and_b32_e32 v27, 0xffff, v27
-; GFX9-NEXT: v_and_b32_e32 v28, 0xffff, v28
-; GFX9-NEXT: v_and_b32_e32 v29, 0xffff, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v55, 16, v55
+; GFX9-NEXT: v_lshlrev_b32_e32 v54, 16, v54
+; GFX9-NEXT: v_lshlrev_b32_e32 v53, 16, v53
+; GFX9-NEXT: v_lshlrev_b32_e32 v52, 16, v52
+; GFX9-NEXT: v_lshlrev_b32_e32 v51, 16, v51
+; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v50
+; GFX9-NEXT: v_lshlrev_b32_e32 v49, 16, v49
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; GFX9-NEXT: v_lshlrev_b32_e32 v39, 16, v39
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; GFX9-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
; GFX9-NEXT: v_readlane_b32 s30, v60, 0
-; GFX9-NEXT: v_lshl_or_b32 v12, v55, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v54, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v53, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v52, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v51, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v50, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v49, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v48, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v39, 16, v20
-; GFX9-NEXT: v_lshl_or_b32 v21, v38, 16, v21
-; GFX9-NEXT: v_lshl_or_b32 v22, v37, 16, v22
-; GFX9-NEXT: v_lshl_or_b32 v23, v36, 16, v23
-; GFX9-NEXT: v_lshl_or_b32 v24, v35, 16, v24
-; GFX9-NEXT: v_lshl_or_b32 v25, v34, 16, v25
-; GFX9-NEXT: v_lshl_or_b32 v26, v33, 16, v26
-; GFX9-NEXT: v_lshl_or_b32 v27, v32, 16, v27
-; GFX9-NEXT: v_lshl_or_b32 v28, v31, 16, v28
-; GFX9-NEXT: v_lshl_or_b32 v29, v30, 16, v29
+; GFX9-NEXT: v_or_b32_sdwa v12, v55, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v54, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v14, v53, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v52, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v16, v51, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v50, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v18, v49, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v48, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v20, v39, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v21, v38, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v22, v37, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v23, v36, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v24, v35, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v25, v34, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v26, v33, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v27, v32, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v28, v31, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v29, v30, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_readlane_b32 s31, v60, 1
; GFX9-NEXT: s_or_saveexec_b64 s[4:5], -1
; GFX9-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
@@ -38248,30 +38248,32 @@ define inreg <60 x i16> @bitcast_v15f64_to_v60i16_scalar(<15 x double> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v35, s47
; GFX9-NEXT: v_mov_b32_e32 v33, s46
; GFX9-NEXT: .LBB49_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_lshl_or_b32 v0, v56, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v59, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v46, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v58, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v44, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v57, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v42, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v47, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v40, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v45, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v11, v43, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v13, v41, 16, v13
+; GFX9-NEXT: v_lshlrev_b32_e32 v56, 16, v56
+; GFX9-NEXT: v_lshlrev_b32_e32 v46, 16, v46
+; GFX9-NEXT: v_lshlrev_b32_e32 v44, 16, v44
+; GFX9-NEXT: v_lshlrev_b32_e32 v42, 16, v42
+; GFX9-NEXT: v_lshlrev_b32_e32 v40, 16, v40
+; GFX9-NEXT: v_or_b32_sdwa v0, v56, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v56, 16, v59
+; GFX9-NEXT: v_or_b32_sdwa v2, v46, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v46, 16, v58
+; GFX9-NEXT: v_or_b32_sdwa v4, v44, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v44, 16, v57
+; GFX9-NEXT: v_or_b32_sdwa v6, v42, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v42, 16, v47
+; GFX9-NEXT: v_or_b32_sdwa v8, v40, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v40, 16, v45
+; GFX9-NEXT: v_lshlrev_b32_e32 v54, 16, v54
+; GFX9-NEXT: v_lshlrev_b32_e32 v52, 16, v52
+; GFX9-NEXT: v_or_b32_sdwa v1, v56, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v46, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v44, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v42, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v40, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v54, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v54, 16, v43
+; GFX9-NEXT: v_or_b32_sdwa v12, v52, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v52, 16, v41
; GFX9-NEXT: buffer_load_dword v59, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
@@ -38284,43 +38286,41 @@ define inreg <60 x i16> @bitcast_v15f64_to_v60i16_scalar(<15 x double> inreg %a,
; GFX9-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_and_b32_e32 v21, 0xffff, v21
-; GFX9-NEXT: v_and_b32_e32 v22, 0xffff, v22
-; GFX9-NEXT: v_and_b32_e32 v23, 0xffff, v23
-; GFX9-NEXT: v_and_b32_e32 v24, 0xffff, v24
-; GFX9-NEXT: v_and_b32_e32 v25, 0xffff, v25
-; GFX9-NEXT: v_and_b32_e32 v26, 0xffff, v26
-; GFX9-NEXT: v_and_b32_e32 v27, 0xffff, v27
-; GFX9-NEXT: v_and_b32_e32 v28, 0xffff, v28
-; GFX9-NEXT: v_and_b32_e32 v29, 0xffff, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v50
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; GFX9-NEXT: v_or_b32_sdwa v14, v50, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v55
+; GFX9-NEXT: v_or_b32_sdwa v16, v48, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v53
+; GFX9-NEXT: v_or_b32_sdwa v18, v38, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v51
+; GFX9-NEXT: v_or_b32_sdwa v20, v36, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v49
+; GFX9-NEXT: v_or_b32_sdwa v22, v34, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v39
+; GFX9-NEXT: v_or_b32_sdwa v24, v32, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v37
+; GFX9-NEXT: v_or_b32_sdwa v26, v31, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v35
+; GFX9-NEXT: v_or_b32_sdwa v28, v30, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v33
; GFX9-NEXT: v_readlane_b32 s30, v60, 0
-; GFX9-NEXT: v_lshl_or_b32 v10, v54, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v12, v52, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v14, v50, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v55, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v48, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v53, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v38, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v51, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v36, 16, v20
-; GFX9-NEXT: v_lshl_or_b32 v21, v49, 16, v21
-; GFX9-NEXT: v_lshl_or_b32 v22, v34, 16, v22
-; GFX9-NEXT: v_lshl_or_b32 v23, v39, 16, v23
-; GFX9-NEXT: v_lshl_or_b32 v24, v32, 16, v24
-; GFX9-NEXT: v_lshl_or_b32 v25, v37, 16, v25
-; GFX9-NEXT: v_lshl_or_b32 v26, v31, 16, v26
-; GFX9-NEXT: v_lshl_or_b32 v27, v35, 16, v27
-; GFX9-NEXT: v_lshl_or_b32 v28, v30, 16, v28
-; GFX9-NEXT: v_lshl_or_b32 v29, v33, 16, v29
+; GFX9-NEXT: v_or_b32_sdwa v11, v54, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v52, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v50, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v48, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v38, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v21, v36, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v23, v34, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v25, v32, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v27, v31, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v29, v30, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_readlane_b32 s31, v60, 1
; GFX9-NEXT: s_or_saveexec_b64 s[4:5], -1
; GFX9-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
@@ -43119,30 +43119,32 @@ define inreg <60 x half> @bitcast_v15f64_to_v60f16_scalar(<15 x double> inreg %a
; GFX9-NEXT: v_mov_b32_e32 v35, s47
; GFX9-NEXT: v_mov_b32_e32 v33, s46
; GFX9-NEXT: .LBB53_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_lshl_or_b32 v0, v56, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v59, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v46, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v58, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v44, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v57, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v42, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v47, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v40, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v45, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v11, v43, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v13, v41, 16, v13
+; GFX9-NEXT: v_lshlrev_b32_e32 v56, 16, v56
+; GFX9-NEXT: v_lshlrev_b32_e32 v46, 16, v46
+; GFX9-NEXT: v_lshlrev_b32_e32 v44, 16, v44
+; GFX9-NEXT: v_lshlrev_b32_e32 v42, 16, v42
+; GFX9-NEXT: v_lshlrev_b32_e32 v40, 16, v40
+; GFX9-NEXT: v_or_b32_sdwa v0, v56, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v56, 16, v59
+; GFX9-NEXT: v_or_b32_sdwa v2, v46, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v46, 16, v58
+; GFX9-NEXT: v_or_b32_sdwa v4, v44, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v44, 16, v57
+; GFX9-NEXT: v_or_b32_sdwa v6, v42, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v42, 16, v47
+; GFX9-NEXT: v_or_b32_sdwa v8, v40, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v40, 16, v45
+; GFX9-NEXT: v_lshlrev_b32_e32 v54, 16, v54
+; GFX9-NEXT: v_lshlrev_b32_e32 v52, 16, v52
+; GFX9-NEXT: v_or_b32_sdwa v1, v56, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v46, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v44, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v42, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v40, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v54, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v54, 16, v43
+; GFX9-NEXT: v_or_b32_sdwa v12, v52, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v52, 16, v41
; GFX9-NEXT: buffer_load_dword v59, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
@@ -43155,43 +43157,41 @@ define inreg <60 x half> @bitcast_v15f64_to_v60f16_scalar(<15 x double> inreg %a
; GFX9-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_and_b32_e32 v21, 0xffff, v21
-; GFX9-NEXT: v_and_b32_e32 v22, 0xffff, v22
-; GFX9-NEXT: v_and_b32_e32 v23, 0xffff, v23
-; GFX9-NEXT: v_and_b32_e32 v24, 0xffff, v24
-; GFX9-NEXT: v_and_b32_e32 v25, 0xffff, v25
-; GFX9-NEXT: v_and_b32_e32 v26, 0xffff, v26
-; GFX9-NEXT: v_and_b32_e32 v27, 0xffff, v27
-; GFX9-NEXT: v_and_b32_e32 v28, 0xffff, v28
-; GFX9-NEXT: v_and_b32_e32 v29, 0xffff, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v50
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; GFX9-NEXT: v_or_b32_sdwa v14, v50, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v55
+; GFX9-NEXT: v_or_b32_sdwa v16, v48, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v53
+; GFX9-NEXT: v_or_b32_sdwa v18, v38, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v51
+; GFX9-NEXT: v_or_b32_sdwa v20, v36, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v49
+; GFX9-NEXT: v_or_b32_sdwa v22, v34, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v39
+; GFX9-NEXT: v_or_b32_sdwa v24, v32, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v37
+; GFX9-NEXT: v_or_b32_sdwa v26, v31, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v35
+; GFX9-NEXT: v_or_b32_sdwa v28, v30, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v33
; GFX9-NEXT: v_readlane_b32 s30, v60, 0
-; GFX9-NEXT: v_lshl_or_b32 v10, v54, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v12, v52, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v14, v50, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v55, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v48, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v53, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v38, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v51, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v36, 16, v20
-; GFX9-NEXT: v_lshl_or_b32 v21, v49, 16, v21
-; GFX9-NEXT: v_lshl_or_b32 v22, v34, 16, v22
-; GFX9-NEXT: v_lshl_or_b32 v23, v39, 16, v23
-; GFX9-NEXT: v_lshl_or_b32 v24, v32, 16, v24
-; GFX9-NEXT: v_lshl_or_b32 v25, v37, 16, v25
-; GFX9-NEXT: v_lshl_or_b32 v26, v31, 16, v26
-; GFX9-NEXT: v_lshl_or_b32 v27, v35, 16, v27
-; GFX9-NEXT: v_lshl_or_b32 v28, v30, 16, v28
-; GFX9-NEXT: v_lshl_or_b32 v29, v33, 16, v29
+; GFX9-NEXT: v_or_b32_sdwa v11, v54, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v52, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v50, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v48, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v38, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v21, v36, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v23, v34, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v25, v32, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v27, v31, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v29, v30, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_readlane_b32 s31, v60, 1
; GFX9-NEXT: s_or_saveexec_b64 s[4:5], -1
; GFX9-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
@@ -49037,30 +49037,30 @@ define inreg <60 x half> @bitcast_v60i16_to_v60f16_scalar(<60 x i16> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v58, s7
; GFX9-NEXT: v_mov_b32_e32 v59, s6
; GFX9-NEXT: .LBB57_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_lshl_or_b32 v0, v59, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v58, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v57, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v56, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v47, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v46, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v45, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v44, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v43, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v42, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v41, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v40, 16, v11
+; GFX9-NEXT: v_lshlrev_b32_e32 v59, 16, v59
+; GFX9-NEXT: v_lshlrev_b32_e32 v58, 16, v58
+; GFX9-NEXT: v_lshlrev_b32_e32 v57, 16, v57
+; GFX9-NEXT: v_lshlrev_b32_e32 v56, 16, v56
+; GFX9-NEXT: v_lshlrev_b32_e32 v47, 16, v47
+; GFX9-NEXT: v_lshlrev_b32_e32 v46, 16, v46
+; GFX9-NEXT: v_lshlrev_b32_e32 v45, 16, v45
+; GFX9-NEXT: v_lshlrev_b32_e32 v44, 16, v44
+; GFX9-NEXT: v_lshlrev_b32_e32 v43, 16, v43
+; GFX9-NEXT: v_lshlrev_b32_e32 v42, 16, v42
+; GFX9-NEXT: v_lshlrev_b32_e32 v41, 16, v41
+; GFX9-NEXT: v_lshlrev_b32_e32 v40, 16, v40
+; GFX9-NEXT: v_or_b32_sdwa v0, v59, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v58, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v57, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v56, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v4, v47, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v46, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v45, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v44, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v43, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v42, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v41, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v40, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: buffer_load_dword v59, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
@@ -49073,43 +49073,43 @@ define inreg <60 x half> @bitcast_v60i16_to_v60f16_scalar(<60 x i16> inreg %a, i
; GFX9-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_and_b32_e32 v21, 0xffff, v21
-; GFX9-NEXT: v_and_b32_e32 v22, 0xffff, v22
-; GFX9-NEXT: v_and_b32_e32 v23, 0xffff, v23
-; GFX9-NEXT: v_and_b32_e32 v24, 0xffff, v24
-; GFX9-NEXT: v_and_b32_e32 v25, 0xffff, v25
-; GFX9-NEXT: v_and_b32_e32 v26, 0xffff, v26
-; GFX9-NEXT: v_and_b32_e32 v27, 0xffff, v27
-; GFX9-NEXT: v_and_b32_e32 v28, 0xffff, v28
-; GFX9-NEXT: v_and_b32_e32 v29, 0xffff, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v55, 16, v55
+; GFX9-NEXT: v_lshlrev_b32_e32 v54, 16, v54
+; GFX9-NEXT: v_lshlrev_b32_e32 v53, 16, v53
+; GFX9-NEXT: v_lshlrev_b32_e32 v52, 16, v52
+; GFX9-NEXT: v_lshlrev_b32_e32 v51, 16, v51
+; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v50
+; GFX9-NEXT: v_lshlrev_b32_e32 v49, 16, v49
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; GFX9-NEXT: v_lshlrev_b32_e32 v39, 16, v39
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; GFX9-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
; GFX9-NEXT: v_readlane_b32 s30, v60, 0
-; GFX9-NEXT: v_lshl_or_b32 v12, v55, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v54, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v53, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v52, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v51, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v50, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v49, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v48, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v39, 16, v20
-; GFX9-NEXT: v_lshl_or_b32 v21, v38, 16, v21
-; GFX9-NEXT: v_lshl_or_b32 v22, v37, 16, v22
-; GFX9-NEXT: v_lshl_or_b32 v23, v36, 16, v23
-; GFX9-NEXT: v_lshl_or_b32 v24, v35, 16, v24
-; GFX9-NEXT: v_lshl_or_b32 v25, v34, 16, v25
-; GFX9-NEXT: v_lshl_or_b32 v26, v33, 16, v26
-; GFX9-NEXT: v_lshl_or_b32 v27, v32, 16, v27
-; GFX9-NEXT: v_lshl_or_b32 v28, v31, 16, v28
-; GFX9-NEXT: v_lshl_or_b32 v29, v30, 16, v29
+; GFX9-NEXT: v_or_b32_sdwa v12, v55, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v54, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v14, v53, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v52, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v16, v51, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v50, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v18, v49, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v48, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v20, v39, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v21, v38, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v22, v37, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v23, v36, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v24, v35, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v25, v34, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v26, v33, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v27, v32, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v28, v31, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v29, v30, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_readlane_b32 s31, v60, 1
; GFX9-NEXT: s_or_saveexec_b64 s[4:5], -1
; GFX9-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
@@ -51721,30 +51721,30 @@ define inreg <60 x i16> @bitcast_v60f16_to_v60i16_scalar(<60 x half> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v58, s7
; GFX9-NEXT: v_mov_b32_e32 v59, s6
; GFX9-NEXT: .LBB59_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_lshl_or_b32 v0, v59, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v58, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v57, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v56, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v47, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v46, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v45, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v44, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v43, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v42, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v41, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v40, 16, v11
+; GFX9-NEXT: v_lshlrev_b32_e32 v59, 16, v59
+; GFX9-NEXT: v_lshlrev_b32_e32 v58, 16, v58
+; GFX9-NEXT: v_lshlrev_b32_e32 v57, 16, v57
+; GFX9-NEXT: v_lshlrev_b32_e32 v56, 16, v56
+; GFX9-NEXT: v_lshlrev_b32_e32 v47, 16, v47
+; GFX9-NEXT: v_lshlrev_b32_e32 v46, 16, v46
+; GFX9-NEXT: v_lshlrev_b32_e32 v45, 16, v45
+; GFX9-NEXT: v_lshlrev_b32_e32 v44, 16, v44
+; GFX9-NEXT: v_lshlrev_b32_e32 v43, 16, v43
+; GFX9-NEXT: v_lshlrev_b32_e32 v42, 16, v42
+; GFX9-NEXT: v_lshlrev_b32_e32 v41, 16, v41
+; GFX9-NEXT: v_lshlrev_b32_e32 v40, 16, v40
+; GFX9-NEXT: v_or_b32_sdwa v0, v59, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v58, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v57, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v56, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v4, v47, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v46, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v45, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v44, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v43, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v42, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v41, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v40, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: buffer_load_dword v59, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
@@ -51757,43 +51757,43 @@ define inreg <60 x i16> @bitcast_v60f16_to_v60i16_scalar(<60 x half> inreg %a, i
; GFX9-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_and_b32_e32 v21, 0xffff, v21
-; GFX9-NEXT: v_and_b32_e32 v22, 0xffff, v22
-; GFX9-NEXT: v_and_b32_e32 v23, 0xffff, v23
-; GFX9-NEXT: v_and_b32_e32 v24, 0xffff, v24
-; GFX9-NEXT: v_and_b32_e32 v25, 0xffff, v25
-; GFX9-NEXT: v_and_b32_e32 v26, 0xffff, v26
-; GFX9-NEXT: v_and_b32_e32 v27, 0xffff, v27
-; GFX9-NEXT: v_and_b32_e32 v28, 0xffff, v28
-; GFX9-NEXT: v_and_b32_e32 v29, 0xffff, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v55, 16, v55
+; GFX9-NEXT: v_lshlrev_b32_e32 v54, 16, v54
+; GFX9-NEXT: v_lshlrev_b32_e32 v53, 16, v53
+; GFX9-NEXT: v_lshlrev_b32_e32 v52, 16, v52
+; GFX9-NEXT: v_lshlrev_b32_e32 v51, 16, v51
+; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v50
+; GFX9-NEXT: v_lshlrev_b32_e32 v49, 16, v49
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; GFX9-NEXT: v_lshlrev_b32_e32 v39, 16, v39
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; GFX9-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
; GFX9-NEXT: v_readlane_b32 s30, v60, 0
-; GFX9-NEXT: v_lshl_or_b32 v12, v55, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v54, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v53, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v52, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v51, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v50, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v49, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v48, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v39, 16, v20
-; GFX9-NEXT: v_lshl_or_b32 v21, v38, 16, v21
-; GFX9-NEXT: v_lshl_or_b32 v22, v37, 16, v22
-; GFX9-NEXT: v_lshl_or_b32 v23, v36, 16, v23
-; GFX9-NEXT: v_lshl_or_b32 v24, v35, 16, v24
-; GFX9-NEXT: v_lshl_or_b32 v25, v34, 16, v25
-; GFX9-NEXT: v_lshl_or_b32 v26, v33, 16, v26
-; GFX9-NEXT: v_lshl_or_b32 v27, v32, 16, v27
-; GFX9-NEXT: v_lshl_or_b32 v28, v31, 16, v28
-; GFX9-NEXT: v_lshl_or_b32 v29, v30, 16, v29
+; GFX9-NEXT: v_or_b32_sdwa v12, v55, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v54, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v14, v53, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v52, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v16, v51, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v50, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v18, v49, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v48, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v20, v39, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v21, v38, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v22, v37, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v23, v36, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v24, v35, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v25, v34, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v26, v33, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v27, v32, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v28, v31, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v29, v30, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_readlane_b32 s31, v60, 1
; GFX9-NEXT: s_or_saveexec_b64 s[4:5], -1
; GFX9-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll
index 0e04f1ebc8e9d..3741816fd4a56 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll
@@ -2423,52 +2423,51 @@ define inreg <3 x i32> @bitcast_v6bf16_to_v3i32_scalar(<6 x bfloat> inreg %a, i3
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX9-NEXT: v_mov_b32_e32 v3, 0xffff
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s17
-; GFX9-NEXT: v_lshl_or_b32 v2, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
-; GFX9-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v4, v4, v1
-; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
-; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v3, v3, v1
+; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: s_lshl_b32 s4, s17, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v5, vcc
-; GFX9-NEXT: v_add_f32_e32 v4, s4, v0
-; GFX9-NEXT: v_bfe_u32 v5, v4, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v5, v5, v4
-; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
-; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v4
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
+; GFX9-NEXT: v_add_f32_e32 v3, s4, v0
+; GFX9-NEXT: v_bfe_u32 v4, v3, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v4, v4, v3
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v4, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s16
-; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v4
-; GFX9-NEXT: v_add_f32_e32 v4, s4, v0
-; GFX9-NEXT: v_bfe_u32 v5, v4, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v5, v5, v4
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f32_e32 v3, s4, v0
+; GFX9-NEXT: v_bfe_u32 v4, v3, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v4, v4, v3
; GFX9-NEXT: s_lshl_b32 s4, s16, 16
-; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
-; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v4
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
+; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
; GFX9-NEXT: v_add_f32_e32 v0, s4, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc
-; GFX9-NEXT: v_bfe_u32 v5, v0, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v5, v5, v0
-; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
-; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
+; GFX9-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v4, v4, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v0
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v5, v6, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX9-NEXT: v_and_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v0, v4, 16, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_setpc_b64 s[30:31]
; GFX9-NEXT: .LBB11_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
@@ -5768,52 +5767,51 @@ define inreg <3 x float> @bitcast_v6bf16_to_v3f32_scalar(<6 x bfloat> inreg %a,
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX9-NEXT: v_mov_b32_e32 v3, 0xffff
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s17
-; GFX9-NEXT: v_lshl_or_b32 v2, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
-; GFX9-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v4, v4, v1
-; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
-; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v3, v3, v1
+; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: s_lshl_b32 s4, s17, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v5, vcc
-; GFX9-NEXT: v_add_f32_e32 v4, s4, v0
-; GFX9-NEXT: v_bfe_u32 v5, v4, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v5, v5, v4
-; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
-; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v4
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
+; GFX9-NEXT: v_add_f32_e32 v3, s4, v0
+; GFX9-NEXT: v_bfe_u32 v4, v3, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v4, v4, v3
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v4, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s16
-; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v4
-; GFX9-NEXT: v_add_f32_e32 v4, s4, v0
-; GFX9-NEXT: v_bfe_u32 v5, v4, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v5, v5, v4
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f32_e32 v3, s4, v0
+; GFX9-NEXT: v_bfe_u32 v4, v3, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v4, v4, v3
; GFX9-NEXT: s_lshl_b32 s4, s16, 16
-; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
-; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v4
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
+; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
; GFX9-NEXT: v_add_f32_e32 v0, s4, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc
-; GFX9-NEXT: v_bfe_u32 v5, v0, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v5, v5, v0
-; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
-; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
+; GFX9-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v4, v4, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v0
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v5, v6, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX9-NEXT: v_and_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v0, v4, 16, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_setpc_b64 s[30:31]
; GFX9-NEXT: .LBB27_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
@@ -8621,9 +8619,9 @@ define inreg <12 x i8> @bitcast_v6bf16_to_v12i8_scalar(<6 x bfloat> inreg %a, i3
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v13, 16, v0
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v13
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v6
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s16
-; GFX9-NEXT: v_lshl_or_b32 v15, v6, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v15, v0, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v0, s4, v1
; GFX9-NEXT: v_bfe_u32 v2, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v0
@@ -8641,9 +8639,9 @@ define inreg <12 x i8> @bitcast_v6bf16_to_v12i8_scalar(<6 x bfloat> inreg %a, i3
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s18
-; GFX9-NEXT: v_lshl_or_b32 v14, v2, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v14, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v3, s4, v1
; GFX9-NEXT: v_bfe_u32 v4, v3, 16, 1
; GFX9-NEXT: v_add_u32_e32 v4, v4, v3
@@ -8661,8 +8659,8 @@ define inreg <12 x i8> @bitcast_v6bf16_to_v12i8_scalar(<6 x bfloat> inreg %a, i3
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v1
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v8
-; GFX9-NEXT: v_lshl_or_b32 v11, v10, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v10
+; GFX9-NEXT: v_or_b32_sdwa v11, v1, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v12, 0x7fc07fc0
; GFX9-NEXT: v_lshrrev_b64 v[3:4], 24, v[14:15]
; GFX9-NEXT: v_lshrrev_b32_e32 v9, 8, v11
@@ -12306,19 +12304,18 @@ define inreg <6 x half> @bitcast_v6bf16_to_v6f16_scalar(<6 x bfloat> inreg %a, i
; GFX9-NEXT: v_cndmask_b32_e32 v2, v6, v7, vcc
; GFX9-NEXT: v_bfe_u32 v6, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v6, v6, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX9-NEXT: v_add_u32_e32 v6, 0x7fff, v6
; GFX9-NEXT: v_or_b32_e32 v7, 0x400000, v0
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v7, vcc
-; GFX9-NEXT: v_mov_b32_e32 v6, 0xffff
-; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX9-NEXT: v_and_b32_sdwa v0, v6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v2, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v0, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v0, v6, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v0, v3, 16, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v7, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_setpc_b64 s[30:31]
; GFX9-NEXT: .LBB49_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll
index 1113c9757b6e8..5b420f5394542 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll
@@ -2474,7 +2474,6 @@ define amdgpu_kernel void @udiv_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_and_b32 s5, s2, 0xffff
; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s5
@@ -2482,41 +2481,40 @@ define amdgpu_kernel void @udiv_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s4
; GFX9-NEXT: s_lshr_b32 s2, s2, 16
; GFX9-NEXT: v_rcp_f32_e32 v0, v0
-; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s2
+; GFX9-NEXT: v_cvt_f32_u32_e32 v2, s2
; GFX9-NEXT: s_lshr_b32 s0, s0, 16
; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
; GFX9-NEXT: s_and_b32 s2, s3, 0xffff
-; GFX9-NEXT: v_cvt_f32_u32_e32 v4, s0
+; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s0
; GFX9-NEXT: v_mul_f32_e32 v0, v1, v0
; GFX9-NEXT: s_and_b32 s0, s1, 0xffff
; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s2
-; GFX9-NEXT: v_rcp_f32_e32 v3, v3
-; GFX9-NEXT: v_cvt_f32_u32_e32 v5, s0
-; GFX9-NEXT: v_add_u32_e32 v4, 1, v4
+; GFX9-NEXT: v_rcp_f32_e32 v2, v2
+; GFX9-NEXT: v_cvt_f32_u32_e32 v4, s0
+; GFX9-NEXT: v_add_u32_e32 v3, 1, v3
; GFX9-NEXT: v_rcp_f32_e32 v1, v1
; GFX9-NEXT: s_lshr_b32 s0, s3, 16
-; GFX9-NEXT: v_mul_f32_e32 v3, v4, v3
-; GFX9-NEXT: v_add_u32_e32 v4, 1, v5
-; GFX9-NEXT: v_cvt_f32_u32_e32 v5, s0
-; GFX9-NEXT: s_lshr_b32 s0, s1, 16
-; GFX9-NEXT: v_mul_f32_e32 v1, v4, v1
+; GFX9-NEXT: v_mul_f32_e32 v2, v3, v2
+; GFX9-NEXT: v_add_u32_e32 v3, 1, v4
; GFX9-NEXT: v_cvt_f32_u32_e32 v4, s0
-; GFX9-NEXT: v_rcp_f32_e32 v5, v5
+; GFX9-NEXT: s_lshr_b32 s0, s1, 16
+; GFX9-NEXT: v_mul_f32_e32 v1, v3, v1
+; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s0
+; GFX9-NEXT: v_rcp_f32_e32 v4, v4
; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: v_trunc_f32_e32 v2, v2
+; GFX9-NEXT: v_add_u32_e32 v3, 1, v3
+; GFX9-NEXT: v_mul_f32_e32 v3, v3, v4
; GFX9-NEXT: v_trunc_f32_e32 v1, v1
-; GFX9-NEXT: v_add_u32_e32 v4, 1, v4
-; GFX9-NEXT: v_mul_f32_e32 v4, v4, v5
+; GFX9-NEXT: v_trunc_f32_e32 v3, v3
; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
; GFX9-NEXT: v_cvt_u32_f32_e32 v1, v1
-; GFX9-NEXT: v_trunc_f32_e32 v4, v4
-; GFX9-NEXT: v_trunc_f32_e32 v3, v3
-; GFX9-NEXT: v_cvt_u32_f32_e32 v4, v4
-; GFX9-NEXT: v_cvt_u32_f32_e32 v3, v3
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v4, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v0, v3, 16, v0
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7]
+; GFX9-NEXT: v_cvt_u32_f32_sdwa v3, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; GFX9-NEXT: v_cvt_u32_f32_sdwa v2, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: global_store_dwordx2 v4, v[0:1], s[6:7]
; GFX9-NEXT: s_endpgm
%r = udiv <4 x i16> %x, %y
store <4 x i16> %r, ptr addrspace(1) %out
@@ -2692,25 +2690,23 @@ define amdgpu_kernel void @urem_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
; GFX9-NEXT: v_trunc_f32_e32 v0, v0
; GFX9-NEXT: v_trunc_f32_e32 v1, v1
; GFX9-NEXT: v_add_u32_e32 v4, 1, v4
-; GFX9-NEXT: v_trunc_f32_e32 v3, v3
; GFX9-NEXT: v_mul_f32_e32 v4, v4, v5
+; GFX9-NEXT: v_trunc_f32_e32 v3, v3
+; GFX9-NEXT: v_trunc_f32_e32 v4, v4
; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
; GFX9-NEXT: v_cvt_u32_f32_e32 v1, v1
; GFX9-NEXT: v_cvt_u32_f32_e32 v3, v3
-; GFX9-NEXT: v_trunc_f32_e32 v4, v4
; GFX9-NEXT: v_cvt_u32_f32_e32 v4, v4
; GFX9-NEXT: v_mul_lo_u32 v0, v0, s5
; GFX9-NEXT: v_mul_lo_u32 v1, v1, s2
; GFX9-NEXT: v_mul_lo_u32 v3, v3, s9
; GFX9-NEXT: v_mul_lo_u32 v4, v4, s3
; GFX9-NEXT: v_sub_u32_e32 v0, s4, v0
-; GFX9-NEXT: v_sub_u32_e32 v5, s0, v1
+; GFX9-NEXT: v_sub_u32_sdwa v5, s0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_sub_u32_e32 v1, s8, v3
-; GFX9-NEXT: v_sub_u32_e32 v3, s1, v4
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v3, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v0, v5, 16, v0
+; GFX9-NEXT: v_sub_u32_sdwa v3, s1, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v0, v5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7]
; GFX9-NEXT: s_endpgm
%r = urem <4 x i16> %x, %y
@@ -2840,7 +2836,6 @@ define amdgpu_kernel void @sdiv_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_sext_i32_i16 s5, s2
; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s5
@@ -2848,41 +2843,40 @@ define amdgpu_kernel void @sdiv_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s4
; GFX9-NEXT: s_ashr_i32 s2, s2, 16
; GFX9-NEXT: v_rcp_f32_e32 v0, v0
-; GFX9-NEXT: v_cvt_f32_i32_e32 v3, s2
+; GFX9-NEXT: v_cvt_f32_i32_e32 v2, s2
; GFX9-NEXT: s_ashr_i32 s0, s0, 16
; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
; GFX9-NEXT: s_sext_i32_i16 s2, s3
-; GFX9-NEXT: v_cvt_f32_i32_e32 v4, s0
+; GFX9-NEXT: v_cvt_f32_i32_e32 v3, s0
; GFX9-NEXT: v_mul_f32_e32 v0, v1, v0
; GFX9-NEXT: s_sext_i32_i16 s0, s1
; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s2
-; GFX9-NEXT: v_rcp_f32_e32 v3, v3
-; GFX9-NEXT: v_cvt_f32_i32_e32 v5, s0
-; GFX9-NEXT: v_add_u32_e32 v4, 1, v4
+; GFX9-NEXT: v_rcp_f32_e32 v2, v2
+; GFX9-NEXT: v_cvt_f32_i32_e32 v4, s0
+; GFX9-NEXT: v_add_u32_e32 v3, 1, v3
; GFX9-NEXT: v_rcp_f32_e32 v1, v1
; GFX9-NEXT: s_ashr_i32 s0, s3, 16
-; GFX9-NEXT: v_mul_f32_e32 v3, v4, v3
-; GFX9-NEXT: v_add_u32_e32 v4, 1, v5
-; GFX9-NEXT: v_cvt_f32_i32_e32 v5, s0
-; GFX9-NEXT: s_ashr_i32 s0, s1, 16
-; GFX9-NEXT: v_mul_f32_e32 v1, v4, v1
+; GFX9-NEXT: v_mul_f32_e32 v2, v3, v2
+; GFX9-NEXT: v_add_u32_e32 v3, 1, v4
; GFX9-NEXT: v_cvt_f32_i32_e32 v4, s0
-; GFX9-NEXT: v_rcp_f32_e32 v5, v5
+; GFX9-NEXT: s_ashr_i32 s0, s1, 16
+; GFX9-NEXT: v_mul_f32_e32 v1, v3, v1
+; GFX9-NEXT: v_cvt_f32_i32_e32 v3, s0
+; GFX9-NEXT: v_rcp_f32_e32 v4, v4
; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: v_trunc_f32_e32 v2, v2
+; GFX9-NEXT: v_add_u32_e32 v3, 1, v3
+; GFX9-NEXT: v_mul_f32_e32 v3, v3, v4
; GFX9-NEXT: v_trunc_f32_e32 v1, v1
-; GFX9-NEXT: v_add_u32_e32 v4, 1, v4
-; GFX9-NEXT: v_mul_f32_e32 v4, v4, v5
+; GFX9-NEXT: v_trunc_f32_e32 v3, v3
; GFX9-NEXT: v_cvt_i32_f32_e32 v0, v0
; GFX9-NEXT: v_cvt_i32_f32_e32 v1, v1
-; GFX9-NEXT: v_trunc_f32_e32 v4, v4
-; GFX9-NEXT: v_trunc_f32_e32 v3, v3
-; GFX9-NEXT: v_cvt_i32_f32_e32 v4, v4
-; GFX9-NEXT: v_cvt_i32_f32_e32 v3, v3
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v4, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v0, v3, 16, v0
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7]
+; GFX9-NEXT: v_cvt_i32_f32_sdwa v3, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; GFX9-NEXT: v_cvt_i32_f32_sdwa v2, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: global_store_dwordx2 v4, v[0:1], s[6:7]
; GFX9-NEXT: s_endpgm
%r = sdiv <4 x i16> %x, %y
store <4 x i16> %r, ptr addrspace(1) %out
@@ -3062,25 +3056,23 @@ define amdgpu_kernel void @srem_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
; GFX9-NEXT: v_trunc_f32_e32 v0, v0
; GFX9-NEXT: v_trunc_f32_e32 v1, v1
; GFX9-NEXT: v_add_u32_e32 v4, 1, v4
-; GFX9-NEXT: v_trunc_f32_e32 v3, v3
; GFX9-NEXT: v_mul_f32_e32 v4, v4, v5
+; GFX9-NEXT: v_trunc_f32_e32 v3, v3
+; GFX9-NEXT: v_trunc_f32_e32 v4, v4
; GFX9-NEXT: v_cvt_i32_f32_e32 v0, v0
; GFX9-NEXT: v_cvt_i32_f32_e32 v1, v1
; GFX9-NEXT: v_cvt_i32_f32_e32 v3, v3
-; GFX9-NEXT: v_trunc_f32_e32 v4, v4
; GFX9-NEXT: v_cvt_i32_f32_e32 v4, v4
; GFX9-NEXT: v_mul_lo_u32 v0, v0, s5
; GFX9-NEXT: v_mul_lo_u32 v1, v1, s2
; GFX9-NEXT: v_mul_lo_u32 v3, v3, s9
; GFX9-NEXT: v_mul_lo_u32 v4, v4, s3
; GFX9-NEXT: v_sub_u32_e32 v0, s4, v0
-; GFX9-NEXT: v_sub_u32_e32 v5, s0, v1
+; GFX9-NEXT: v_sub_u32_sdwa v5, s0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_sub_u32_e32 v1, s8, v3
-; GFX9-NEXT: v_sub_u32_e32 v3, s1, v4
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v3, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v0, v5, 16, v0
+; GFX9-NEXT: v_sub_u32_sdwa v3, s1, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v0, v5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7]
; GFX9-NEXT: s_endpgm
%r = srem <4 x i16> %x, %y
@@ -3471,16 +3463,15 @@ define amdgpu_kernel void @udiv_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s0
; GFX9-NEXT: v_rcp_f32_e32 v1, v1
; GFX9-NEXT: v_trunc_f32_e32 v0, v0
-; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX9-NEXT: v_trunc_f32_e32 v2, v2
; GFX9-NEXT: v_add_u32_e32 v3, 1, v3
; GFX9-NEXT: v_mul_f32_e32 v1, v3, v1
-; GFX9-NEXT: v_trunc_f32_e32 v2, v2
; GFX9-NEXT: v_trunc_f32_e32 v1, v1
-; GFX9-NEXT: v_cvt_u32_f32_e32 v2, v2
+; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_u32_f32_sdwa v2, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
; GFX9-NEXT: v_cvt_u32_f32_e32 v1, v1
; GFX9-NEXT: v_mov_b32_e32 v3, 0
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v0, v2, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: global_store_short v3, v1, s[6:7] offset:4
; GFX9-NEXT: global_store_dword v3, v0, s[6:7]
; GFX9-NEXT: s_endpgm
@@ -3617,23 +3608,22 @@ define amdgpu_kernel void @urem_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s9
; GFX9-NEXT: v_rcp_f32_e32 v1, v1
; GFX9-NEXT: v_trunc_f32_e32 v0, v0
-; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX9-NEXT: v_trunc_f32_e32 v2, v2
; GFX9-NEXT: v_add_u32_e32 v3, 1, v3
; GFX9-NEXT: v_mul_f32_e32 v1, v3, v1
-; GFX9-NEXT: v_trunc_f32_e32 v2, v2
; GFX9-NEXT: v_trunc_f32_e32 v1, v1
+; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
; GFX9-NEXT: v_cvt_u32_f32_e32 v2, v2
; GFX9-NEXT: v_cvt_u32_f32_e32 v1, v1
-; GFX9-NEXT: v_mul_lo_u32 v0, v0, s7
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: v_mul_lo_u32 v0, v0, s7
; GFX9-NEXT: v_mul_lo_u32 v2, v2, s2
; GFX9-NEXT: v_mul_lo_u32 v1, v1, s3
-; GFX9-NEXT: v_sub_u32_e32 v0, s6, v0
; GFX9-NEXT: v_mov_b32_e32 v3, 0
-; GFX9-NEXT: v_sub_u32_e32 v2, s8, v2
+; GFX9-NEXT: v_sub_u32_e32 v0, s6, v0
+; GFX9-NEXT: v_sub_u32_sdwa v2, s8, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_sub_u32_e32 v1, s9, v1
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v0, v2, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_store_short v3, v1, s[0:1] offset:4
; GFX9-NEXT: global_store_dword v3, v0, s[0:1]
@@ -3760,16 +3750,15 @@ define amdgpu_kernel void @sdiv_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
; GFX9-NEXT: v_cvt_f32_i32_e32 v3, s0
; GFX9-NEXT: v_rcp_f32_e32 v1, v1
; GFX9-NEXT: v_trunc_f32_e32 v0, v0
-; GFX9-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX9-NEXT: v_trunc_f32_e32 v2, v2
; GFX9-NEXT: v_add_u32_e32 v3, 1, v3
; GFX9-NEXT: v_mul_f32_e32 v1, v3, v1
-; GFX9-NEXT: v_trunc_f32_e32 v2, v2
; GFX9-NEXT: v_trunc_f32_e32 v1, v1
-; GFX9-NEXT: v_cvt_i32_f32_e32 v2, v2
+; GFX9-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_i32_f32_sdwa v2, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
; GFX9-NEXT: v_cvt_i32_f32_e32 v1, v1
; GFX9-NEXT: v_mov_b32_e32 v3, 0
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v0, v2, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: global_store_short v3, v1, s[6:7] offset:4
; GFX9-NEXT: global_store_dword v3, v0, s[6:7]
; GFX9-NEXT: s_endpgm
@@ -3908,23 +3897,22 @@ define amdgpu_kernel void @srem_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
; GFX9-NEXT: v_cvt_f32_i32_e32 v3, s9
; GFX9-NEXT: v_rcp_f32_e32 v1, v1
; GFX9-NEXT: v_trunc_f32_e32 v0, v0
-; GFX9-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX9-NEXT: v_trunc_f32_e32 v2, v2
; GFX9-NEXT: v_add_u32_e32 v3, 1, v3
; GFX9-NEXT: v_mul_f32_e32 v1, v3, v1
-; GFX9-NEXT: v_trunc_f32_e32 v2, v2
; GFX9-NEXT: v_trunc_f32_e32 v1, v1
+; GFX9-NEXT: v_cvt_i32_f32_e32 v0, v0
; GFX9-NEXT: v_cvt_i32_f32_e32 v2, v2
; GFX9-NEXT: v_cvt_i32_f32_e32 v1, v1
-; GFX9-NEXT: v_mul_lo_u32 v0, v0, s7
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: v_mul_lo_u32 v0, v0, s7
; GFX9-NEXT: v_mul_lo_u32 v2, v2, s2
; GFX9-NEXT: v_mul_lo_u32 v1, v1, s3
-; GFX9-NEXT: v_sub_u32_e32 v0, s6, v0
; GFX9-NEXT: v_mov_b32_e32 v3, 0
-; GFX9-NEXT: v_sub_u32_e32 v2, s8, v2
+; GFX9-NEXT: v_sub_u32_e32 v0, s6, v0
+; GFX9-NEXT: v_sub_u32_sdwa v2, s8, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_sub_u32_e32 v1, s9, v1
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v0, v2, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_store_short v3, v1, s[0:1] offset:4
; GFX9-NEXT: global_store_dword v3, v0, s[0:1]
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
index 45e3cee156901..f4fd16eb83a80 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
@@ -6959,13 +6959,13 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no
; GFX10-NEXT: v_mov_b32_e32 v1, s20
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: buffer_load_ushort v3, v0, s[16:19], 0 offen offset:1024
-; GFX10-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v3, v0, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX10-NEXT: v_mov_b32_e32 v3, s20
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshl_or_b32 v0, v4, 16, v0
+; GFX10-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_mov_b32_e32 v5, v0
@@ -6997,16 +6997,16 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX908-NEXT: v_mov_b32_e32 v1, s20
; GFX908-NEXT: v_mov_b32_e32 v2, v0
; GFX908-NEXT: v_mov_b32_e32 v0, s20
-; GFX908-NEXT: v_mov_b32_e32 v1, s20
-; GFX908-NEXT: buffer_load_ushort v3, v0, s[16:19], 0 offen offset:1024
-; GFX908-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: buffer_load_ushort v4, v0, s[16:19], 0 offen offset:1024
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: s_waitcnt vmcnt(1)
-; GFX908-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshl_or_b32 v0, v4, 16, v0
+; GFX908-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX908-NEXT: v_mov_b32_e32 v3, s20
; GFX908-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -7249,13 +7249,13 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_fin
; GFX10-NEXT: v_mov_b32_e32 v2, s20
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1024
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v3
; GFX10-NEXT: v_mov_b32_e32 v3, s20
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshl_or_b32 v2, v4, 16, v1
+; GFX10-NEXT: v_or_b32_sdwa v2, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_pk_add_f16 v1, v2, v0
@@ -8158,13 +8158,13 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset(ptr addrsp
; GFX10-NEXT: v_mov_b32_e32 v1, s20
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: buffer_load_ushort v3, v0, s[16:19], 0 offen offset:1024
-; GFX10-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v3, v0, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX10-NEXT: v_mov_b32_e32 v3, s20
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshl_or_b32 v0, v4, 16, v0
+; GFX10-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_mov_b32_e32 v5, v0
@@ -8187,16 +8187,16 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset(ptr addrsp
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_mov_b32_e32 v1, s20
; GFX90A-NEXT: v_mov_b32_e32 v2, v0
; GFX90A-NEXT: v_mov_b32_e32 v0, s20
-; GFX90A-NEXT: v_mov_b32_e32 v1, s20
-; GFX90A-NEXT: buffer_load_ushort v3, v0, s[16:19], 0 offen offset:1024
-; GFX90A-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: buffer_load_ushort v4, v0, s[16:19], 0 offen offset:1024
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: s_waitcnt vmcnt(1)
-; GFX90A-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshl_or_b32 v0, v4, 16, v0
+; GFX90A-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX90A-NEXT: v_mov_b32_e32 v3, s20
; GFX90A-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -8217,16 +8217,16 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset(ptr addrsp
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX908-NEXT: v_mov_b32_e32 v1, s20
; GFX908-NEXT: v_mov_b32_e32 v2, v0
; GFX908-NEXT: v_mov_b32_e32 v0, s20
-; GFX908-NEXT: v_mov_b32_e32 v1, s20
-; GFX908-NEXT: buffer_load_ushort v3, v0, s[16:19], 0 offen offset:1024
-; GFX908-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: buffer_load_ushort v4, v0, s[16:19], 0 offen offset:1024
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: s_waitcnt vmcnt(1)
-; GFX908-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshl_or_b32 v0, v4, 16, v0
+; GFX908-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX908-NEXT: v_mov_b32_e32 v3, s20
; GFX908-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -8469,13 +8469,13 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset(ptr addrspace(
; GFX10-NEXT: v_mov_b32_e32 v2, s20
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1024
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v3
; GFX10-NEXT: v_mov_b32_e32 v3, s20
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshl_or_b32 v2, v4, 16, v1
+; GFX10-NEXT: v_or_b32_sdwa v2, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: .LBB23_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_pk_add_f16 v1, v2, v0
@@ -8498,15 +8498,15 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset(ptr addrspace(
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v1, s20
; GFX90A-NEXT: v_mov_b32_e32 v2, s20
-; GFX90A-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX90A-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: v_mov_b32_e32 v1, s20
+; GFX90A-NEXT: buffer_load_ushort v3, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: s_waitcnt vmcnt(1)
-; GFX90A-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v3
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshl_or_b32 v3, v4, 16, v1
+; GFX90A-NEXT: v_or_b32_sdwa v3, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX90A-NEXT: v_mov_b32_e32 v1, s20
; GFX90A-NEXT: .LBB23_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -8527,15 +8527,15 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset(ptr addrspace(
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v1, s20
; GFX908-NEXT: v_mov_b32_e32 v2, s20
-; GFX908-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX908-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: v_mov_b32_e32 v1, s20
+; GFX908-NEXT: buffer_load_ushort v3, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: s_waitcnt vmcnt(1)
-; GFX908-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v1, 16, v3
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshl_or_b32 v2, v4, 16, v1
+; GFX908-NEXT: v_or_b32_sdwa v2, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX908-NEXT: v_mov_b32_e32 v3, s20
; GFX908-NEXT: .LBB23_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -8772,13 +8772,13 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no
; GFX10-NEXT: v_mov_b32_e32 v1, s20
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: buffer_load_ushort v3, v0, s[16:19], 0 offen offset:1024
-; GFX10-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v3, v0, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX10-NEXT: v_mov_b32_e32 v3, s20
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshl_or_b32 v0, v4, 16, v0
+; GFX10-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_mov_b32_e32 v5, v0
@@ -8801,16 +8801,16 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no_remote_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_mov_b32_e32 v1, s20
; GFX90A-NEXT: v_mov_b32_e32 v2, v0
; GFX90A-NEXT: v_mov_b32_e32 v0, s20
-; GFX90A-NEXT: v_mov_b32_e32 v1, s20
-; GFX90A-NEXT: buffer_load_ushort v3, v0, s[16:19], 0 offen offset:1024
-; GFX90A-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: buffer_load_ushort v4, v0, s[16:19], 0 offen offset:1024
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: s_waitcnt vmcnt(1)
-; GFX90A-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshl_or_b32 v0, v4, 16, v0
+; GFX90A-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX90A-NEXT: v_mov_b32_e32 v3, s20
; GFX90A-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -8831,16 +8831,16 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no_remote_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX908-NEXT: v_mov_b32_e32 v1, s20
; GFX908-NEXT: v_mov_b32_e32 v2, v0
; GFX908-NEXT: v_mov_b32_e32 v0, s20
-; GFX908-NEXT: v_mov_b32_e32 v1, s20
-; GFX908-NEXT: buffer_load_ushort v3, v0, s[16:19], 0 offen offset:1024
-; GFX908-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: buffer_load_ushort v4, v0, s[16:19], 0 offen offset:1024
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: s_waitcnt vmcnt(1)
-; GFX908-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshl_or_b32 v0, v4, 16, v0
+; GFX908-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX908-NEXT: v_mov_b32_e32 v3, s20
; GFX908-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -9083,13 +9083,13 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_rem
; GFX10-NEXT: v_mov_b32_e32 v2, s20
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1024
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v3
; GFX10-NEXT: v_mov_b32_e32 v3, s20
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshl_or_b32 v2, v4, 16, v1
+; GFX10-NEXT: v_or_b32_sdwa v2, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_pk_add_f16 v1, v2, v0
@@ -9112,15 +9112,15 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_rem
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_remote_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v1, s20
; GFX90A-NEXT: v_mov_b32_e32 v2, s20
-; GFX90A-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX90A-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: v_mov_b32_e32 v1, s20
+; GFX90A-NEXT: buffer_load_ushort v3, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: s_waitcnt vmcnt(1)
-; GFX90A-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v3
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshl_or_b32 v3, v4, 16, v1
+; GFX90A-NEXT: v_or_b32_sdwa v3, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX90A-NEXT: v_mov_b32_e32 v1, s20
; GFX90A-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -9141,15 +9141,15 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_rem
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_remote_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v1, s20
; GFX908-NEXT: v_mov_b32_e32 v2, s20
-; GFX908-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX908-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: v_mov_b32_e32 v1, s20
+; GFX908-NEXT: buffer_load_ushort v3, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: s_waitcnt vmcnt(1)
-; GFX908-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v1, 16, v3
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshl_or_b32 v2, v4, 16, v1
+; GFX908-NEXT: v_or_b32_sdwa v2, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX908-NEXT: v_mov_b32_e32 v3, s20
; GFX908-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -9307,19 +9307,19 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v1, s16
; GFX942-NEXT: v_mov_b32_e32 v2, s16
-; GFX942-NEXT: buffer_load_ushort v4, v1, s[0:3], 0 offen offset:1024
-; GFX942-NEXT: buffer_load_ushort v5, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: v_mov_b32_e32 v1, s16
+; GFX942-NEXT: buffer_load_ushort v4, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: buffer_load_ushort v5, v1, s[0:3], 0 offen offset:1024
; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX942-NEXT: s_mov_b64 s[6:7], 0
; GFX942-NEXT: s_movk_i32 s8, 0x7fff
; GFX942-NEXT: s_mov_b32 s9, 0x7060302
; GFX942-NEXT: s_waitcnt vmcnt(1)
-; GFX942-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshl_or_b32 v0, v5, 16, v0
+; GFX942-NEXT: v_or_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX942-NEXT: v_mov_b32_e32 v4, s16
; GFX942-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -9472,14 +9472,14 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX10-NEXT: v_mov_b32_e32 v2, s20
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1024
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v3
; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshl_or_b32 v0, v4, 16, v1
+; GFX10-NEXT: v_or_b32_sdwa v0, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_mov_b32_e32 v4, s20
; GFX10-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -9517,19 +9517,19 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v1, s20
; GFX90A-NEXT: v_mov_b32_e32 v2, s20
-; GFX90A-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
-; GFX90A-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: v_mov_b32_e32 v1, s20
+; GFX90A-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: s_waitcnt vmcnt(1)
-; GFX90A-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshl_or_b32 v0, v5, 16, v0
+; GFX90A-NEXT: v_or_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX90A-NEXT: v_mov_b32_e32 v4, s20
; GFX90A-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -9564,19 +9564,19 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v1, s20
; GFX908-NEXT: v_mov_b32_e32 v2, s20
-; GFX908-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
-; GFX908-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: v_mov_b32_e32 v1, s20
+; GFX908-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX908-NEXT: s_mov_b64 s[6:7], 0
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: s_waitcnt vmcnt(1)
-; GFX908-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshl_or_b32 v0, v5, 16, v0
+; GFX908-NEXT: v_or_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX908-NEXT: v_mov_b32_e32 v4, s20
; GFX908-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -9782,19 +9782,19 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v1, s16
; GFX942-NEXT: v_mov_b32_e32 v2, s16
-; GFX942-NEXT: buffer_load_ushort v4, v1, s[0:3], 0 offen offset:1024
-; GFX942-NEXT: buffer_load_ushort v5, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: v_mov_b32_e32 v1, s16
+; GFX942-NEXT: buffer_load_ushort v4, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: buffer_load_ushort v5, v1, s[0:3], 0 offen offset:1024
; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX942-NEXT: s_mov_b64 s[6:7], 0
; GFX942-NEXT: s_movk_i32 s8, 0x7fff
; GFX942-NEXT: s_mov_b32 s9, 0x7060302
; GFX942-NEXT: s_waitcnt vmcnt(1)
-; GFX942-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshl_or_b32 v1, v5, 16, v0
+; GFX942-NEXT: v_or_b32_sdwa v1, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX942-NEXT: v_mov_b32_e32 v4, s16
; GFX942-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -9941,14 +9941,14 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX10-NEXT: v_mov_b32_e32 v2, s20
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1024
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v3
; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshl_or_b32 v1, v4, 16, v1
+; GFX10-NEXT: v_or_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_mov_b32_e32 v4, s20
; GFX10-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -9986,19 +9986,19 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v1, s20
; GFX90A-NEXT: v_mov_b32_e32 v2, s20
-; GFX90A-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
-; GFX90A-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: v_mov_b32_e32 v1, s20
+; GFX90A-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: s_waitcnt vmcnt(1)
-; GFX90A-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshl_or_b32 v1, v5, 16, v0
+; GFX90A-NEXT: v_or_b32_sdwa v1, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX90A-NEXT: v_mov_b32_e32 v4, s20
; GFX90A-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -10033,19 +10033,19 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v1, s20
; GFX908-NEXT: v_mov_b32_e32 v2, s20
-; GFX908-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
-; GFX908-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: v_mov_b32_e32 v1, s20
+; GFX908-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX908-NEXT: s_mov_b64 s[6:7], 0
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: s_waitcnt vmcnt(1)
-; GFX908-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshl_or_b32 v1, v5, 16, v0
+; GFX908-NEXT: v_or_b32_sdwa v1, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX908-NEXT: v_mov_b32_e32 v4, s20
; GFX908-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -11123,19 +11123,19 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset(ptr add
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v1, s16
; GFX942-NEXT: v_mov_b32_e32 v2, s16
-; GFX942-NEXT: buffer_load_ushort v4, v1, s[0:3], 0 offen offset:1024
-; GFX942-NEXT: buffer_load_ushort v5, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: v_mov_b32_e32 v1, s16
+; GFX942-NEXT: buffer_load_ushort v4, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: buffer_load_ushort v5, v1, s[0:3], 0 offen offset:1024
; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX942-NEXT: s_mov_b64 s[6:7], 0
; GFX942-NEXT: s_movk_i32 s8, 0x7fff
; GFX942-NEXT: s_mov_b32 s9, 0x7060302
; GFX942-NEXT: s_waitcnt vmcnt(1)
-; GFX942-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshl_or_b32 v0, v5, 16, v0
+; GFX942-NEXT: v_or_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX942-NEXT: v_mov_b32_e32 v4, s16
; GFX942-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -11288,14 +11288,14 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset(ptr add
; GFX10-NEXT: v_mov_b32_e32 v2, s20
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1024
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v3
; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshl_or_b32 v0, v4, 16, v1
+; GFX10-NEXT: v_or_b32_sdwa v0, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_mov_b32_e32 v4, s20
; GFX10-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -11333,19 +11333,19 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset(ptr add
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v1, s20
; GFX90A-NEXT: v_mov_b32_e32 v2, s20
-; GFX90A-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
-; GFX90A-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: v_mov_b32_e32 v1, s20
+; GFX90A-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: s_waitcnt vmcnt(1)
-; GFX90A-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshl_or_b32 v0, v5, 16, v0
+; GFX90A-NEXT: v_or_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX90A-NEXT: v_mov_b32_e32 v4, s20
; GFX90A-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -11380,19 +11380,19 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset(ptr add
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v1, s20
; GFX908-NEXT: v_mov_b32_e32 v2, s20
-; GFX908-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
-; GFX908-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: v_mov_b32_e32 v1, s20
+; GFX908-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX908-NEXT: s_mov_b64 s[6:7], 0
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: s_waitcnt vmcnt(1)
-; GFX908-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshl_or_b32 v0, v5, 16, v0
+; GFX908-NEXT: v_or_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX908-NEXT: v_mov_b32_e32 v4, s20
; GFX908-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -11598,19 +11598,19 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset(ptr addrspace
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v1, s16
; GFX942-NEXT: v_mov_b32_e32 v2, s16
-; GFX942-NEXT: buffer_load_ushort v4, v1, s[0:3], 0 offen offset:1024
-; GFX942-NEXT: buffer_load_ushort v5, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: v_mov_b32_e32 v1, s16
+; GFX942-NEXT: buffer_load_ushort v4, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: buffer_load_ushort v5, v1, s[0:3], 0 offen offset:1024
; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX942-NEXT: s_mov_b64 s[6:7], 0
; GFX942-NEXT: s_movk_i32 s8, 0x7fff
; GFX942-NEXT: s_mov_b32 s9, 0x7060302
; GFX942-NEXT: s_waitcnt vmcnt(1)
-; GFX942-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshl_or_b32 v1, v5, 16, v0
+; GFX942-NEXT: v_or_b32_sdwa v1, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX942-NEXT: v_mov_b32_e32 v4, s16
; GFX942-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -11757,14 +11757,14 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset(ptr addrspace
; GFX10-NEXT: v_mov_b32_e32 v2, s20
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1024
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v3
; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshl_or_b32 v1, v4, 16, v1
+; GFX10-NEXT: v_or_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_mov_b32_e32 v4, s20
; GFX10-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -11802,19 +11802,19 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset(ptr addrspace
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v1, s20
; GFX90A-NEXT: v_mov_b32_e32 v2, s20
-; GFX90A-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
-; GFX90A-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: v_mov_b32_e32 v1, s20
+; GFX90A-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: s_waitcnt vmcnt(1)
-; GFX90A-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshl_or_b32 v1, v5, 16, v0
+; GFX90A-NEXT: v_or_b32_sdwa v1, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX90A-NEXT: v_mov_b32_e32 v4, s20
; GFX90A-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -11849,19 +11849,19 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset(ptr addrspace
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v1, s20
; GFX908-NEXT: v_mov_b32_e32 v2, s20
-; GFX908-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
-; GFX908-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: v_mov_b32_e32 v1, s20
+; GFX908-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX908-NEXT: s_mov_b64 s[6:7], 0
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: s_waitcnt vmcnt(1)
-; GFX908-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshl_or_b32 v1, v5, 16, v0
+; GFX908-NEXT: v_or_b32_sdwa v1, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX908-NEXT: v_mov_b32_e32 v4, s20
; GFX908-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -12059,19 +12059,19 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_remote_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v1, s16
; GFX942-NEXT: v_mov_b32_e32 v2, s16
-; GFX942-NEXT: buffer_load_ushort v4, v1, s[0:3], 0 offen offset:1024
-; GFX942-NEXT: buffer_load_ushort v5, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: v_mov_b32_e32 v1, s16
+; GFX942-NEXT: buffer_load_ushort v4, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: buffer_load_ushort v5, v1, s[0:3], 0 offen offset:1024
; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX942-NEXT: s_mov_b64 s[6:7], 0
; GFX942-NEXT: s_movk_i32 s8, 0x7fff
; GFX942-NEXT: s_mov_b32 s9, 0x7060302
; GFX942-NEXT: s_waitcnt vmcnt(1)
-; GFX942-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshl_or_b32 v0, v5, 16, v0
+; GFX942-NEXT: v_or_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX942-NEXT: v_mov_b32_e32 v4, s16
; GFX942-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -12224,14 +12224,14 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX10-NEXT: v_mov_b32_e32 v2, s20
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1024
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v3
; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshl_or_b32 v0, v4, 16, v1
+; GFX10-NEXT: v_or_b32_sdwa v0, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_mov_b32_e32 v4, s20
; GFX10-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -12269,19 +12269,19 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_remote_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v1, s20
; GFX90A-NEXT: v_mov_b32_e32 v2, s20
-; GFX90A-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
-; GFX90A-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: v_mov_b32_e32 v1, s20
+; GFX90A-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: s_waitcnt vmcnt(1)
-; GFX90A-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshl_or_b32 v0, v5, 16, v0
+; GFX90A-NEXT: v_or_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX90A-NEXT: v_mov_b32_e32 v4, s20
; GFX90A-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -12316,19 +12316,19 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_remote_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v1, s20
; GFX908-NEXT: v_mov_b32_e32 v2, s20
-; GFX908-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
-; GFX908-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: v_mov_b32_e32 v1, s20
+; GFX908-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX908-NEXT: s_mov_b64 s[6:7], 0
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: s_waitcnt vmcnt(1)
-; GFX908-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshl_or_b32 v0, v5, 16, v0
+; GFX908-NEXT: v_or_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX908-NEXT: v_mov_b32_e32 v4, s20
; GFX908-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -12534,19 +12534,19 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_re
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_remote_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v1, s16
; GFX942-NEXT: v_mov_b32_e32 v2, s16
-; GFX942-NEXT: buffer_load_ushort v4, v1, s[0:3], 0 offen offset:1024
-; GFX942-NEXT: buffer_load_ushort v5, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: v_mov_b32_e32 v1, s16
+; GFX942-NEXT: buffer_load_ushort v4, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: buffer_load_ushort v5, v1, s[0:3], 0 offen offset:1024
; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX942-NEXT: s_mov_b64 s[6:7], 0
; GFX942-NEXT: s_movk_i32 s8, 0x7fff
; GFX942-NEXT: s_mov_b32 s9, 0x7060302
; GFX942-NEXT: s_waitcnt vmcnt(1)
-; GFX942-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshl_or_b32 v1, v5, 16, v0
+; GFX942-NEXT: v_or_b32_sdwa v1, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX942-NEXT: v_mov_b32_e32 v4, s16
; GFX942-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -12693,14 +12693,14 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_re
; GFX10-NEXT: v_mov_b32_e32 v2, s20
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1024
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v3
; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshl_or_b32 v1, v4, 16, v1
+; GFX10-NEXT: v_or_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_mov_b32_e32 v4, s20
; GFX10-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -12738,19 +12738,19 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_re
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_remote_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v1, s20
; GFX90A-NEXT: v_mov_b32_e32 v2, s20
-; GFX90A-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
-; GFX90A-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: v_mov_b32_e32 v1, s20
+; GFX90A-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: s_waitcnt vmcnt(1)
-; GFX90A-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshl_or_b32 v1, v5, 16, v0
+; GFX90A-NEXT: v_or_b32_sdwa v1, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX90A-NEXT: v_mov_b32_e32 v4, s20
; GFX90A-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -12785,19 +12785,19 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_re
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_remote_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v1, s20
; GFX908-NEXT: v_mov_b32_e32 v2, s20
-; GFX908-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
-; GFX908-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: v_mov_b32_e32 v1, s20
+; GFX908-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX908-NEXT: s_mov_b64 s[6:7], 0
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: s_waitcnt vmcnt(1)
-; GFX908-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshl_or_b32 v1, v5, 16, v0
+; GFX908-NEXT: v_or_b32_sdwa v1, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX908-NEXT: v_mov_b32_e32 v4, s20
; GFX908-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -12995,19 +12995,19 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v1, s16
; GFX942-NEXT: v_mov_b32_e32 v2, s16
-; GFX942-NEXT: buffer_load_ushort v4, v1, s[0:3], 0 offen offset:1024
-; GFX942-NEXT: buffer_load_ushort v5, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: v_mov_b32_e32 v1, s16
+; GFX942-NEXT: buffer_load_ushort v4, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: buffer_load_ushort v5, v1, s[0:3], 0 offen offset:1024
; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX942-NEXT: s_mov_b64 s[6:7], 0
; GFX942-NEXT: s_movk_i32 s8, 0x7fff
; GFX942-NEXT: s_mov_b32 s9, 0x7060302
; GFX942-NEXT: s_waitcnt vmcnt(1)
-; GFX942-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshl_or_b32 v1, v5, 16, v0
+; GFX942-NEXT: v_or_b32_sdwa v1, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX942-NEXT: v_mov_b32_e32 v4, s16
; GFX942-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -13154,14 +13154,14 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX10-NEXT: v_mov_b32_e32 v2, s20
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1024
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v3
; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshl_or_b32 v1, v4, 16, v1
+; GFX10-NEXT: v_or_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_mov_b32_e32 v4, s20
; GFX10-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -13199,19 +13199,19 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v1, s20
; GFX90A-NEXT: v_mov_b32_e32 v2, s20
-; GFX90A-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
-; GFX90A-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: v_mov_b32_e32 v1, s20
+; GFX90A-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: s_waitcnt vmcnt(1)
-; GFX90A-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshl_or_b32 v1, v5, 16, v0
+; GFX90A-NEXT: v_or_b32_sdwa v1, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX90A-NEXT: v_mov_b32_e32 v4, s20
; GFX90A-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -13246,19 +13246,19 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v1, s20
; GFX908-NEXT: v_mov_b32_e32 v2, s20
-; GFX908-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
-; GFX908-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: v_mov_b32_e32 v1, s20
+; GFX908-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX908-NEXT: s_mov_b64 s[6:7], 0
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: s_waitcnt vmcnt(1)
-; GFX908-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshl_or_b32 v1, v5, 16, v0
+; GFX908-NEXT: v_or_b32_sdwa v1, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX908-NEXT: v_mov_b32_e32 v4, s20
; GFX908-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
index 1078b38f52f4c..7a0f3a40af673 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
@@ -6074,16 +6074,16 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v1, s16
; GFX942-NEXT: v_mov_b32_e32 v2, s16
-; GFX942-NEXT: buffer_load_ushort v3, v1, s[0:3], 0 offen offset:1024
-; GFX942-NEXT: buffer_load_ushort v4, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: v_mov_b32_e32 v1, s16
+; GFX942-NEXT: buffer_load_ushort v3, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: buffer_load_ushort v4, v1, s[0:3], 0 offen offset:1024
; GFX942-NEXT: v_pk_max_f16 v2, v0, v0
; GFX942-NEXT: s_mov_b64 s[4:5], 0
; GFX942-NEXT: s_waitcnt vmcnt(1)
-; GFX942-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshl_or_b32 v0, v4, 16, v0
+; GFX942-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX942-NEXT: v_mov_b32_e32 v3, s16
; GFX942-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -6183,14 +6183,14 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no
; GFX10-NEXT: v_mov_b32_e32 v2, s20
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1024
; GFX10-NEXT: v_pk_max_f16 v2, v0, v0
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v3
; GFX10-NEXT: v_mov_b32_e32 v3, s20
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshl_or_b32 v0, v4, 16, v1
+; GFX10-NEXT: v_or_b32_sdwa v0, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_mov_b32_e32 v5, v0
@@ -6214,16 +6214,16 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v1, s20
; GFX90A-NEXT: v_mov_b32_e32 v2, s20
-; GFX90A-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX90A-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: v_mov_b32_e32 v1, s20
+; GFX90A-NEXT: buffer_load_ushort v3, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
; GFX90A-NEXT: v_pk_max_f16 v2, v0, v0
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: s_waitcnt vmcnt(1)
-; GFX90A-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshl_or_b32 v0, v4, 16, v0
+; GFX90A-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX90A-NEXT: v_mov_b32_e32 v3, s20
; GFX90A-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -6245,16 +6245,16 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v1, s20
; GFX908-NEXT: v_mov_b32_e32 v2, s20
-; GFX908-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX908-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: v_mov_b32_e32 v1, s20
+; GFX908-NEXT: buffer_load_ushort v3, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
; GFX908-NEXT: v_pk_max_f16 v2, v0, v0
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: s_waitcnt vmcnt(1)
-; GFX908-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshl_or_b32 v0, v4, 16, v0
+; GFX908-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX908-NEXT: v_mov_b32_e32 v3, s20
; GFX908-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -6482,16 +6482,16 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fin
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v1, s16
; GFX942-NEXT: v_mov_b32_e32 v2, s16
-; GFX942-NEXT: buffer_load_ushort v3, v1, s[0:3], 0 offen offset:1024
-; GFX942-NEXT: buffer_load_ushort v4, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: v_mov_b32_e32 v1, s16
+; GFX942-NEXT: buffer_load_ushort v3, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: buffer_load_ushort v4, v1, s[0:3], 0 offen offset:1024
; GFX942-NEXT: v_pk_max_f16 v2, v0, v0
; GFX942-NEXT: s_mov_b64 s[4:5], 0
; GFX942-NEXT: s_waitcnt vmcnt(1)
-; GFX942-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshl_or_b32 v1, v4, 16, v0
+; GFX942-NEXT: v_or_b32_sdwa v1, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX942-NEXT: v_mov_b32_e32 v3, s16
; GFX942-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -6591,14 +6591,14 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fin
; GFX10-NEXT: v_mov_b32_e32 v2, s20
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1024
; GFX10-NEXT: v_pk_max_f16 v2, v0, v0
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v3
; GFX10-NEXT: v_mov_b32_e32 v3, s20
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshl_or_b32 v1, v4, 16, v1
+; GFX10-NEXT: v_or_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_pk_max_f16 v0, v1, v1
@@ -6622,16 +6622,16 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fin
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v1, s20
; GFX90A-NEXT: v_mov_b32_e32 v2, s20
-; GFX90A-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX90A-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: v_mov_b32_e32 v1, s20
+; GFX90A-NEXT: buffer_load_ushort v3, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
; GFX90A-NEXT: v_pk_max_f16 v2, v0, v0
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: s_waitcnt vmcnt(1)
-; GFX90A-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshl_or_b32 v1, v4, 16, v0
+; GFX90A-NEXT: v_or_b32_sdwa v1, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX90A-NEXT: v_mov_b32_e32 v3, s20
; GFX90A-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -6653,16 +6653,16 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fin
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v1, s20
; GFX908-NEXT: v_mov_b32_e32 v2, s20
-; GFX908-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX908-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: v_mov_b32_e32 v1, s20
+; GFX908-NEXT: buffer_load_ushort v3, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
; GFX908-NEXT: v_pk_max_f16 v2, v0, v0
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: s_waitcnt vmcnt(1)
-; GFX908-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshl_or_b32 v1, v4, 16, v0
+; GFX908-NEXT: v_or_b32_sdwa v1, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX908-NEXT: v_mov_b32_e32 v3, s20
; GFX908-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -7830,19 +7830,19 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v1, s16
; GFX942-NEXT: v_mov_b32_e32 v2, s16
-; GFX942-NEXT: buffer_load_ushort v4, v1, s[0:3], 0 offen offset:1024
-; GFX942-NEXT: buffer_load_ushort v5, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: v_mov_b32_e32 v1, s16
+; GFX942-NEXT: buffer_load_ushort v4, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: buffer_load_ushort v5, v1, s[0:3], 0 offen offset:1024
; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX942-NEXT: s_mov_b64 s[6:7], 0
; GFX942-NEXT: s_movk_i32 s8, 0x7fff
; GFX942-NEXT: s_mov_b32 s9, 0x7060302
; GFX942-NEXT: s_waitcnt vmcnt(1)
-; GFX942-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshl_or_b32 v0, v5, 16, v0
+; GFX942-NEXT: v_or_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX942-NEXT: v_mov_b32_e32 v4, s16
; GFX942-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -7995,14 +7995,14 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu
; GFX10-NEXT: v_mov_b32_e32 v2, s20
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1024
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v3
; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshl_or_b32 v0, v4, 16, v1
+; GFX10-NEXT: v_or_b32_sdwa v0, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_mov_b32_e32 v4, s20
; GFX10-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -8040,19 +8040,19 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v1, s20
; GFX90A-NEXT: v_mov_b32_e32 v2, s20
-; GFX90A-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
-; GFX90A-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: v_mov_b32_e32 v1, s20
+; GFX90A-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: s_waitcnt vmcnt(1)
-; GFX90A-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshl_or_b32 v0, v5, 16, v0
+; GFX90A-NEXT: v_or_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX90A-NEXT: v_mov_b32_e32 v4, s20
; GFX90A-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -8087,19 +8087,19 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v1, s20
; GFX908-NEXT: v_mov_b32_e32 v2, s20
-; GFX908-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
-; GFX908-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: v_mov_b32_e32 v1, s20
+; GFX908-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX908-NEXT: s_mov_b64 s[6:7], 0
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: s_waitcnt vmcnt(1)
-; GFX908-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshl_or_b32 v0, v5, 16, v0
+; GFX908-NEXT: v_or_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX908-NEXT: v_mov_b32_e32 v4, s20
; GFX908-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -8401,19 +8401,19 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fi
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v1, s16
; GFX942-NEXT: v_mov_b32_e32 v2, s16
-; GFX942-NEXT: buffer_load_ushort v4, v1, s[0:3], 0 offen offset:1024
-; GFX942-NEXT: buffer_load_ushort v5, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: v_mov_b32_e32 v1, s16
+; GFX942-NEXT: buffer_load_ushort v4, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: buffer_load_ushort v5, v1, s[0:3], 0 offen offset:1024
; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX942-NEXT: s_mov_b64 s[6:7], 0
; GFX942-NEXT: s_movk_i32 s8, 0x7fff
; GFX942-NEXT: s_mov_b32 s9, 0x7060302
; GFX942-NEXT: s_waitcnt vmcnt(1)
-; GFX942-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshl_or_b32 v1, v5, 16, v0
+; GFX942-NEXT: v_or_b32_sdwa v1, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX942-NEXT: v_mov_b32_e32 v4, s16
; GFX942-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -8560,14 +8560,14 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fi
; GFX10-NEXT: v_mov_b32_e32 v2, s20
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1024
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v3
; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshl_or_b32 v1, v4, 16, v1
+; GFX10-NEXT: v_or_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_mov_b32_e32 v4, s20
; GFX10-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -8605,19 +8605,19 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fi
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v1, s20
; GFX90A-NEXT: v_mov_b32_e32 v2, s20
-; GFX90A-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
-; GFX90A-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: v_mov_b32_e32 v1, s20
+; GFX90A-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: s_waitcnt vmcnt(1)
-; GFX90A-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshl_or_b32 v1, v5, 16, v0
+; GFX90A-NEXT: v_or_b32_sdwa v1, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX90A-NEXT: v_mov_b32_e32 v4, s20
; GFX90A-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -8652,19 +8652,19 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fi
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v1, s20
; GFX908-NEXT: v_mov_b32_e32 v2, s20
-; GFX908-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
-; GFX908-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: v_mov_b32_e32 v1, s20
+; GFX908-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX908-NEXT: s_mov_b64 s[6:7], 0
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: s_waitcnt vmcnt(1)
-; GFX908-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshl_or_b32 v1, v5, 16, v0
+; GFX908-NEXT: v_or_b32_sdwa v1, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX908-NEXT: v_mov_b32_e32 v4, s20
; GFX908-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
index e5a87c78d54d2..4eab0eafcb13a 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
@@ -6074,16 +6074,16 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v1, s16
; GFX942-NEXT: v_mov_b32_e32 v2, s16
-; GFX942-NEXT: buffer_load_ushort v3, v1, s[0:3], 0 offen offset:1024
-; GFX942-NEXT: buffer_load_ushort v4, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: v_mov_b32_e32 v1, s16
+; GFX942-NEXT: buffer_load_ushort v3, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: buffer_load_ushort v4, v1, s[0:3], 0 offen offset:1024
; GFX942-NEXT: v_pk_max_f16 v2, v0, v0
; GFX942-NEXT: s_mov_b64 s[4:5], 0
; GFX942-NEXT: s_waitcnt vmcnt(1)
-; GFX942-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshl_or_b32 v0, v4, 16, v0
+; GFX942-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX942-NEXT: v_mov_b32_e32 v3, s16
; GFX942-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -6183,14 +6183,14 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no
; GFX10-NEXT: v_mov_b32_e32 v2, s20
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1024
; GFX10-NEXT: v_pk_max_f16 v2, v0, v0
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v3
; GFX10-NEXT: v_mov_b32_e32 v3, s20
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshl_or_b32 v0, v4, 16, v1
+; GFX10-NEXT: v_or_b32_sdwa v0, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_mov_b32_e32 v5, v0
@@ -6214,16 +6214,16 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v1, s20
; GFX90A-NEXT: v_mov_b32_e32 v2, s20
-; GFX90A-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX90A-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: v_mov_b32_e32 v1, s20
+; GFX90A-NEXT: buffer_load_ushort v3, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
; GFX90A-NEXT: v_pk_max_f16 v2, v0, v0
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: s_waitcnt vmcnt(1)
-; GFX90A-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshl_or_b32 v0, v4, 16, v0
+; GFX90A-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX90A-NEXT: v_mov_b32_e32 v3, s20
; GFX90A-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -6245,16 +6245,16 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v1, s20
; GFX908-NEXT: v_mov_b32_e32 v2, s20
-; GFX908-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX908-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: v_mov_b32_e32 v1, s20
+; GFX908-NEXT: buffer_load_ushort v3, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
; GFX908-NEXT: v_pk_max_f16 v2, v0, v0
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: s_waitcnt vmcnt(1)
-; GFX908-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshl_or_b32 v0, v4, 16, v0
+; GFX908-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX908-NEXT: v_mov_b32_e32 v3, s20
; GFX908-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -6482,16 +6482,16 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fin
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v1, s16
; GFX942-NEXT: v_mov_b32_e32 v2, s16
-; GFX942-NEXT: buffer_load_ushort v3, v1, s[0:3], 0 offen offset:1024
-; GFX942-NEXT: buffer_load_ushort v4, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: v_mov_b32_e32 v1, s16
+; GFX942-NEXT: buffer_load_ushort v3, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: buffer_load_ushort v4, v1, s[0:3], 0 offen offset:1024
; GFX942-NEXT: v_pk_max_f16 v2, v0, v0
; GFX942-NEXT: s_mov_b64 s[4:5], 0
; GFX942-NEXT: s_waitcnt vmcnt(1)
-; GFX942-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshl_or_b32 v1, v4, 16, v0
+; GFX942-NEXT: v_or_b32_sdwa v1, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX942-NEXT: v_mov_b32_e32 v3, s16
; GFX942-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -6591,14 +6591,14 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fin
; GFX10-NEXT: v_mov_b32_e32 v2, s20
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1024
; GFX10-NEXT: v_pk_max_f16 v2, v0, v0
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v3
; GFX10-NEXT: v_mov_b32_e32 v3, s20
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshl_or_b32 v1, v4, 16, v1
+; GFX10-NEXT: v_or_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_pk_max_f16 v0, v1, v1
@@ -6622,16 +6622,16 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fin
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v1, s20
; GFX90A-NEXT: v_mov_b32_e32 v2, s20
-; GFX90A-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX90A-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: v_mov_b32_e32 v1, s20
+; GFX90A-NEXT: buffer_load_ushort v3, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
; GFX90A-NEXT: v_pk_max_f16 v2, v0, v0
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: s_waitcnt vmcnt(1)
-; GFX90A-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshl_or_b32 v1, v4, 16, v0
+; GFX90A-NEXT: v_or_b32_sdwa v1, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX90A-NEXT: v_mov_b32_e32 v3, s20
; GFX90A-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -6653,16 +6653,16 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fin
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v1, s20
; GFX908-NEXT: v_mov_b32_e32 v2, s20
-; GFX908-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX908-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: v_mov_b32_e32 v1, s20
+; GFX908-NEXT: buffer_load_ushort v3, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
; GFX908-NEXT: v_pk_max_f16 v2, v0, v0
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: s_waitcnt vmcnt(1)
-; GFX908-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshl_or_b32 v1, v4, 16, v0
+; GFX908-NEXT: v_or_b32_sdwa v1, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX908-NEXT: v_mov_b32_e32 v3, s20
; GFX908-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -7830,19 +7830,19 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v1, s16
; GFX942-NEXT: v_mov_b32_e32 v2, s16
-; GFX942-NEXT: buffer_load_ushort v4, v1, s[0:3], 0 offen offset:1024
-; GFX942-NEXT: buffer_load_ushort v5, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: v_mov_b32_e32 v1, s16
+; GFX942-NEXT: buffer_load_ushort v4, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: buffer_load_ushort v5, v1, s[0:3], 0 offen offset:1024
; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX942-NEXT: s_mov_b64 s[6:7], 0
; GFX942-NEXT: s_movk_i32 s8, 0x7fff
; GFX942-NEXT: s_mov_b32 s9, 0x7060302
; GFX942-NEXT: s_waitcnt vmcnt(1)
-; GFX942-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshl_or_b32 v0, v5, 16, v0
+; GFX942-NEXT: v_or_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX942-NEXT: v_mov_b32_e32 v4, s16
; GFX942-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -7995,14 +7995,14 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu
; GFX10-NEXT: v_mov_b32_e32 v2, s20
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1024
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v3
; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshl_or_b32 v0, v4, 16, v1
+; GFX10-NEXT: v_or_b32_sdwa v0, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_mov_b32_e32 v4, s20
; GFX10-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -8040,19 +8040,19 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v1, s20
; GFX90A-NEXT: v_mov_b32_e32 v2, s20
-; GFX90A-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
-; GFX90A-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: v_mov_b32_e32 v1, s20
+; GFX90A-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: s_waitcnt vmcnt(1)
-; GFX90A-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshl_or_b32 v0, v5, 16, v0
+; GFX90A-NEXT: v_or_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX90A-NEXT: v_mov_b32_e32 v4, s20
; GFX90A-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -8087,19 +8087,19 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v1, s20
; GFX908-NEXT: v_mov_b32_e32 v2, s20
-; GFX908-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
-; GFX908-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: v_mov_b32_e32 v1, s20
+; GFX908-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX908-NEXT: s_mov_b64 s[6:7], 0
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: s_waitcnt vmcnt(1)
-; GFX908-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshl_or_b32 v0, v5, 16, v0
+; GFX908-NEXT: v_or_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX908-NEXT: v_mov_b32_e32 v4, s20
; GFX908-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -8401,19 +8401,19 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fi
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v1, s16
; GFX942-NEXT: v_mov_b32_e32 v2, s16
-; GFX942-NEXT: buffer_load_ushort v4, v1, s[0:3], 0 offen offset:1024
-; GFX942-NEXT: buffer_load_ushort v5, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: v_mov_b32_e32 v1, s16
+; GFX942-NEXT: buffer_load_ushort v4, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: buffer_load_ushort v5, v1, s[0:3], 0 offen offset:1024
; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX942-NEXT: s_mov_b64 s[6:7], 0
; GFX942-NEXT: s_movk_i32 s8, 0x7fff
; GFX942-NEXT: s_mov_b32 s9, 0x7060302
; GFX942-NEXT: s_waitcnt vmcnt(1)
-; GFX942-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshl_or_b32 v1, v5, 16, v0
+; GFX942-NEXT: v_or_b32_sdwa v1, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX942-NEXT: v_mov_b32_e32 v4, s16
; GFX942-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -8560,14 +8560,14 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fi
; GFX10-NEXT: v_mov_b32_e32 v2, s20
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1024
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v3
; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshl_or_b32 v1, v4, 16, v1
+; GFX10-NEXT: v_or_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_mov_b32_e32 v4, s20
; GFX10-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -8605,19 +8605,19 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fi
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v1, s20
; GFX90A-NEXT: v_mov_b32_e32 v2, s20
-; GFX90A-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
-; GFX90A-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: v_mov_b32_e32 v1, s20
+; GFX90A-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: s_waitcnt vmcnt(1)
-; GFX90A-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshl_or_b32 v1, v5, 16, v0
+; GFX90A-NEXT: v_or_b32_sdwa v1, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX90A-NEXT: v_mov_b32_e32 v4, s20
; GFX90A-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -8652,19 +8652,19 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fi
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v1, s20
; GFX908-NEXT: v_mov_b32_e32 v2, s20
-; GFX908-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
-; GFX908-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: v_mov_b32_e32 v1, s20
+; GFX908-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX908-NEXT: s_mov_b64 s[6:7], 0
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: s_waitcnt vmcnt(1)
-; GFX908-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshl_or_b32 v1, v5, 16, v0
+; GFX908-NEXT: v_or_b32_sdwa v1, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX908-NEXT: v_mov_b32_e32 v4, s20
; GFX908-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-contents-legalization.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-contents-legalization.ll
index b8b323aa83851..fd469c3f069bf 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-contents-legalization.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-contents-legalization.ll
@@ -2333,11 +2333,10 @@ define void @store_v6i8(<6 x i8> %data, ptr addrspace(8) inreg %buf) {
; GISEL-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GISEL-NEXT: v_and_b32_e32 v1, 0xff, v3
; GISEL-NEXT: v_lshlrev_b16_e32 v1, 8, v1
-; GISEL-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GISEL-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GISEL-NEXT: v_and_b32_e32 v2, 0xff, v5
-; GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GISEL-NEXT: v_lshlrev_b16_e32 v2, 8, v2
-; GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GISEL-NEXT: v_or_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GISEL-NEXT: buffer_store_dword v0, off, s[16:19], 0
; GISEL-NEXT: buffer_store_short v2, off, s[16:19], 0 offset:4
@@ -3862,11 +3861,10 @@ define void @volatile_store_v6i8(<6 x i8> %data, ptr addrspace(8) inreg %buf) {
; GISEL-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GISEL-NEXT: v_and_b32_e32 v1, 0xff, v3
; GISEL-NEXT: v_lshlrev_b16_e32 v1, 8, v1
-; GISEL-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GISEL-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GISEL-NEXT: v_and_b32_e32 v2, 0xff, v5
-; GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GISEL-NEXT: v_lshlrev_b16_e32 v2, 8, v2
-; GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GISEL-NEXT: v_or_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GISEL-NEXT: buffer_store_dword v0, off, s[16:19], 0
; GISEL-NEXT: s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/build-vector-packed-partial-undef.ll b/llvm/test/CodeGen/AMDGPU/build-vector-packed-partial-undef.ll
index 39f482ff71a22..466b475ed487d 100644
--- a/llvm/test/CodeGen/AMDGPU/build-vector-packed-partial-undef.ll
+++ b/llvm/test/CodeGen/AMDGPU/build-vector-packed-partial-undef.ll
@@ -624,8 +624,8 @@ define void @undef_hi_v2i16(i16 %arg0) {
; GFX9-GISEL-LABEL: undef_hi_v2i16:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, s4, 16, v0
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e64 v1, 16, s4
+; GFX9-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-GISEL-NEXT: ;;#ASMSTART
; GFX9-GISEL-NEXT: ; use v0
; GFX9-GISEL-NEXT: ;;#ASMEND
@@ -691,8 +691,8 @@ define void @undef_hi_v2f16(half %arg0) {
; GFX9-GISEL-LABEL: undef_hi_v2f16:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, s4, 16, v0
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e64 v1, 16, s4
+; GFX9-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-GISEL-NEXT: ;;#ASMSTART
; GFX9-GISEL-NEXT: ; use v0
; GFX9-GISEL-NEXT: ;;#ASMEND
@@ -767,8 +767,8 @@ define void @undef_hi_op_v2f16(half %arg0) {
; GFX9-GISEL-LABEL: undef_hi_op_v2f16:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, s4, 16, v0
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e64 v1, 16, s4
+; GFX9-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-GISEL-NEXT: v_pk_add_f16 v0, v0, 1.0 op_sel_hi:[1,0]
; GFX9-GISEL-NEXT: ;;#ASMSTART
; GFX9-GISEL-NEXT: ; use v0
@@ -855,8 +855,8 @@ define void @undef_hi_op_v2i16(i16 %arg0) {
; GFX9-GISEL-LABEL: undef_hi_op_v2i16:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, s4, 16, v0
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e64 v1, 16, s4
+; GFX9-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0x630063
; GFX9-GISEL-NEXT: v_pk_add_u16 v0, v0, v1
; GFX9-GISEL-NEXT: ;;#ASMSTART
@@ -938,8 +938,8 @@ define void @undef_hi3_v4i16(i16 %arg0) {
; GFX9-GISEL-LABEL: undef_hi3_v4i16:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, s4, 16, v0
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e64 v1, 16, s4
+; GFX9-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-GISEL-NEXT: ;;#ASMSTART
; GFX9-GISEL-NEXT: ; use v[0:1]
; GFX9-GISEL-NEXT: ;;#ASMEND
@@ -1006,8 +1006,8 @@ define void @undef_hi3_v4f16(half %arg0) {
; GFX9-GISEL-LABEL: undef_hi3_v4f16:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, s4, 16, v0
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e64 v1, 16, s4
+; GFX9-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-GISEL-NEXT: ;;#ASMSTART
; GFX9-GISEL-NEXT: ; use v[0:1]
; GFX9-GISEL-NEXT: ;;#ASMEND
diff --git a/llvm/test/CodeGen/AMDGPU/constant-address-space-32bit.ll b/llvm/test/CodeGen/AMDGPU/constant-address-space-32bit.ll
index 22fbbe3fe33b8..32bfce4a2fc58 100644
--- a/llvm/test/CodeGen/AMDGPU/constant-address-space-32bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/constant-address-space-32bit.ll
@@ -2543,14 +2543,13 @@ define amdgpu_vs <2 x bfloat> @load_v2bf16(ptr addrspace(6) inreg %p0, ptr addrs
; GFX9-NEXT: v_add_f32_e32 v1, s1, v1
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
-; GFX9-NEXT: v_mov_b32_e32 v2, 0xffff
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: ; return to shader part epilog
%gep1 = getelementptr inbounds <2 x bfloat>, ptr addrspace(6) %p1, i32 2
%r0 = load <2 x bfloat>, ptr addrspace(6) %p0
diff --git a/llvm/test/CodeGen/AMDGPU/ctlz_zero_poison.ll b/llvm/test/CodeGen/AMDGPU/ctlz_zero_poison.ll
index 240cf3d042250..b9a47766cb089 100644
--- a/llvm/test/CodeGen/AMDGPU/ctlz_zero_poison.ll
+++ b/llvm/test/CodeGen/AMDGPU/ctlz_zero_poison.ll
@@ -2419,11 +2419,10 @@ define <2 x i16> @v_ctlz_zero_poison_v2i16(<2 x i16> %val) {
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v0, v0
; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v1, v1
-; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v0, v0
+; GFX9-GISEL-NEXT: v_ffbh_u32_sdwa v1, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; GFX9-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
%ctlz = call <2 x i16> @llvm.ctlz.v2i16(<2 x i16> %val, i1 true)
ret <2 x i16> %ctlz
@@ -2465,15 +2464,14 @@ define <3 x i16> @v_ctlz_zero_poison_v3i16(<3 x i16> %val) {
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v0, v0
; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v0, v0
+; GFX9-GISEL-NEXT: v_ffbh_u32_sdwa v2, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v1, v1
-; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v2, v2
-; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v2, 16, v0
-; GFX9-GISEL-NEXT: v_lshl_or_b32 v1, s4, 16, v1
+; GFX9-GISEL-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e64 v2, 16, s4
+; GFX9-GISEL-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
%ctlz = call <3 x i16> @llvm.ctlz.v3i16(<3 x i16> %val, i1 true)
ret <3 x i16> %ctlz
@@ -2524,17 +2522,15 @@ define <4 x i16> @v_ctlz_zero_poison_v4i16(<4 x i16> %val) {
; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v0, v0
-; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-GISEL-NEXT: v_ffbh_u32_sdwa v2, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v1, v1
-; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v2, v2
-; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v3, v3
-; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v2, 16, v0
-; GFX9-GISEL-NEXT: v_lshl_or_b32 v1, v3, 16, v1
+; GFX9-GISEL-NEXT: v_ffbh_u32_sdwa v3, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; GFX9-GISEL-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-GISEL-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
%ctlz = call <4 x i16> @llvm.ctlz.v4i16(<4 x i16> %val, i1 true)
ret <4 x i16> %ctlz
diff --git a/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll b/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
index 179eeb2cdf3aa..644d5074278f9 100644
--- a/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
+++ b/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
@@ -1676,27 +1676,27 @@ define amdgpu_kernel void @load_v4i8_to_v4f32_2_uses(ptr addrspace(1) noalias %o
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX10-NEXT: v_mov_b32_e32 v4, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: global_load_dword v0, v0, s[0:1]
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffffff00, v0
-; GFX10-NEXT: v_add_nc_u16 v2, v0, 9
-; GFX10-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; GFX10-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffffff00, v3
-; GFX10-NEXT: v_add_nc_u16 v3, v3, 9
-; GFX10-NEXT: v_add_nc_u16 v1, 0x900, v1
-; GFX10-NEXT: v_or_b32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX10-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX10-NEXT: v_and_b32_e32 v3, 0xffffff00, v0
+; GFX10-NEXT: v_add_nc_u16 v4, v0, 9
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffffff00, v1
+; GFX10-NEXT: v_add_nc_u16 v1, v1, 9
+; GFX10-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX10-NEXT: v_or_b32_sdwa v2, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX10-NEXT: v_mov_b32_e32 v4, 0
; GFX10-NEXT: v_cvt_f32_ubyte3_e32 v3, v0
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff, v1
-; GFX10-NEXT: v_cvt_f32_ubyte1_e32 v1, v0
-; GFX10-NEXT: v_add_nc_u16 v6, 0x900, v2
+; GFX10-NEXT: v_add_nc_u16 v1, 0x900, v1
+; GFX10-NEXT: v_add_nc_u16 v5, 0x900, v2
; GFX10-NEXT: v_cvt_f32_ubyte2_e32 v2, v0
+; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v1
+; GFX10-NEXT: v_cvt_f32_ubyte1_e32 v1, v0
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
-; GFX10-NEXT: v_lshl_or_b32 v5, v6, 16, v5
+; GFX10-NEXT: v_or_b32_sdwa v5, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX10-NEXT: global_store_dword v4, v5, s[2:3]
diff --git a/llvm/test/CodeGen/AMDGPU/fmaximum.ll b/llvm/test/CodeGen/AMDGPU/fmaximum.ll
index 98e24aa3340a3..213d6ffeb9f00 100644
--- a/llvm/test/CodeGen/AMDGPU/fmaximum.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmaximum.ll
@@ -511,13 +511,12 @@ define amdgpu_ps <2 x half> @test_fmaximum_v2f16_vv(<2 x half> %a, <2 x half> %b
; GFX9-GISEL-LABEL: test_fmaximum_v2f16_vv:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: v_pk_max_f16 v2, v0, v1
+; GFX9-GISEL-NEXT: v_cmp_o_f16_sdwa vcc, v0, v1 src0_sel:WORD_1 src1_sel:WORD_1
; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, 0x7e00
; GFX9-GISEL-NEXT: v_cmp_o_f16_e64 s[0:1], v0, v1
-; GFX9-GISEL-NEXT: v_cmp_o_f16_sdwa vcc, v0, v1 src0_sel:WORD_1 src1_sel:WORD_1
; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v0, v3, v2, s[0:1]
-; GFX9-GISEL-NEXT: v_cndmask_b32_sdwa v1, v3, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX9-GISEL-NEXT: v_cndmask_b32_sdwa v1, v3, v2, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-GISEL-NEXT: ; return to shader part epilog
;
; GFX1170-LABEL: test_fmaximum_v2f16_vv:
@@ -546,9 +545,8 @@ define amdgpu_ps <2 x half> @test_fmaximum_v2f16_ss(<2 x half> inreg %a, <2 x ha
; GFX9-SDAG-NEXT: v_mov_b32_e32 v3, s1
; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
; GFX9-SDAG-NEXT: v_cmp_o_f16_e32 vcc, s0, v3
-; GFX9-SDAG-NEXT: v_cndmask_b32_sdwa v1, v2, v1, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-SDAG-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX9-SDAG-NEXT: v_cndmask_b32_sdwa v1, v2, v1, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-SDAG-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-SDAG-NEXT: ; return to shader part epilog
;
; GFX9-GISEL-LABEL: test_fmaximum_v2f16_ss:
@@ -615,10 +613,10 @@ define amdgpu_ps <3 x half> @test_fmaximum_v3f16_vv(<3 x half> %a, <3 x half> %b
; GFX9-GISEL-NEXT: v_pk_max_f16 v2, v1, v3
; GFX9-GISEL-NEXT: v_cmp_o_f16_e32 vcc, v1, v3
; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v1, v6, v2, vcc
-; GFX9-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v4
-; GFX9-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v0, 16, v2
-; GFX9-GISEL-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e64 v2, 16, s0
+; GFX9-GISEL-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-GISEL-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-GISEL-NEXT: ; return to shader part epilog
;
; GFX1170-SDAG-LABEL: test_fmaximum_v3f16_vv:
@@ -684,9 +682,8 @@ define amdgpu_ps <3 x half> @test_fmaximum_v3f16_ss(<3 x half> inreg %a, <3 x ha
; GFX9-SDAG-NEXT: v_mov_b32_e32 v4, s1
; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
; GFX9-SDAG-NEXT: v_cmp_o_f16_e32 vcc, s0, v4
-; GFX9-SDAG-NEXT: v_cndmask_b32_sdwa v2, v2, v3, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-SDAG-NEXT: v_lshl_or_b32 v0, v2, 16, v0
+; GFX9-SDAG-NEXT: v_cndmask_b32_sdwa v2, v2, v3, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-SDAG-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-SDAG-NEXT: ; return to shader part epilog
;
; GFX9-GISEL-LABEL: test_fmaximum_v3f16_ss:
@@ -791,15 +788,14 @@ define amdgpu_ps <4 x half> @test_fmaximum_v4f16(<4 x half> %a, <4 x half> %b) {
; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc
; GFX9-GISEL-NEXT: v_cmp_o_f16_sdwa vcc, v0, v2 src0_sel:WORD_1 src1_sel:WORD_1
; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v0, v6, v5, vcc
-; GFX9-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v4
-; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v0, 16, v2
; GFX9-GISEL-NEXT: v_pk_max_f16 v2, v1, v3
-; GFX9-GISEL-NEXT: v_cmp_o_f16_e64 s[0:1], v1, v3
; GFX9-GISEL-NEXT: v_cmp_o_f16_sdwa vcc, v1, v3 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-GISEL-NEXT: v_cmp_o_f16_e64 s[0:1], v1, v3
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v1, v6, v2, s[0:1]
-; GFX9-GISEL-NEXT: v_cndmask_b32_sdwa v2, v6, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-GISEL-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX9-GISEL-NEXT: v_cndmask_b32_sdwa v2, v6, v2, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-GISEL-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-GISEL-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-GISEL-NEXT: ; return to shader part epilog
;
; GFX1170-LABEL: test_fmaximum_v4f16:
@@ -841,11 +837,10 @@ define amdgpu_ps <4 x half> @test_fmaximum_v4f16_ss(<4 x half> inreg %a, <4 x ha
; GFX9-SDAG-NEXT: v_mov_b32_e32 v5, s1
; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc
; GFX9-SDAG-NEXT: v_cmp_o_f16_e32 vcc, s0, v5
-; GFX9-SDAG-NEXT: v_cndmask_b32_sdwa v2, v2, v4, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-SDAG-NEXT: v_lshl_or_b32 v0, v2, 16, v0
-; GFX9-SDAG-NEXT: v_and_b32_e32 v2, 0xffff, v3
-; GFX9-SDAG-NEXT: v_lshl_or_b32 v1, v1, 16, v2
+; GFX9-SDAG-NEXT: v_cndmask_b32_sdwa v2, v2, v4, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-SDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-SDAG-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-SDAG-NEXT: v_or_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-SDAG-NEXT: ; return to shader part epilog
;
; GFX9-GISEL-LABEL: test_fmaximum_v4f16_ss:
diff --git a/llvm/test/CodeGen/AMDGPU/fminimum.ll b/llvm/test/CodeGen/AMDGPU/fminimum.ll
index 8ba95ee3550c0..094dd76e0f945 100644
--- a/llvm/test/CodeGen/AMDGPU/fminimum.ll
+++ b/llvm/test/CodeGen/AMDGPU/fminimum.ll
@@ -511,13 +511,12 @@ define amdgpu_ps <2 x half> @test_fminimum_v2f16_vv(<2 x half> %a, <2 x half> %b
; GFX9-GISEL-LABEL: test_fminimum_v2f16_vv:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: v_pk_min_f16 v2, v0, v1
+; GFX9-GISEL-NEXT: v_cmp_o_f16_sdwa vcc, v0, v1 src0_sel:WORD_1 src1_sel:WORD_1
; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, 0x7e00
; GFX9-GISEL-NEXT: v_cmp_o_f16_e64 s[0:1], v0, v1
-; GFX9-GISEL-NEXT: v_cmp_o_f16_sdwa vcc, v0, v1 src0_sel:WORD_1 src1_sel:WORD_1
; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v0, v3, v2, s[0:1]
-; GFX9-GISEL-NEXT: v_cndmask_b32_sdwa v1, v3, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX9-GISEL-NEXT: v_cndmask_b32_sdwa v1, v3, v2, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-GISEL-NEXT: ; return to shader part epilog
;
; GFX1170-LABEL: test_fminimum_v2f16_vv:
@@ -546,9 +545,8 @@ define amdgpu_ps <2 x half> @test_fminimum_v2f16_ss(<2 x half> inreg %a, <2 x ha
; GFX9-SDAG-NEXT: v_mov_b32_e32 v3, s1
; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
; GFX9-SDAG-NEXT: v_cmp_o_f16_e32 vcc, s0, v3
-; GFX9-SDAG-NEXT: v_cndmask_b32_sdwa v1, v2, v1, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-SDAG-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX9-SDAG-NEXT: v_cndmask_b32_sdwa v1, v2, v1, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-SDAG-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-SDAG-NEXT: ; return to shader part epilog
;
; GFX9-GISEL-LABEL: test_fminimum_v2f16_ss:
@@ -615,10 +613,10 @@ define amdgpu_ps <3 x half> @test_fminimum_v3f16_vv(<3 x half> %a, <3 x half> %b
; GFX9-GISEL-NEXT: v_pk_min_f16 v2, v1, v3
; GFX9-GISEL-NEXT: v_cmp_o_f16_e32 vcc, v1, v3
; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v1, v6, v2, vcc
-; GFX9-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v4
-; GFX9-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v0, 16, v2
-; GFX9-GISEL-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e64 v2, 16, s0
+; GFX9-GISEL-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-GISEL-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-GISEL-NEXT: ; return to shader part epilog
;
; GFX1170-SDAG-LABEL: test_fminimum_v3f16_vv:
@@ -684,9 +682,8 @@ define amdgpu_ps <3 x half> @test_fminimum_v3f16_ss(<3 x half> inreg %a, <3 x ha
; GFX9-SDAG-NEXT: v_mov_b32_e32 v4, s1
; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
; GFX9-SDAG-NEXT: v_cmp_o_f16_e32 vcc, s0, v4
-; GFX9-SDAG-NEXT: v_cndmask_b32_sdwa v2, v2, v3, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-SDAG-NEXT: v_lshl_or_b32 v0, v2, 16, v0
+; GFX9-SDAG-NEXT: v_cndmask_b32_sdwa v2, v2, v3, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-SDAG-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-SDAG-NEXT: ; return to shader part epilog
;
; GFX9-GISEL-LABEL: test_fminimum_v3f16_ss:
@@ -791,15 +788,14 @@ define amdgpu_ps <4 x half> @test_fminimum_v4f16(<4 x half> %a, <4 x half> %b) {
; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc
; GFX9-GISEL-NEXT: v_cmp_o_f16_sdwa vcc, v0, v2 src0_sel:WORD_1 src1_sel:WORD_1
; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v0, v6, v5, vcc
-; GFX9-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v4
-; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v0, 16, v2
; GFX9-GISEL-NEXT: v_pk_min_f16 v2, v1, v3
-; GFX9-GISEL-NEXT: v_cmp_o_f16_e64 s[0:1], v1, v3
; GFX9-GISEL-NEXT: v_cmp_o_f16_sdwa vcc, v1, v3 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-GISEL-NEXT: v_cmp_o_f16_e64 s[0:1], v1, v3
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v1, v6, v2, s[0:1]
-; GFX9-GISEL-NEXT: v_cndmask_b32_sdwa v2, v6, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-GISEL-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX9-GISEL-NEXT: v_cndmask_b32_sdwa v2, v6, v2, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-GISEL-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-GISEL-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-GISEL-NEXT: ; return to shader part epilog
;
; GFX1170-LABEL: test_fminimum_v4f16:
@@ -841,11 +837,10 @@ define amdgpu_ps <4 x half> @test_fminimum_v4f16_ss(<4 x half> inreg %a, <4 x ha
; GFX9-SDAG-NEXT: v_mov_b32_e32 v5, s1
; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc
; GFX9-SDAG-NEXT: v_cmp_o_f16_e32 vcc, s0, v5
-; GFX9-SDAG-NEXT: v_cndmask_b32_sdwa v2, v2, v4, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-SDAG-NEXT: v_lshl_or_b32 v0, v2, 16, v0
-; GFX9-SDAG-NEXT: v_and_b32_e32 v2, 0xffff, v3
-; GFX9-SDAG-NEXT: v_lshl_or_b32 v1, v1, 16, v2
+; GFX9-SDAG-NEXT: v_cndmask_b32_sdwa v2, v2, v4, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-SDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-SDAG-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-SDAG-NEXT: v_or_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-SDAG-NEXT: ; return to shader part epilog
;
; GFX9-GISEL-LABEL: test_fminimum_v4f16_ss:
diff --git a/llvm/test/CodeGen/AMDGPU/fmul-to-ldexp.ll b/llvm/test/CodeGen/AMDGPU/fmul-to-ldexp.ll
index 2ce2efcfe49ec..4980d15b0c8be 100644
--- a/llvm/test/CodeGen/AMDGPU/fmul-to-ldexp.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmul-to-ldexp.ll
@@ -6639,9 +6639,8 @@ define <2 x half> @v_mul_v2f16_select_64_1(<2 x i32> %arg, <2 x half> %x) {
; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
; GFX10-GISEL-NEXT: v_ldexp_f16_e32 v0, v2, v0
; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 6, vcc_lo
-; GFX10-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: v_mul_v2f16_select_64_1:
@@ -6743,9 +6742,8 @@ define <2 x half> @v_mul_v2f16_select_1_64(<2 x i32> %arg, <2 x half> %x) {
; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
; GFX10-GISEL-NEXT: v_ldexp_f16_e32 v0, v2, v0
; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v1, 6, 0, vcc_lo
-; GFX10-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: v_mul_v2f16_select_1_64:
@@ -6849,9 +6847,8 @@ define <2 x half> @v_mul_v2f16_select_n1_n64(<2 x i32> %arg, <2 x half> %x) {
; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
; GFX10-GISEL-NEXT: v_ldexp_f16_e32 v0, v2, v0
; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v1, 6, 0, vcc_lo
-; GFX10-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: v_mul_v2f16_select_n1_n64:
@@ -7111,9 +7108,8 @@ define <2 x half> @v_mul_v2f16_select_n128_n16(<2 x i32> %arg, <2 x half> %x) {
; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
; GFX10-GISEL-NEXT: v_ldexp_f16_e32 v0, v2, v0
; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v1, 4, 7, vcc_lo
-; GFX10-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: v_mul_v2f16_select_n128_n16:
@@ -7218,9 +7214,8 @@ define <2 x half> @v_contract_mul_add_v2f16_select_64_1(<2 x i32> %arg, <2 x hal
; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
; GFX10-GISEL-NEXT: v_ldexp_f16_e32 v0, v2, v0
; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 6, vcc_lo
-; GFX10-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-GISEL-NEXT: v_pk_add_f16 v0, v0, v3
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
@@ -7327,9 +7322,8 @@ define <2 x half> @v_contract_mul_add_v2f16_select_1_64(<2 x i32> %arg, <2 x hal
; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
; GFX10-GISEL-NEXT: v_ldexp_f16_e32 v0, v2, v0
; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v1, 6, 0, vcc_lo
-; GFX10-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-GISEL-NEXT: v_pk_add_f16 v0, v0, v3
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
@@ -7438,9 +7432,8 @@ define <2 x half> @v_contract_mul_add_v2f16_select_n64_n1(<2 x i32> %arg, <2 x h
; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
; GFX10-GISEL-NEXT: v_ldexp_f16_e32 v0, v2, v0
; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 6, vcc_lo
-; GFX10-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-GISEL-NEXT: v_pk_add_f16 v0, v0, v3
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
@@ -7551,9 +7544,8 @@ define <2 x half> @v_contract_mul_add_v2f16_select_n1_n64(<2 x i32> %arg, <2 x h
; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
; GFX10-GISEL-NEXT: v_ldexp_f16_e32 v0, v2, v0
; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v1, 6, 0, vcc_lo
-; GFX10-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-GISEL-NEXT: v_pk_add_f16 v0, v0, v3
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
@@ -7740,9 +7732,8 @@ define <2 x half> @v_contract_mul_add_v2f16_select_128_4(<2 x i32> %arg, <2 x ha
; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
; GFX10-GISEL-NEXT: v_ldexp_f16_e32 v0, v2, v0
; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v1, 2, 7, vcc_lo
-; GFX10-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-GISEL-NEXT: v_pk_add_f16 v0, v0, v3
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
@@ -7927,9 +7918,8 @@ define <2 x half> @v_contract_mul_add_v2f16_select_4_128(<2 x i32> %arg, <2 x ha
; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
; GFX10-GISEL-NEXT: v_ldexp_f16_e32 v0, v2, v0
; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v1, 7, 2, vcc_lo
-; GFX10-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-GISEL-NEXT: v_pk_add_f16 v0, v0, v3
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/fneg-fabs.bf16.ll b/llvm/test/CodeGen/AMDGPU/fneg-fabs.bf16.ll
index 59793344be87c..9b6fd0bcf703b 100644
--- a/llvm/test/CodeGen/AMDGPU/fneg-fabs.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fneg-fabs.bf16.ll
@@ -466,14 +466,13 @@ define amdgpu_kernel void @s_fneg_fabs_v2bf16_non_bc_src(ptr addrspace(1) %out,
; GFX9-NEXT: v_add_f32_e64 v2, s2, 1.0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX9-NEXT: v_mov_b32_e32 v3, 0xffff
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_or_b32_e32 v1, 0x80008000, v1
; GFX9-NEXT: global_store_dword v0, v1, s[0:1]
; GFX9-NEXT: s_endpgm
@@ -780,10 +779,9 @@ define amdgpu_kernel void @fold_user_fneg_fabs_v2bf16(ptr addrspace(1) %out, <2
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX9-NEXT: v_mov_b32_e32 v3, 0xffff
; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX9-NEXT: v_and_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: global_store_dword v0, v1, s[0:1]
; GFX9-NEXT: s_endpgm
;
@@ -1037,10 +1035,9 @@ define amdgpu_kernel void @s_fneg_multi_use_fabs_foldable_neg_v2bf16(ptr addrspa
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX9-NEXT: v_mov_b32_e32 v3, 0xffff
; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX9-NEXT: v_and_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_mov_b32_e32 v2, s5
; GFX9-NEXT: global_store_dword v0, v2, s[0:1]
; GFX9-NEXT: global_store_dword v0, v1, s[2:3]
diff --git a/llvm/test/CodeGen/AMDGPU/fneg.bf16.ll b/llvm/test/CodeGen/AMDGPU/fneg.bf16.ll
index bed203ebaf8dd..07d243c393d31 100644
--- a/llvm/test/CodeGen/AMDGPU/fneg.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fneg.bf16.ll
@@ -683,32 +683,31 @@ define amdgpu_kernel void @v_fneg_fold_v2bf16(ptr addrspace(1) %out, ptr addrspa
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: v_mov_b32_e32 v2, 0xffff
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_load_dword v1, v0, s[2:3]
; GFX9-NEXT: s_mov_b32 s2, 0x8000
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX9-NEXT: v_xor_b32_sdwa v4, v1, s2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX9-NEXT: v_xor_b32_sdwa v3, v1, s2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
; GFX9-NEXT: v_xor_b32_sdwa v1, v1, s2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: v_mul_f32_e32 v3, v4, v3
-; GFX9-NEXT: v_mul_f32_e32 v1, v1, v5
-; GFX9-NEXT: v_bfe_u32 v4, v3, 16, 1
-; GFX9-NEXT: v_bfe_u32 v6, v1, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v4, v4, v3
-; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v3
-; GFX9-NEXT: v_add_u32_e32 v6, v6, v1
-; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX9-NEXT: v_or_b32_e32 v7, 0x400000, v1
-; GFX9-NEXT: v_add_u32_e32 v6, 0x7fff, v6
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
+; GFX9-NEXT: v_mul_f32_e32 v2, v3, v2
+; GFX9-NEXT: v_mul_f32_e32 v1, v1, v4
+; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
+; GFX9-NEXT: v_bfe_u32 v5, v1, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
+; GFX9-NEXT: v_add_u32_e32 v5, v5, v1
+; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v1
+; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v7, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v6, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: global_store_dword v0, v1, s[0:1]
; GFX9-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/fptosi-sat-vector.ll b/llvm/test/CodeGen/AMDGPU/fptosi-sat-vector.ll
index 402c10f4e3ab2..58d1786901eba 100644
--- a/llvm/test/CodeGen/AMDGPU/fptosi-sat-vector.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptosi-sat-vector.ll
@@ -2104,14 +2104,14 @@ define <2 x i16> @test_s_signed_v2f64_v2i16(<2 x double> inreg %f) {
; GFX9-LABEL: test_s_signed_v2f64_v2i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_cvt_i32_f64_e32 v1, s[16:17]
; GFX9-NEXT: v_cvt_i32_f64_e32 v0, s[18:19]
+; GFX9-NEXT: v_cvt_i32_f64_e32 v1, s[16:17]
; GFX9-NEXT: s_movk_i32 s4, 0x8000
; GFX9-NEXT: v_mov_b32_e32 v2, 0x7fff
-; GFX9-NEXT: v_med3_i32 v1, v1, s4, v2
; GFX9-NEXT: v_med3_i32 v0, v0, s4, v2
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v1
+; GFX9-NEXT: v_med3_i32 v1, v1, s4, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: test_s_signed_v2f64_v2i16:
@@ -3813,15 +3813,13 @@ define <4 x i16> @test_s_signed_v4f16_v4i16(<4 x half> inreg %f) {
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_lshr_b32 s4, s17, 16
-; GFX9-NEXT: v_cvt_i16_f16_e32 v1, s4
+; GFX9-NEXT: v_cvt_i16_f16_sdwa v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
; GFX9-NEXT: s_lshr_b32 s4, s16, 16
+; GFX9-NEXT: v_cvt_i16_f16_sdwa v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
; GFX9-NEXT: v_cvt_i16_f16_e32 v2, s17
; GFX9-NEXT: v_cvt_i16_f16_e32 v3, s16
-; GFX9-NEXT: v_cvt_i16_f16_e32 v0, s4
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: test_s_signed_v4f16_v4i16:
@@ -4392,8 +4390,6 @@ define <8 x i8> @test_signed_v8f16_v8i8(<8 x half> %f) {
; GFX9-NEXT: v_med3_i16 v3, v3, s4, v7
; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: v_or_b32_sdwa v8, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v8
-; GFX9-NEXT: v_lshl_or_b32 v4, v6, 16, v2
; GFX9-NEXT: v_cvt_i16_f16_e32 v2, v1
; GFX9-NEXT: v_cvt_i16_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
; GFX9-NEXT: v_med3_i16 v1, v1, s4, v7
@@ -4402,13 +4398,14 @@ define <8 x i8> @test_signed_v8f16_v8i8(<8 x half> %f) {
; GFX9-NEXT: v_or_b32_sdwa v2, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_cvt_i16_f16_e32 v1, v0
; GFX9-NEXT: v_cvt_i16_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v6
; GFX9-NEXT: v_med3_i16 v0, v0, s4, v7
+; GFX9-NEXT: v_or_b32_sdwa v4, v9, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX9-NEXT: v_med3_i16 v1, v1, s4, v7
; GFX9-NEXT: v_lshlrev_b16_e32 v7, 8, v0
; GFX9-NEXT: v_lshrrev_b32_e32 v5, 8, v4
; GFX9-NEXT: v_or_b32_sdwa v0, v1, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v6
; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v7
; GFX9-NEXT: v_lshrrev_b64 v[3:4], 24, v[3:4]
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 8, v1
@@ -6307,25 +6304,21 @@ define <8 x i16> @test_s_signed_v8f16_v8i16(<8 x half> inreg %f) {
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_lshr_b32 s4, s19, 16
-; GFX9-NEXT: v_cvt_i16_f16_e32 v3, s4
+; GFX9-NEXT: v_cvt_i16_f16_sdwa v3, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
; GFX9-NEXT: s_lshr_b32 s4, s18, 16
-; GFX9-NEXT: v_cvt_i16_f16_e32 v2, s4
+; GFX9-NEXT: v_cvt_i16_f16_sdwa v2, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
; GFX9-NEXT: s_lshr_b32 s4, s17, 16
-; GFX9-NEXT: v_cvt_i16_f16_e32 v1, s4
+; GFX9-NEXT: v_cvt_i16_f16_sdwa v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
; GFX9-NEXT: s_lshr_b32 s4, s16, 16
+; GFX9-NEXT: v_cvt_i16_f16_sdwa v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
; GFX9-NEXT: v_cvt_i16_f16_e32 v4, s19
; GFX9-NEXT: v_cvt_i16_f16_e32 v5, s18
; GFX9-NEXT: v_cvt_i16_f16_e32 v6, s17
; GFX9-NEXT: v_cvt_i16_f16_e32 v7, s16
-; GFX9-NEXT: v_cvt_i16_f16_e32 v0, s4
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v2, v2, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v3, v3, 16, v4
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: test_s_signed_v8f16_v8i16:
@@ -6826,19 +6819,19 @@ define <4 x i16> @test_s_signed_v4f32_v4i16(<4 x float> inreg %f) {
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_cvt_i32_f32_e32 v0, s19
-; GFX9-NEXT: v_cvt_i32_f32_e32 v2, s18
; GFX9-NEXT: v_cvt_i32_f32_e32 v3, s17
+; GFX9-NEXT: v_cvt_i32_f32_e32 v2, s18
; GFX9-NEXT: v_cvt_i32_f32_e32 v4, s16
; GFX9-NEXT: s_movk_i32 s4, 0x7fff
; GFX9-NEXT: v_mov_b32_e32 v1, 0x8000
; GFX9-NEXT: v_med3_i32 v5, v0, v1, s4
-; GFX9-NEXT: v_med3_i32 v2, v2, v1, s4
; GFX9-NEXT: v_med3_i32 v0, v3, v1, s4
+; GFX9-NEXT: v_med3_i32 v2, v2, v1, s4
; GFX9-NEXT: v_med3_i32 v1, v4, v1, s4
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v1
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v2
-; GFX9-NEXT: v_lshl_or_b32 v1, v5, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v5
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: test_s_signed_v4f32_v4i16:
diff --git a/llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll b/llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll
index 69eec3819ec4f..8a4dfd06190bd 100644
--- a/llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll
@@ -1974,12 +1974,12 @@ define <2 x i16> @test_s_unsigned_v2f64_v2i16(<2 x double> inreg %f) {
; GFX9-LABEL: test_s_unsigned_v2f64_v2i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_cvt_u32_f64_e32 v1, s[16:17]
; GFX9-NEXT: v_cvt_u32_f64_e32 v0, s[18:19]
+; GFX9-NEXT: v_cvt_u32_f64_e32 v1, s[16:17]
+; GFX9-NEXT: s_mov_b32 s4, 0xffff
+; GFX9-NEXT: v_min_u32_sdwa v0, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_min_u32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_min_u32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: test_s_unsigned_v2f64_v2i16:
@@ -3517,15 +3517,13 @@ define <4 x i16> @test_s_unsigned_v4f16_v4i16(<4 x half> inreg %f) {
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_lshr_b32 s4, s17, 16
-; GFX9-NEXT: v_cvt_u16_f16_e32 v1, s4
+; GFX9-NEXT: v_cvt_u16_f16_sdwa v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
; GFX9-NEXT: s_lshr_b32 s4, s16, 16
+; GFX9-NEXT: v_cvt_u16_f16_sdwa v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
; GFX9-NEXT: v_cvt_u16_f16_e32 v2, s17
; GFX9-NEXT: v_cvt_u16_f16_e32 v3, s16
-; GFX9-NEXT: v_cvt_u16_f16_e32 v0, s4
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: test_s_unsigned_v4f16_v4i16:
@@ -4077,25 +4075,24 @@ define <8 x i8> @test_unsigned_v8f16_v8i8(<8 x half> %f) {
; GFX9-NEXT: v_min_u16_e32 v3, 0xff, v3
; GFX9-NEXT: v_min_u16_sdwa v2, v2, s4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_or_b32_e32 v8, v3, v2
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v8
-; GFX9-NEXT: v_lshl_or_b32 v4, v6, 16, v2
; GFX9-NEXT: v_cvt_u16_f16_e32 v2, v1
; GFX9-NEXT: v_cvt_u16_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
; GFX9-NEXT: v_min_u16_e32 v2, 0xff, v2
; GFX9-NEXT: v_min_u16_sdwa v1, v1, s4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v6
; GFX9-NEXT: v_or_b32_e32 v2, v2, v1
; GFX9-NEXT: v_cvt_u16_f16_e32 v1, v0
; GFX9-NEXT: v_cvt_u16_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-NEXT: v_or_b32_sdwa v4, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX9-NEXT: v_min_u16_e32 v1, 0xff, v1
-; GFX9-NEXT: v_min_u16_sdwa v7, v0, s4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_min_u16_sdwa v9, v0, s4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_lshrrev_b32_e32 v5, 8, v4
-; GFX9-NEXT: v_or_b32_e32 v0, v1, v7
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v7
+; GFX9-NEXT: v_or_b32_e32 v0, v1, v9
+; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v9
; GFX9-NEXT: v_lshrrev_b64 v[3:4], 24, v[3:4]
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 8, v1
-; GFX9-NEXT: v_lshrrev_b32_e32 v7, 24, v9
+; GFX9-NEXT: v_lshrrev_b32_e32 v7, 24, v7
; GFX9-NEXT: v_mov_b32_e32 v4, v8
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -5833,25 +5830,21 @@ define <8 x i16> @test_s_unsigned_v8f16_v8i16(<8 x half> inreg %f) {
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_lshr_b32 s4, s19, 16
-; GFX9-NEXT: v_cvt_u16_f16_e32 v3, s4
+; GFX9-NEXT: v_cvt_u16_f16_sdwa v3, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
; GFX9-NEXT: s_lshr_b32 s4, s18, 16
-; GFX9-NEXT: v_cvt_u16_f16_e32 v2, s4
+; GFX9-NEXT: v_cvt_u16_f16_sdwa v2, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
; GFX9-NEXT: s_lshr_b32 s4, s17, 16
-; GFX9-NEXT: v_cvt_u16_f16_e32 v1, s4
+; GFX9-NEXT: v_cvt_u16_f16_sdwa v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
; GFX9-NEXT: s_lshr_b32 s4, s16, 16
+; GFX9-NEXT: v_cvt_u16_f16_sdwa v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
; GFX9-NEXT: v_cvt_u16_f16_e32 v4, s19
; GFX9-NEXT: v_cvt_u16_f16_e32 v5, s18
; GFX9-NEXT: v_cvt_u16_f16_e32 v6, s17
; GFX9-NEXT: v_cvt_u16_f16_e32 v7, s16
-; GFX9-NEXT: v_cvt_u16_f16_e32 v0, s4
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v2, v2, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v3, v3, 16, v4
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: test_s_unsigned_v8f16_v8i16:
@@ -6332,14 +6325,13 @@ define <4 x i16> @test_s_unsigned_v4f32_v4i16(<4 x float> inreg %f) {
; GFX9-NEXT: v_cvt_u32_f32_e32 v1, s18
; GFX9-NEXT: v_cvt_u32_f32_e32 v2, s17
; GFX9-NEXT: v_cvt_u32_f32_e32 v3, s16
-; GFX9-NEXT: v_min_u32_e32 v4, 0xffff, v0
+; GFX9-NEXT: s_mov_b32 s4, 0xffff
+; GFX9-NEXT: v_min_u32_sdwa v4, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_min_u32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_min_u32_e32 v0, 0xffff, v2
+; GFX9-NEXT: v_min_u32_sdwa v0, v2, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_min_u32_e32 v2, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v1, v4, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: test_s_unsigned_v4f32_v4i16:
diff --git a/llvm/test/CodeGen/AMDGPU/freeze.ll b/llvm/test/CodeGen/AMDGPU/freeze.ll
index 55f78749569b4..433dc96366e62 100644
--- a/llvm/test/CodeGen/AMDGPU/freeze.ll
+++ b/llvm/test/CodeGen/AMDGPU/freeze.ll
@@ -14409,13 +14409,13 @@ define void @freeze_v3i8(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0xff
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v4, 8, v0
+; GFX9-GISEL-NEXT: v_and_b32_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX9-GISEL-NEXT: v_and_b32_e32 v4, 0xff, v4
+; GFX9-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-GISEL-NEXT: v_lshlrev_b16_e32 v4, 8, v4
-; GFX9-GISEL-NEXT: v_and_b32_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-GISEL-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX9-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-GISEL-NEXT: global_store_short v[2:3], v0, off
; GFX9-GISEL-NEXT: global_store_byte_d16_hi v[2:3], v0, off offset:2
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
@@ -14437,12 +14437,12 @@ define void @freeze_v3i8(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0xff
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10-GISEL-NEXT: v_lshrrev_b32_e32 v4, 8, v0
-; GFX10-GISEL-NEXT: v_and_b32_sdwa v4, v4, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-GISEL-NEXT: v_and_b32_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-GISEL-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10-GISEL-NEXT: v_and_b32_sdwa v5, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-GISEL-NEXT: v_and_b32_sdwa v1, v4, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v5
+; GFX10-GISEL-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX10-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-GISEL-NEXT: global_store_short v[2:3], v0, off
; GFX10-GISEL-NEXT: global_store_byte_d16_hi v[2:3], v0, off offset:2
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/frem.ll b/llvm/test/CodeGen/AMDGPU/frem.ll
index 8700059729501..1425240721c49 100644
--- a/llvm/test/CodeGen/AMDGPU/frem.ll
+++ b/llvm/test/CodeGen/AMDGPU/frem.ll
@@ -5898,9 +5898,8 @@ define amdgpu_kernel void @frem_v2f16(ptr addrspace(1) %out, ptr addrspace(1) %i
; GFX9-NEXT: v_cmp_nge_f16_e64 s[4:5], |v3|, s4
; GFX9-NEXT: v_cndmask_b32_e32 v2, v1, v2, vcc
; GFX9-NEXT: s_and_b64 vcc, s[4:5], s[2:3]
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v4, vcc
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v2
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v1
+; GFX9-NEXT: v_cndmask_b32_sdwa v0, v1, v4, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: global_store_dword v5, v0, s[0:1]
; GFX9-NEXT: s_endpgm
;
@@ -6077,11 +6076,11 @@ define amdgpu_kernel void @frem_v2f16(ptr addrspace(1) %out, ptr addrspace(1) %i
; GFX10-NEXT: s_and_b32 vcc_lo, s2, vcc_lo
; GFX10-NEXT: v_cndmask_b32_e32 v1, 0x7e00, v2, vcc_lo
; GFX10-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX10-NEXT: v_cmp_lg_f16_sdwa s2, v0, v2 src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-NEXT: v_mov_b32_e32 v0, 0x7e00
; GFX10-NEXT: s_and_b32 vcc_lo, s3, s2
-; GFX10-NEXT: v_cndmask_b32_e32 v0, 0x7e00, v4, vcc_lo
-; GFX10-NEXT: v_lshl_or_b32 v0, v0, 16, v1
+; GFX10-NEXT: v_cndmask_b32_sdwa v0, v0, v4, vcc_lo dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: global_store_dword v2, v0, s[0:1]
; GFX10-NEXT: s_endpgm
;
@@ -9035,7 +9034,7 @@ define amdgpu_kernel void @frem_v4f16(ptr addrspace(1) %out, ptr addrspace(1) %i
; GFX9-NEXT: v_cmp_nge_f16_e64 s[4:5], |v5|, s6
; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v4, vcc
; GFX9-NEXT: s_and_b64 vcc, s[4:5], s[2:3]
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v6, vcc
+; GFX9-NEXT: v_cndmask_b32_sdwa v0, v2, v6, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_cmp_lg_f16_e32 vcc, 0, v1
; GFX9-NEXT: v_cmp_nge_f16_e64 s[2:3], |v3|, s6
; GFX9-NEXT: s_and_b64 vcc, s[2:3], vcc
@@ -9043,11 +9042,9 @@ define amdgpu_kernel void @frem_v4f16(ptr addrspace(1) %out, ptr addrspace(1) %i
; GFX9-NEXT: v_cmp_nge_f16_e64 s[4:5], |v8|, s6
; GFX9-NEXT: v_cndmask_b32_e32 v3, v2, v7, vcc
; GFX9-NEXT: s_and_b64 vcc, s[4:5], s[2:3]
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v9, vcc
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v3
-; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v2
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v4
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v2
+; GFX9-NEXT: v_cndmask_b32_sdwa v1, v2, v9, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: global_store_dwordx2 v10, v[0:1], s[0:1]
; GFX9-NEXT: s_endpgm
;
@@ -9379,21 +9376,20 @@ define amdgpu_kernel void @frem_v4f16(ptr addrspace(1) %out, ptr addrspace(1) %i
; GFX10-NEXT: s_and_b32 vcc_lo, s2, vcc_lo
; GFX10-NEXT: v_cndmask_b32_e32 v2, 0x7e00, v4, vcc_lo
; GFX10-NEXT: v_mov_b32_e32 v4, 0
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX10-NEXT: v_cmp_lg_f16_sdwa s2, v0, v4 src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-NEXT: v_mov_b32_e32 v0, 0x7e00
; GFX10-NEXT: s_and_b32 vcc_lo, s3, s2
; GFX10-NEXT: v_cmp_nle_f16_e64 s2, 0x7c00, |v3|
-; GFX10-NEXT: v_cndmask_b32_e32 v0, 0x7e00, v6, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_sdwa v5, v0, v6, vcc_lo dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX10-NEXT: v_cmp_lg_f16_e32 vcc_lo, 0, v1
; GFX10-NEXT: v_cmp_nle_f16_e64 s3, 0x7c00, |v8|
-; GFX10-NEXT: v_lshl_or_b32 v0, v0, 16, v2
; GFX10-NEXT: s_and_b32 vcc_lo, s2, vcc_lo
; GFX10-NEXT: v_cmp_lg_f16_sdwa s2, v1, v4 src0_sel:WORD_1 src1_sel:DWORD
; GFX10-NEXT: v_cndmask_b32_e32 v3, 0x7e00, v7, vcc_lo
; GFX10-NEXT: s_and_b32 vcc_lo, s3, s2
-; GFX10-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX10-NEXT: v_cndmask_b32_e32 v1, 0x7e00, v9, vcc_lo
-; GFX10-NEXT: v_lshl_or_b32 v1, v1, 16, v3
+; GFX10-NEXT: v_cndmask_b32_sdwa v0, v0, v9, vcc_lo dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v1, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v0, v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: global_store_dwordx2 v4, v[0:1], s[0:1]
; GFX10-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/idot4u.ll b/llvm/test/CodeGen/AMDGPU/idot4u.ll
index c3603090a3f16..3f2c70790fd56 100644
--- a/llvm/test/CodeGen/AMDGPU/idot4u.ll
+++ b/llvm/test/CodeGen/AMDGPU/idot4u.ll
@@ -2663,27 +2663,26 @@ define amdgpu_kernel void @udot4_acc8_vecMul(ptr addrspace(1) %src1,
; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX10-DL-NEXT: global_load_ubyte v3, v0, s[6:7]
; GFX10-DL-NEXT: s_waitcnt vmcnt(2)
-; GFX10-DL-NEXT: v_lshrrev_b16 v4, 8, v1
-; GFX10-DL-NEXT: v_lshrrev_b32_e32 v5, 24, v1
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v4, 24, v1
; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
-; GFX10-DL-NEXT: v_lshrrev_b32_e32 v6, 24, v2
-; GFX10-DL-NEXT: v_lshrrev_b16 v7, 8, v2
-; GFX10-DL-NEXT: v_lshrrev_b32_e32 v8, 16, v1
-; GFX10-DL-NEXT: v_lshrrev_b32_e32 v9, 16, v2
-; GFX10-DL-NEXT: v_mul_lo_u16 v5, v5, v6
-; GFX10-DL-NEXT: v_mul_lo_u16 v4, v4, v7
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v5, 24, v2
+; GFX10-DL-NEXT: v_lshrrev_b16 v6, 8, v1
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v7, 16, v1
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v8, 16, v2
+; GFX10-DL-NEXT: v_lshrrev_b16 v9, 8, v2
+; GFX10-DL-NEXT: v_mul_lo_u16 v4, v4, v5
; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
; GFX10-DL-NEXT: v_mad_u16 v1, v1, v2, v3
-; GFX10-DL-NEXT: v_mul_lo_u16 v6, v8, v9
-; GFX10-DL-NEXT: v_lshlrev_b16 v5, 8, v5
+; GFX10-DL-NEXT: v_mul_lo_u16 v5, v7, v8
+; GFX10-DL-NEXT: v_mul_lo_u16 v6, v6, v9
; GFX10-DL-NEXT: v_lshlrev_b16 v4, 8, v4
-; GFX10-DL-NEXT: v_or_b32_sdwa v6, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-DL-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX10-DL-NEXT: v_lshrrev_b32_e32 v2, 8, v5
-; GFX10-DL-NEXT: v_lshl_or_b32 v4, v6, 16, v4
-; GFX10-DL-NEXT: v_lshrrev_b32_e32 v4, 8, v4
-; GFX10-DL-NEXT: v_add_nc_u16 v1, v1, v4
-; GFX10-DL-NEXT: v_mad_u16 v1, v8, v9, v1
+; GFX10-DL-NEXT: v_lshlrev_b16 v6, 8, v6
+; GFX10-DL-NEXT: v_or_b32_sdwa v5, v5, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v2, 8, v4
+; GFX10-DL-NEXT: v_or_b32_sdwa v5, v5, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v5, 8, v5
+; GFX10-DL-NEXT: v_add_nc_u16 v1, v1, v5
+; GFX10-DL-NEXT: v_mad_u16 v1, v7, v8, v1
; GFX10-DL-NEXT: v_add_nc_u16 v1, v1, v2
; GFX10-DL-NEXT: global_store_byte v0, v1, s[6:7]
; GFX10-DL-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/idot8s.ll b/llvm/test/CodeGen/AMDGPU/idot8s.ll
index 684a21f7ac077..112c26aad518a 100644
--- a/llvm/test/CodeGen/AMDGPU/idot8s.ll
+++ b/llvm/test/CodeGen/AMDGPU/idot8s.ll
@@ -2729,53 +2729,52 @@ define amdgpu_kernel void @idot8_acc8_vecMul(ptr addrspace(1) %src1,
; GFX9-NEXT: v_lshlrev_b16_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
; GFX9-NEXT: v_lshlrev_b16_e32 v4, 12, v9
; GFX9-NEXT: v_ashrrev_i16_e32 v9, 12, v15
+; GFX9-NEXT: v_lshlrev_b16_e32 v8, 12, v8
+; GFX9-NEXT: v_lshlrev_b16_e32 v7, 12, v7
; GFX9-NEXT: v_ashrrev_i16_e32 v15, 12, v16
; GFX9-NEXT: v_lshlrev_b16_e32 v6, 12, v6
; GFX9-NEXT: v_ashrrev_i16_e32 v16, 12, v1
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 12, v5
; GFX9-NEXT: v_lshlrev_b16_e32 v5, 12, v14
; GFX9-NEXT: v_ashrrev_i16_e32 v14, 12, v17
+; GFX9-NEXT: v_lshlrev_b16_e32 v13, 12, v13
+; GFX9-NEXT: v_lshlrev_b16_e32 v12, 12, v12
; GFX9-NEXT: v_ashrrev_i16_e32 v17, 12, v18
; GFX9-NEXT: v_lshlrev_b16_e32 v11, 12, v11
; GFX9-NEXT: v_ashrrev_i16_e32 v18, 12, v2
; GFX9-NEXT: v_lshlrev_b16_e32 v2, 12, v10
-; GFX9-NEXT: v_lshlrev_b16_e32 v8, 12, v8
-; GFX9-NEXT: v_lshlrev_b16_e32 v7, 12, v7
-; GFX9-NEXT: v_lshlrev_b16_e32 v13, 12, v13
-; GFX9-NEXT: v_lshlrev_b16_e32 v12, 12, v12
-; GFX9-NEXT: v_ashrrev_i16_e32 v6, 12, v6
-; GFX9-NEXT: v_ashrrev_i16_e32 v1, 12, v1
-; GFX9-NEXT: v_ashrrev_i16_e32 v11, 12, v11
-; GFX9-NEXT: v_ashrrev_i16_e32 v2, 12, v2
; GFX9-NEXT: v_ashrrev_i16_e32 v8, 12, v8
; GFX9-NEXT: v_ashrrev_i16_e32 v7, 12, v7
+; GFX9-NEXT: v_ashrrev_i16_e32 v6, 12, v6
+; GFX9-NEXT: v_ashrrev_i16_e32 v1, 12, v1
; GFX9-NEXT: v_ashrrev_i16_e32 v10, 12, v13
; GFX9-NEXT: v_ashrrev_i16_e32 v12, 12, v12
-; GFX9-NEXT: v_mul_lo_u16_e32 v19, v15, v17
-; GFX9-NEXT: v_mul_lo_u16_sdwa v1, v1, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: v_mul_lo_u16_sdwa v2, v6, v11 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_ashrrev_i16_e32 v11, 12, v11
+; GFX9-NEXT: v_ashrrev_i16_e32 v2, 12, v2
; GFX9-NEXT: v_ashrrev_i16_e32 v4, 12, v4
; GFX9-NEXT: v_ashrrev_i16_e32 v5, 12, v5
; GFX9-NEXT: v_mul_lo_u16_e32 v13, v16, v18
+; GFX9-NEXT: v_mul_lo_u16_e32 v19, v15, v17
+; GFX9-NEXT: v_mul_lo_u16_sdwa v1, v1, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_mul_lo_u16_sdwa v2, v6, v11 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mul_lo_u16_sdwa v6, v7, v12 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mul_lo_u16_e32 v7, v8, v10
-; GFX9-NEXT: v_or_b32_sdwa v2, v19, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_mul_lo_u16_e32 v9, v9, v14
; GFX9-NEXT: v_mul_lo_u16_sdwa v4, v4, v5 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: v_or_b32_sdwa v5, v13, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v5, v13, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v2, v19, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v6, v7, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_or_b32_sdwa v7, v9, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_mul_lo_u16_e32 v9, v9, v14
; GFX9-NEXT: v_lshrrev_b32_e32 v8, 8, v1
+; GFX9-NEXT: v_or_b32_sdwa v2, v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v6
+; GFX9-NEXT: v_or_b32_sdwa v7, v9, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_lshl_or_b32 v4, v6, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v2, v5, 16, v2
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_add_u16_e32 v3, v7, v3
-; GFX9-NEXT: v_lshrrev_b32_e32 v4, 8, v4
; GFX9-NEXT: v_lshrrev_b32_e32 v5, 8, v2
; GFX9-NEXT: v_lshrrev_b64 v[1:2], 24, v[1:2]
-; GFX9-NEXT: v_add_u16_e32 v2, v3, v4
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 8, v4
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_add_u16_e32 v2, v7, v3
+; GFX9-NEXT: v_add_u16_e32 v2, v2, v4
; GFX9-NEXT: v_add_u16_e32 v2, v2, v6
; GFX9-NEXT: v_add_u16_e32 v1, v2, v1
; GFX9-NEXT: v_mad_legacy_u16 v1, v15, v17, v1
@@ -2822,53 +2821,52 @@ define amdgpu_kernel void @idot8_acc8_vecMul(ptr addrspace(1) %src1,
; GFX9-DL-NEXT: v_lshlrev_b16_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
; GFX9-DL-NEXT: v_lshlrev_b16_e32 v4, 12, v9
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v9, 12, v15
+; GFX9-DL-NEXT: v_lshlrev_b16_e32 v8, 12, v8
+; GFX9-DL-NEXT: v_lshlrev_b16_e32 v7, 12, v7
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v15, 12, v16
; GFX9-DL-NEXT: v_lshlrev_b16_e32 v6, 12, v6
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v16, 12, v1
; GFX9-DL-NEXT: v_lshlrev_b16_e32 v1, 12, v5
; GFX9-DL-NEXT: v_lshlrev_b16_e32 v5, 12, v14
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v14, 12, v17
+; GFX9-DL-NEXT: v_lshlrev_b16_e32 v13, 12, v13
+; GFX9-DL-NEXT: v_lshlrev_b16_e32 v12, 12, v12
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v17, 12, v18
; GFX9-DL-NEXT: v_lshlrev_b16_e32 v11, 12, v11
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v18, 12, v2
; GFX9-DL-NEXT: v_lshlrev_b16_e32 v2, 12, v10
-; GFX9-DL-NEXT: v_lshlrev_b16_e32 v8, 12, v8
-; GFX9-DL-NEXT: v_lshlrev_b16_e32 v7, 12, v7
-; GFX9-DL-NEXT: v_lshlrev_b16_e32 v13, 12, v13
-; GFX9-DL-NEXT: v_lshlrev_b16_e32 v12, 12, v12
-; GFX9-DL-NEXT: v_ashrrev_i16_e32 v6, 12, v6
-; GFX9-DL-NEXT: v_ashrrev_i16_e32 v1, 12, v1
-; GFX9-DL-NEXT: v_ashrrev_i16_e32 v11, 12, v11
-; GFX9-DL-NEXT: v_ashrrev_i16_e32 v2, 12, v2
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v8, 12, v8
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v7, 12, v7
+; GFX9-DL-NEXT: v_ashrrev_i16_e32 v6, 12, v6
+; GFX9-DL-NEXT: v_ashrrev_i16_e32 v1, 12, v1
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v10, 12, v13
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v12, 12, v12
-; GFX9-DL-NEXT: v_mul_lo_u16_e32 v19, v15, v17
-; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v1, v1, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v2, v6, v11 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-DL-NEXT: v_ashrrev_i16_e32 v11, 12, v11
+; GFX9-DL-NEXT: v_ashrrev_i16_e32 v2, 12, v2
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v4, 12, v4
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v5, 12, v5
; GFX9-DL-NEXT: v_mul_lo_u16_e32 v13, v16, v18
+; GFX9-DL-NEXT: v_mul_lo_u16_e32 v19, v15, v17
+; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v1, v1, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v2, v6, v11 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v6, v7, v12 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-DL-NEXT: v_mul_lo_u16_e32 v7, v8, v10
-; GFX9-DL-NEXT: v_or_b32_sdwa v2, v19, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-DL-NEXT: v_mul_lo_u16_e32 v9, v9, v14
; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v4, v4, v5 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-DL-NEXT: v_or_b32_sdwa v5, v13, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-DL-NEXT: v_or_b32_sdwa v5, v13, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-DL-NEXT: v_or_b32_sdwa v2, v19, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-DL-NEXT: v_or_b32_sdwa v6, v7, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-DL-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-DL-NEXT: v_or_b32_sdwa v7, v9, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-DL-NEXT: v_mul_lo_u16_e32 v9, v9, v14
; GFX9-DL-NEXT: v_lshrrev_b32_e32 v8, 8, v1
+; GFX9-DL-NEXT: v_or_b32_sdwa v2, v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-DL-NEXT: v_lshlrev_b32_e32 v1, 16, v6
+; GFX9-DL-NEXT: v_or_b32_sdwa v7, v9, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-DL-NEXT: v_lshl_or_b32 v4, v6, 16, v4
-; GFX9-DL-NEXT: v_lshl_or_b32 v2, v5, 16, v2
-; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DL-NEXT: v_add_u16_e32 v3, v7, v3
-; GFX9-DL-NEXT: v_lshrrev_b32_e32 v4, 8, v4
; GFX9-DL-NEXT: v_lshrrev_b32_e32 v5, 8, v2
; GFX9-DL-NEXT: v_lshrrev_b64 v[1:2], 24, v[1:2]
-; GFX9-DL-NEXT: v_add_u16_e32 v2, v3, v4
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v4, 8, v4
+; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-DL-NEXT: v_add_u16_e32 v2, v7, v3
+; GFX9-DL-NEXT: v_add_u16_e32 v2, v2, v4
; GFX9-DL-NEXT: v_add_u16_e32 v2, v2, v6
; GFX9-DL-NEXT: v_add_u16_e32 v1, v2, v1
; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v15, v17, v1
@@ -2895,80 +2893,78 @@ define amdgpu_kernel void @idot8_acc8_vecMul(ptr addrspace(1) %src1,
; GFX10-DL-XNACK-NEXT: s_clause 0x1
; GFX10-DL-XNACK-NEXT: global_load_dword v1, v0, s[8:9]
; GFX10-DL-XNACK-NEXT: global_load_dword v2, v0, s[10:11]
-; GFX10-DL-XNACK-NEXT: global_load_ubyte v0, v4, s[0:1]
+; GFX10-DL-XNACK-NEXT: global_load_ubyte v3, v4, s[0:1]
; GFX10-DL-XNACK-NEXT: s_waitcnt vmcnt(2)
; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v8, 12, v1
-; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v10, 4, v1
; GFX10-DL-XNACK-NEXT: s_waitcnt vmcnt(1)
; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v15, 12, v2
-; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v17, 4, v2
-; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v6, 20, v1
-; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v13, 20, v2
; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v9, 8, v1
; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v16, 8, v2
-; GFX10-DL-XNACK-NEXT: v_lshlrev_b16 v10, 12, v10
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v10, 4, v1
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16 v8, 12, v8
-; GFX10-DL-XNACK-NEXT: v_lshlrev_b16 v17, 12, v17
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16 v15, 12, v15
-; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v3, 28, v1
-; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v7, 16, v1
-; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v11, 28, v2
-; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v14, 16, v2
-; GFX10-DL-XNACK-NEXT: v_lshlrev_b16 v6, 12, v6
-; GFX10-DL-XNACK-NEXT: v_lshlrev_b16 v13, 12, v13
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v17, 4, v2
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16 v9, 12, v9
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16 v16, 12, v16
-; GFX10-DL-XNACK-NEXT: v_ashrrev_i16 v10, 12, v10
; GFX10-DL-XNACK-NEXT: v_ashrrev_i16 v8, 12, v8
-; GFX10-DL-XNACK-NEXT: v_ashrrev_i16 v17, 12, v17
; GFX10-DL-XNACK-NEXT: v_ashrrev_i16 v15, 12, v15
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v0, 28, v1
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v6, 20, v1
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v11, 28, v2
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v13, 20, v2
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16 v10, 12, v10
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16 v17, 12, v17
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16 v9, 12, v9
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16 v16, 12, v16
+; GFX10-DL-XNACK-NEXT: v_mul_lo_u16 v8, v8, v15
; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v5, 24, v1
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v7, 16, v1
; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v12, 24, v2
-; GFX10-DL-XNACK-NEXT: v_lshlrev_b16 v7, 12, v7
-; GFX10-DL-XNACK-NEXT: v_lshlrev_b16 v3, 12, v3
-; GFX10-DL-XNACK-NEXT: v_lshlrev_b16 v14, 12, v14
-; GFX10-DL-XNACK-NEXT: v_lshlrev_b16 v11, 12, v11
-; GFX10-DL-XNACK-NEXT: v_ashrrev_i16 v6, 12, v6
-; GFX10-DL-XNACK-NEXT: v_ashrrev_i16 v13, 12, v13
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v14, 16, v2
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16 v1, 12, v1
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16 v2, 12, v2
-; GFX10-DL-XNACK-NEXT: v_ashrrev_i16 v9, 12, v9
-; GFX10-DL-XNACK-NEXT: v_ashrrev_i16 v16, 12, v16
-; GFX10-DL-XNACK-NEXT: v_mul_lo_u16 v8, v8, v15
-; GFX10-DL-XNACK-NEXT: v_mul_lo_u16 v10, v10, v17
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16 v6, 12, v6
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16 v0, 12, v0
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16 v13, 12, v13
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16 v11, 12, v11
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16 v10, 12, v10
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16 v15, 12, v17
+; GFX10-DL-XNACK-NEXT: v_mul_lo_u16 v9, v9, v16
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16 v8, 8, v8
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16 v1, 12, v1
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16 v7, 12, v7
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16 v5, 12, v5
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16 v2, 12, v2
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16 v14, 12, v14
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16 v12, 12, v12
-; GFX10-DL-XNACK-NEXT: v_ashrrev_i16 v7, 12, v7
-; GFX10-DL-XNACK-NEXT: v_ashrrev_i16 v3, 12, v3
-; GFX10-DL-XNACK-NEXT: v_ashrrev_i16 v14, 12, v14
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16 v6, 12, v6
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16 v0, 12, v0
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16 v13, 12, v13
; GFX10-DL-XNACK-NEXT: v_ashrrev_i16 v11, 12, v11
-; GFX10-DL-XNACK-NEXT: v_mul_lo_u16 v6, v6, v13
-; GFX10-DL-XNACK-NEXT: v_ashrrev_i16 v1, 12, v1
-; GFX10-DL-XNACK-NEXT: v_ashrrev_i16 v2, 12, v2
-; GFX10-DL-XNACK-NEXT: v_mul_lo_u16 v9, v9, v16
-; GFX10-DL-XNACK-NEXT: v_lshlrev_b16 v8, 8, v8
-; GFX10-DL-XNACK-NEXT: v_lshlrev_b16 v10, 8, v10
+; GFX10-DL-XNACK-NEXT: v_mul_lo_u16 v10, v10, v15
+; GFX10-DL-XNACK-NEXT: v_or_b32_sdwa v8, v9, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16 v7, 12, v7
; GFX10-DL-XNACK-NEXT: v_ashrrev_i16 v5, 12, v5
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16 v14, 12, v14
; GFX10-DL-XNACK-NEXT: v_ashrrev_i16 v12, 12, v12
-; GFX10-DL-XNACK-NEXT: v_mul_lo_u16 v3, v3, v11
-; GFX10-DL-XNACK-NEXT: v_mul_lo_u16 v11, v7, v14
-; GFX10-DL-XNACK-NEXT: v_lshlrev_b16 v6, 8, v6
-; GFX10-DL-XNACK-NEXT: v_or_b32_sdwa v8, v9, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-DL-XNACK-NEXT: v_and_b32_e32 v9, 0xffff, v10
; GFX10-DL-XNACK-NEXT: v_mul_lo_u16 v1, v1, v2
-; GFX10-DL-XNACK-NEXT: v_mul_lo_u16 v2, v5, v12
-; GFX10-DL-XNACK-NEXT: v_lshlrev_b16 v13, 8, v3
-; GFX10-DL-XNACK-NEXT: v_or_b32_sdwa v3, v11, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-DL-XNACK-NEXT: v_lshl_or_b32 v6, v8, 16, v9
-; GFX10-DL-XNACK-NEXT: v_or_b32_sdwa v1, v1, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-DL-XNACK-NEXT: v_or_b32_sdwa v2, v2, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-DL-XNACK-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX10-DL-XNACK-NEXT: v_mul_lo_u16 v2, v0, v11
+; GFX10-DL-XNACK-NEXT: v_mul_lo_u16 v6, v6, v13
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16 v9, 8, v10
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b32_e32 v0, 16, v8
+; GFX10-DL-XNACK-NEXT: v_mul_lo_u16 v10, v5, v12
+; GFX10-DL-XNACK-NEXT: v_mul_lo_u16 v11, v7, v14
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16 v13, 8, v2
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16 v2, 8, v6
+; GFX10-DL-XNACK-NEXT: v_or_b32_sdwa v6, v0, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-DL-XNACK-NEXT: v_or_b32_sdwa v1, v1, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-DL-XNACK-NEXT: v_or_b32_sdwa v9, v10, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-DL-XNACK-NEXT: v_or_b32_sdwa v2, v11, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v6, 8, v6
; GFX10-DL-XNACK-NEXT: s_waitcnt vmcnt(0)
-; GFX10-DL-XNACK-NEXT: v_add_nc_u16 v9, v1, v0
-; GFX10-DL-XNACK-NEXT: v_lshlrev_b32_e32 v0, 16, v8
-; GFX10-DL-XNACK-NEXT: v_lshl_or_b32 v1, v2, 16, v3
-; GFX10-DL-XNACK-NEXT: v_add_nc_u16 v6, v9, v6
+; GFX10-DL-XNACK-NEXT: v_add_nc_u16 v3, v1, v3
+; GFX10-DL-XNACK-NEXT: v_or_b32_sdwa v1, v9, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-DL-XNACK-NEXT: v_add_nc_u16 v6, v3, v6
; GFX10-DL-XNACK-NEXT: v_lshrrev_b64 v[2:3], 24, v[0:1]
; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; GFX10-DL-XNACK-NEXT: v_add_nc_u16 v0, v6, v8
@@ -3001,76 +2997,74 @@ define amdgpu_kernel void @idot8_acc8_vecMul(ptr addrspace(1) %src1,
; GFX10-DL-NOXNACK-NEXT: global_load_ubyte v2, v4, s[0:1]
; GFX10-DL-NOXNACK-NEXT: s_waitcnt vmcnt(2)
; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v8, 12, v1
-; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v10, 4, v1
; GFX10-DL-NOXNACK-NEXT: s_waitcnt vmcnt(1)
; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v15, 12, v0
-; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v17, 4, v0
-; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v6, 20, v1
-; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v13, 20, v0
; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v9, 8, v1
; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v16, 8, v0
-; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16 v10, 12, v10
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v10, 4, v1
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16 v8, 12, v8
-; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16 v17, 12, v17
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16 v15, 12, v15
-; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v3, 28, v1
-; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v7, 16, v1
-; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v11, 28, v0
-; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v14, 16, v0
-; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16 v6, 12, v6
-; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16 v13, 12, v13
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v17, 4, v0
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16 v9, 12, v9
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16 v16, 12, v16
-; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16 v10, 12, v10
; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16 v8, 12, v8
-; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16 v17, 12, v17
; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16 v15, 12, v15
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v3, 28, v1
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v6, 20, v1
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v11, 28, v0
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v13, 20, v0
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16 v10, 12, v10
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16 v17, 12, v17
+; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16 v9, 12, v9
+; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16 v16, 12, v16
+; GFX10-DL-NOXNACK-NEXT: v_mul_lo_u16 v8, v8, v15
; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v5, 24, v1
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v7, 16, v1
; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v12, 24, v0
-; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16 v7, 12, v7
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v14, 16, v0
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16 v6, 12, v6
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16 v3, 12, v3
-; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16 v14, 12, v14
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16 v13, 12, v13
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16 v11, 12, v11
-; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16 v6, 12, v6
-; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16 v13, 12, v13
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16 v1, 12, v1
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16 v0, 12, v0
-; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16 v9, 12, v9
-; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16 v16, 12, v16
-; GFX10-DL-NOXNACK-NEXT: v_mul_lo_u16 v8, v8, v15
-; GFX10-DL-NOXNACK-NEXT: v_mul_lo_u16 v10, v10, v17
+; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16 v10, 12, v10
+; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16 v15, 12, v17
+; GFX10-DL-NOXNACK-NEXT: v_mul_lo_u16 v9, v9, v16
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16 v8, 8, v8
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16 v7, 12, v7
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16 v5, 12, v5
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16 v14, 12, v14
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16 v12, 12, v12
-; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16 v7, 12, v7
+; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16 v6, 12, v6
; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16 v3, 12, v3
-; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16 v14, 12, v14
+; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16 v13, 12, v13
; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16 v11, 12, v11
-; GFX10-DL-NOXNACK-NEXT: v_mul_lo_u16 v6, v6, v13
; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16 v1, 12, v1
; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16 v0, 12, v0
-; GFX10-DL-NOXNACK-NEXT: v_mul_lo_u16 v9, v9, v16
-; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16 v8, 8, v8
-; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16 v10, 8, v10
+; GFX10-DL-NOXNACK-NEXT: v_mul_lo_u16 v10, v10, v15
+; GFX10-DL-NOXNACK-NEXT: v_or_b32_sdwa v8, v9, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16 v7, 12, v7
; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16 v5, 12, v5
+; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16 v14, 12, v14
; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16 v12, 12, v12
; GFX10-DL-NOXNACK-NEXT: v_mul_lo_u16 v3, v3, v11
+; GFX10-DL-NOXNACK-NEXT: v_mul_lo_u16 v6, v6, v13
+; GFX10-DL-NOXNACK-NEXT: v_mul_lo_u16 v1, v1, v0
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16 v9, 8, v10
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b32_e32 v0, 16, v8
+; GFX10-DL-NOXNACK-NEXT: v_mul_lo_u16 v10, v5, v12
; GFX10-DL-NOXNACK-NEXT: v_mul_lo_u16 v11, v7, v14
-; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16 v6, 8, v6
-; GFX10-DL-NOXNACK-NEXT: v_or_b32_sdwa v8, v9, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-DL-NOXNACK-NEXT: v_and_b32_e32 v9, 0xffff, v10
-; GFX10-DL-NOXNACK-NEXT: v_mul_lo_u16 v0, v1, v0
-; GFX10-DL-NOXNACK-NEXT: v_mul_lo_u16 v1, v5, v12
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16 v13, 8, v3
-; GFX10-DL-NOXNACK-NEXT: v_or_b32_sdwa v3, v11, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-DL-NOXNACK-NEXT: v_lshl_or_b32 v6, v8, 16, v9
-; GFX10-DL-NOXNACK-NEXT: v_or_b32_sdwa v0, v0, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-DL-NOXNACK-NEXT: v_or_b32_sdwa v1, v1, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-DL-NOXNACK-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16 v3, 8, v6
+; GFX10-DL-NOXNACK-NEXT: v_or_b32_sdwa v6, v0, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-DL-NOXNACK-NEXT: v_or_b32_sdwa v1, v1, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-DL-NOXNACK-NEXT: v_or_b32_sdwa v9, v10, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-DL-NOXNACK-NEXT: v_or_b32_sdwa v3, v11, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v6, 8, v6
; GFX10-DL-NOXNACK-NEXT: s_waitcnt vmcnt(0)
-; GFX10-DL-NOXNACK-NEXT: v_add_nc_u16 v2, v0, v2
-; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b32_e32 v0, 16, v8
-; GFX10-DL-NOXNACK-NEXT: v_lshl_or_b32 v1, v1, 16, v3
+; GFX10-DL-NOXNACK-NEXT: v_add_nc_u16 v2, v1, v2
+; GFX10-DL-NOXNACK-NEXT: v_or_b32_sdwa v1, v9, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-DL-NOXNACK-NEXT: v_add_nc_u16 v6, v2, v6
; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b64 v[2:3], 24, v[0:1]
; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v1, 8, v1
diff --git a/llvm/test/CodeGen/AMDGPU/idot8u.ll b/llvm/test/CodeGen/AMDGPU/idot8u.ll
index 646e9508164d4..5f1b3696c6663 100644
--- a/llvm/test/CodeGen/AMDGPU/idot8u.ll
+++ b/llvm/test/CodeGen/AMDGPU/idot8u.ll
@@ -2538,134 +2538,132 @@ define amdgpu_kernel void @udot8_acc8_vecMul(ptr addrspace(1) %src1,
;
; GFX9-LABEL: udot8_acc8_vecMul:
; GFX9: ; %bb.0: ; %entry
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NEXT: v_mov_b32_e32 v3, 0
; GFX9-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_load_dword v1, v0, s[0:1]
+; GFX9-NEXT: global_load_dword v2, v0, s[2:3]
+; GFX9-NEXT: global_load_ubyte v4, v3, s[6:7]
; GFX9-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX9-NEXT: s_mov_b32 s14, -1
; GFX9-NEXT: s_mov_b32 s15, 0xe00000
; GFX9-NEXT: s_add_u32 s12, s12, s11
-; GFX9-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v1, v0, s[8:9]
-; GFX9-NEXT: global_load_dword v2, v0, s[10:11]
-; GFX9-NEXT: global_load_ubyte v4, v3, s[0:1]
; GFX9-NEXT: s_addc_u32 s13, s13, 0
; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_bfe_u32 v8, v1, 20, 4
-; GFX9-NEXT: v_bfe_u32 v9, v1, 16, 4
-; GFX9-NEXT: v_lshrrev_b32_e32 v10, 28, v1
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_bfe_u32 v15, v2, 20, 4
-; GFX9-NEXT: v_bfe_u32 v16, v2, 16, 4
-; GFX9-NEXT: v_lshrrev_b32_e32 v17, 28, v2
; GFX9-NEXT: v_bfe_u32 v0, v1, 4, 4
; GFX9-NEXT: v_and_b32_e32 v5, 15, v1
; GFX9-NEXT: v_bfe_u32 v6, v1, 12, 4
; GFX9-NEXT: v_bfe_u32 v7, v1, 8, 4
+; GFX9-NEXT: v_bfe_u32 v8, v1, 20, 4
+; GFX9-NEXT: v_bfe_u32 v9, v1, 16, 4
+; GFX9-NEXT: v_lshrrev_b32_e32 v10, 28, v1
; GFX9-NEXT: v_bfe_u32 v11, v1, 24, 4
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_bfe_u32 v1, v2, 4, 4
; GFX9-NEXT: v_and_b32_e32 v12, 15, v2
; GFX9-NEXT: v_bfe_u32 v13, v2, 12, 4
; GFX9-NEXT: v_bfe_u32 v14, v2, 8, 4
+; GFX9-NEXT: v_bfe_u32 v15, v2, 20, 4
+; GFX9-NEXT: v_bfe_u32 v16, v2, 16, 4
+; GFX9-NEXT: v_lshrrev_b32_e32 v17, 28, v2
; GFX9-NEXT: v_bfe_u32 v2, v2, 24, 4
+; GFX9-NEXT: v_mul_lo_u16_e32 v18, v11, v2
; GFX9-NEXT: v_mul_lo_u16_sdwa v10, v10, v17 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mul_lo_u16_e32 v17, v9, v16
; GFX9-NEXT: v_mul_lo_u16_sdwa v8, v8, v15 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: v_mul_lo_u16_e32 v18, v11, v2
; GFX9-NEXT: v_mul_lo_u16_e32 v7, v7, v14
; GFX9-NEXT: v_mul_lo_u16_sdwa v6, v6, v13 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: v_mul_lo_u16_sdwa v1, v0, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: v_or_b32_e32 v0, v17, v8
; GFX9-NEXT: v_mul_lo_u16_e32 v5, v5, v12
-; GFX9-NEXT: v_or_b32_e32 v12, v18, v10
+; GFX9-NEXT: v_mul_lo_u16_sdwa v12, v0, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v0, v18, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_e32 v1, v17, v8
; GFX9-NEXT: v_or_b32_e32 v6, v7, v6
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v0
-; GFX9-NEXT: v_or_b32_e32 v5, v5, v1
-; GFX9-NEXT: v_lshrrev_b32_e32 v7, 8, v10
+; GFX9-NEXT: v_or_b32_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v10, v6, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v1, v12, 16, v8
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_add_u16_e32 v4, v5, v4
-; GFX9-NEXT: v_lshrrev_b32_e32 v5, 8, v10
-; GFX9-NEXT: v_lshrrev_b32_e32 v8, 8, v1
+; GFX9-NEXT: v_or_b32_e32 v5, v5, v12
+; GFX9-NEXT: v_lshrrev_b32_e32 v7, 8, v10
+; GFX9-NEXT: v_lshl_or_b32 v8, v6, 16, v12
+; GFX9-NEXT: v_lshrrev_b32_e32 v10, 8, v1
; GFX9-NEXT: v_lshrrev_b64 v[0:1], 24, v[0:1]
-; GFX9-NEXT: v_add_u16_e32 v1, v4, v5
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 8, v8
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_add_u16_e32 v1, v5, v4
+; GFX9-NEXT: v_add_u16_e32 v1, v1, v8
; GFX9-NEXT: v_add_u16_e32 v1, v1, v6
; GFX9-NEXT: v_add_u16_e32 v0, v1, v0
; GFX9-NEXT: v_mad_legacy_u16 v0, v9, v16, v0
-; GFX9-NEXT: v_add_u16_e32 v0, v0, v8
+; GFX9-NEXT: v_add_u16_e32 v0, v0, v10
; GFX9-NEXT: v_mad_legacy_u16 v0, v11, v2, v0
; GFX9-NEXT: v_add_u16_e32 v0, v0, v7
-; GFX9-NEXT: global_store_byte v3, v0, s[0:1]
+; GFX9-NEXT: global_store_byte v3, v0, s[6:7]
; GFX9-NEXT: s_endpgm
;
; GFX9-DL-LABEL: udot8_acc8_vecMul:
; GFX9-DL: ; %bb.0: ; %entry
+; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-DL-NEXT: v_mov_b32_e32 v3, 0
; GFX9-DL-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-DL-NEXT: global_load_dword v1, v0, s[0:1]
+; GFX9-DL-NEXT: global_load_dword v2, v0, s[2:3]
+; GFX9-DL-NEXT: global_load_ubyte v4, v3, s[6:7]
; GFX9-DL-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX9-DL-NEXT: s_mov_b32 s14, -1
; GFX9-DL-NEXT: s_mov_b32 s15, 0xe00000
; GFX9-DL-NEXT: s_add_u32 s12, s12, s11
-; GFX9-DL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
-; GFX9-DL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX9-DL-NEXT: v_mov_b32_e32 v3, 0
-; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: global_load_dword v1, v0, s[8:9]
-; GFX9-DL-NEXT: global_load_dword v2, v0, s[10:11]
-; GFX9-DL-NEXT: global_load_ubyte v4, v3, s[0:1]
; GFX9-DL-NEXT: s_addc_u32 s13, s13, 0
; GFX9-DL-NEXT: s_waitcnt vmcnt(2)
-; GFX9-DL-NEXT: v_bfe_u32 v8, v1, 20, 4
-; GFX9-DL-NEXT: v_bfe_u32 v9, v1, 16, 4
-; GFX9-DL-NEXT: v_lshrrev_b32_e32 v10, 28, v1
-; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
-; GFX9-DL-NEXT: v_bfe_u32 v15, v2, 20, 4
-; GFX9-DL-NEXT: v_bfe_u32 v16, v2, 16, 4
-; GFX9-DL-NEXT: v_lshrrev_b32_e32 v17, 28, v2
; GFX9-DL-NEXT: v_bfe_u32 v0, v1, 4, 4
; GFX9-DL-NEXT: v_and_b32_e32 v5, 15, v1
; GFX9-DL-NEXT: v_bfe_u32 v6, v1, 12, 4
; GFX9-DL-NEXT: v_bfe_u32 v7, v1, 8, 4
+; GFX9-DL-NEXT: v_bfe_u32 v8, v1, 20, 4
+; GFX9-DL-NEXT: v_bfe_u32 v9, v1, 16, 4
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v10, 28, v1
; GFX9-DL-NEXT: v_bfe_u32 v11, v1, 24, 4
+; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
; GFX9-DL-NEXT: v_bfe_u32 v1, v2, 4, 4
; GFX9-DL-NEXT: v_and_b32_e32 v12, 15, v2
; GFX9-DL-NEXT: v_bfe_u32 v13, v2, 12, 4
; GFX9-DL-NEXT: v_bfe_u32 v14, v2, 8, 4
+; GFX9-DL-NEXT: v_bfe_u32 v15, v2, 20, 4
+; GFX9-DL-NEXT: v_bfe_u32 v16, v2, 16, 4
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v17, 28, v2
; GFX9-DL-NEXT: v_bfe_u32 v2, v2, 24, 4
+; GFX9-DL-NEXT: v_mul_lo_u16_e32 v18, v11, v2
; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v10, v10, v17 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-DL-NEXT: v_mul_lo_u16_e32 v17, v9, v16
; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v8, v8, v15 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-DL-NEXT: v_mul_lo_u16_e32 v18, v11, v2
; GFX9-DL-NEXT: v_mul_lo_u16_e32 v7, v7, v14
; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v6, v6, v13 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v1, v0, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-DL-NEXT: v_or_b32_e32 v0, v17, v8
; GFX9-DL-NEXT: v_mul_lo_u16_e32 v5, v5, v12
-; GFX9-DL-NEXT: v_or_b32_e32 v12, v18, v10
+; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v12, v0, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-DL-NEXT: v_or_b32_sdwa v0, v18, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-DL-NEXT: v_or_b32_e32 v1, v17, v8
; GFX9-DL-NEXT: v_or_b32_e32 v6, v7, v6
-; GFX9-DL-NEXT: v_and_b32_e32 v8, 0xffff, v0
-; GFX9-DL-NEXT: v_or_b32_e32 v5, v5, v1
-; GFX9-DL-NEXT: v_lshrrev_b32_e32 v7, 8, v10
+; GFX9-DL-NEXT: v_or_b32_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 16, v6
-; GFX9-DL-NEXT: v_lshl_or_b32 v10, v6, 16, v1
-; GFX9-DL-NEXT: v_lshl_or_b32 v1, v12, 16, v8
-; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DL-NEXT: v_add_u16_e32 v4, v5, v4
-; GFX9-DL-NEXT: v_lshrrev_b32_e32 v5, 8, v10
-; GFX9-DL-NEXT: v_lshrrev_b32_e32 v8, 8, v1
+; GFX9-DL-NEXT: v_or_b32_e32 v5, v5, v12
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v7, 8, v10
+; GFX9-DL-NEXT: v_lshl_or_b32 v8, v6, 16, v12
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v10, 8, v1
; GFX9-DL-NEXT: v_lshrrev_b64 v[0:1], 24, v[0:1]
-; GFX9-DL-NEXT: v_add_u16_e32 v1, v4, v5
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v8, 8, v8
+; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-DL-NEXT: v_add_u16_e32 v1, v5, v4
+; GFX9-DL-NEXT: v_add_u16_e32 v1, v1, v8
; GFX9-DL-NEXT: v_add_u16_e32 v1, v1, v6
; GFX9-DL-NEXT: v_add_u16_e32 v0, v1, v0
; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v9, v16, v0
-; GFX9-DL-NEXT: v_add_u16_e32 v0, v0, v8
+; GFX9-DL-NEXT: v_add_u16_e32 v0, v0, v10
; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v11, v2, v0
; GFX9-DL-NEXT: v_add_u16_e32 v0, v0, v7
-; GFX9-DL-NEXT: global_store_byte v3, v0, s[0:1]
+; GFX9-DL-NEXT: global_store_byte v3, v0, s[6:7]
; GFX9-DL-NEXT: s_endpgm
;
; GFX10-DL-LABEL: udot8_acc8_vecMul:
@@ -2685,58 +2683,56 @@ define amdgpu_kernel void @udot8_acc8_vecMul(ptr addrspace(1) %src1,
; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: global_load_dword v1, v0, s[0:1]
; GFX10-DL-NEXT: global_load_dword v2, v0, s[2:3]
-; GFX10-DL-NEXT: global_load_ubyte v0, v4, s[6:7]
+; GFX10-DL-NEXT: global_load_ubyte v3, v4, s[6:7]
; GFX10-DL-NEXT: s_waitcnt vmcnt(2)
-; GFX10-DL-NEXT: v_bfe_u32 v3, v1, 4, 4
-; GFX10-DL-NEXT: v_and_b32_e32 v5, 15, v1
; GFX10-DL-NEXT: v_bfe_u32 v6, v1, 12, 4
+; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX10-DL-NEXT: v_bfe_u32 v9, v2, 12, 4
+; GFX10-DL-NEXT: v_bfe_u32 v0, v1, 4, 4
+; GFX10-DL-NEXT: v_and_b32_e32 v5, 15, v1
; GFX10-DL-NEXT: v_bfe_u32 v7, v1, 8, 4
; GFX10-DL-NEXT: v_bfe_u32 v8, v1, 20, 4
-; GFX10-DL-NEXT: v_bfe_u32 v9, v1, 16, 4
-; GFX10-DL-NEXT: v_lshrrev_b32_e32 v10, 28, v1
-; GFX10-DL-NEXT: v_bfe_u32 v11, v1, 24, 4
-; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
-; GFX10-DL-NEXT: v_bfe_u32 v1, v2, 4, 4
-; GFX10-DL-NEXT: v_bfe_u32 v13, v2, 12, 4
-; GFX10-DL-NEXT: v_bfe_u32 v15, v2, 20, 4
-; GFX10-DL-NEXT: v_bfe_u32 v14, v2, 8, 4
-; GFX10-DL-NEXT: v_and_b32_e32 v12, 15, v2
-; GFX10-DL-NEXT: v_mul_lo_u16 v1, v3, v1
-; GFX10-DL-NEXT: v_mul_lo_u16 v6, v6, v13
-; GFX10-DL-NEXT: v_bfe_u32 v16, v2, 16, 4
-; GFX10-DL-NEXT: v_lshrrev_b32_e32 v17, 28, v2
-; GFX10-DL-NEXT: v_bfe_u32 v13, v2, 24, 4
-; GFX10-DL-NEXT: v_mul_lo_u16 v2, v8, v15
-; GFX10-DL-NEXT: v_mul_lo_u16 v3, v7, v14
+; GFX10-DL-NEXT: v_bfe_u32 v10, v1, 16, 4
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v11, 28, v1
+; GFX10-DL-NEXT: v_bfe_u32 v12, v1, 24, 4
+; GFX10-DL-NEXT: v_bfe_u32 v1, v2, 8, 4
+; GFX10-DL-NEXT: v_mul_lo_u16 v6, v6, v9
+; GFX10-DL-NEXT: v_bfe_u32 v9, v2, 4, 4
+; GFX10-DL-NEXT: v_bfe_u32 v14, v2, 20, 4
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v15, 28, v2
+; GFX10-DL-NEXT: v_mul_lo_u16 v1, v7, v1
; GFX10-DL-NEXT: v_lshlrev_b16 v6, 8, v6
-; GFX10-DL-NEXT: v_lshlrev_b16 v1, 8, v1
-; GFX10-DL-NEXT: v_mul_lo_u16 v7, v10, v17
-; GFX10-DL-NEXT: v_mul_lo_u16 v8, v9, v16
-; GFX10-DL-NEXT: v_lshlrev_b16 v2, 8, v2
-; GFX10-DL-NEXT: v_or_b32_e32 v6, v3, v6
-; GFX10-DL-NEXT: v_and_b32_e32 v3, 0xffff, v1
-; GFX10-DL-NEXT: v_mul_lo_u16 v5, v5, v12
-; GFX10-DL-NEXT: v_mul_lo_u16 v10, v11, v13
-; GFX10-DL-NEXT: v_lshlrev_b16 v7, 8, v7
-; GFX10-DL-NEXT: v_or_b32_e32 v2, v8, v2
-; GFX10-DL-NEXT: v_lshl_or_b32 v3, v6, 16, v3
-; GFX10-DL-NEXT: v_or_b32_e32 v1, v5, v1
-; GFX10-DL-NEXT: v_or_b32_e32 v5, v10, v7
-; GFX10-DL-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10-DL-NEXT: v_lshrrev_b32_e32 v3, 8, v3
-; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
-; GFX10-DL-NEXT: v_add_nc_u16 v8, v1, v0
+; GFX10-DL-NEXT: v_and_b32_e32 v13, 15, v2
+; GFX10-DL-NEXT: v_mul_lo_u16 v0, v0, v9
+; GFX10-DL-NEXT: v_bfe_u32 v7, v2, 24, 4
+; GFX10-DL-NEXT: v_bfe_u32 v16, v2, 16, 4
+; GFX10-DL-NEXT: v_or_b32_e32 v6, v1, v6
+; GFX10-DL-NEXT: v_mul_lo_u16 v2, v11, v15
+; GFX10-DL-NEXT: v_mul_lo_u16 v8, v8, v14
+; GFX10-DL-NEXT: v_lshlrev_b16 v9, 8, v0
+; GFX10-DL-NEXT: v_mul_lo_u16 v5, v5, v13
; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 16, v6
-; GFX10-DL-NEXT: v_lshl_or_b32 v1, v5, 16, v2
-; GFX10-DL-NEXT: v_add_nc_u16 v5, v8, v3
+; GFX10-DL-NEXT: v_mul_lo_u16 v1, v12, v7
+; GFX10-DL-NEXT: v_mul_lo_u16 v11, v10, v16
+; GFX10-DL-NEXT: v_lshlrev_b16 v13, 8, v2
+; GFX10-DL-NEXT: v_lshlrev_b16 v2, 8, v8
+; GFX10-DL-NEXT: v_or_b32_sdwa v8, v0, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-DL-NEXT: v_or_b32_e32 v5, v5, v9
+; GFX10-DL-NEXT: v_or_b32_sdwa v1, v1, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-DL-NEXT: v_or_b32_e32 v2, v11, v2
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v8, 8, v8
+; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
+; GFX10-DL-NEXT: v_add_nc_u16 v3, v5, v3
+; GFX10-DL-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-DL-NEXT: v_add_nc_u16 v5, v3, v8
; GFX10-DL-NEXT: v_lshrrev_b64 v[2:3], 24, v[0:1]
; GFX10-DL-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; GFX10-DL-NEXT: v_add_nc_u16 v0, v5, v6
; GFX10-DL-NEXT: v_add_nc_u16 v0, v0, v2
-; GFX10-DL-NEXT: v_mad_u16 v0, v9, v16, v0
+; GFX10-DL-NEXT: v_mad_u16 v0, v10, v16, v0
; GFX10-DL-NEXT: v_add_nc_u16 v0, v0, v1
-; GFX10-DL-NEXT: v_lshrrev_b32_e32 v1, 8, v7
-; GFX10-DL-NEXT: v_mad_u16 v0, v11, v13, v0
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v1, 8, v13
+; GFX10-DL-NEXT: v_mad_u16 v0, v12, v7, v0
; GFX10-DL-NEXT: v_add_nc_u16 v0, v0, v1
; GFX10-DL-NEXT: global_store_byte v4, v0, s[6:7]
; GFX10-DL-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll b/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll
index 8c89db87ebd78..0a3874e6f70c0 100644
--- a/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll
+++ b/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll
@@ -11665,10 +11665,10 @@ define <2 x i16> @multi_use_mul_mad_i16_var(i16 %x, i16 %y, i16 %z0, i16 %z1) {
; GFX10-GISEL-LABEL: multi_use_mul_mad_i16_var:
; GFX10-GISEL: ; %bb.0: ; %entry
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mad_u16 v2, v0, v1, v2
-; GFX10-GISEL-NEXT: v_mad_u16 v0, v0, v1, v3
-; GFX10-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v2
-; GFX10-GISEL-NEXT: v_lshl_or_b32 v0, v0, 16, v1
+; GFX10-GISEL-NEXT: v_mad_u16 v3, v0, v1, v3
+; GFX10-GISEL-NEXT: v_mad_u16 v0, v0, v1, v2
+; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX10-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: multi_use_mul_mad_i16_var:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.a16.dim.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.a16.dim.ll
index 556fb9647309c..8920ba521b3b3 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.a16.dim.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.a16.dim.ll
@@ -13,8 +13,8 @@ define amdgpu_ps <4 x float> @sample_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX10GISEL-LABEL: sample_d_1d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10GISEL-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX10GISEL-NEXT: v_lshlrev_b32_e64 v3, 16, s0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_d v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -33,8 +33,8 @@ define amdgpu_ps <4 x float> @sample_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX10GISEL-LABEL: sample_d_2d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX10GISEL-NEXT: v_lshl_or_b32 v4, v5, 16, v4
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX10GISEL-NEXT: v_or_b32_sdwa v4, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_d v[0:3], v[0:4], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -60,10 +60,10 @@ define amdgpu_ps <4 x float> @sample_d_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX10GISEL-LABEL: sample_d_3d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX10GISEL-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX10GISEL-NEXT: v_lshl_or_b32 v6, v7, 16, v6
-; GFX10GISEL-NEXT: v_lshl_or_b32 v7, s0, 16, v8
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX10GISEL-NEXT: v_lshlrev_b32_e64 v9, 16, s0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v6, v7, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v7, v9, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_d v[0:3], v[0:7], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_3D a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -81,8 +81,8 @@ define amdgpu_ps <4 x float> @sample_c_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inr
;
; GFX10GISEL-LABEL: sample_c_d_1d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX10GISEL-NEXT: v_lshl_or_b32 v3, s0, 16, v3
+; GFX10GISEL-NEXT: v_lshlrev_b32_e64 v4, 16, s0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_c_d v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -101,8 +101,8 @@ define amdgpu_ps <4 x float> @sample_c_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
;
; GFX10GISEL-LABEL: sample_c_d_2d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX10GISEL-NEXT: v_lshl_or_b32 v5, v6, 16, v5
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX10GISEL-NEXT: v_or_b32_sdwa v5, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_c_d v[0:3], v[0:5], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -121,8 +121,8 @@ define amdgpu_ps <4 x float> @sample_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX10GISEL-LABEL: sample_d_cl_1d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10GISEL-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX10GISEL-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_d_cl v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -146,10 +146,10 @@ define amdgpu_ps <4 x float> @sample_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX10GISEL-LABEL: sample_d_cl_2d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX10GISEL-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX10GISEL-NEXT: v_lshl_or_b32 v4, v5, 16, v4
-; GFX10GISEL-NEXT: v_lshl_or_b32 v5, s0, 16, v6
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX10GISEL-NEXT: v_lshlrev_b32_e64 v7, 16, s0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v4, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v5, v7, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_d_cl v[0:3], v[0:5], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -168,8 +168,8 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX10GISEL-LABEL: sample_c_d_cl_1d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX10GISEL-NEXT: v_lshl_or_b32 v3, v4, 16, v3
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX10GISEL-NEXT: v_or_b32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_c_d_cl v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -194,10 +194,10 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX10GISEL-LABEL: sample_c_d_cl_2d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX10GISEL-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX10GISEL-NEXT: v_lshl_or_b32 v5, v6, 16, v5
-; GFX10GISEL-NEXT: v_lshl_or_b32 v6, s0, 16, v7
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX10GISEL-NEXT: v_lshlrev_b32_e64 v8, 16, s0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v5, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v6, v8, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_c_d_cl v[0:3], v[0:6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -215,8 +215,8 @@ define amdgpu_ps <4 x float> @sample_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> inre
;
; GFX10GISEL-LABEL: sample_cd_1d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10GISEL-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX10GISEL-NEXT: v_lshlrev_b32_e64 v3, 16, s0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_cd v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -235,8 +235,8 @@ define amdgpu_ps <4 x float> @sample_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
;
; GFX10GISEL-LABEL: sample_cd_2d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX10GISEL-NEXT: v_lshl_or_b32 v4, v5, 16, v4
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX10GISEL-NEXT: v_or_b32_sdwa v4, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_cd v[0:3], v[0:4], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -254,8 +254,8 @@ define amdgpu_ps <4 x float> @sample_c_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX10GISEL-LABEL: sample_c_cd_1d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX10GISEL-NEXT: v_lshl_or_b32 v3, s0, 16, v3
+; GFX10GISEL-NEXT: v_lshlrev_b32_e64 v4, 16, s0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_c_cd v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -274,8 +274,8 @@ define amdgpu_ps <4 x float> @sample_c_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX10GISEL-LABEL: sample_c_cd_2d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX10GISEL-NEXT: v_lshl_or_b32 v5, v6, 16, v5
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX10GISEL-NEXT: v_or_b32_sdwa v5, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_c_cd v[0:3], v[0:5], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -294,8 +294,8 @@ define amdgpu_ps <4 x float> @sample_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> i
;
; GFX10GISEL-LABEL: sample_cd_cl_1d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10GISEL-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX10GISEL-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_cd_cl v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -319,10 +319,10 @@ define amdgpu_ps <4 x float> @sample_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
;
; GFX10GISEL-LABEL: sample_cd_cl_2d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX10GISEL-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX10GISEL-NEXT: v_lshl_or_b32 v4, v5, 16, v4
-; GFX10GISEL-NEXT: v_lshl_or_b32 v5, s0, 16, v6
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX10GISEL-NEXT: v_lshlrev_b32_e64 v7, 16, s0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v4, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v5, v7, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_cd_cl v[0:3], v[0:5], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -341,8 +341,8 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX10GISEL-LABEL: sample_c_cd_cl_1d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX10GISEL-NEXT: v_lshl_or_b32 v3, v4, 16, v3
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX10GISEL-NEXT: v_or_b32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_c_cd_cl v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -367,10 +367,10 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX10GISEL-LABEL: sample_c_cd_cl_2d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX10GISEL-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX10GISEL-NEXT: v_lshl_or_b32 v5, v6, 16, v5
-; GFX10GISEL-NEXT: v_lshl_or_b32 v6, s0, 16, v7
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX10GISEL-NEXT: v_lshlrev_b32_e64 v8, 16, s0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v5, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v6, v8, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_c_cd_cl v[0:3], v[0:6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -396,10 +396,10 @@ define amdgpu_ps float @sample_c_d_o_2darray_V1(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX10GISEL-LABEL: sample_c_d_o_2darray_V1:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX10GISEL-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX10GISEL-NEXT: v_lshl_or_b32 v6, v7, 16, v6
-; GFX10GISEL-NEXT: v_lshl_or_b32 v7, s0, 16, v8
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX10GISEL-NEXT: v_lshlrev_b32_e64 v9, 16, s0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v6, v7, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v7, v9, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_c_d_o v0, v[0:7], s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_2D_ARRAY a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -425,10 +425,10 @@ define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4
;
; GFX10GISEL-LABEL: sample_c_d_o_2darray_V2:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX10GISEL-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX10GISEL-NEXT: v_lshl_or_b32 v6, v7, 16, v6
-; GFX10GISEL-NEXT: v_lshl_or_b32 v7, s0, 16, v8
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX10GISEL-NEXT: v_lshlrev_b32_e64 v9, 16, s0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v6, v7, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v7, v9, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_c_d_o v[0:1], v[0:7], s[0:7], s[8:11] dmask:0x6 dim:SQ_RSRC_IMG_2D_ARRAY a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -468,10 +468,9 @@ define amdgpu_ps <4 x float> @sample_g16_noa16_d_1d(<8 x i32> inreg %rsrc, <4 x
;
; GFX10GISEL-LABEL: sample_g16_noa16_d_1d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10GISEL-NEXT: v_lshl_or_b32 v0, s0, 16, v0
-; GFX10GISEL-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX10GISEL-NEXT: v_lshlrev_b32_e64 v3, 16, s0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_d_g16 v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -491,10 +490,10 @@ define amdgpu_ps <4 x float> @sample_g16_noa16_d_2d(<8 x i32> inreg %rsrc, <4 x
;
; GFX10GISEL-LABEL: sample_g16_noa16_d_2d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX10GISEL-NEXT: v_lshl_or_b32 v1, v3, 16, v2
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX10GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v1, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_d_g16 v[0:3], [v0, v1, v4, v5], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -516,14 +515,15 @@ define amdgpu_ps <4 x float> @sample_g16_noa16_d_3d(<8 x i32> inreg %rsrc, <4 x
;
; GFX10GISEL-LABEL: sample_g16_noa16_d_3d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10GISEL-NEXT: v_and_b32_e32 v9, 0xffff, v2
-; GFX10GISEL-NEXT: v_and_b32_e32 v10, 0xffff, v3
-; GFX10GISEL-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX10GISEL-NEXT: v_lshl_or_b32 v2, v1, 16, v0
-; GFX10GISEL-NEXT: v_lshl_or_b32 v3, s0, 16, v9
-; GFX10GISEL-NEXT: v_lshl_or_b32 v4, v4, 16, v10
-; GFX10GISEL-NEXT: v_lshl_or_b32 v5, s0, 16, v5
+; GFX10GISEL-NEXT: v_mov_b32_e32 v9, v2
+; GFX10GISEL-NEXT: v_mov_b32_e32 v10, v3
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX10GISEL-NEXT: v_lshlrev_b32_e64 v11, 16, s0
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX10GISEL-NEXT: v_or_b32_sdwa v2, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v3, v11, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v4, v4, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v5, v11, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_d_g16 v[0:3], v[2:8], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_3D
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -541,10 +541,9 @@ define amdgpu_ps <4 x float> @sample_g16_noa16_c_d_1d(<8 x i32> inreg %rsrc, <4
;
; GFX10GISEL-LABEL: sample_g16_noa16_c_d_1d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10GISEL-NEXT: v_lshl_or_b32 v1, s0, 16, v1
-; GFX10GISEL-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX10GISEL-NEXT: v_lshlrev_b32_e64 v4, 16, s0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_c_d_g16 v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -564,10 +563,10 @@ define amdgpu_ps <4 x float> @sample_g16_noa16_c_d_2d(<8 x i32> inreg %rsrc, <4
;
; GFX10GISEL-LABEL: sample_g16_noa16_c_d_2d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX10GISEL-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX10GISEL-NEXT: v_lshl_or_b32 v2, v4, 16, v3
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX10GISEL-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v2, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_c_d_g16 v[0:3], [v0, v1, v2, v5, v6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -585,10 +584,9 @@ define amdgpu_ps <4 x float> @sample_g16_noa16_d_cl_1d(<8 x i32> inreg %rsrc, <4
;
; GFX10GISEL-LABEL: sample_g16_noa16_d_cl_1d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10GISEL-NEXT: v_lshl_or_b32 v0, s0, 16, v0
-; GFX10GISEL-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX10GISEL-NEXT: v_lshlrev_b32_e64 v4, 16, s0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_d_cl_g16 v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -608,10 +606,10 @@ define amdgpu_ps <4 x float> @sample_g16_noa16_d_cl_2d(<8 x i32> inreg %rsrc, <4
;
; GFX10GISEL-LABEL: sample_g16_noa16_d_cl_2d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX10GISEL-NEXT: v_lshl_or_b32 v1, v3, 16, v2
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX10GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v1, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_d_cl_g16 v[0:3], [v0, v1, v4, v5, v6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -629,10 +627,9 @@ define amdgpu_ps <4 x float> @sample_g16_noa16_c_d_cl_1d(<8 x i32> inreg %rsrc,
;
; GFX10GISEL-LABEL: sample_g16_noa16_c_d_cl_1d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10GISEL-NEXT: v_lshl_or_b32 v1, s0, 16, v1
-; GFX10GISEL-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX10GISEL-NEXT: v_lshlrev_b32_e64 v5, 16, s0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v1, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v2, v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_c_d_cl_g16 v[0:3], v[0:4], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -655,11 +652,12 @@ define amdgpu_ps <4 x float> @sample_g16_noa16_c_d_cl_2d(<8 x i32> inreg %rsrc,
; GFX10GISEL-LABEL: sample_g16_noa16_c_d_cl_2d:
; GFX10GISEL: ; %bb.0: ; %main_body
; GFX10GISEL-NEXT: v_mov_b32_e32 v8, v2
+; GFX10GISEL-NEXT: v_mov_b32_e32 v9, v3
; GFX10GISEL-NEXT: v_mov_b32_e32 v2, v0
-; GFX10GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v1
-; GFX10GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v3
-; GFX10GISEL-NEXT: v_lshl_or_b32 v3, v8, 16, v0
-; GFX10GISEL-NEXT: v_lshl_or_b32 v4, v4, 16, v1
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v8
+; GFX10GISEL-NEXT: v_or_b32_sdwa v4, v4, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v3, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_c_d_cl_g16 v[0:3], v[2:7], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -677,10 +675,9 @@ define amdgpu_ps <4 x float> @sample_g16_noa16_cd_1d(<8 x i32> inreg %rsrc, <4 x
;
; GFX10GISEL-LABEL: sample_g16_noa16_cd_1d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10GISEL-NEXT: v_lshl_or_b32 v0, s0, 16, v0
-; GFX10GISEL-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX10GISEL-NEXT: v_lshlrev_b32_e64 v3, 16, s0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_cd_g16 v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -700,10 +697,10 @@ define amdgpu_ps <4 x float> @sample_g16_noa16_cd_2d(<8 x i32> inreg %rsrc, <4 x
;
; GFX10GISEL-LABEL: sample_g16_noa16_cd_2d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX10GISEL-NEXT: v_lshl_or_b32 v1, v3, 16, v2
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX10GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v1, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_cd_g16 v[0:3], [v0, v1, v4, v5], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -721,10 +718,9 @@ define amdgpu_ps <4 x float> @sample_g16_noa16_c_cd_1d(<8 x i32> inreg %rsrc, <4
;
; GFX10GISEL-LABEL: sample_g16_noa16_c_cd_1d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10GISEL-NEXT: v_lshl_or_b32 v1, s0, 16, v1
-; GFX10GISEL-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX10GISEL-NEXT: v_lshlrev_b32_e64 v4, 16, s0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_c_cd_g16 v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -744,10 +740,10 @@ define amdgpu_ps <4 x float> @sample_g16_noa16_c_cd_2d(<8 x i32> inreg %rsrc, <4
;
; GFX10GISEL-LABEL: sample_g16_noa16_c_cd_2d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX10GISEL-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX10GISEL-NEXT: v_lshl_or_b32 v2, v4, 16, v3
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX10GISEL-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v2, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_c_cd_g16 v[0:3], [v0, v1, v2, v5, v6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -765,10 +761,9 @@ define amdgpu_ps <4 x float> @sample_g16_noa16_cd_cl_1d(<8 x i32> inreg %rsrc, <
;
; GFX10GISEL-LABEL: sample_g16_noa16_cd_cl_1d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10GISEL-NEXT: v_lshl_or_b32 v0, s0, 16, v0
-; GFX10GISEL-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX10GISEL-NEXT: v_lshlrev_b32_e64 v4, 16, s0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_cd_cl_g16 v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -788,10 +783,10 @@ define amdgpu_ps <4 x float> @sample_g16_noa16_cd_cl_2d(<8 x i32> inreg %rsrc, <
;
; GFX10GISEL-LABEL: sample_g16_noa16_cd_cl_2d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX10GISEL-NEXT: v_lshl_or_b32 v1, v3, 16, v2
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX10GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v1, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_cd_cl_g16 v[0:3], [v0, v1, v4, v5, v6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -809,10 +804,9 @@ define amdgpu_ps <4 x float> @sample_g16_noa16_c_cd_cl_1d(<8 x i32> inreg %rsrc,
;
; GFX10GISEL-LABEL: sample_g16_noa16_c_cd_cl_1d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10GISEL-NEXT: v_lshl_or_b32 v1, s0, 16, v1
-; GFX10GISEL-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX10GISEL-NEXT: v_lshlrev_b32_e64 v5, 16, s0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v1, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v2, v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_c_cd_cl_g16 v[0:3], v[0:4], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -835,11 +829,12 @@ define amdgpu_ps <4 x float> @sample_g16_noa16_c_cd_cl_2d(<8 x i32> inreg %rsrc,
; GFX10GISEL-LABEL: sample_g16_noa16_c_cd_cl_2d:
; GFX10GISEL: ; %bb.0: ; %main_body
; GFX10GISEL-NEXT: v_mov_b32_e32 v8, v2
+; GFX10GISEL-NEXT: v_mov_b32_e32 v9, v3
; GFX10GISEL-NEXT: v_mov_b32_e32 v2, v0
-; GFX10GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v1
-; GFX10GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v3
-; GFX10GISEL-NEXT: v_lshl_or_b32 v3, v8, 16, v0
-; GFX10GISEL-NEXT: v_lshl_or_b32 v4, v4, 16, v1
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v8
+; GFX10GISEL-NEXT: v_or_b32_sdwa v4, v4, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v3, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_c_cd_cl_g16 v[0:3], v[2:7], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -863,14 +858,15 @@ define amdgpu_ps float @sample_g16_noa16_c_d_o_2darray_V1(<8 x i32> inreg %rsrc,
;
; GFX10GISEL-LABEL: sample_g16_noa16_c_d_o_2darray_V1:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_mov_b32_e32 v9, v2
-; GFX10GISEL-NEXT: v_mov_b32_e32 v10, v3
+; GFX10GISEL-NEXT: v_mov_b32_e32 v9, v3
+; GFX10GISEL-NEXT: v_mov_b32_e32 v10, v2
+; GFX10GISEL-NEXT: v_mov_b32_e32 v11, v4
; GFX10GISEL-NEXT: v_mov_b32_e32 v2, v0
; GFX10GISEL-NEXT: v_mov_b32_e32 v3, v1
-; GFX10GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v4
-; GFX10GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v9
-; GFX10GISEL-NEXT: v_lshl_or_b32 v5, v5, 16, v1
-; GFX10GISEL-NEXT: v_lshl_or_b32 v4, v10, 16, v0
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v9
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v5
+; GFX10GISEL-NEXT: v_or_b32_sdwa v4, v0, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v5, v1, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_c_d_o_g16 v0, v[2:8], s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_2D_ARRAY
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -894,14 +890,15 @@ define amdgpu_ps <2 x float> @sample_g16_noa16_c_d_o_2darray_V2(<8 x i32> inreg
;
; GFX10GISEL-LABEL: sample_g16_noa16_c_d_o_2darray_V2:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_mov_b32_e32 v9, v2
-; GFX10GISEL-NEXT: v_mov_b32_e32 v10, v3
+; GFX10GISEL-NEXT: v_mov_b32_e32 v9, v3
+; GFX10GISEL-NEXT: v_mov_b32_e32 v10, v2
+; GFX10GISEL-NEXT: v_mov_b32_e32 v11, v4
; GFX10GISEL-NEXT: v_mov_b32_e32 v2, v0
; GFX10GISEL-NEXT: v_mov_b32_e32 v3, v1
-; GFX10GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v4
-; GFX10GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v9
-; GFX10GISEL-NEXT: v_lshl_or_b32 v5, v5, 16, v1
-; GFX10GISEL-NEXT: v_lshl_or_b32 v4, v10, 16, v0
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v9
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v5
+; GFX10GISEL-NEXT: v_or_b32_sdwa v4, v0, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v5, v1, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_c_d_o_g16 v[0:1], v[2:8], s[0:7], s[8:11] dmask:0x6 dim:SQ_RSRC_IMG_2D_ARRAY
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -941,12 +938,10 @@ define amdgpu_ps <4 x float> @sample_d_1d_g16_a16(<8 x i32> inreg %rsrc, <4 x i3
;
; GFX10GISEL-LABEL: sample_d_1d_g16_a16:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10GISEL-NEXT: v_lshl_or_b32 v0, s0, 16, v0
-; GFX10GISEL-NEXT: v_lshl_or_b32 v1, s0, 16, v1
-; GFX10GISEL-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX10GISEL-NEXT: v_lshlrev_b32_e64 v3, 16, s0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_d_g16 v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -967,12 +962,12 @@ define amdgpu_ps <4 x float> @sample_d_2d_g16_a16(<8 x i32> inreg %rsrc, <4 x i3
;
; GFX10GISEL-LABEL: sample_d_2d_g16_a16:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX10GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX10GISEL-NEXT: v_lshl_or_b32 v1, v3, 16, v2
-; GFX10GISEL-NEXT: v_lshl_or_b32 v2, v5, 16, v4
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX10GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v1, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v2, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_d_g16 v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -996,18 +991,16 @@ define amdgpu_ps <4 x float> @sample_d_3d_g16_a16(<8 x i32> inreg %rsrc, <4 x i3
;
; GFX10GISEL-LABEL: sample_d_3d_g16_a16:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX10GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v2
-; GFX10GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v3
-; GFX10GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v5
-; GFX10GISEL-NEXT: v_and_b32_e32 v5, 0xffff, v6
-; GFX10GISEL-NEXT: v_and_b32_e32 v6, 0xffff, v8
-; GFX10GISEL-NEXT: v_lshl_or_b32 v1, s0, 16, v1
-; GFX10GISEL-NEXT: v_lshl_or_b32 v2, v4, 16, v2
-; GFX10GISEL-NEXT: v_lshl_or_b32 v3, s0, 16, v3
-; GFX10GISEL-NEXT: v_lshl_or_b32 v4, v7, 16, v5
-; GFX10GISEL-NEXT: v_lshl_or_b32 v5, s0, 16, v6
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX10GISEL-NEXT: v_lshlrev_b32_e64 v9, 16, s0
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX10GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v1, v9, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v2, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v3, v9, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v4, v7, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v5, v9, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_d_g16 v[0:3], v[0:5], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_3D a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.noret.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.noret.ll
index b913a41094b48..2f0c53c9a5642 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.noret.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.noret.ll
@@ -444,10 +444,9 @@ define amdgpu_ps void @sample_d_1d_g16_nortn(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX10_11-GISEL-LABEL: sample_d_1d_g16_nortn:
; GFX10_11-GISEL: ; %bb.0: ; %main_body
-; GFX10_11-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10_11-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10_11-GISEL-NEXT: v_lshl_or_b32 v0, s0, 16, v0
-; GFX10_11-GISEL-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX10_11-GISEL-NEXT: v_lshlrev_b32_e64 v3, 16, s0
+; GFX10_11-GISEL-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10_11-GISEL-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10_11-GISEL-NEXT: image_sample_d_g16 off, v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX10_11-GISEL-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.fptrunc.round.ll b/llvm/test/CodeGen/AMDGPU/llvm.fptrunc.round.ll
index a84942e06b5a0..2ab76c99095f8 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.fptrunc.round.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.fptrunc.round.ll
@@ -191,8 +191,8 @@ define amdgpu_gs <2 x half> @v_fptrunc_round_poison_to_v2f16_towardzero(float %a
; GISEL-LABEL: v_fptrunc_round_poison_to_v2f16_towardzero:
; GISEL: ; %bb.0:
; GISEL-NEXT: v_cvt_pkrtz_f16_f32_e32 v0, v0, v0
-; GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GISEL-NEXT: v_lshl_or_b32 v0, s0, 16, v0
+; GISEL-NEXT: v_lshlrev_b32_e64 v1, 16, s0
+; GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GISEL-NEXT: ; return to shader part epilog
;
; GFX12-SDAG-LABEL: v_fptrunc_round_poison_to_v2f16_towardzero:
@@ -1133,10 +1133,10 @@ define amdgpu_gs <2 x i32> @s_fptrunc_round_v2f32_to_v2f16_towardzero(<2 x float
define amdgpu_gs void @s_fptrunc_round_v2f32_to_v2f16_upward_multiple_calls(<2 x float> inreg %a, <2 x float> inreg %b, ptr addrspace(1) %out) {
; SDAG-LABEL: s_fptrunc_round_v2f32_to_v2f16_upward_multiple_calls:
; SDAG: ; %bb.0:
-; SDAG-NEXT: v_mov_b32_e32 v2, s0
-; SDAG-NEXT: v_mov_b32_e32 v3, s2
-; SDAG-NEXT: v_mov_b32_e32 v4, s1
-; SDAG-NEXT: v_mov_b32_e32 v5, s3
+; SDAG-NEXT: v_mov_b32_e32 v2, s1
+; SDAG-NEXT: v_mov_b32_e32 v3, s3
+; SDAG-NEXT: v_mov_b32_e32 v4, s0
+; SDAG-NEXT: v_mov_b32_e32 v5, s2
; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 1
; SDAG-NEXT: v_cvt_f16_f32_e32 v2, v2
; SDAG-NEXT: v_cvt_f16_f32_e32 v6, v3
@@ -1144,13 +1144,13 @@ define amdgpu_gs void @s_fptrunc_round_v2f32_to_v2f16_upward_multiple_calls(<2 x
; SDAG-NEXT: v_cvt_f16_f32_e32 v7, v5
; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 2
; SDAG-NEXT: v_cvt_f16_f32_e32 v3, v3
-; SDAG-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; SDAG-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; SDAG-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; SDAG-NEXT: v_lshl_or_b32 v2, v4, 16, v2
+; SDAG-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SDAG-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; SDAG-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SDAG-NEXT: v_or_b32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; SDAG-NEXT: v_cvt_f16_f32_e32 v4, v5
-; SDAG-NEXT: v_lshl_or_b32 v5, v7, 16, v6
-; SDAG-NEXT: v_lshl_or_b32 v3, v4, 16, v3
+; SDAG-NEXT: v_or_b32_sdwa v5, v6, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; SDAG-NEXT: v_or_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 3, 1), 0
; SDAG-NEXT: v_pk_add_f16 v2, v2, v5
; SDAG-NEXT: v_pk_add_f16 v2, v3, v2
@@ -1315,13 +1315,13 @@ define amdgpu_gs <3 x half> @v_fptrunc_round_v3f32_to_v3f16_upward(<3 x float> %
;
; GISEL-LABEL: v_fptrunc_round_v3f32_to_v3f16_upward:
; GISEL: ; %bb.0:
+; GISEL-NEXT: v_lshlrev_b32_e64 v3, 16, s0
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 1
-; GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
; GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
; GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
; GISEL-NEXT: v_pack_b32_f16 v0, v0, v1
-; GISEL-NEXT: v_lshl_or_b32 v1, s0, 16, v2
+; GISEL-NEXT: v_or_b32_sdwa v1, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GISEL-NEXT: ; return to shader part epilog
;
; GFX12-SDAG-LABEL: v_fptrunc_round_v3f32_to_v3f16_upward:
@@ -1375,13 +1375,13 @@ define amdgpu_gs <3 x half> @v_fptrunc_round_v3f32_to_v3f16_downward(<3 x float>
;
; GISEL-LABEL: v_fptrunc_round_v3f32_to_v3f16_downward:
; GISEL: ; %bb.0:
+; GISEL-NEXT: v_lshlrev_b32_e64 v3, 16, s0
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 3, 1), 1
-; GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
; GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
; GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
; GISEL-NEXT: v_pack_b32_f16 v0, v0, v1
-; GISEL-NEXT: v_lshl_or_b32 v1, s0, 16, v2
+; GISEL-NEXT: v_or_b32_sdwa v1, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GISEL-NEXT: ; return to shader part epilog
;
; GFX12-SDAG-LABEL: v_fptrunc_round_v3f32_to_v3f16_downward:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.maximum.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.maximum.f16.ll
index 1988ced3bef34..207f1432504c8 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.maximum.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.maximum.f16.ll
@@ -1143,9 +1143,8 @@ define void @s_maximum_v2f16(<2 x half> inreg %src0, <2 x half> inreg %src1) {
; GFX900-NEXT: v_mov_b32_e32 v3, s4
; GFX900-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
; GFX900-NEXT: v_cmp_o_f16_e32 vcc, s5, v3
-; GFX900-NEXT: v_cndmask_b32_sdwa v1, v2, v1, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX900-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX900-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX900-NEXT: v_cndmask_b32_sdwa v1, v2, v1, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX900-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX900-NEXT: ;;#ASMSTART
; GFX900-NEXT: ; use v0
; GFX900-NEXT: ;;#ASMEND
@@ -1169,12 +1168,11 @@ define void @s_maximum_v2f16(<2 x half> inreg %src0, <2 x half> inreg %src1) {
; GFX10-NEXT: s_lshr_b32 s5, s16, 16
; GFX10-NEXT: v_pk_max_f16 v0, s16, s17
; GFX10-NEXT: v_cmp_o_f16_e64 vcc_lo, s5, s4
-; GFX10-NEXT: v_cmp_o_f16_e64 s4, s16, s17
; GFX10-NEXT: v_mov_b32_e32 v1, 0x7e00
+; GFX10-NEXT: v_cmp_o_f16_e64 s4, s16, s17
; GFX10-NEXT: v_cndmask_b32_e64 v2, 0x7e00, v0, s4
-; GFX10-NEXT: v_cndmask_b32_sdwa v0, v1, v0, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v2
-; GFX10-NEXT: v_lshl_or_b32 v0, v0, 16, v1
+; GFX10-NEXT: v_cndmask_b32_sdwa v0, v1, v0, vcc_lo dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: ;;#ASMSTART
; GFX10-NEXT: ; use v0
; GFX10-NEXT: ;;#ASMEND
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.minimum.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.minimum.f16.ll
index f6c3cfaaebe8c..c1d182730d3d9 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.minimum.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.minimum.f16.ll
@@ -953,9 +953,8 @@ define void @s_minimum_v2f16(<2 x half> inreg %src0, <2 x half> inreg %src1) {
; GFX900-NEXT: v_mov_b32_e32 v3, s4
; GFX900-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
; GFX900-NEXT: v_cmp_o_f16_e32 vcc, s5, v3
-; GFX900-NEXT: v_cndmask_b32_sdwa v1, v2, v1, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX900-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX900-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX900-NEXT: v_cndmask_b32_sdwa v1, v2, v1, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX900-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX900-NEXT: ;;#ASMSTART
; GFX900-NEXT: ; use v0
; GFX900-NEXT: ;;#ASMEND
@@ -979,12 +978,11 @@ define void @s_minimum_v2f16(<2 x half> inreg %src0, <2 x half> inreg %src1) {
; GFX10-NEXT: s_lshr_b32 s5, s16, 16
; GFX10-NEXT: v_pk_min_f16 v0, s16, s17
; GFX10-NEXT: v_cmp_o_f16_e64 vcc_lo, s5, s4
-; GFX10-NEXT: v_cmp_o_f16_e64 s4, s16, s17
; GFX10-NEXT: v_mov_b32_e32 v1, 0x7e00
+; GFX10-NEXT: v_cmp_o_f16_e64 s4, s16, s17
; GFX10-NEXT: v_cndmask_b32_e64 v2, 0x7e00, v0, s4
-; GFX10-NEXT: v_cndmask_b32_sdwa v0, v1, v0, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v2
-; GFX10-NEXT: v_lshl_or_b32 v0, v0, 16, v1
+; GFX10-NEXT: v_cndmask_b32_sdwa v0, v1, v0, vcc_lo dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: ;;#ASMSTART
; GFX10-NEXT: ; use v0
; GFX10-NEXT: ;;#ASMEND
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.modf.ll b/llvm/test/CodeGen/AMDGPU/llvm.modf.ll
index 64154d25aa552..e79726614c629 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.modf.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.modf.ll
@@ -120,9 +120,8 @@ define { <2 x half>, <2 x half> } @test_modf_v2f16(<2 x half> %x) {
; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v3, v6, 0, s[4:5]
; GFX9-GISEL-NEXT: v_and_b32_e32 v3, 0x7fff, v3
; GFX9-GISEL-NEXT: v_and_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-GISEL-NEXT: v_or_b32_e32 v0, v3, v0
-; GFX9-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v0, 16, v2
+; GFX9-GISEL-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-GISEL-NEXT: v_pack_b32_f16 v1, v1, v5
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
%result = call { <2 x half>, <2 x half> } @llvm.modf.v2f16(<2 x half> %x)
@@ -167,9 +166,8 @@ define <2 x half> @test_modf_v2f16_only_use_fract(<2 x half> %x) {
; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v2, v4, 0, s[4:5]
; GFX9-GISEL-NEXT: v_and_b32_e32 v2, 0x7fff, v2
; GFX9-GISEL-NEXT: v_and_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-GISEL-NEXT: v_or_b32_e32 v0, v2, v0
-; GFX9-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v0, 16, v1
+; GFX9-GISEL-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
%result = call { <2 x half>, <2 x half> } @llvm.modf.v2f16(<2 x half> %x)
%result.0 = extractvalue { <2 x half>, <2 x half> } %result, 0
diff --git a/llvm/test/CodeGen/AMDGPU/mad-mix-lo.ll b/llvm/test/CodeGen/AMDGPU/mad-mix-lo.ll
index 88f482a0ac889..db0e6fb66dc67 100644
--- a/llvm/test/CodeGen/AMDGPU/mad-mix-lo.ll
+++ b/llvm/test/CodeGen/AMDGPU/mad-mix-lo.ll
@@ -646,8 +646,8 @@ define <3 x half> @v_mad_mix_v3f32(<3 x half> %src0, <3 x half> %src1, <3 x half
; GISEL-GFX900-NEXT: v_mad_mixlo_f16 v6, v0, v2, v4 op_sel_hi:[1,1,1]
; GISEL-GFX900-NEXT: v_mad_mixlo_f16 v1, v1, v3, v5 op_sel_hi:[1,1,1]
; GISEL-GFX900-NEXT: v_mad_mixhi_f16 v6, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1]
-; GISEL-GFX900-NEXT: v_and_b32_e32 v0, 0xffff, v1
-; GISEL-GFX900-NEXT: v_lshl_or_b32 v1, s4, 16, v0
+; GISEL-GFX900-NEXT: v_lshlrev_b32_e64 v0, 16, s4
+; GISEL-GFX900-NEXT: v_or_b32_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GISEL-GFX900-NEXT: v_mov_b32_e32 v0, v6
; GISEL-GFX900-NEXT: s_setpc_b64 s[30:31]
;
@@ -657,8 +657,8 @@ define <3 x half> @v_mad_mix_v3f32(<3 x half> %src0, <3 x half> %src1, <3 x half
; GISEL-GFX906-NEXT: v_fma_mixlo_f16 v6, v0, v2, v4 op_sel_hi:[1,1,1]
; GISEL-GFX906-NEXT: v_fma_mixlo_f16 v1, v1, v3, v5 op_sel_hi:[1,1,1]
; GISEL-GFX906-NEXT: v_fma_mixhi_f16 v6, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1]
-; GISEL-GFX906-NEXT: v_and_b32_e32 v0, 0xffff, v1
-; GISEL-GFX906-NEXT: v_lshl_or_b32 v1, s4, 16, v0
+; GISEL-GFX906-NEXT: v_lshlrev_b32_e64 v0, 16, s4
+; GISEL-GFX906-NEXT: v_or_b32_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GISEL-GFX906-NEXT: v_mov_b32_e32 v0, v6
; GISEL-GFX906-NEXT: s_setpc_b64 s[30:31]
;
@@ -1229,9 +1229,9 @@ define <3 x half> @v_mad_mix_v3f32_clamp_postcvt(<3 x half> %src0, <3 x half> %s
; GISEL-GFX900: ; %bb.0:
; GISEL-GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GISEL-GFX900-NEXT: v_mad_mixlo_f16 v1, v1, v3, v5 op_sel_hi:[1,1,1]
-; GISEL-GFX900-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GISEL-GFX900-NEXT: v_lshlrev_b32_e64 v3, 16, s4
+; GISEL-GFX900-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GISEL-GFX900-NEXT: v_mad_mixlo_f16 v3, v0, v2, v4 op_sel_hi:[1,1,1] clamp
-; GISEL-GFX900-NEXT: v_lshl_or_b32 v1, s4, 16, v1
; GISEL-GFX900-NEXT: v_mad_mixhi_f16 v3, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp
; GISEL-GFX900-NEXT: v_pk_max_f16 v1, v1, v1 clamp
; GISEL-GFX900-NEXT: v_mov_b32_e32 v0, v3
@@ -1241,9 +1241,9 @@ define <3 x half> @v_mad_mix_v3f32_clamp_postcvt(<3 x half> %src0, <3 x half> %s
; GISEL-GFX906: ; %bb.0:
; GISEL-GFX906-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GISEL-GFX906-NEXT: v_fma_mixlo_f16 v1, v1, v3, v5 op_sel_hi:[1,1,1]
-; GISEL-GFX906-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GISEL-GFX906-NEXT: v_lshlrev_b32_e64 v3, 16, s4
+; GISEL-GFX906-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GISEL-GFX906-NEXT: v_fma_mixlo_f16 v3, v0, v2, v4 op_sel_hi:[1,1,1] clamp
-; GISEL-GFX906-NEXT: v_lshl_or_b32 v1, s4, 16, v1
; GISEL-GFX906-NEXT: v_fma_mixhi_f16 v3, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp
; GISEL-GFX906-NEXT: v_pk_max_f16 v1, v1, v1 clamp
; GISEL-GFX906-NEXT: v_mov_b32_e32 v0, v3
diff --git a/llvm/test/CodeGen/AMDGPU/packed-op-sel.ll b/llvm/test/CodeGen/AMDGPU/packed-op-sel.ll
index 0ee380f690492..84ad5a592dd4a 100644
--- a/llvm/test/CodeGen/AMDGPU/packed-op-sel.ll
+++ b/llvm/test/CodeGen/AMDGPU/packed-op-sel.ll
@@ -223,6 +223,7 @@ define amdgpu_kernel void @fma_vector_vector_scalar_lo_neg_scalar_hi(ptr addrspa
; GCN-LABEL: fma_vector_vector_scalar_lo_neg_scalar_hi:
; GCN: ; %bb.0: ; %bb
; GCN-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GCN-NEXT: s_mov_b32 s0, 0x8000
; GCN-NEXT: v_mov_b32_e32 v4, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: v_mov_b32_e32 v0, s6
@@ -231,11 +232,9 @@ define amdgpu_kernel void @fma_vector_vector_scalar_lo_neg_scalar_hi(ptr addrspa
; GCN-NEXT: ds_read_b32 v0, v0 offset:4
; GCN-NEXT: ds_read_u16 v3, v1
; GCN-NEXT: ds_read_u16 v1, v1 offset:4
-; GCN-NEXT: s_waitcnt lgkmcnt(1)
-; GCN-NEXT: v_and_b32_e32 v3, 0xffff, v3
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_xor_b32_e32 v1, 0x8000, v1
-; GCN-NEXT: v_lshl_or_b32 v1, v1, 16, v3
+; GCN-NEXT: v_xor_b32_sdwa v1, v1, s0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GCN-NEXT: v_or_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GCN-NEXT: v_pk_fma_f16 v0, v2, v0, v1
; GCN-NEXT: global_store_dword v4, v0, s[4:5]
; GCN-NEXT: s_endpgm
@@ -688,7 +687,6 @@ define amdgpu_kernel void @extract_from_i64(ptr addrspace(1) %out, ptr addrspace
; GCN-LABEL: extract_from_i64:
; GCN: ; %bb.0: ; %bb
; GCN-NEXT: s_load_dword s2, s[0:1], 0x2c
-; GCN-NEXT: v_mov_b32_e32 v3, 0xffff
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: v_mov_b32_e32 v0, s2
; GCN-NEXT: ds_read_b32 v2, v0
@@ -696,8 +694,8 @@ define amdgpu_kernel void @extract_from_i64(ptr addrspace(1) %out, ptr addrspace
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24
; GCN-NEXT: v_mov_b32_e32 v1, 0
-; GCN-NEXT: v_and_b32_sdwa v3, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GCN-NEXT: v_lshl_or_b32 v0, v0, 16, v3
+; GCN-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GCN-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: v_pk_add_u16 v0, v2, v0
; GCN-NEXT: global_store_dword v1, v0, s[2:3]
diff --git a/llvm/test/CodeGen/AMDGPU/permute_i8.ll b/llvm/test/CodeGen/AMDGPU/permute_i8.ll
index e098fd19dd437..a7aadc64d750d 100644
--- a/llvm/test/CodeGen/AMDGPU/permute_i8.ll
+++ b/llvm/test/CodeGen/AMDGPU/permute_i8.ll
@@ -567,17 +567,16 @@ define hidden void @addUsesOr(ptr addrspace(1) %in0, ptr addrspace(1) %in1, i8 %
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 24, v4
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_e32 v1, 24, v7
-; GFX10-NEXT: v_lshrrev_b16 v2, 8, v7
-; GFX10-NEXT: v_lshrrev_b32_e32 v3, 16, v4
-; GFX10-NEXT: v_lshrrev_b32_e32 v7, 16, v7
+; GFX10-NEXT: v_lshrrev_b32_e32 v2, 16, v4
+; GFX10-NEXT: v_lshrrev_b32_e32 v3, 16, v7
; GFX10-NEXT: v_add_nc_u16 v0, v0, v1
-; GFX10-NEXT: v_add_nc_u16 v1, v4, v2
-; GFX10-NEXT: v_add_nc_u16 v2, v3, v7
+; GFX10-NEXT: v_lshrrev_b16 v1, 8, v7
+; GFX10-NEXT: v_add_nc_u16 v2, v2, v3
; GFX10-NEXT: v_lshlrev_b16 v0, 8, v0
+; GFX10-NEXT: v_add_nc_u16 v1, v4, v1
+; GFX10-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX10-NEXT: v_lshlrev_b16 v1, 8, v1
-; GFX10-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_lshl_or_b32 v0, v0, 16, v1
+; GFX10-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: global_store_dword v[5:6], v0, off
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -689,20 +688,19 @@ define hidden void @add(ptr addrspace(1) %in0, ptr addrspace(1) %in1, i8 %elt, p
; GFX10-NEXT: s_waitcnt vmcnt(1)
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 24, v4
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshrrev_b16 v1, 8, v7
-; GFX10-NEXT: v_lshrrev_b32_e32 v2, 24, v7
-; GFX10-NEXT: v_lshrrev_b16 v3, 8, v4
-; GFX10-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; GFX10-NEXT: v_add_nc_u16 v0, v0, v1
-; GFX10-NEXT: v_add_nc_u16 v2, v7, v2
-; GFX10-NEXT: v_add_nc_u16 v3, v3, v7
-; GFX10-NEXT: v_add_nc_u16 v1, v1, v4
+; GFX10-NEXT: v_lshrrev_b32_e32 v1, 24, v7
+; GFX10-NEXT: v_lshrrev_b16 v2, 8, v7
+; GFX10-NEXT: v_lshrrev_b32_e32 v3, 16, v7
+; GFX10-NEXT: v_lshrrev_b16 v4, 8, v4
+; GFX10-NEXT: v_add_nc_u16 v1, v7, v1
+; GFX10-NEXT: v_add_nc_u16 v0, v0, v2
+; GFX10-NEXT: v_add_nc_u16 v2, v2, v3
+; GFX10-NEXT: v_add_nc_u16 v3, v4, v7
+; GFX10-NEXT: v_lshlrev_b16 v1, 8, v1
; GFX10-NEXT: v_lshlrev_b16 v0, 8, v0
-; GFX10-NEXT: v_lshlrev_b16 v2, 8, v2
+; GFX10-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX10-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: global_store_dword v[5:6], v0, off
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -717,10 +715,9 @@ define hidden void @add(ptr addrspace(1) %in0, ptr addrspace(1) %in1, i8 %elt, p
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_add_u16_sdwa v2, v7, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_1
; GFX9-NEXT: v_add_u16_sdwa v3, v7, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
-; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: global_store_dword v[5:6], v0, off
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -925,20 +922,19 @@ define hidden void @add_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1,
; GFX10-NEXT: s_waitcnt vmcnt(1)
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 24, v4
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshrrev_b16 v1, 8, v9
-; GFX10-NEXT: v_lshrrev_b32_e32 v2, 24, v9
-; GFX10-NEXT: v_lshrrev_b16 v3, 8, v4
-; GFX10-NEXT: v_lshrrev_b32_e32 v10, 16, v9
-; GFX10-NEXT: v_add_nc_u16 v0, v0, v1
-; GFX10-NEXT: v_add_nc_u16 v2, v9, v2
-; GFX10-NEXT: v_add_nc_u16 v3, v3, v9
-; GFX10-NEXT: v_add_nc_u16 v1, v1, v10
+; GFX10-NEXT: v_lshrrev_b32_e32 v1, 24, v9
+; GFX10-NEXT: v_lshrrev_b16 v2, 8, v9
+; GFX10-NEXT: v_lshrrev_b32_e32 v3, 16, v9
+; GFX10-NEXT: v_lshrrev_b16 v10, 8, v4
+; GFX10-NEXT: v_add_nc_u16 v1, v9, v1
+; GFX10-NEXT: v_add_nc_u16 v0, v0, v2
+; GFX10-NEXT: v_add_nc_u16 v2, v2, v3
+; GFX10-NEXT: v_add_nc_u16 v3, v10, v9
+; GFX10-NEXT: v_lshlrev_b16 v1, 8, v1
; GFX10-NEXT: v_lshlrev_b16 v0, 8, v0
-; GFX10-NEXT: v_lshlrev_b16 v2, 8, v2
+; GFX10-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX10-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_perm_b32 v1, v4, v9, 0x10705
; GFX10-NEXT: global_store_dword v[5:6], v0, off
; GFX10-NEXT: global_store_dword v[7:8], v1, off
@@ -962,10 +958,9 @@ define hidden void @add_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1,
; GFX9-NEXT: v_add_u16_sdwa v2, v9, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:WORD_1
; GFX9-NEXT: v_add_u16_sdwa v3, v4, v9 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_1
; GFX9-NEXT: v_add_u16_sdwa v4, v4, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
-; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v2, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: global_store_dword v[5:6], v1, off
; GFX9-NEXT: global_store_dword v[7:8], v0, off
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -988,25 +983,24 @@ define hidden void @and_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1,
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_and_b32_e32 v4, 0x3ff, v31
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 2, v4
-; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, v2, v4
-; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
-; GFX10-NEXT: global_load_dword v4, v[2:3], off
-; GFX10-NEXT: global_load_dword v9, v[0:1], off
+; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, v2, v4
+; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
+; GFX10-NEXT: global_load_dword v4, v[0:1], off
+; GFX10-NEXT: global_load_dword v9, v[2:3], off
; GFX10-NEXT: v_mov_b32_e32 v0, 2
; GFX10-NEXT: v_mov_b32_e32 v1, 1
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_and_b32_sdwa v2, v4, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_and_b32_e32 v2, 0x100, v4
+; GFX10-NEXT: v_and_b32_sdwa v3, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_and_b32_sdwa v1, v9, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-NEXT: v_and_b32_e32 v3, 0x100, v9
-; GFX10-NEXT: v_and_b32_sdwa v0, v9, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
-; GFX10-NEXT: v_or_b32_e32 v1, v1, v2
-; GFX10-NEXT: v_or_b32_e32 v0, v0, v3
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_lshl_or_b32 v0, v0, 16, v1
-; GFX10-NEXT: v_perm_b32 v1, v9, v4, 0x5070006
+; GFX10-NEXT: v_and_b32_sdwa v0, v9, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_and_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX10-NEXT: v_perm_b32 v1, v4, v9, 0x5070006
+; GFX10-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: global_store_dword v[5:6], v0, off
; GFX10-NEXT: global_store_dword v[7:8], v1, off
; GFX10-NEXT: s_setpc_b64 s[30:31]
@@ -1026,15 +1020,14 @@ define hidden void @and_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1,
; GFX9-NEXT: v_mov_b32_e32 v1, 1
; GFX9-NEXT: s_mov_b32 s4, 0x5070006
; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_and_b32_e32 v3, 0x100, v4
; GFX9-NEXT: v_and_b32_sdwa v10, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_and_b32_sdwa v0, v9, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_and_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v3, 0x100, v4
+; GFX9-NEXT: v_or_b32_sdwa v3, v10, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX9-NEXT: v_or_b32_e32 v3, v10, v3
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v0, v3, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_perm_b32 v2, v4, v9, s4
; GFX9-NEXT: global_store_dword v[5:6], v0, off
; GFX9-NEXT: global_store_dword v[7:8], v2, off
@@ -1063,21 +1056,20 @@ define hidden void @ashr_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1,
; GFX10-NEXT: global_load_dword v9, v[0:1], off
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v2, v4
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v3, vcc_lo
+; GFX10-NEXT: v_mov_b32_e32 v2, 26
; GFX10-NEXT: global_load_dword v0, v[0:1], off
-; GFX10-NEXT: v_mov_b32_e32 v1, 26
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_bfe_i32 v2, v9, 0, 8
-; GFX10-NEXT: v_ashrrev_i32_sdwa v1, v1, v9 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_bfe_i32 v1, v9, 0, 8
+; GFX10-NEXT: v_ashrrev_i32_sdwa v2, v2, v9 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX10-NEXT: v_ashrrev_i32_e32 v3, 25, v9
-; GFX10-NEXT: v_lshlrev_b16 v2, 7, v2
-; GFX10-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT: v_lshlrev_b16 v1, 7, v1
+; GFX10-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_ashrrev_i16 v4, 10, v0
; GFX10-NEXT: v_perm_b32 v0, v9, v0, 0x4010707
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffffff00, v2
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_or_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffffff00, v1
+; GFX10-NEXT: v_or_b32_sdwa v1, v4, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: global_store_dword v[5:6], v1, off
; GFX10-NEXT: global_store_dword v[7:8], v0, off
; GFX10-NEXT: s_setpc_b64 s[30:31]
@@ -1093,21 +1085,20 @@ define hidden void @ashr_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1,
; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
; GFX9-NEXT: global_load_dword v4, v[0:1], off
; GFX9-NEXT: global_load_dword v9, v[2:3], off
-; GFX9-NEXT: v_mov_b32_e32 v0, 26
; GFX9-NEXT: v_mov_b32_e32 v1, 7
; GFX9-NEXT: s_mov_b32 s4, 0x4010707
+; GFX9-NEXT: v_mov_b32_e32 v0, 26
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_ashrrev_i32_sdwa v0, v0, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: v_ashrrev_i32_e32 v3, 25, v4
; GFX9-NEXT: v_lshlrev_b16_sdwa v1, v1, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v2, v4, v9, s4
+; GFX9-NEXT: v_ashrrev_i32_sdwa v0, v0, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_ashrrev_i32_e32 v3, 25, v4
; GFX9-NEXT: v_ashrrev_i16_e32 v9, 10, v9
-; GFX9-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_and_b32_e32 v1, 0xffffff00, v1
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_or_b32_sdwa v1, v9, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v1, v9, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: global_store_dword v[5:6], v0, off
; GFX9-NEXT: global_store_dword v[7:8], v2, off
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -1303,15 +1294,14 @@ define hidden void @lhsr_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1,
; GFX10-NEXT: v_mov_b32_e32 v0, 26
; GFX10-NEXT: s_waitcnt vmcnt(1)
; GFX10-NEXT: v_lshrrev_b16 v1, 1, v4
+; GFX10-NEXT: v_lshrrev_b32_e32 v2, 26, v4
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_sdwa v0, v0, v9 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT: v_lshrrev_b32_e32 v2, 25, v9
-; GFX10-NEXT: v_lshrrev_b32_e32 v3, 26, v4
+; GFX10-NEXT: v_lshrrev_b32_e32 v3, 25, v9
; GFX10-NEXT: v_and_b32_e32 v1, 0x7f00, v1
-; GFX10-NEXT: v_or_b32_e32 v0, v2, v0
-; GFX10-NEXT: v_or_b32_e32 v1, v3, v1
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10-NEXT: v_or_b32_e32 v0, v3, v0
+; GFX10-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_perm_b32 v1, v9, v4, 0x1030707
; GFX10-NEXT: global_store_dword v[5:6], v0, off
; GFX10-NEXT: global_store_dword v[7:8], v1, off
@@ -1339,9 +1329,8 @@ define hidden void @lhsr_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1,
; GFX9-NEXT: v_lshrrev_b32_e32 v4, 25, v9
; GFX9-NEXT: v_and_b32_e32 v2, 0x7f00, v2
; GFX9-NEXT: v_or_b32_e32 v0, v4, v0
-; GFX9-NEXT: v_or_b32_e32 v2, v3, v2
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v0, v2, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: global_store_dword v[5:6], v0, off
; GFX9-NEXT: global_store_dword v[7:8], v1, off
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -1374,19 +1363,18 @@ define hidden void @mul_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1,
; GFX10-NEXT: s_waitcnt vmcnt(1)
; GFX10-NEXT: v_lshrrev_b16 v0, 8, v4
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshrrev_b16 v1, 8, v9
-; GFX10-NEXT: v_lshrrev_b32_e32 v2, 24, v9
-; GFX10-NEXT: v_lshrrev_b32_e32 v3, 16, v9
-; GFX10-NEXT: v_mul_lo_u16 v0, v0, v1
-; GFX10-NEXT: v_mul_lo_u16 v1, v4, v9
-; GFX10-NEXT: v_mul_lo_u16 v2, v3, v2
-; GFX10-NEXT: v_mul_lo_u16 v3, v9, v3
+; GFX10-NEXT: v_lshrrev_b32_e32 v1, 24, v9
+; GFX10-NEXT: v_lshrrev_b32_e32 v2, 16, v9
+; GFX10-NEXT: v_lshrrev_b16 v3, 8, v9
+; GFX10-NEXT: v_mul_lo_u16 v1, v2, v1
+; GFX10-NEXT: v_mul_lo_u16 v0, v0, v3
+; GFX10-NEXT: v_mul_lo_u16 v3, v4, v9
+; GFX10-NEXT: v_mul_lo_u16 v2, v9, v2
+; GFX10-NEXT: v_lshlrev_b16 v1, 8, v1
; GFX10-NEXT: v_lshlrev_b16 v0, 8, v0
-; GFX10-NEXT: v_lshlrev_b16 v2, 8, v2
-; GFX10-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_or_b32_sdwa v1, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_perm_b32 v1, v4, v9, 0x2000504
; GFX10-NEXT: global_store_dword v[5:6], v0, off
; GFX10-NEXT: global_store_dword v[7:8], v1, off
@@ -1410,10 +1398,9 @@ define hidden void @mul_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1,
; GFX9-NEXT: v_mul_lo_u16_sdwa v2, v9, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_mul_lo_u16_e32 v3, v4, v9
; GFX9-NEXT: v_mul_lo_u16_sdwa v4, v4, v9 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
-; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v2, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: global_store_dword v[5:6], v1, off
; GFX9-NEXT: global_store_dword v[7:8], v0, off
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -1444,6 +1431,7 @@ define hidden void @or_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1, i
; GFX10-NEXT: global_load_dword v4, v[2:3], off
; GFX10-NEXT: global_load_dword v9, v[0:1], off
; GFX10-NEXT: v_mov_b32_e32 v0, 16
+; GFX10-NEXT: v_bfrev_b32_e32 v2, 4.0
; GFX10-NEXT: s_waitcnt vmcnt(1)
; GFX10-NEXT: v_lshlrev_b16 v1, 8, v4
; GFX10-NEXT: v_lshrrev_b32_sdwa v0, v0, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -1451,9 +1439,8 @@ define hidden void @or_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1, i
; GFX10-NEXT: v_or_b32_sdwa v1, v9, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
; GFX10-NEXT: v_or_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
; GFX10-NEXT: v_or_b32_e32 v1, 0x201, v1
-; GFX10-NEXT: v_or_b32_e32 v0, 0x102, v0
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_lshl_or_b32 v0, v0, 16, v1
+; GFX10-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_perm_b32 v1, v9, v4, 0x2010005
; GFX10-NEXT: global_store_dword v[5:6], v0, off
; GFX10-NEXT: global_store_dword v[7:8], v1, off
@@ -1466,23 +1453,23 @@ define hidden void @or_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1, i
; GFX9-NEXT: v_lshlrev_b32_e32 v4, 2, v4
; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v2, v4
; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v4
; GFX9-NEXT: global_load_dword v2, v[2:3], off
+; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v4
; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX9-NEXT: global_load_dword v0, v[0:1], off
; GFX9-NEXT: s_mov_b32 s4, 0x2010005
+; GFX9-NEXT: s_movk_i32 s5, 0x102
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; GFX9-NEXT: v_lshlrev_b16_e32 v3, 8, v2
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v4, v0, v2, s4
; GFX9-NEXT: v_or_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
-; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_e32 v0, 0x201, v0
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_or_b32_e32 v1, 0x102, v1
-; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: global_store_dword v[5:6], v0, off
; GFX9-NEXT: global_store_dword v[7:8], v4, off
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -1513,35 +1500,34 @@ define hidden void @sdiv_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1,
; GFX10-NEXT: global_load_dword v9, v[0:1], off
; GFX10-NEXT: s_waitcnt vmcnt(1)
; GFX10-NEXT: v_cvt_f32_i32_sdwa v0, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v2, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v2, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v3, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v10, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cvt_f32_i32_sdwa v1, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v3, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v10, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v11, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v11, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
; GFX10-NEXT: v_rcp_f32_e32 v12, v0
; GFX10-NEXT: v_rcp_f32_e32 v2, v2
-; GFX10-NEXT: v_add_nc_u32_e32 v1, 1, v1
+; GFX10-NEXT: v_rcp_f32_e32 v3, v3
; GFX10-NEXT: v_rcp_f32_e32 v10, v10
-; GFX10-NEXT: v_rcp_f32_e32 v11, v11
-; GFX10-NEXT: v_add_nc_u32_e32 v3, 1, v3
+; GFX10-NEXT: v_add_nc_u32_e32 v1, 1, v1
; GFX10-NEXT: v_add_nc_u32_e32 v0, 1, v0
+; GFX10-NEXT: v_add_nc_u32_e32 v11, 1, v11
; GFX10-NEXT: v_mul_f32_e32 v12, v1, v12
-; GFX10-NEXT: v_mul_f32_e32 v2, v3, v2
-; GFX10-NEXT: v_mul_f32_e32 v1, v1, v10
-; GFX10-NEXT: v_mul_f32_e32 v0, v0, v11
+; GFX10-NEXT: v_mul_f32_e32 v1, v1, v2
+; GFX10-NEXT: v_mul_f32_e32 v0, v0, v3
+; GFX10-NEXT: v_mul_f32_e32 v2, v11, v10
; GFX10-NEXT: v_trunc_f32_e32 v3, v12
-; GFX10-NEXT: v_trunc_f32_e32 v2, v2
; GFX10-NEXT: v_trunc_f32_e32 v1, v1
; GFX10-NEXT: v_trunc_f32_e32 v0, v0
+; GFX10-NEXT: v_trunc_f32_e32 v2, v2
; GFX10-NEXT: v_cvt_i32_f32_e32 v3, v3
-; GFX10-NEXT: v_cvt_i32_f32_sdwa v2, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD
; GFX10-NEXT: v_cvt_i32_f32_e32 v1, v1
; GFX10-NEXT: v_cvt_i32_f32_sdwa v0, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD
-; GFX10-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v2
-; GFX10-NEXT: v_lshl_or_b32 v0, v0, 16, v1
+; GFX10-NEXT: v_cvt_i32_f32_sdwa v2, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v1, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_perm_b32 v1, v9, v4, 0x60706
; GFX10-NEXT: global_store_dword v[5:6], v0, off
; GFX10-NEXT: global_store_dword v[7:8], v1, off
@@ -1587,10 +1573,9 @@ define hidden void @sdiv_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1,
; GFX9-NEXT: v_cvt_i32_f32_sdwa v0, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD
; GFX9-NEXT: v_cvt_i32_f32_sdwa v2, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD
; GFX9-NEXT: v_perm_b32 v4, v9, v4, s4
-; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: global_store_dword v[5:6], v0, off
; GFX9-NEXT: global_store_dword v[7:8], v4, off
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -1678,25 +1663,24 @@ define hidden void @shl_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1,
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_and_b32_e32 v4, 0x3ff, v31
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 2, v4
-; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
-; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, v2, v4
; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
-; GFX10-NEXT: global_load_dword v4, v[0:1], off
-; GFX10-NEXT: global_load_dword v9, v[2:3], off
+; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
+; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT: global_load_dword v4, v[2:3], off
+; GFX10-NEXT: global_load_dword v9, v[0:1], off
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_lshlrev_b16 v0, 2, v4
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshlrev_b16 v1, 1, v9
-; GFX10-NEXT: v_and_b32_e32 v2, 0xfffffc00, v0
-; GFX10-NEXT: v_and_b32_e32 v3, 0xfe, v1
-; GFX10-NEXT: v_and_b32_e32 v1, 0xfffffe00, v1
-; GFX10-NEXT: v_and_b32_e32 v0, 0xfc, v0
-; GFX10-NEXT: v_or_b32_e32 v2, v3, v2
+; GFX10-NEXT: v_lshlrev_b16 v0, 1, v4
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_lshlrev_b16 v1, 2, v9
+; GFX10-NEXT: v_and_b32_e32 v2, 0xfffffe00, v0
+; GFX10-NEXT: v_and_b32_e32 v3, 0xfc, v1
+; GFX10-NEXT: v_and_b32_e32 v1, 0xfffffc00, v1
+; GFX10-NEXT: v_and_b32_e32 v0, 0xfe, v0
+; GFX10-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX10-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v2
-; GFX10-NEXT: v_lshl_or_b32 v0, v0, 16, v1
-; GFX10-NEXT: v_perm_b32 v1, v9, v4, 0x5000104
+; GFX10-NEXT: v_perm_b32 v1, v4, v9, 0x5000104
+; GFX10-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: global_store_dword v[5:6], v0, off
; GFX10-NEXT: global_store_dword v[7:8], v1, off
; GFX10-NEXT: s_setpc_b64 s[30:31]
@@ -1722,10 +1706,9 @@ define hidden void @shl_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1,
; GFX9-NEXT: v_and_b32_e32 v4, 0xfc, v2
; GFX9-NEXT: v_and_b32_e32 v2, 0xfffffc00, v2
; GFX9-NEXT: v_and_b32_e32 v1, 0xfe, v1
+; GFX9-NEXT: v_or_b32_sdwa v3, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_or_b32_e32 v1, v1, v2
-; GFX9-NEXT: v_or_b32_e32 v3, v4, v3
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_lshl_or_b32 v1, v3, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: global_store_dword v[5:6], v1, off
; GFX9-NEXT: global_store_dword v[7:8], v0, off
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -1820,47 +1803,46 @@ define hidden void @srem_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1,
; GFX10-NEXT: global_load_dword v9, v[0:1], off
; GFX10-NEXT: s_waitcnt vmcnt(1)
; GFX10-NEXT: v_cvt_f32_i32_sdwa v0, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v1, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v2, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v1, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v3, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v10, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v3, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v10, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
-; GFX10-NEXT: v_rcp_f32_e32 v0, v0
-; GFX10-NEXT: v_rcp_f32_e32 v1, v1
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v2, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
; GFX10-NEXT: v_cvt_f32_i32_sdwa v11, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
-; GFX10-NEXT: v_rcp_f32_e32 v12, v2
-; GFX10-NEXT: v_rcp_f32_e32 v13, v10
-; GFX10-NEXT: v_add_nc_u32_e32 v2, 1, v2
+; GFX10-NEXT: v_rcp_f32_e32 v0, v0
+; GFX10-NEXT: v_rcp_f32_e32 v12, v1
+; GFX10-NEXT: v_rcp_f32_e32 v13, v3
+; GFX10-NEXT: v_rcp_f32_e32 v10, v10
+; GFX10-NEXT: v_add_nc_u32_e32 v1, 1, v1
; GFX10-NEXT: v_add_nc_u32_e32 v3, 1, v3
-; GFX10-NEXT: v_add_nc_u32_e32 v10, 1, v10
; GFX10-NEXT: v_add_nc_u32_e32 v11, 1, v11
-; GFX10-NEXT: v_mul_f32_e32 v0, v2, v0
-; GFX10-NEXT: v_mul_f32_e32 v1, v3, v1
-; GFX10-NEXT: v_lshrrev_b32_e32 v2, 16, v4
-; GFX10-NEXT: v_mul_f32_e32 v3, v10, v12
-; GFX10-NEXT: v_mul_f32_e32 v10, v11, v13
+; GFX10-NEXT: v_add_nc_u32_e32 v2, 1, v2
+; GFX10-NEXT: v_lshrrev_b32_e32 v14, 16, v4
+; GFX10-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX10-NEXT: v_mul_f32_e32 v1, v3, v12
+; GFX10-NEXT: v_mul_f32_e32 v3, v11, v13
+; GFX10-NEXT: v_mul_f32_e32 v2, v2, v10
+; GFX10-NEXT: v_lshrrev_b32_e32 v10, 8, v4
; GFX10-NEXT: v_trunc_f32_e32 v0, v0
; GFX10-NEXT: v_trunc_f32_e32 v1, v1
-; GFX10-NEXT: v_lshrrev_b32_e32 v11, 8, v4
; GFX10-NEXT: v_trunc_f32_e32 v3, v3
-; GFX10-NEXT: v_trunc_f32_e32 v10, v10
+; GFX10-NEXT: v_trunc_f32_e32 v2, v2
+; GFX10-NEXT: v_lshrrev_b32_e32 v11, 24, v4
; GFX10-NEXT: v_cvt_i32_f32_e32 v0, v0
; GFX10-NEXT: v_cvt_i32_f32_e32 v1, v1
-; GFX10-NEXT: v_lshrrev_b32_e32 v12, 24, v4
; GFX10-NEXT: v_cvt_i32_f32_e32 v3, v3
-; GFX10-NEXT: v_cvt_i32_f32_e32 v10, v10
+; GFX10-NEXT: v_cvt_i32_f32_e32 v2, v2
; GFX10-NEXT: v_mul_lo_u32 v0, v0, v4
-; GFX10-NEXT: v_mul_lo_u32 v1, v1, v11
-; GFX10-NEXT: v_mul_lo_u32 v3, v3, v2
-; GFX10-NEXT: v_mul_lo_u32 v10, v10, v12
-; GFX10-NEXT: v_sub_nc_u32_e32 v0, v2, v0
-; GFX10-NEXT: v_sub_nc_u32_sdwa v1, v9, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
-; GFX10-NEXT: v_sub_nc_u32_e32 v2, v12, v3
-; GFX10-NEXT: v_sub_nc_u32_sdwa v3, v9, v10 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_or_b32_sdwa v1, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10-NEXT: v_mul_lo_u32 v1, v1, v14
+; GFX10-NEXT: v_mul_lo_u32 v3, v3, v11
+; GFX10-NEXT: v_mul_lo_u32 v2, v2, v10
+; GFX10-NEXT: v_sub_nc_u32_e32 v0, v14, v0
+; GFX10-NEXT: v_sub_nc_u32_e32 v1, v11, v1
+; GFX10-NEXT: v_sub_nc_u32_sdwa v3, v9, v3 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-NEXT: v_sub_nc_u32_sdwa v2, v9, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v1, v1, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_perm_b32 v1, v4, v9, 0x2070306
; GFX10-NEXT: global_store_dword v[5:6], v0, off
; GFX10-NEXT: global_store_dword v[7:8], v1, off
@@ -1880,15 +1862,15 @@ define hidden void @srem_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1,
; GFX9-NEXT: s_mov_b32 s4, 0x2070306
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_cvt_f32_i32_sdwa v2, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0
-; GFX9-NEXT: v_cvt_f32_i32_sdwa v12, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1
; GFX9-NEXT: v_cvt_f32_i32_sdwa v3, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
+; GFX9-NEXT: v_cvt_f32_i32_sdwa v12, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1
; GFX9-NEXT: v_cvt_f32_i32_sdwa v13, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cvt_f32_i32_sdwa v11, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
; GFX9-NEXT: v_cvt_f32_i32_sdwa v15, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
; GFX9-NEXT: v_rcp_f32_e32 v2, v2
-; GFX9-NEXT: v_rcp_f32_e32 v12, v12
; GFX9-NEXT: v_add_u32_e32 v16, 1, v3
+; GFX9-NEXT: v_rcp_f32_e32 v12, v12
; GFX9-NEXT: v_rcp_f32_e32 v3, v3
; GFX9-NEXT: v_add_u32_e32 v17, 1, v13
; GFX9-NEXT: v_rcp_f32_e32 v13, v13
@@ -1906,10 +1888,10 @@ define hidden void @srem_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1,
; GFX9-NEXT: v_cvt_i32_f32_e32 v11, v11
; GFX9-NEXT: v_cvt_i32_f32_e32 v3, v3
; GFX9-NEXT: v_cvt_i32_f32_e32 v12, v12
-; GFX9-NEXT: v_lshrrev_b32_e32 v10, 8, v4
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v4
-; GFX9-NEXT: v_perm_b32 v1, v4, v9, s4
+; GFX9-NEXT: v_lshrrev_b32_e32 v10, 8, v4
; GFX9-NEXT: v_lshrrev_b32_e32 v14, 24, v4
+; GFX9-NEXT: v_perm_b32 v1, v4, v9, s4
; GFX9-NEXT: v_mul_lo_u32 v2, v2, v4
; GFX9-NEXT: v_mul_lo_u32 v4, v11, v10
; GFX9-NEXT: v_mul_lo_u32 v3, v3, v0
@@ -1918,10 +1900,9 @@ define hidden void @srem_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1,
; GFX9-NEXT: v_sub_u32_sdwa v2, v9, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
; GFX9-NEXT: v_sub_u32_e32 v3, v14, v3
; GFX9-NEXT: v_sub_u32_sdwa v4, v9, v10 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v3, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_or_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v0, v3, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: global_store_dword v[5:6], v0, off
; GFX9-NEXT: global_store_dword v[7:8], v1, off
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -1949,26 +1930,25 @@ define hidden void @sub_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1,
; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
-; GFX10-NEXT: global_load_dword v4, v[2:3], off
-; GFX10-NEXT: global_load_dword v9, v[0:1], off
+; GFX10-NEXT: global_load_dword v2, v[2:3], off
+; GFX10-NEXT: global_load_dword v0, v[0:1], off
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_lshrrev_b16 v0, 8, v4
-; GFX10-NEXT: v_lshrrev_b32_e32 v1, 24, v4
-; GFX10-NEXT: v_lshrrev_b32_e32 v2, 16, v4
+; GFX10-NEXT: v_lshrrev_b32_e32 v1, 24, v2
+; GFX10-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; GFX10-NEXT: v_lshrrev_b16 v4, 8, v2
+; GFX10-NEXT: v_sub_nc_u16 v9, v3, v1
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_sub_nc_u16 v0, v9, v0
-; GFX10-NEXT: v_sub_nc_u16 v10, v1, v4
-; GFX10-NEXT: v_sub_nc_u16 v3, v2, v1
+; GFX10-NEXT: v_sub_nc_u16 v4, v0, v4
+; GFX10-NEXT: v_sub_nc_u16 v3, v1, v3
; GFX10-NEXT: v_sub_nc_u16 v1, v1, v2
-; GFX10-NEXT: v_lshlrev_b16 v0, 8, v0
-; GFX10-NEXT: v_lshlrev_b16 v2, 8, v3
-; GFX10-NEXT: v_or_b32_sdwa v0, v10, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX10-NEXT: v_perm_b32 v1, v4, v9, 0x6070007
-; GFX10-NEXT: global_store_dword v[5:6], v0, off
-; GFX10-NEXT: global_store_dword v[7:8], v1, off
+; GFX10-NEXT: v_perm_b32 v0, v2, v0, 0x6070007
+; GFX10-NEXT: v_lshlrev_b16 v9, 8, v9
+; GFX10-NEXT: v_lshlrev_b16 v4, 8, v4
+; GFX10-NEXT: v_or_b32_sdwa v3, v3, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: global_store_dword v[5:6], v1, off
+; GFX10-NEXT: global_store_dword v[7:8], v0, off
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: sub_store_div:
@@ -1989,10 +1969,9 @@ define hidden void @sub_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1,
; GFX9-NEXT: v_sub_u16_sdwa v2, v9, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:WORD_1
; GFX9-NEXT: v_sub_u16_sdwa v3, v4, v9 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_1
; GFX9-NEXT: v_sub_u16_sdwa v4, v9, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
-; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v2, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: global_store_dword v[5:6], v1, off
; GFX9-NEXT: global_store_dword v[7:8], v0, off
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -2133,45 +2112,44 @@ define hidden void @udiv(ptr addrspace(1) %in0, ptr addrspace(1) %in1, i8 %elt,
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_and_b32_e32 v4, 0x3ff, v31
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 2, v4
-; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, v2, v4
-; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
-; GFX10-NEXT: global_load_dword v4, v[2:3], off
-; GFX10-NEXT: global_load_dword v9, v[0:1], off
+; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, v2, v4
+; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
+; GFX10-NEXT: global_load_dword v4, v[0:1], off
+; GFX10-NEXT: global_load_dword v9, v[2:3], off
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, v4
-; GFX10-NEXT: v_cvt_f32_ubyte1_e32 v1, v4
+; GFX10-NEXT: v_cvt_f32_ubyte3_e32 v0, v4
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_cvt_f32_ubyte3_e32 v2, v9
-; GFX10-NEXT: v_cvt_f32_ubyte2_e32 v3, v4
-; GFX10-NEXT: v_cvt_f32_ubyte3_e32 v10, v4
-; GFX10-NEXT: v_rcp_f32_e32 v11, v0
-; GFX10-NEXT: v_rcp_f32_e32 v1, v1
-; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v12, v9
-; GFX10-NEXT: v_rcp_f32_e32 v13, v3
+; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v1, v9
+; GFX10-NEXT: v_cvt_f32_ubyte2_e32 v2, v9
+; GFX10-NEXT: v_cvt_f32_ubyte3_e32 v3, v9
+; GFX10-NEXT: v_cvt_f32_ubyte1_e32 v10, v9
+; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v11, v4
+; GFX10-NEXT: v_rcp_f32_e32 v12, v1
+; GFX10-NEXT: v_rcp_f32_e32 v13, v2
+; GFX10-NEXT: v_rcp_f32_e32 v3, v3
; GFX10-NEXT: v_rcp_f32_e32 v10, v10
-; GFX10-NEXT: v_add_nc_u32_e32 v2, 1, v2
-; GFX10-NEXT: v_add_nc_u32_e32 v3, 1, v3
-; GFX10-NEXT: v_add_nc_u32_e32 v12, 1, v12
; GFX10-NEXT: v_add_nc_u32_e32 v0, 1, v0
-; GFX10-NEXT: v_mul_f32_e32 v2, v2, v11
-; GFX10-NEXT: v_mul_f32_e32 v1, v3, v1
-; GFX10-NEXT: v_mul_f32_e32 v3, v12, v13
-; GFX10-NEXT: v_mul_f32_e32 v0, v0, v10
-; GFX10-NEXT: v_trunc_f32_e32 v2, v2
-; GFX10-NEXT: v_trunc_f32_e32 v1, v1
-; GFX10-NEXT: v_trunc_f32_e32 v3, v3
+; GFX10-NEXT: v_add_nc_u32_e32 v11, 1, v11
+; GFX10-NEXT: v_add_nc_u32_e32 v1, 1, v1
+; GFX10-NEXT: v_add_nc_u32_e32 v2, 1, v2
+; GFX10-NEXT: v_mul_f32_e32 v0, v0, v12
+; GFX10-NEXT: v_mul_f32_e32 v11, v11, v13
+; GFX10-NEXT: v_mul_f32_e32 v1, v1, v3
+; GFX10-NEXT: v_mul_f32_e32 v2, v2, v10
; GFX10-NEXT: v_trunc_f32_e32 v0, v0
-; GFX10-NEXT: v_cvt_u32_f32_e32 v2, v2
-; GFX10-NEXT: v_cvt_u32_f32_sdwa v1, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; GFX10-NEXT: v_trunc_f32_e32 v3, v11
+; GFX10-NEXT: v_trunc_f32_e32 v1, v1
+; GFX10-NEXT: v_trunc_f32_e32 v2, v2
+; GFX10-NEXT: v_cvt_u32_f32_e32 v0, v0
; GFX10-NEXT: v_cvt_u32_f32_e32 v3, v3
-; GFX10-NEXT: v_cvt_u32_f32_sdwa v0, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD
-; GFX10-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_lshl_or_b32 v0, v0, 16, v1
-; GFX10-NEXT: v_perm_b32 v1, v9, v4, 0x40207
+; GFX10-NEXT: v_cvt_u32_f32_sdwa v1, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; GFX10-NEXT: v_cvt_u32_f32_sdwa v2, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_perm_b32 v1, v4, v9, 0x40207
; GFX10-NEXT: global_store_dword v[5:6], v0, off
; GFX10-NEXT: global_store_dword v[7:8], v1, off
; GFX10-NEXT: s_setpc_b64 s[30:31]
@@ -2213,14 +2191,13 @@ define hidden void @udiv(ptr addrspace(1) %in0, ptr addrspace(1) %in1, i8 %elt,
; GFX9-NEXT: v_trunc_f32_e32 v3, v3
; GFX9-NEXT: v_trunc_f32_e32 v1, v1
; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX9-NEXT: v_cvt_u32_f32_sdwa v2, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD
; GFX9-NEXT: v_cvt_u32_f32_e32 v3, v3
; GFX9-NEXT: v_cvt_u32_f32_sdwa v1, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; GFX9-NEXT: v_cvt_u32_f32_sdwa v2, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD
; GFX9-NEXT: v_perm_b32 v4, v4, v9, s4
+; GFX9-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: global_store_dword v[5:6], v0, off
; GFX9-NEXT: global_store_dword v[7:8], v4, off
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -2307,51 +2284,50 @@ define hidden void @urem_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1,
; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
-; GFX10-NEXT: global_load_dword v2, v[2:3], off
-; GFX10-NEXT: global_load_dword v0, v[0:1], off
+; GFX10-NEXT: global_load_dword v4, v[2:3], off
+; GFX10-NEXT: global_load_dword v9, v[0:1], off
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v1, v2
-; GFX10-NEXT: v_cvt_f32_ubyte1_e32 v3, v2
-; GFX10-NEXT: v_cvt_f32_ubyte2_e32 v4, v2
-; GFX10-NEXT: v_cvt_f32_ubyte3_e32 v9, v2
+; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, v4
+; GFX10-NEXT: v_cvt_f32_ubyte2_e32 v1, v4
+; GFX10-NEXT: v_cvt_f32_ubyte3_e32 v2, v4
+; GFX10-NEXT: v_cvt_f32_ubyte1_e32 v3, v4
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_cvt_f32_ubyte2_e32 v11, v0
+; GFX10-NEXT: v_cvt_f32_ubyte2_e32 v10, v9
+; GFX10-NEXT: v_rcp_f32_e32 v0, v0
; GFX10-NEXT: v_rcp_f32_e32 v1, v1
-; GFX10-NEXT: v_rcp_f32_e32 v10, v3
-; GFX10-NEXT: v_rcp_f32_e32 v4, v4
-; GFX10-NEXT: v_rcp_f32_e32 v9, v9
+; GFX10-NEXT: v_rcp_f32_e32 v2, v2
+; GFX10-NEXT: v_rcp_f32_e32 v11, v3
; GFX10-NEXT: v_add_nc_u32_e32 v3, 1, v3
-; GFX10-NEXT: v_add_nc_u32_e32 v11, 1, v11
-; GFX10-NEXT: v_lshrrev_b32_e32 v12, 16, v2
+; GFX10-NEXT: v_add_nc_u32_e32 v10, 1, v10
+; GFX10-NEXT: v_lshrrev_b32_e32 v12, 16, v4
+; GFX10-NEXT: v_mul_f32_e32 v0, v3, v0
; GFX10-NEXT: v_mul_f32_e32 v1, v3, v1
-; GFX10-NEXT: v_mul_f32_e32 v10, v3, v10
-; GFX10-NEXT: v_mul_f32_e32 v3, v3, v4
-; GFX10-NEXT: v_mul_f32_e32 v4, v11, v9
-; GFX10-NEXT: v_lshrrev_b32_e32 v11, 24, v2
+; GFX10-NEXT: v_mul_f32_e32 v2, v10, v2
+; GFX10-NEXT: v_mul_f32_e32 v3, v3, v11
+; GFX10-NEXT: v_lshrrev_b32_e32 v10, 8, v4
+; GFX10-NEXT: v_trunc_f32_e32 v0, v0
; GFX10-NEXT: v_trunc_f32_e32 v1, v1
-; GFX10-NEXT: v_trunc_f32_e32 v9, v10
-; GFX10-NEXT: v_lshrrev_b32_e32 v10, 8, v2
+; GFX10-NEXT: v_trunc_f32_e32 v2, v2
; GFX10-NEXT: v_trunc_f32_e32 v3, v3
-; GFX10-NEXT: v_trunc_f32_e32 v4, v4
+; GFX10-NEXT: v_lshrrev_b32_e32 v11, 24, v4
+; GFX10-NEXT: v_cvt_u32_f32_e32 v0, v0
; GFX10-NEXT: v_cvt_u32_f32_e32 v1, v1
-; GFX10-NEXT: v_cvt_u32_f32_e32 v9, v9
+; GFX10-NEXT: v_cvt_u32_f32_e32 v2, v2
; GFX10-NEXT: v_cvt_u32_f32_e32 v3, v3
-; GFX10-NEXT: v_cvt_u32_f32_e32 v4, v4
-; GFX10-NEXT: v_mul_lo_u32 v1, v1, v2
-; GFX10-NEXT: v_mul_lo_u32 v9, v9, v10
-; GFX10-NEXT: v_mul_lo_u32 v3, v3, v12
-; GFX10-NEXT: v_mul_lo_u32 v4, v4, v11
+; GFX10-NEXT: v_mul_lo_u32 v0, v0, v4
+; GFX10-NEXT: v_mul_lo_u32 v1, v1, v12
+; GFX10-NEXT: v_mul_lo_u32 v2, v2, v11
+; GFX10-NEXT: v_mul_lo_u32 v3, v3, v10
+; GFX10-NEXT: v_sub_nc_u32_e32 v0, v10, v0
; GFX10-NEXT: v_sub_nc_u32_e32 v1, v10, v1
-; GFX10-NEXT: v_sub_nc_u32_sdwa v9, v10, v9 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT: v_sub_nc_u32_e32 v3, v10, v3
-; GFX10-NEXT: v_sub_nc_u32_sdwa v4, v0, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-NEXT: v_perm_b32 v0, v2, v0, 0x2050505
-; GFX10-NEXT: v_or_b32_sdwa v1, v1, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_or_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_lshl_or_b32 v1, v3, 16, v1
-; GFX10-NEXT: global_store_dword v[5:6], v1, off
-; GFX10-NEXT: global_store_dword v[7:8], v0, off
+; GFX10-NEXT: v_sub_nc_u32_sdwa v2, v9, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-NEXT: v_sub_nc_u32_sdwa v3, v10, v3 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_perm_b32 v1, v4, v9, 0x2050505
+; GFX10-NEXT: global_store_dword v[5:6], v0, off
+; GFX10-NEXT: global_store_dword v[7:8], v1, off
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: urem_store_div:
@@ -2391,22 +2367,21 @@ define hidden void @urem_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1,
; GFX9-NEXT: v_cvt_u32_f32_e32 v3, v3
; GFX9-NEXT: v_cvt_u32_f32_e32 v11, v11
; GFX9-NEXT: v_cvt_u32_f32_e32 v13, v13
-; GFX9-NEXT: v_lshrrev_b32_e32 v10, 8, v4
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v4
+; GFX9-NEXT: v_lshrrev_b32_e32 v10, 8, v4
; GFX9-NEXT: v_lshrrev_b32_e32 v12, 24, v4
+; GFX9-NEXT: v_perm_b32 v1, v4, v9, s4
; GFX9-NEXT: v_mul_lo_u32 v2, v2, v4
; GFX9-NEXT: v_mul_lo_u32 v3, v3, v10
-; GFX9-NEXT: v_perm_b32 v1, v4, v9, s4
; GFX9-NEXT: v_mul_lo_u32 v0, v11, v0
; GFX9-NEXT: v_mul_lo_u32 v4, v13, v12
; GFX9-NEXT: v_sub_u32_e32 v2, v10, v2
; GFX9-NEXT: v_sub_u32_sdwa v3, v10, v3 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_sub_u32_e32 v0, v10, v0
; GFX9-NEXT: v_sub_u32_sdwa v4, v9, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: global_store_dword v[5:6], v0, off
; GFX9-NEXT: global_store_dword v[7:8], v1, off
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -2430,28 +2405,27 @@ define hidden void @xor_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1,
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_and_b32_e32 v4, 0x3ff, v31
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 2, v4
-; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
-; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, v2, v4
; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
-; GFX10-NEXT: global_load_dword v4, v[0:1], off
-; GFX10-NEXT: global_load_dword v9, v[2:3], off
+; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
+; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT: global_load_dword v4, v[2:3], off
+; GFX10-NEXT: global_load_dword v9, v[0:1], off
; GFX10-NEXT: v_mov_b32_e32 v0, 0xffffff00
-; GFX10-NEXT: v_mov_b32_e32 v1, 1
-; GFX10-NEXT: v_mov_b32_e32 v2, 2
+; GFX10-NEXT: v_mov_b32_e32 v1, 2
+; GFX10-NEXT: v_mov_b32_e32 v2, 1
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_and_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-NEXT: v_and_b32_e32 v3, 0xffffff00, v4
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_and_b32_e32 v3, 0xffffff00, v9
-; GFX10-NEXT: v_xor_b32_sdwa v1, v9, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
-; GFX10-NEXT: v_xor_b32_sdwa v2, v9, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-NEXT: v_xor_b32_e32 v0, 0x200, v0
+; GFX10-NEXT: v_and_b32_sdwa v0, v9, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-NEXT: v_xor_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-NEXT: v_xor_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
; GFX10-NEXT: v_xor_b32_e32 v3, 0x100, v3
-; GFX10-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX10-NEXT: v_or_b32_sdwa v1, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX10-NEXT: v_perm_b32 v1, v9, v4, 0x5060307
+; GFX10-NEXT: v_xor_b32_e32 v0, 0x200, v0
+; GFX10-NEXT: v_or_b32_sdwa v1, v1, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX10-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_perm_b32 v1, v4, v9, 0x5060307
; GFX10-NEXT: global_store_dword v[5:6], v0, off
; GFX10-NEXT: global_store_dword v[7:8], v1, off
; GFX10-NEXT: s_setpc_b64 s[30:31]
@@ -2468,21 +2442,20 @@ define hidden void @xor_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1,
; GFX9-NEXT: global_load_dword v4, v[0:1], off
; GFX9-NEXT: global_load_dword v9, v[2:3], off
; GFX9-NEXT: s_movk_i32 s4, 0xff00
-; GFX9-NEXT: v_mov_b32_e32 v1, 1
; GFX9-NEXT: v_mov_b32_e32 v0, 2
+; GFX9-NEXT: v_mov_b32_e32 v1, 1
; GFX9-NEXT: s_mov_b32 s5, 0x5060307
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_and_b32_sdwa v2, v4, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_and_b32_e32 v3, 0xffffff00, v9
-; GFX9-NEXT: v_xor_b32_sdwa v1, v9, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
-; GFX9-NEXT: v_xor_b32_e32 v2, 0x200, v2
; GFX9-NEXT: v_xor_b32_sdwa v0, v9, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT: v_xor_b32_sdwa v1, v9, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
; GFX9-NEXT: v_xor_b32_e32 v3, 0x100, v3
+; GFX9-NEXT: v_xor_b32_e32 v2, 0x200, v2
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_e32 v1, v1, v2
-; GFX9-NEXT: v_or_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_perm_b32 v4, v9, v4, s5
; GFX9-NEXT: global_store_dword v[5:6], v0, off
; GFX9-NEXT: global_store_dword v[7:8], v4, off
diff --git a/llvm/test/CodeGen/AMDGPU/scalar_to_vector.ll b/llvm/test/CodeGen/AMDGPU/scalar_to_vector.ll
index 6b1a46d0fe10c..3bbc5b9113672 100644
--- a/llvm/test/CodeGen/AMDGPU/scalar_to_vector.ll
+++ b/llvm/test/CodeGen/AMDGPU/scalar_to_vector.ll
@@ -458,11 +458,10 @@ define i64 @bitcast_combine_scalar_to_vector_v4i16(i16 %arg) {
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_and_b32_e32 v1, 0xffffff00, v0
-; GFX9-NEXT: v_or_b32_sdwa v2, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v0, v2, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: bitcast_combine_scalar_to_vector_v4i16:
diff --git a/llvm/test/CodeGen/AMDGPU/sdwa-peephole-lshl-or-hi16pack.mir b/llvm/test/CodeGen/AMDGPU/sdwa-peephole-lshl-or-hi16pack.mir
new file mode 100644
index 0000000000000..256fcfbee9483
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/sdwa-peephole-lshl-or-hi16pack.mir
@@ -0,0 +1,156 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -run-pass=si-peephole-sdwa -o - %s | FileCheck -check-prefix=GFX9 %s
+
+# The (hi << 16) | (z & 0xffff) high-half pack is selected to a fused
+# V_LSHL_OR_B32 preceded by an explicit V_AND_B32 0xffff mask (V_LSHL_OR_B32
+# has no SDWA form). This peephole rewrites the pair back into a
+# V_LSHLREV_B32 + V_OR_B32_sdwa with src1_sel:WORD_0 (the WORD_0 select does
+# the masking for free) and erases the dead V_AND, restoring the SDWA fold.
+
+---
+name: pack_mask_on_and_src0
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr1
+ ; GFX9-LABEL: name: pack_mask_on_and_src0
+ ; GFX9: liveins: $vgpr0, $vgpr1
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX9-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX9-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 65535, implicit $exec
+ ; GFX9-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 16, [[COPY]], implicit $exec
+ ; GFX9-NEXT: [[V_OR_B32_sdwa:%[0-9]+]]:vgpr_32 = V_OR_B32_sdwa 0, [[V_LSHLREV_B32_e64_]], 0, [[COPY1]], 0, 6, 0, 6, 4, implicit $exec
+ ; GFX9-NEXT: $vgpr0 = COPY [[V_OR_B32_sdwa]]
+ ; GFX9-NEXT: SI_RETURN implicit $vgpr0
+ %0:vgpr_32 = COPY $vgpr0
+ %1:vgpr_32 = COPY $vgpr1
+ %2:vgpr_32 = V_MOV_B32_e32 65535, implicit $exec
+ %3:vgpr_32 = V_AND_B32_e64 %2, %1, implicit $exec
+ %4:vgpr_32 = V_LSHL_OR_B32_e64 %0, 16, %3, implicit $exec
+ $vgpr0 = COPY %4
+ SI_RETURN implicit $vgpr0
+...
+
+---
+name: pack_mask_on_and_src1
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr1
+ ; GFX9-LABEL: name: pack_mask_on_and_src1
+ ; GFX9: liveins: $vgpr0, $vgpr1
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX9-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX9-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 65535, implicit $exec
+ ; GFX9-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 16, [[COPY]], implicit $exec
+ ; GFX9-NEXT: [[V_OR_B32_sdwa:%[0-9]+]]:vgpr_32 = V_OR_B32_sdwa 0, [[V_LSHLREV_B32_e64_]], 0, [[COPY1]], 0, 6, 0, 6, 4, implicit $exec
+ ; GFX9-NEXT: $vgpr0 = COPY [[V_OR_B32_sdwa]]
+ ; GFX9-NEXT: SI_RETURN implicit $vgpr0
+ %0:vgpr_32 = COPY $vgpr0
+ %1:vgpr_32 = COPY $vgpr1
+ %2:vgpr_32 = V_MOV_B32_e32 65535, implicit $exec
+ %3:vgpr_32 = V_AND_B32_e64 %1, %2, implicit $exec
+ %4:vgpr_32 = V_LSHL_OR_B32_e64 %0, 16, %3, implicit $exec
+ $vgpr0 = COPY %4
+ SI_RETURN implicit $vgpr0
+...
+
+---
+name: pack_mask_and_e32
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr1
+ ; GFX9-LABEL: name: pack_mask_and_e32
+ ; GFX9: liveins: $vgpr0, $vgpr1
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX9-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX9-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 16, [[COPY]], implicit $exec
+ ; GFX9-NEXT: [[V_OR_B32_sdwa:%[0-9]+]]:vgpr_32 = V_OR_B32_sdwa 0, [[V_LSHLREV_B32_e64_]], 0, [[COPY1]], 0, 6, 0, 6, 4, implicit $exec
+ ; GFX9-NEXT: $vgpr0 = COPY [[V_OR_B32_sdwa]]
+ ; GFX9-NEXT: SI_RETURN implicit $vgpr0
+ %0:vgpr_32 = COPY $vgpr0
+ %1:vgpr_32 = COPY $vgpr1
+ %2:vgpr_32 = V_AND_B32_e32 65535, %1, implicit $exec
+ %3:vgpr_32 = V_LSHL_OR_B32_e64 %0, 16, %2, implicit $exec
+ $vgpr0 = COPY %3
+ SI_RETURN implicit $vgpr0
+...
+
+# Negative: the V_AND has a second use, so it cannot be removed and the
+# V_LSHL_OR_B32 must be left untouched.
+---
+name: no_rewrite_multiuse_and
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr1
+ ; GFX9-LABEL: name: no_rewrite_multiuse_and
+ ; GFX9: liveins: $vgpr0, $vgpr1
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX9-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX9-NEXT: [[V_AND_B32_e32_:%[0-9]+]]:vgpr_32 = V_AND_B32_e32 65535, [[COPY1]], implicit $exec
+ ; GFX9-NEXT: [[V_LSHL_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHL_OR_B32_e64 [[COPY]], 16, [[V_AND_B32_e32_]], implicit $exec
+ ; GFX9-NEXT: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[V_AND_B32_e32_]], [[V_LSHL_OR_B32_e64_]], 0, implicit $exec
+ ; GFX9-NEXT: $vgpr0 = COPY [[V_ADD_U32_e64_]]
+ ; GFX9-NEXT: SI_RETURN implicit $vgpr0
+ %0:vgpr_32 = COPY $vgpr0
+ %1:vgpr_32 = COPY $vgpr1
+ %2:vgpr_32 = V_AND_B32_e32 65535, %1, implicit $exec
+ %3:vgpr_32 = V_LSHL_OR_B32_e64 %0, 16, %2, implicit $exec
+ %4:vgpr_32 = V_ADD_U32_e64 %2, %3, 0, implicit $exec
+ $vgpr0 = COPY %4
+ SI_RETURN implicit $vgpr0
+...
+
+# Negative: the mask is not 0xffff, so WORD_0 would not be equivalent.
+---
+name: no_rewrite_wrong_mask
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr1
+ ; GFX9-LABEL: name: no_rewrite_wrong_mask
+ ; GFX9: liveins: $vgpr0, $vgpr1
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX9-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX9-NEXT: [[V_AND_B32_e32_:%[0-9]+]]:vgpr_32 = V_AND_B32_e32 32767, [[COPY1]], implicit $exec
+ ; GFX9-NEXT: [[V_LSHL_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHL_OR_B32_e64 [[COPY]], 16, [[V_AND_B32_e32_]], implicit $exec
+ ; GFX9-NEXT: $vgpr0 = COPY [[V_LSHL_OR_B32_e64_]]
+ ; GFX9-NEXT: SI_RETURN implicit $vgpr0
+ %0:vgpr_32 = COPY $vgpr0
+ %1:vgpr_32 = COPY $vgpr1
+ %2:vgpr_32 = V_AND_B32_e32 32767, %1, implicit $exec
+ %3:vgpr_32 = V_LSHL_OR_B32_e64 %0, 16, %2, implicit $exec
+ $vgpr0 = COPY %3
+ SI_RETURN implicit $vgpr0
+...
+
+# Negative: the shift amount is not 16.
+---
+name: no_rewrite_wrong_shift
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr1
+ ; GFX9-LABEL: name: no_rewrite_wrong_shift
+ ; GFX9: liveins: $vgpr0, $vgpr1
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX9-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX9-NEXT: [[V_AND_B32_e32_:%[0-9]+]]:vgpr_32 = V_AND_B32_e32 65535, [[COPY1]], implicit $exec
+ ; GFX9-NEXT: [[V_LSHL_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHL_OR_B32_e64 [[COPY]], 8, [[V_AND_B32_e32_]], implicit $exec
+ ; GFX9-NEXT: $vgpr0 = COPY [[V_LSHL_OR_B32_e64_]]
+ ; GFX9-NEXT: SI_RETURN implicit $vgpr0
+ %0:vgpr_32 = COPY $vgpr0
+ %1:vgpr_32 = COPY $vgpr1
+ %2:vgpr_32 = V_AND_B32_e32 65535, %1, implicit $exec
+ %3:vgpr_32 = V_LSHL_OR_B32_e64 %0, 8, %2, implicit $exec
+ $vgpr0 = COPY %3
+ SI_RETURN implicit $vgpr0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/sdwa-peephole.ll b/llvm/test/CodeGen/AMDGPU/sdwa-peephole.ll
index b596d9d4ad6e6..992f2db2ce48c 100644
--- a/llvm/test/CodeGen/AMDGPU/sdwa-peephole.ll
+++ b/llvm/test/CodeGen/AMDGPU/sdwa-peephole.ll
@@ -1221,10 +1221,9 @@ define amdgpu_kernel void @mul_v4i8(ptr addrspace(1) %out, ptr addrspace(1) %ina
; GFX9-NEXT: v_mul_lo_u16_sdwa v4, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX9-NEXT: v_mul_lo_u16_e32 v5, v1, v2
; GFX9-NEXT: v_mul_lo_u16_sdwa v1, v1, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
+; GFX9-NEXT: v_or_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_or_b32_sdwa v2, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: global_store_dword v0, v1, s[0:1]
; GFX9-NEXT: s_endpgm
;
@@ -1240,24 +1239,23 @@ define amdgpu_kernel void @mul_v4i8(ptr addrspace(1) %out, ptr addrspace(1) %ina
; GFX10-NEXT: global_load_dword v2, v0, s[6:7]
; GFX10-NEXT: s_waitcnt vmcnt(1)
; GFX10-NEXT: v_lshrrev_b16 v0, 8, v1
+; GFX10-NEXT: v_lshrrev_b32_e32 v3, 24, v1
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshrrev_b16 v3, 8, v2
-; GFX10-NEXT: v_lshrrev_b32_e32 v4, 24, v1
-; GFX10-NEXT: v_lshrrev_b32_e32 v5, 24, v2
-; GFX10-NEXT: v_lshrrev_b32_e32 v6, 16, v2
-; GFX10-NEXT: v_mul_lo_u16 v0, v0, v3
-; GFX10-NEXT: v_lshrrev_b32_e32 v3, 16, v1
-; GFX10-NEXT: v_mul_lo_u16 v4, v4, v5
+; GFX10-NEXT: v_lshrrev_b32_e32 v4, 24, v2
+; GFX10-NEXT: v_lshrrev_b16 v5, 8, v2
+; GFX10-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX10-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX10-NEXT: v_mul_lo_u16 v1, v1, v2
+; GFX10-NEXT: v_mul_lo_u16 v3, v3, v4
+; GFX10-NEXT: v_mul_lo_u16 v0, v0, v5
+; GFX10-NEXT: v_mul_lo_u16 v2, v6, v7
+; GFX10-NEXT: v_lshlrev_b16 v3, 8, v3
; GFX10-NEXT: v_lshlrev_b16 v0, 8, v0
-; GFX10-NEXT: v_mul_lo_u16 v3, v3, v6
-; GFX10-NEXT: v_lshlrev_b16 v2, 8, v4
+; GFX10-NEXT: v_or_b32_sdwa v2, v2, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX10-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_or_b32_sdwa v1, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX10-NEXT: global_store_dword v2, v0, s[0:1]
+; GFX10-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: global_store_dword v1, v0, s[0:1]
; GFX10-NEXT: s_endpgm
entry:
%idx = call i32 @llvm.amdgcn.workitem.id.x()
@@ -1383,13 +1381,11 @@ define amdgpu_kernel void @mul_v8i8(ptr addrspace(1) %out, ptr addrspace(1) %ina
; GFX9-NEXT: v_mul_lo_u16_e32 v9, v1, v3
; GFX9-NEXT: v_mul_lo_u16_sdwa v1, v1, v3 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
; GFX9-NEXT: v_or_b32_sdwa v3, v5, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v2, v8, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v9, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_or_b32_sdwa v0, v0, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_or_b32_sdwa v2, v8, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: global_store_dwordx2 v4, v[0:1], s[0:1]
; GFX9-NEXT: s_endpgm
;
@@ -1405,39 +1401,37 @@ define amdgpu_kernel void @mul_v8i8(ptr addrspace(1) %out, ptr addrspace(1) %ina
; GFX10-NEXT: global_load_dwordx2 v[2:3], v4, s[6:7]
; GFX10-NEXT: s_waitcnt vmcnt(1)
; GFX10-NEXT: v_lshrrev_b16 v4, 8, v1
+; GFX10-NEXT: v_lshrrev_b32_e32 v6, 24, v1
+; GFX10-NEXT: v_lshrrev_b32_e32 v8, 24, v0
; GFX10-NEXT: v_lshrrev_b16 v9, 8, v0
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshrrev_b16 v10, 8, v3
+; GFX10-NEXT: v_lshrrev_b32_e32 v12, 24, v3
; GFX10-NEXT: v_lshrrev_b16 v13, 8, v2
+; GFX10-NEXT: v_lshrrev_b32_e32 v14, 24, v2
; GFX10-NEXT: v_lshrrev_b32_e32 v5, 16, v1
-; GFX10-NEXT: v_lshrrev_b32_e32 v6, 24, v1
; GFX10-NEXT: v_lshrrev_b32_e32 v7, 16, v0
-; GFX10-NEXT: v_lshrrev_b32_e32 v8, 24, v0
; GFX10-NEXT: v_lshrrev_b32_e32 v11, 16, v3
-; GFX10-NEXT: v_lshrrev_b32_e32 v12, 24, v3
-; GFX10-NEXT: v_lshrrev_b32_e32 v14, 16, v2
-; GFX10-NEXT: v_lshrrev_b32_e32 v15, 24, v2
+; GFX10-NEXT: v_lshrrev_b32_e32 v15, 16, v2
; GFX10-NEXT: v_mul_lo_u16 v0, v0, v2
; GFX10-NEXT: v_mul_lo_u16 v1, v1, v3
; GFX10-NEXT: v_mul_lo_u16 v2, v9, v13
-; GFX10-NEXT: v_mul_lo_u16 v3, v4, v10
-; GFX10-NEXT: v_mul_lo_u16 v4, v8, v15
+; GFX10-NEXT: v_mul_lo_u16 v3, v8, v14
; GFX10-NEXT: v_mul_lo_u16 v6, v6, v12
-; GFX10-NEXT: v_mul_lo_u16 v7, v7, v14
+; GFX10-NEXT: v_mul_lo_u16 v4, v4, v10
+; GFX10-NEXT: v_mul_lo_u16 v7, v7, v15
+; GFX10-NEXT: v_mul_lo_u16 v5, v5, v11
; GFX10-NEXT: v_lshlrev_b16 v2, 8, v2
; GFX10-NEXT: v_lshlrev_b16 v3, 8, v3
-; GFX10-NEXT: v_mul_lo_u16 v5, v5, v11
-; GFX10-NEXT: v_lshlrev_b16 v4, 8, v4
; GFX10-NEXT: v_lshlrev_b16 v6, 8, v6
+; GFX10-NEXT: v_lshlrev_b16 v4, 8, v4
; GFX10-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_or_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_or_b32_sdwa v2, v7, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_or_b32_sdwa v3, v5, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX10-NEXT: v_or_b32_sdwa v2, v7, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v3, v5, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX10-NEXT: v_mov_b32_e32 v4, 0
-; GFX10-NEXT: v_lshl_or_b32 v0, v2, 16, v0
-; GFX10-NEXT: v_lshl_or_b32 v1, v3, 16, v1
+; GFX10-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: global_store_dwordx2 v4, v[0:1], s[0:1]
; GFX10-NEXT: s_endpgm
entry:
@@ -1508,9 +1502,8 @@ define amdgpu_kernel void @sitofp_v2i16_to_v2f16(
; GFX10-NEXT: global_load_dword v1, v0, s[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cvt_f16_i16_e32 v2, v1
-; GFX10-NEXT: v_cvt_f16_i16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10-NEXT: v_lshl_or_b32 v1, v1, 16, v2
+; GFX10-NEXT: v_cvt_f16_i16_sdwa v1, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX10-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: global_store_dword v0, v1, s[0:1]
; GFX10-NEXT: s_endpgm
ptr addrspace(1) %r,
diff --git a/llvm/test/CodeGen/AMDGPU/sminmax.v2i16.ll b/llvm/test/CodeGen/AMDGPU/sminmax.v2i16.ll
index ea91318c97abd..c432a815070a1 100644
--- a/llvm/test/CodeGen/AMDGPU/sminmax.v2i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/sminmax.v2i16.ll
@@ -861,20 +861,20 @@ define amdgpu_kernel void @v_min_max_v2i16_user(ptr addrspace(1) %out0, ptr addr
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v2
-; GFX9-NEXT: v_cmp_gt_i32_sdwa vcc, sext(v1), sext(v2) src0_sel:WORD_0 src1_sel:WORD_0
; GFX9-NEXT: v_cmp_gt_i32_sdwa s[0:1], sext(v1), sext(v2) src0_sel:WORD_1 src1_sel:WORD_1
-; GFX9-NEXT: v_cndmask_b32_e32 v5, v2, v1, vcc
+; GFX9-NEXT: v_cmp_gt_i32_sdwa vcc, sext(v1), sext(v2) src0_sel:WORD_0 src1_sel:WORD_0
; GFX9-NEXT: v_cndmask_b32_e64 v6, v4, v3, s[0:1]
; GFX9-NEXT: v_cndmask_b32_e64 v3, v3, v4, s[0:1]
; GFX9-NEXT: v_cndmask_b32_e64 v4, 0, 1, s[0:1]
+; GFX9-NEXT: v_cndmask_b32_e32 v5, v2, v1, vcc
; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
; GFX9-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v4, 1, v4
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_lshl_or_b32 v5, v6, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v1, v3, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v5, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_or_b32_e32 v2, v2, v4
+; GFX9-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: global_store_dword v0, v5, s[8:9]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: global_store_dword v0, v1, s[10:11]
diff --git a/llvm/test/CodeGen/AMDGPU/strict_fsub.f16.ll b/llvm/test/CodeGen/AMDGPU/strict_fsub.f16.ll
index c8ff9b3a3d614..1f7f8e8521189 100644
--- a/llvm/test/CodeGen/AMDGPU/strict_fsub.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/strict_fsub.f16.ll
@@ -552,9 +552,9 @@ define <3 x half> @v_constained_fsub_v3f16_fpexcept_strict(<3 x half> %x, <3 x h
; GFX10-GISEL: ; %bb.0:
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-GISEL-NEXT: v_sub_f16_e32 v1, v1, v3
+; GFX10-GISEL-NEXT: v_lshlrev_b32_e64 v3, 16, s4
; GFX10-GISEL-NEXT: v_pk_add_f16 v0, v0, v2 neg_lo:[0,1] neg_hi:[0,1]
-; GFX10-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-GISEL-NEXT: v_lshl_or_b32 v1, s4, 16, v1
+; GFX10-GISEL-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: v_constained_fsub_v3f16_fpexcept_strict:
@@ -887,12 +887,12 @@ define amdgpu_ps <2 x half> @s_constained_fsub_v2f16_fpexcept_strict(<2 x half>
;
; GFX10-SDAG-LABEL: s_constained_fsub_v2f16_fpexcept_strict:
; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: v_sub_f16_e64 v0, s2, s3
; GFX10-SDAG-NEXT: s_lshr_b32 s0, s3, 16
-; GFX10-SDAG-NEXT: s_lshr_b32 s1, s2, 16
-; GFX10-SDAG-NEXT: v_sub_f16_e64 v1, s1, s0
-; GFX10-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-SDAG-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10-SDAG-NEXT: v_sub_f16_e64 v1, s2, s3
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s0
+; GFX10-SDAG-NEXT: s_lshr_b32 s0, s2, 16
+; GFX10-SDAG-NEXT: v_sub_f16_sdwa v0, s0, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-SDAG-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-SDAG-NEXT: ; return to shader part epilog
;
; GFX10-GISEL-LABEL: s_constained_fsub_v2f16_fpexcept_strict:
diff --git a/llvm/test/CodeGen/AMDGPU/v_sat_pk_u8_i16.ll b/llvm/test/CodeGen/AMDGPU/v_sat_pk_u8_i16.ll
index dc9c7f3429464..305d5a27cc374 100644
--- a/llvm/test/CodeGen/AMDGPU/v_sat_pk_u8_i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/v_sat_pk_u8_i16.ll
@@ -98,10 +98,10 @@ define <2 x i16> @basic_smax_smin(i16 %src0, i16 %src1) {
; GISEL-GFX9: ; %bb.0:
; GISEL-GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GISEL-GFX9-NEXT: v_mov_b32_e32 v2, 0xff
-; GISEL-GFX9-NEXT: v_med3_i16 v0, v0, 0, v2
; GISEL-GFX9-NEXT: v_med3_i16 v1, v1, 0, v2
-; GISEL-GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GISEL-GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GISEL-GFX9-NEXT: v_med3_i16 v0, v0, 0, v2
+; GISEL-GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GISEL-GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GISEL-GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GISEL-GFX11-TRUE16-LABEL: basic_smax_smin:
@@ -181,8 +181,8 @@ define amdgpu_kernel void @basic_smax_smin_sgpr(ptr addrspace(1) %out, i32 inreg
; SDAG-GFX9-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX9-NEXT: v_med3_i16 v2, s2, 0, v1
; SDAG-GFX9-NEXT: v_med3_i16 v1, s3, 0, v1
-; SDAG-GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; SDAG-GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v2
+; SDAG-GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SDAG-GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; SDAG-GFX9-NEXT: global_store_dword v0, v1, s[0:1]
; SDAG-GFX9-NEXT: s_endpgm
;
@@ -399,10 +399,10 @@ define <2 x i16> @basic_smin_smax(i16 %src0, i16 %src1) {
; GISEL-GFX9: ; %bb.0:
; GISEL-GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GISEL-GFX9-NEXT: v_mov_b32_e32 v2, 0xff
-; GISEL-GFX9-NEXT: v_med3_i16 v0, v0, 0, v2
; GISEL-GFX9-NEXT: v_med3_i16 v1, v1, 0, v2
-; GISEL-GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GISEL-GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GISEL-GFX9-NEXT: v_med3_i16 v0, v0, 0, v2
+; GISEL-GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GISEL-GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GISEL-GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GISEL-GFX11-TRUE16-LABEL: basic_smin_smax:
@@ -533,10 +533,10 @@ define <2 x i16> @basic_smin_smax_combined(i16 %src0, i16 %src1) {
; GISEL-GFX9: ; %bb.0:
; GISEL-GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GISEL-GFX9-NEXT: v_mov_b32_e32 v2, 0xff
-; GISEL-GFX9-NEXT: v_med3_i16 v0, v0, 0, v2
; GISEL-GFX9-NEXT: v_med3_i16 v1, v1, 0, v2
-; GISEL-GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GISEL-GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GISEL-GFX9-NEXT: v_med3_i16 v0, v0, 0, v2
+; GISEL-GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GISEL-GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GISEL-GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GISEL-GFX11-TRUE16-LABEL: basic_smin_smax_combined:
More information about the llvm-commits
mailing list