[llvm] e477a99 - [AMDGPU] Re-fold masked hi16 pack into v_or_b32_sdwa (#210735)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Jul 30 03:14:46 PDT 2026
Author: michaelselehov
Date: 2026-07-30T12:14:37+02:00
New Revision: e477a9950ed219a02f4b33b35c4bee24f4c87b00
URL: https://github.com/llvm/llvm-project/commit/e477a9950ed219a02f4b33b35c4bee24f4c87b00
DIFF: https://github.com/llvm/llvm-project/commit/e477a9950ed219a02f4b33b35c4bee24f4c87b00.diff
LOG: [AMDGPU] Re-fold masked hi16 pack into v_or_b32_sdwa (#210735)
Follow-up to #206058, which selects the (hi << 16) | (z & 0xffff)
high-half pack to a fused v_lshl_or_b32. When z's high bits are not
known zero this needs an explicit v_and_b32 0xffff, and since
v_lshl_or_b32 has no SDWA form the mask can no longer fold into a single
v_or_b32_sdwa (src1_sel:WORD_0) as it did before -- a small but real
perf hole (~5% on a VALU-bound char4 kernel on gfx942).
Add a SIPeepholeSDWA rewrite of the masked pack back into v_lshlrev_b32
+ v_or_b32_sdwa, erasing the dead v_and. It fires only on the masked
form, so the mask-free #206058 win is preserved, and it never increases
the instruction count.
Assisted-by: Claude Opus
Added:
llvm/test/CodeGen/AMDGPU/sdwa-peephole-lshl-or-hi16pack.ll
llvm/test/CodeGen/AMDGPU/sdwa-peephole-lshl-or-hi16pack.mir
Modified:
llvm/lib/Target/AMDGPU/SIPeepholeSDWA.cpp
llvm/test/CodeGen/AMDGPU/GlobalISel/add.vni16.ll
llvm/test/CodeGen/AMDGPU/GlobalISel/fpow.ll
llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.abs.ll
llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.atomic.dim.a16.ll
llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.gather4.a16.dim.ll
llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.getresinfo.a16.ll
llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.2darraymsaa.a16.ll
llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.3d.a16.ll
llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.sample.cd.g16.ll
llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.sample.g16.ll
llvm/test/CodeGen/AMDGPU/GlobalISel/saddsat.ll
llvm/test/CodeGen/AMDGPU/GlobalISel/select-to-fmin-fmax.ll
llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll
llvm/test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll
llvm/test/CodeGen/AMDGPU/GlobalISel/usubsat.ll
llvm/test/CodeGen/AMDGPU/GlobalISel/vni8-across-blocks.ll
llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll
llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll
llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.320bit.ll
llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.32bit.ll
llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.48bit.ll
llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll
llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.576bit.ll
llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.640bit.ll
llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll
llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.704bit.ll
llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.768bit.ll
llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.832bit.ll
llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.896bit.ll
llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.960bit.ll
llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll
llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll
llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-contents-legalization.ll
llvm/test/CodeGen/AMDGPU/build-vector-packed-partial-undef.ll
llvm/test/CodeGen/AMDGPU/constant-address-space-32bit.ll
llvm/test/CodeGen/AMDGPU/ctlz_zero_poison.ll
llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
llvm/test/CodeGen/AMDGPU/dagcombine-fmul-sel.ll
llvm/test/CodeGen/AMDGPU/fmaximum.ll
llvm/test/CodeGen/AMDGPU/fminimum.ll
llvm/test/CodeGen/AMDGPU/fmul-to-ldexp.ll
llvm/test/CodeGen/AMDGPU/fneg-fabs.bf16.ll
llvm/test/CodeGen/AMDGPU/fneg.bf16.ll
llvm/test/CodeGen/AMDGPU/fptosi-sat-vector.ll
llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll
llvm/test/CodeGen/AMDGPU/fptrunc.v2f16.no.fast.math.ll
llvm/test/CodeGen/AMDGPU/freeze.ll
llvm/test/CodeGen/AMDGPU/frem.ll
llvm/test/CodeGen/AMDGPU/idot4u.ll
llvm/test/CodeGen/AMDGPU/idot8s.ll
llvm/test/CodeGen/AMDGPU/idot8u.ll
llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.a16.dim.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.noret.ll
llvm/test/CodeGen/AMDGPU/llvm.fptrunc.round.ll
llvm/test/CodeGen/AMDGPU/llvm.maximum.f16.ll
llvm/test/CodeGen/AMDGPU/llvm.minimum.f16.ll
llvm/test/CodeGen/AMDGPU/llvm.modf.ll
llvm/test/CodeGen/AMDGPU/mad-mix-lo.ll
llvm/test/CodeGen/AMDGPU/packed-op-sel.ll
llvm/test/CodeGen/AMDGPU/permute_i8.ll
llvm/test/CodeGen/AMDGPU/scalar_to_vector.ll
llvm/test/CodeGen/AMDGPU/sdwa-peephole.ll
llvm/test/CodeGen/AMDGPU/sminmax.v2i16.ll
llvm/test/CodeGen/AMDGPU/strict_fsub.f16.ll
llvm/test/CodeGen/AMDGPU/v_sat_pk_u8_i16.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/AMDGPU/SIPeepholeSDWA.cpp b/llvm/lib/Target/AMDGPU/SIPeepholeSDWA.cpp
index acd9dce08a692..cbb1efee7e559 100644
--- a/llvm/lib/Target/AMDGPU/SIPeepholeSDWA.cpp
+++ b/llvm/lib/Target/AMDGPU/SIPeepholeSDWA.cpp
@@ -59,6 +59,11 @@ class SIPeepholeSDWA {
std::optional<int64_t> foldToImm(const MachineOperand &Op) const;
+ // If MI is a v_and_b32 with a 0xffff or 0xff immediate, return the masked
+ // value operand and the matching SDWA selector (WORD_0 / BYTE_0).
+ std::optional<std::pair<MachineOperand *, AMDGPU::SDWA::SdwaSel>>
+ matchAndMask(MachineInstr &MI) const;
+
void matchSDWAOperands(MachineBasicBlock &MBB);
std::unique_ptr<SDWAOperand> matchSDWAOperand(MachineInstr &MI);
void pseudoOpConvertToVOP2(MachineInstr &MI,
@@ -67,6 +72,7 @@ class SIPeepholeSDWA {
MachineInstr *createSDWAVersion(MachineInstr &MI);
bool convertToSDWA(MachineInstr &MI, const SDWAOperandsVector &SDWAOperands);
void legalizeScalarOperands(MachineInstr &MI, const GCNSubtarget &ST) const;
+ bool splitLshlOrForSDWA(MachineBasicBlock &MBB);
public:
bool run(MachineFunction &MF);
@@ -667,6 +673,26 @@ SIPeepholeSDWA::foldToImm(const MachineOperand &Op) const {
return std::nullopt;
}
+std::optional<std::pair<MachineOperand *, SdwaSel>>
+SIPeepholeSDWA::matchAndMask(MachineInstr &MI) const {
+ if (MI.getOpcode() != AMDGPU::V_AND_B32_e32 &&
+ MI.getOpcode() != AMDGPU::V_AND_B32_e64)
+ return std::nullopt;
+
+ MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0);
+ MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1);
+ MachineOperand *ValSrc = Src1;
+ std::optional<int64_t> Imm = foldToImm(*Src0);
+ if (!Imm) {
+ Imm = foldToImm(*Src1);
+ ValSrc = Src0;
+ }
+ if (!Imm || (*Imm != 0x0000ffff && *Imm != 0x000000ff))
+ return std::nullopt;
+
+ return std::make_pair(ValSrc, *Imm == 0x0000ffff ? WORD_0 : BYTE_0);
+}
+
std::unique_ptr<SDWAOperand>
SIPeepholeSDWA::matchSDWAOperand(MachineInstr &MI) {
unsigned Opcode = MI.getOpcode();
@@ -807,19 +833,10 @@ SIPeepholeSDWA::matchSDWAOperand(MachineInstr &MI) {
// e.g.:
// from: v_and_b32_e32 v1, 0x0000ffff/0x000000ff, v0
// to SDWA src:v0 src_sel:WORD_0/BYTE_0
-
- MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0);
- MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1);
- auto *ValSrc = Src1;
- auto Imm = foldToImm(*Src0);
-
- if (!Imm) {
- Imm = foldToImm(*Src1);
- ValSrc = Src0;
- }
-
- if (!Imm || (*Imm != 0x0000ffff && *Imm != 0x000000ff))
+ auto Mask = matchAndMask(MI);
+ if (!Mask)
break;
+ MachineOperand *ValSrc = Mask->first;
MachineOperand *Dst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst);
@@ -827,8 +844,7 @@ SIPeepholeSDWA::matchSDWAOperand(MachineInstr &MI) {
Dst->getReg().isPhysical())
break;
- return std::make_unique<SDWASrcOperand>(
- ValSrc, Dst, *Imm == 0x0000ffff ? WORD_0 : BYTE_0);
+ return std::make_unique<SDWASrcOperand>(ValSrc, Dst, Mask->second);
}
case AMDGPU::V_OR_B32_e32:
@@ -1360,6 +1376,81 @@ void SIPeepholeSDWA::legalizeScalarOperands(MachineInstr &MI,
}
}
+// Re-fold the masked high-half pack (hi << 16) | (z & 0xffff) into a single
+// v_or_b32_sdwa src1_sel:WORD_0, which ISel's fused v_lshl_or_b32 blocks.
+bool SIPeepholeSDWA::splitLshlOrForSDWA(MachineBasicBlock &MBB) {
+ struct Candidate {
+ MachineInstr *LshlOr;
+ MachineInstr *AndMI;
+ MachineOperand *Hi;
+ MachineOperand *ValSrc;
+ };
+ SmallVector<Candidate, 4> Candidates;
+
+ for (MachineInstr &MI : MBB) {
+ if (MI.getOpcode() != AMDGPU::V_LSHL_OR_B32_e64)
+ continue;
+
+ MachineOperand *Shift = TII->getNamedOperand(MI, AMDGPU::OpName::src1);
+ std::optional<int64_t> ShiftImm = foldToImm(*Shift);
+ if (!ShiftImm || *ShiftImm != 16)
+ continue;
+
+ MachineOperand *Hi = TII->getNamedOperand(MI, AMDGPU::OpName::src0);
+ MachineOperand *Src2 = TII->getNamedOperand(MI, AMDGPU::OpName::src2);
+ // Src2 must be a virtual reg so getVRegDef below is valid.
+ if (!Hi->isReg() || !Src2->isReg() || !Src2->getReg().isVirtual())
+ continue;
+
+ // The 0xffff mask must come from a single-use v_and so it can be dropped.
+ if (!MRI->hasOneNonDBGUse(Src2->getReg()))
+ continue;
+ MachineInstr *AndMI = MRI->getVRegDef(Src2->getReg());
+ if (!AndMI)
+ continue;
+ std::optional<std::pair<MachineOperand *, SdwaSel>> Mask =
+ matchAndMask(*AndMI);
+ if (!Mask || Mask->second != WORD_0)
+ continue;
+ MachineOperand *ValSrc = Mask->first;
+ if (!ValSrc->isReg() || !TRI->isVGPR(*MRI, ValSrc->getReg()))
+ continue;
+
+ Candidates.push_back({&MI, AndMI, Hi, ValSrc});
+ }
+
+ for (const Candidate &C : Candidates) {
+ MachineOperand *Dst = TII->getNamedOperand(*C.LshlOr, AMDGPU::OpName::vdst);
+
+ Register ShiftReg = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
+ BuildMI(*C.LshlOr->getParent(), *C.LshlOr, C.LshlOr->getDebugLoc(),
+ TII->get(AMDGPU::V_LSHLREV_B32_e64), ShiftReg)
+ .addImm(16)
+ .add(*C.Hi);
+
+ // vdst, src0_mods, src0, src1_mods, src1, clamp, dst_sel, dst_unused,
+ // src0_sel, src1_sel.
+ BuildMI(*C.LshlOr->getParent(), *C.LshlOr, C.LshlOr->getDebugLoc(),
+ TII->get(AMDGPU::V_OR_B32_sdwa))
+ .add(*Dst)
+ .addImm(0)
+ .addReg(ShiftReg)
+ .addImm(0)
+ .add(*C.ValSrc)
+ .addImm(0)
+ .addImm(DWORD)
+ .addImm(UNUSED_PAD)
+ .addImm(DWORD)
+ .addImm(WORD_0);
+
+ MRI->clearKillFlags(C.ValSrc->getReg());
+ C.LshlOr->eraseFromParent();
+ C.AndMI->eraseFromParent();
+ }
+
+ return !Candidates.empty();
+}
+
bool SIPeepholeSDWALegacy::runOnMachineFunction(MachineFunction &MF) {
if (skipFunction(MF.getFunction()))
return false;
@@ -1382,6 +1473,8 @@ bool SIPeepholeSDWA::run(MachineFunction &MF) {
for (MachineBasicBlock &MBB : MF) {
bool Changed = false;
do {
+ Ret |= splitLshlOrForSDWA(MBB);
+
// Preprocess the ADD/SUB pairs so they could be SDWA'ed.
// Look for a possible ADD or SUB that resulted from a previously lowered
// V_{ADD|SUB}_U64_PSEUDO. The function pseudoOpConvertToVOP2
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/add.vni16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/add.vni16.ll
index 3a8d2a90a7a84..d3eed3e7b4fd9 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/add.vni16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/add.vni16.ll
@@ -43,31 +43,30 @@ define void @add_v3i16(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb, ptr addrs
; GFX9-LABEL: add_v3i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_ushort v6, v[0:1], off
+; GFX9-NEXT: global_load_ushort v6, v[0:1], off offset:2
; GFX9-NEXT: global_load_ushort v7, v[0:1], off offset:4
-; GFX9-NEXT: global_load_ushort v8, v[2:3], off
+; GFX9-NEXT: global_load_ushort v8, v[2:3], off offset:2
; GFX9-NEXT: global_load_ushort v9, v[2:3], off offset:4
-; GFX9-NEXT: global_load_ushort v10, v[0:1], off offset:2
-; GFX9-NEXT: global_load_ushort v11, v[2:3], off offset:2
+; GFX9-NEXT: global_load_ushort v10, v[0:1], off
+; GFX9-NEXT: global_load_ushort v11, v[2:3], off
+; GFX9-NEXT: v_lshlrev_b32_e64 v0, 16, s4
; GFX9-NEXT: s_waitcnt vmcnt(5)
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v6
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v6
; GFX9-NEXT: s_waitcnt vmcnt(4)
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v7
+; GFX9-NEXT: v_or_b32_sdwa v2, v0, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(3)
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v8
; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v9
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_lshl_or_b32 v0, v10, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_lshl_or_b32 v2, v11, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v1, s4, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v3, s4, 16, v3
-; GFX9-NEXT: v_pk_add_u16 v0, v0, v2
+; GFX9-NEXT: v_or_b32_sdwa v3, v3, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_pk_add_u16 v1, v1, v3
-; GFX9-NEXT: global_store_short v[4:5], v0, off
-; GFX9-NEXT: global_store_short_d16_hi v[4:5], v0, off offset:2
-; GFX9-NEXT: global_store_short v[4:5], v1, off offset:4
+; GFX9-NEXT: v_pk_add_u16 v0, v2, v0
+; GFX9-NEXT: global_store_short v[4:5], v1, off
+; GFX9-NEXT: global_store_short_d16_hi v[4:5], v1, off offset:2
+; GFX9-NEXT: global_store_short v[4:5], v0, off offset:4
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
%a = load <3 x i16>, ptr addrspace(1) %ptra, align 4
@@ -213,46 +212,45 @@ define void @add_v5i16(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb, ptr addrs
; GFX9-LABEL: add_v5i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_ushort v6, v[0:1], off
-; GFX9-NEXT: global_load_ushort v7, v[0:1], off offset:4
+; GFX9-NEXT: global_load_ushort v6, v[0:1], off offset:2
+; GFX9-NEXT: global_load_ushort v7, v[0:1], off offset:6
; GFX9-NEXT: global_load_ushort v8, v[0:1], off offset:8
-; GFX9-NEXT: global_load_ushort v9, v[2:3], off
-; GFX9-NEXT: global_load_ushort v10, v[2:3], off offset:4
+; GFX9-NEXT: global_load_ushort v9, v[2:3], off offset:2
+; GFX9-NEXT: global_load_ushort v10, v[2:3], off offset:6
; GFX9-NEXT: global_load_ushort v11, v[2:3], off offset:8
-; GFX9-NEXT: global_load_ushort v12, v[0:1], off offset:2
-; GFX9-NEXT: global_load_ushort v13, v[0:1], off offset:6
-; GFX9-NEXT: global_load_ushort v14, v[2:3], off offset:2
-; GFX9-NEXT: global_load_ushort v15, v[2:3], off offset:6
+; GFX9-NEXT: global_load_ushort v12, v[0:1], off
+; GFX9-NEXT: global_load_ushort v13, v[0:1], off offset:4
+; GFX9-NEXT: global_load_ushort v14, v[2:3], off
+; GFX9-NEXT: global_load_ushort v15, v[2:3], off offset:4
+; GFX9-NEXT: v_lshlrev_b32_e64 v0, 16, s4
; GFX9-NEXT: s_waitcnt vmcnt(9)
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v6
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v6
; GFX9-NEXT: s_waitcnt vmcnt(8)
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v7
; GFX9-NEXT: s_waitcnt vmcnt(7)
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v8
+; GFX9-NEXT: v_or_b32_sdwa v3, v0, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(6)
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v9
+; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v9
; GFX9-NEXT: s_waitcnt vmcnt(5)
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v10
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v10
; GFX9-NEXT: s_waitcnt vmcnt(4)
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v11
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(3)
-; GFX9-NEXT: v_lshl_or_b32 v0, v12, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_lshl_or_b32 v1, v13, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v2, v2, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_lshl_or_b32 v3, v14, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v2, s4, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v6, v6, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_lshl_or_b32 v6, v15, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, s4, 16, v7
-; GFX9-NEXT: v_pk_add_u16 v0, v0, v3
+; GFX9-NEXT: v_or_b32_sdwa v7, v7, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_pk_add_u16 v1, v1, v6
+; GFX9-NEXT: v_pk_add_u16 v0, v3, v0
; GFX9-NEXT: v_pk_add_u16 v2, v2, v7
-; GFX9-NEXT: global_store_short v[4:5], v0, off
-; GFX9-NEXT: global_store_short_d16_hi v[4:5], v0, off offset:2
-; GFX9-NEXT: global_store_short v[4:5], v1, off offset:4
-; GFX9-NEXT: global_store_short_d16_hi v[4:5], v1, off offset:6
-; GFX9-NEXT: global_store_short v[4:5], v2, off offset:8
+; GFX9-NEXT: global_store_short v[4:5], v1, off
+; GFX9-NEXT: global_store_short_d16_hi v[4:5], v1, off offset:2
+; GFX9-NEXT: global_store_short v[4:5], v2, off offset:4
+; GFX9-NEXT: global_store_short_d16_hi v[4:5], v2, off offset:6
+; GFX9-NEXT: global_store_short v[4:5], v0, off offset:8
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
%a = load <5 x i16>, ptr addrspace(1) %ptra, align 4
@@ -432,61 +430,60 @@ define void @addv_7i16(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb, ptr addrs
; GFX9-LABEL: addv_7i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_ushort v6, v[0:1], off
-; GFX9-NEXT: global_load_ushort v7, v[0:1], off offset:4
-; GFX9-NEXT: global_load_ushort v8, v[0:1], off offset:8
+; GFX9-NEXT: global_load_ushort v6, v[0:1], off offset:2
+; GFX9-NEXT: global_load_ushort v7, v[0:1], off offset:6
+; GFX9-NEXT: global_load_ushort v8, v[0:1], off offset:10
; GFX9-NEXT: global_load_ushort v9, v[0:1], off offset:12
-; GFX9-NEXT: global_load_ushort v10, v[2:3], off
-; GFX9-NEXT: global_load_ushort v11, v[2:3], off offset:4
-; GFX9-NEXT: global_load_ushort v12, v[2:3], off offset:8
+; GFX9-NEXT: global_load_ushort v10, v[2:3], off offset:2
+; GFX9-NEXT: global_load_ushort v11, v[2:3], off offset:6
+; GFX9-NEXT: global_load_ushort v12, v[2:3], off offset:10
; GFX9-NEXT: global_load_ushort v13, v[2:3], off offset:12
-; GFX9-NEXT: global_load_ushort v14, v[0:1], off offset:2
-; GFX9-NEXT: global_load_ushort v15, v[0:1], off offset:6
-; GFX9-NEXT: global_load_ushort v16, v[0:1], off offset:10
-; GFX9-NEXT: global_load_ushort v17, v[2:3], off offset:2
-; GFX9-NEXT: global_load_ushort v18, v[2:3], off offset:6
-; GFX9-NEXT: global_load_ushort v19, v[2:3], off offset:10
+; GFX9-NEXT: global_load_ushort v14, v[0:1], off
+; GFX9-NEXT: global_load_ushort v15, v[0:1], off offset:4
+; GFX9-NEXT: global_load_ushort v16, v[0:1], off offset:8
+; GFX9-NEXT: global_load_ushort v17, v[2:3], off
+; GFX9-NEXT: global_load_ushort v18, v[2:3], off offset:4
+; GFX9-NEXT: global_load_ushort v19, v[2:3], off offset:8
+; GFX9-NEXT: v_lshlrev_b32_e64 v0, 16, s4
; GFX9-NEXT: s_waitcnt vmcnt(13)
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v6
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v6
; GFX9-NEXT: s_waitcnt vmcnt(12)
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v7
; GFX9-NEXT: s_waitcnt vmcnt(11)
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v8
; GFX9-NEXT: s_waitcnt vmcnt(10)
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v9
+; GFX9-NEXT: v_or_b32_sdwa v6, v0, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(9)
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v10
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v10
; GFX9-NEXT: s_waitcnt vmcnt(8)
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v11
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v11
; GFX9-NEXT: s_waitcnt vmcnt(7)
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v12
+; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v12
; GFX9-NEXT: s_waitcnt vmcnt(6)
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v13
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(5)
-; GFX9-NEXT: v_lshl_or_b32 v0, v14, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(4)
-; GFX9-NEXT: v_lshl_or_b32 v1, v15, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v2, v2, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(3)
-; GFX9-NEXT: v_lshl_or_b32 v2, v16, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v3, v3, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_lshl_or_b32 v6, v17, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v3, s4, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v7, v7, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_lshl_or_b32 v7, v18, 16, v7
+; GFX9-NEXT: v_or_b32_sdwa v8, v8, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_lshl_or_b32 v8, v19, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, s4, 16, v9
-; GFX9-NEXT: v_pk_add_u16 v0, v0, v6
+; GFX9-NEXT: v_or_b32_sdwa v9, v9, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_pk_add_u16 v1, v1, v7
+; GFX9-NEXT: v_pk_add_u16 v0, v6, v0
; GFX9-NEXT: v_pk_add_u16 v2, v2, v8
; GFX9-NEXT: v_pk_add_u16 v3, v3, v9
-; GFX9-NEXT: global_store_short v[4:5], v0, off
-; GFX9-NEXT: global_store_short_d16_hi v[4:5], v0, off offset:2
-; GFX9-NEXT: global_store_short v[4:5], v1, off offset:4
-; GFX9-NEXT: global_store_short_d16_hi v[4:5], v1, off offset:6
-; GFX9-NEXT: global_store_short v[4:5], v2, off offset:8
-; GFX9-NEXT: global_store_short_d16_hi v[4:5], v2, off offset:10
-; GFX9-NEXT: global_store_short v[4:5], v3, off offset:12
+; GFX9-NEXT: global_store_short v[4:5], v1, off
+; GFX9-NEXT: global_store_short_d16_hi v[4:5], v1, off offset:2
+; GFX9-NEXT: global_store_short v[4:5], v2, off offset:4
+; GFX9-NEXT: global_store_short_d16_hi v[4:5], v2, off offset:6
+; GFX9-NEXT: global_store_short v[4:5], v3, off offset:8
+; GFX9-NEXT: global_store_short_d16_hi v[4:5], v3, off offset:10
+; GFX9-NEXT: global_store_short v[4:5], v0, off offset:12
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
%a = load <7 x i16>, ptr addrspace(1) %ptra, align 4
@@ -565,18 +562,17 @@ define void @add_v9i16(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb, ptr addrs
; GFX9-NEXT: global_load_ushort v14, v[0:1], off offset:16
; GFX9-NEXT: global_load_ushort v15, v[2:3], off offset:16
; GFX9-NEXT: global_load_dwordx4 v[10:13], v[2:3], off
+; GFX9-NEXT: v_lshlrev_b32_e64 v0, 16, s4
; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
+; GFX9-NEXT: v_or_b32_sdwa v14, v0, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
+; GFX9-NEXT: v_or_b32_sdwa v15, v0, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_pk_add_u16 v0, v6, v10
; GFX9-NEXT: v_pk_add_u16 v1, v7, v11
; GFX9-NEXT: v_pk_add_u16 v2, v8, v12
; GFX9-NEXT: v_pk_add_u16 v3, v9, v13
-; GFX9-NEXT: v_lshl_or_b32 v6, s4, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v7, s4, 16, v15
-; GFX9-NEXT: v_pk_add_u16 v6, v6, v7
+; GFX9-NEXT: v_pk_add_u16 v6, v14, v15
; GFX9-NEXT: global_store_dwordx4 v[4:5], v[0:3], off
; GFX9-NEXT: global_store_short v[4:5], v6, off offset:16
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -740,40 +736,38 @@ define void @add_v11i16(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb, ptr addr
; GFX9-LABEL: add_v11i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_ushort v14, v[0:1], off offset:16
+; GFX9-NEXT: global_load_ushort v14, v[0:1], off offset:18
; GFX9-NEXT: global_load_ushort v15, v[0:1], off offset:20
; GFX9-NEXT: global_load_dwordx4 v[6:9], v[2:3], off
-; GFX9-NEXT: global_load_ushort v16, v[2:3], off offset:16
+; GFX9-NEXT: global_load_ushort v16, v[2:3], off offset:18
; GFX9-NEXT: global_load_ushort v17, v[2:3], off offset:20
; GFX9-NEXT: global_load_dwordx4 v[10:13], v[0:1], off
-; GFX9-NEXT: global_load_ushort v18, v[0:1], off offset:18
-; GFX9-NEXT: global_load_ushort v19, v[2:3], off offset:18
+; GFX9-NEXT: global_load_ushort v18, v[0:1], off offset:16
+; GFX9-NEXT: global_load_ushort v19, v[2:3], off offset:16
+; GFX9-NEXT: v_lshlrev_b32_e64 v0, 16, s4
; GFX9-NEXT: s_waitcnt vmcnt(7)
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
+; GFX9-NEXT: v_lshlrev_b32_e32 v14, 16, v14
; GFX9-NEXT: s_waitcnt vmcnt(6)
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
+; GFX9-NEXT: v_or_b32_sdwa v15, v0, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(4)
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
+; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v16
; GFX9-NEXT: s_waitcnt vmcnt(3)
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
+; GFX9-NEXT: v_or_b32_sdwa v17, v0, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(2)
; GFX9-NEXT: v_pk_add_u16 v0, v10, v6
; GFX9-NEXT: v_pk_add_u16 v1, v11, v7
; GFX9-NEXT: v_pk_add_u16 v2, v12, v8
; GFX9-NEXT: v_pk_add_u16 v3, v13, v9
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_lshl_or_b32 v6, v18, 16, v14
+; GFX9-NEXT: v_or_b32_sdwa v6, v14, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_lshl_or_b32 v8, v19, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v7, s4, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v9, s4, 16, v17
+; GFX9-NEXT: v_or_b32_sdwa v7, v16, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: global_store_dwordx4 v[4:5], v[0:3], off
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: v_pk_add_u16 v0, v6, v8
-; GFX9-NEXT: v_pk_add_u16 v1, v7, v9
+; GFX9-NEXT: v_pk_add_u16 v8, v15, v17
+; GFX9-NEXT: v_pk_add_u16 v0, v6, v7
; GFX9-NEXT: global_store_short v[4:5], v0, off offset:16
; GFX9-NEXT: global_store_short_d16_hi v[4:5], v0, off offset:18
-; GFX9-NEXT: global_store_short v[4:5], v1, off offset:20
+; GFX9-NEXT: global_store_short v[4:5], v8, off offset:20
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
%a = load <11 x i16>, ptr addrspace(1) %ptra, align 4
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fpow.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fpow.ll
index 0e5b3b25c51b0..b84ec70d3e35d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fpow.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fpow.ll
@@ -489,12 +489,11 @@ define <2 x half> @v_pow_v2f16(<2 x half> %x, <2 x half> %y) {
; GFX10-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX10-NEXT: v_mul_legacy_f32_e32 v2, v2, v3
; GFX10-NEXT: v_mul_legacy_f32_e32 v0, v0, v1
-; GFX10-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX10-NEXT: v_cvt_f16_f32_e32 v1, v2
; GFX10-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX10-NEXT: v_exp_f16_e32 v1, v2
-; GFX10-NEXT: v_exp_f16_e32 v0, v0
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_lshl_or_b32 v0, v0, 16, v1
+; GFX10-NEXT: v_exp_f16_e32 v1, v1
+; GFX10-NEXT: v_exp_f16_sdwa v0, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: v_pow_v2f16:
@@ -636,12 +635,11 @@ define <2 x half> @v_pow_v2f16_fneg_lhs(<2 x half> %x, <2 x half> %y) {
; GFX10-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX10-NEXT: v_mul_legacy_f32_e32 v2, v2, v3
; GFX10-NEXT: v_mul_legacy_f32_e32 v0, v0, v1
-; GFX10-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX10-NEXT: v_cvt_f16_f32_e32 v1, v2
; GFX10-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX10-NEXT: v_exp_f16_e32 v1, v2
-; GFX10-NEXT: v_exp_f16_e32 v0, v0
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_lshl_or_b32 v0, v0, 16, v1
+; GFX10-NEXT: v_exp_f16_e32 v1, v1
+; GFX10-NEXT: v_exp_f16_sdwa v0, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: v_pow_v2f16_fneg_lhs:
@@ -780,20 +778,19 @@ define <2 x half> @v_pow_v2f16_fneg_rhs(<2 x half> %x, <2 x half> %y) {
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_log_f16_e32 v2, v0
-; GFX10-NEXT: v_xor_b32_e32 v1, 0x80008000, v1
; GFX10-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX10-NEXT: v_xor_b32_e32 v1, 0x80008000, v1
; GFX10-NEXT: v_cvt_f32_f16_e32 v3, v1
; GFX10-NEXT: v_cvt_f32_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
; GFX10-NEXT: v_cvt_f32_f16_e32 v2, v2
; GFX10-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX10-NEXT: v_mul_legacy_f32_e32 v2, v2, v3
; GFX10-NEXT: v_mul_legacy_f32_e32 v0, v0, v1
-; GFX10-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX10-NEXT: v_cvt_f16_f32_e32 v1, v2
; GFX10-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX10-NEXT: v_exp_f16_e32 v1, v2
-; GFX10-NEXT: v_exp_f16_e32 v0, v0
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_lshl_or_b32 v0, v0, 16, v1
+; GFX10-NEXT: v_exp_f16_e32 v1, v1
+; GFX10-NEXT: v_exp_f16_sdwa v0, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: v_pow_v2f16_fneg_rhs:
@@ -943,12 +940,11 @@ define <2 x half> @v_pow_v2f16_fneg_lhs_rhs(<2 x half> %x, <2 x half> %y) {
; GFX10-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX10-NEXT: v_mul_legacy_f32_e32 v2, v2, v3
; GFX10-NEXT: v_mul_legacy_f32_e32 v0, v0, v1
-; GFX10-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX10-NEXT: v_cvt_f16_f32_e32 v1, v2
; GFX10-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX10-NEXT: v_exp_f16_e32 v1, v2
-; GFX10-NEXT: v_exp_f16_e32 v0, v0
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_lshl_or_b32 v0, v0, 16, v1
+; GFX10-NEXT: v_exp_f16_e32 v1, v1
+; GFX10-NEXT: v_exp_f16_sdwa v0, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: v_pow_v2f16_fneg_lhs_rhs:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.abs.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.abs.ll
index b44e7633d568d..fb0c7e3d8c971 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.abs.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.abs.ll
@@ -655,11 +655,11 @@ define <3 x i16> @abs_vgpr_v3i16(<3 x i16> %arg) {
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_sub_nc_u16 v2, 0, v1
+; GFX10-NEXT: v_pk_sub_i16 v3, 0, v0
; GFX10-NEXT: v_max_i16 v1, v1, v2
-; GFX10-NEXT: v_pk_sub_i16 v2, 0, v0
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_pk_max_i16 v0, v0, v2
-; GFX10-NEXT: v_lshl_or_b32 v1, s4, 16, v1
+; GFX10-NEXT: v_lshlrev_b32_e64 v2, 16, s4
+; GFX10-NEXT: v_pk_max_i16 v0, v0, v3
+; GFX10-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-FAKE16-LABEL: abs_vgpr_v3i16:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.atomic.dim.a16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.atomic.dim.a16.ll
index fce5fd01065cc..394d145f6064d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.atomic.dim.a16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.atomic.dim.a16.ll
@@ -10,7 +10,7 @@ define amdgpu_ps float @atomic_swap_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX9-LABEL: atomic_swap_i32_1d:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT: v_lshlrev_b32_e64 v2, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -18,21 +18,21 @@ define amdgpu_ps float @atomic_swap_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_swap v0, v1, s[0:7] dmask:0x1 unorm glc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: atomic_swap_i32_1d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX10-NEXT: s_mov_b32 s0, s2
; GFX10-NEXT: s_mov_b32 s1, s3
+; GFX10-NEXT: v_lshlrev_b32_e64 v2, 16, s0
; GFX10-NEXT: s_mov_b32 s2, s4
; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v1, s0, 16, v1
; GFX10-NEXT: s_mov_b32 s4, s6
; GFX10-NEXT: s_mov_b32 s5, s7
+; GFX10-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: s_mov_b32 s6, s8
; GFX10-NEXT: s_mov_b32 s7, s9
; GFX10-NEXT: image_atomic_swap v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
@@ -108,7 +108,7 @@ define amdgpu_ps float @atomic_add_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX9-LABEL: atomic_add_i32_1d:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT: v_lshlrev_b32_e64 v2, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -116,21 +116,21 @@ define amdgpu_ps float @atomic_add_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_add v0, v1, s[0:7] dmask:0x1 unorm glc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: atomic_add_i32_1d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX10-NEXT: s_mov_b32 s0, s2
; GFX10-NEXT: s_mov_b32 s1, s3
+; GFX10-NEXT: v_lshlrev_b32_e64 v2, 16, s0
; GFX10-NEXT: s_mov_b32 s2, s4
; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v1, s0, 16, v1
; GFX10-NEXT: s_mov_b32 s4, s6
; GFX10-NEXT: s_mov_b32 s5, s7
+; GFX10-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: s_mov_b32 s6, s8
; GFX10-NEXT: s_mov_b32 s7, s9
; GFX10-NEXT: image_atomic_add v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
@@ -206,7 +206,7 @@ define amdgpu_ps float @atomic_sub_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX9-LABEL: atomic_sub_i32_1d:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT: v_lshlrev_b32_e64 v2, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -214,21 +214,21 @@ define amdgpu_ps float @atomic_sub_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_sub v0, v1, s[0:7] dmask:0x1 unorm glc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: atomic_sub_i32_1d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX10-NEXT: s_mov_b32 s0, s2
; GFX10-NEXT: s_mov_b32 s1, s3
+; GFX10-NEXT: v_lshlrev_b32_e64 v2, 16, s0
; GFX10-NEXT: s_mov_b32 s2, s4
; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v1, s0, 16, v1
; GFX10-NEXT: s_mov_b32 s4, s6
; GFX10-NEXT: s_mov_b32 s5, s7
+; GFX10-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: s_mov_b32 s6, s8
; GFX10-NEXT: s_mov_b32 s7, s9
; GFX10-NEXT: image_atomic_sub v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
@@ -304,7 +304,7 @@ define amdgpu_ps float @atomic_smin_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX9-LABEL: atomic_smin_i32_1d:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT: v_lshlrev_b32_e64 v2, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -312,21 +312,21 @@ define amdgpu_ps float @atomic_smin_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_smin v0, v1, s[0:7] dmask:0x1 unorm glc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: atomic_smin_i32_1d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX10-NEXT: s_mov_b32 s0, s2
; GFX10-NEXT: s_mov_b32 s1, s3
+; GFX10-NEXT: v_lshlrev_b32_e64 v2, 16, s0
; GFX10-NEXT: s_mov_b32 s2, s4
; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v1, s0, 16, v1
; GFX10-NEXT: s_mov_b32 s4, s6
; GFX10-NEXT: s_mov_b32 s5, s7
+; GFX10-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: s_mov_b32 s6, s8
; GFX10-NEXT: s_mov_b32 s7, s9
; GFX10-NEXT: image_atomic_smin v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
@@ -402,7 +402,7 @@ define amdgpu_ps float @atomic_umin_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX9-LABEL: atomic_umin_i32_1d:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT: v_lshlrev_b32_e64 v2, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -410,21 +410,21 @@ define amdgpu_ps float @atomic_umin_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_umin v0, v1, s[0:7] dmask:0x1 unorm glc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: atomic_umin_i32_1d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX10-NEXT: s_mov_b32 s0, s2
; GFX10-NEXT: s_mov_b32 s1, s3
+; GFX10-NEXT: v_lshlrev_b32_e64 v2, 16, s0
; GFX10-NEXT: s_mov_b32 s2, s4
; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v1, s0, 16, v1
; GFX10-NEXT: s_mov_b32 s4, s6
; GFX10-NEXT: s_mov_b32 s5, s7
+; GFX10-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: s_mov_b32 s6, s8
; GFX10-NEXT: s_mov_b32 s7, s9
; GFX10-NEXT: image_atomic_umin v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
@@ -500,7 +500,7 @@ define amdgpu_ps float @atomic_smax_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX9-LABEL: atomic_smax_i32_1d:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT: v_lshlrev_b32_e64 v2, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -508,21 +508,21 @@ define amdgpu_ps float @atomic_smax_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_smax v0, v1, s[0:7] dmask:0x1 unorm glc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: atomic_smax_i32_1d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX10-NEXT: s_mov_b32 s0, s2
; GFX10-NEXT: s_mov_b32 s1, s3
+; GFX10-NEXT: v_lshlrev_b32_e64 v2, 16, s0
; GFX10-NEXT: s_mov_b32 s2, s4
; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v1, s0, 16, v1
; GFX10-NEXT: s_mov_b32 s4, s6
; GFX10-NEXT: s_mov_b32 s5, s7
+; GFX10-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: s_mov_b32 s6, s8
; GFX10-NEXT: s_mov_b32 s7, s9
; GFX10-NEXT: image_atomic_smax v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
@@ -598,7 +598,7 @@ define amdgpu_ps float @atomic_umax_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX9-LABEL: atomic_umax_i32_1d:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT: v_lshlrev_b32_e64 v2, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -606,21 +606,21 @@ define amdgpu_ps float @atomic_umax_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_umax v0, v1, s[0:7] dmask:0x1 unorm glc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: atomic_umax_i32_1d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX10-NEXT: s_mov_b32 s0, s2
; GFX10-NEXT: s_mov_b32 s1, s3
+; GFX10-NEXT: v_lshlrev_b32_e64 v2, 16, s0
; GFX10-NEXT: s_mov_b32 s2, s4
; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v1, s0, 16, v1
; GFX10-NEXT: s_mov_b32 s4, s6
; GFX10-NEXT: s_mov_b32 s5, s7
+; GFX10-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: s_mov_b32 s6, s8
; GFX10-NEXT: s_mov_b32 s7, s9
; GFX10-NEXT: image_atomic_umax v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
@@ -696,7 +696,7 @@ define amdgpu_ps float @atomic_and_i321d(<8 x i32> inreg %rsrc, i32 %data, i16 %
; GFX9-LABEL: atomic_and_i321d:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT: v_lshlrev_b32_e64 v2, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -704,21 +704,21 @@ define amdgpu_ps float @atomic_and_i321d(<8 x i32> inreg %rsrc, i32 %data, i16 %
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_and v0, v1, s[0:7] dmask:0x1 unorm glc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: atomic_and_i321d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX10-NEXT: s_mov_b32 s0, s2
; GFX10-NEXT: s_mov_b32 s1, s3
+; GFX10-NEXT: v_lshlrev_b32_e64 v2, 16, s0
; GFX10-NEXT: s_mov_b32 s2, s4
; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v1, s0, 16, v1
; GFX10-NEXT: s_mov_b32 s4, s6
; GFX10-NEXT: s_mov_b32 s5, s7
+; GFX10-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: s_mov_b32 s6, s8
; GFX10-NEXT: s_mov_b32 s7, s9
; GFX10-NEXT: image_atomic_and v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
@@ -794,7 +794,7 @@ define amdgpu_ps float @atomic_or_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %
; GFX9-LABEL: atomic_or_i32_1d:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT: v_lshlrev_b32_e64 v2, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -802,21 +802,21 @@ define amdgpu_ps float @atomic_or_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_or v0, v1, s[0:7] dmask:0x1 unorm glc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: atomic_or_i32_1d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX10-NEXT: s_mov_b32 s0, s2
; GFX10-NEXT: s_mov_b32 s1, s3
+; GFX10-NEXT: v_lshlrev_b32_e64 v2, 16, s0
; GFX10-NEXT: s_mov_b32 s2, s4
; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v1, s0, 16, v1
; GFX10-NEXT: s_mov_b32 s4, s6
; GFX10-NEXT: s_mov_b32 s5, s7
+; GFX10-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: s_mov_b32 s6, s8
; GFX10-NEXT: s_mov_b32 s7, s9
; GFX10-NEXT: image_atomic_or v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
@@ -892,7 +892,7 @@ define amdgpu_ps float @atomic_xor_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX9-LABEL: atomic_xor_i32_1d:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT: v_lshlrev_b32_e64 v2, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -900,21 +900,21 @@ define amdgpu_ps float @atomic_xor_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_xor v0, v1, s[0:7] dmask:0x1 unorm glc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: atomic_xor_i32_1d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX10-NEXT: s_mov_b32 s0, s2
; GFX10-NEXT: s_mov_b32 s1, s3
+; GFX10-NEXT: v_lshlrev_b32_e64 v2, 16, s0
; GFX10-NEXT: s_mov_b32 s2, s4
; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v1, s0, 16, v1
; GFX10-NEXT: s_mov_b32 s4, s6
; GFX10-NEXT: s_mov_b32 s5, s7
+; GFX10-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: s_mov_b32 s6, s8
; GFX10-NEXT: s_mov_b32 s7, s9
; GFX10-NEXT: image_atomic_xor v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
@@ -990,7 +990,7 @@ define amdgpu_ps float @atomic_inc_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX9-LABEL: atomic_inc_i32_1d:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT: v_lshlrev_b32_e64 v2, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -998,21 +998,21 @@ define amdgpu_ps float @atomic_inc_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_inc v0, v1, s[0:7] dmask:0x1 unorm glc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: atomic_inc_i32_1d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX10-NEXT: s_mov_b32 s0, s2
; GFX10-NEXT: s_mov_b32 s1, s3
+; GFX10-NEXT: v_lshlrev_b32_e64 v2, 16, s0
; GFX10-NEXT: s_mov_b32 s2, s4
; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v1, s0, 16, v1
; GFX10-NEXT: s_mov_b32 s4, s6
; GFX10-NEXT: s_mov_b32 s5, s7
+; GFX10-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: s_mov_b32 s6, s8
; GFX10-NEXT: s_mov_b32 s7, s9
; GFX10-NEXT: image_atomic_inc v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
@@ -1088,7 +1088,7 @@ define amdgpu_ps float @atomic_dec_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX9-LABEL: atomic_dec_i32_1d:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT: v_lshlrev_b32_e64 v2, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -1096,21 +1096,21 @@ define amdgpu_ps float @atomic_dec_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_dec v0, v1, s[0:7] dmask:0x1 unorm glc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: atomic_dec_i32_1d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX10-NEXT: s_mov_b32 s0, s2
; GFX10-NEXT: s_mov_b32 s1, s3
+; GFX10-NEXT: v_lshlrev_b32_e64 v2, 16, s0
; GFX10-NEXT: s_mov_b32 s2, s4
; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v1, s0, 16, v1
; GFX10-NEXT: s_mov_b32 s4, s6
; GFX10-NEXT: s_mov_b32 s5, s7
+; GFX10-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: s_mov_b32 s6, s8
; GFX10-NEXT: s_mov_b32 s7, s9
; GFX10-NEXT: image_atomic_dec v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
@@ -1186,7 +1186,7 @@ define amdgpu_ps float @atomic_cmpswap_i32_1d(<8 x i32> inreg %rsrc, i32 %cmp, i
; GFX9-LABEL: atomic_cmpswap_i32_1d:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX9-NEXT: v_lshlrev_b32_e64 v3, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -1194,21 +1194,21 @@ define amdgpu_ps float @atomic_cmpswap_i32_1d(<8 x i32> inreg %rsrc, i32 %cmp, i
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_cmpswap v[0:1], v2, s[0:7] dmask:0x3 unorm glc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: atomic_cmpswap_i32_1d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX10-NEXT: s_mov_b32 s0, s2
; GFX10-NEXT: s_mov_b32 s1, s3
+; GFX10-NEXT: v_lshlrev_b32_e64 v3, 16, s0
; GFX10-NEXT: s_mov_b32 s2, s4
; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v2, s0, 16, v2
; GFX10-NEXT: s_mov_b32 s4, s6
; GFX10-NEXT: s_mov_b32 s5, s7
+; GFX10-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: s_mov_b32 s6, s8
; GFX10-NEXT: s_mov_b32 s7, s9
; GFX10-NEXT: image_atomic_cmpswap v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
@@ -1283,7 +1283,7 @@ main_body:
define amdgpu_ps float @atomic_add_i32_2d(<8 x i32> inreg %rsrc, i32 %data, i16 %s, i16 %t) {
; GFX9-LABEL: atomic_add_i32_2d:
; GFX9: ; %bb.0: ; %main_body
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX9-NEXT: s_mov_b32 s0, s2
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
@@ -1292,19 +1292,19 @@ define amdgpu_ps float @atomic_add_i32_2d(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_add v0, v1, s[0:7] dmask:0x1 unorm glc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: atomic_add_i32_2d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX10-NEXT: s_mov_b32 s0, s2
; GFX10-NEXT: s_mov_b32 s1, s3
; GFX10-NEXT: s_mov_b32 s2, s4
; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX10-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: s_mov_b32 s4, s6
; GFX10-NEXT: s_mov_b32 s5, s7
; GFX10-NEXT: s_mov_b32 s6, s8
@@ -1383,10 +1383,10 @@ main_body:
define amdgpu_ps float @atomic_add_i32_3d(<8 x i32> inreg %rsrc, i32 %data, i16 %s, i16 %t, i16 %r) {
; GFX9-LABEL: atomic_add_i32_3d:
; GFX9: ; %bb.0: ; %main_body
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v3
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e64 v2, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -1394,21 +1394,21 @@ define amdgpu_ps float @atomic_add_i32_3d(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 unorm glc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: atomic_add_i32_3d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_and_b32_e32 v3, 0xffff, v3
; GFX10-NEXT: s_mov_b32 s0, s2
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX10-NEXT: v_lshlrev_b32_e64 v4, 16, s0
; GFX10-NEXT: s_mov_b32 s1, s3
; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX10-NEXT: v_lshl_or_b32 v2, s0, 16, v3
; GFX10-NEXT: s_mov_b32 s3, s5
+; GFX10-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v2, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: s_mov_b32 s4, s6
; GFX10-NEXT: s_mov_b32 s5, s7
; GFX10-NEXT: s_mov_b32 s6, s8
@@ -1491,10 +1491,10 @@ main_body:
define amdgpu_ps float @atomic_add_i32_cube(<8 x i32> inreg %rsrc, i32 %data, i16 %s, i16 %t, i16 %face) {
; GFX9-LABEL: atomic_add_i32_cube:
; GFX9: ; %bb.0: ; %main_body
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v3
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e64 v2, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -1502,21 +1502,21 @@ define amdgpu_ps float @atomic_add_i32_cube(<8 x i32> inreg %rsrc, i32 %data, i1
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 unorm glc a16 da
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: atomic_add_i32_cube:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_and_b32_e32 v3, 0xffff, v3
; GFX10-NEXT: s_mov_b32 s0, s2
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX10-NEXT: v_lshlrev_b32_e64 v4, 16, s0
; GFX10-NEXT: s_mov_b32 s1, s3
; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX10-NEXT: v_lshl_or_b32 v2, s0, 16, v3
; GFX10-NEXT: s_mov_b32 s3, s5
+; GFX10-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v2, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: s_mov_b32 s4, s6
; GFX10-NEXT: s_mov_b32 s5, s7
; GFX10-NEXT: s_mov_b32 s6, s8
@@ -1599,7 +1599,7 @@ main_body:
define amdgpu_ps float @atomic_add_i32_1darray(<8 x i32> inreg %rsrc, i32 %data, i16 %s, i16 %slice) {
; GFX9-LABEL: atomic_add_i32_1darray:
; GFX9: ; %bb.0: ; %main_body
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX9-NEXT: s_mov_b32 s0, s2
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
@@ -1608,19 +1608,19 @@ define amdgpu_ps float @atomic_add_i32_1darray(<8 x i32> inreg %rsrc, i32 %data,
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_add v0, v1, s[0:7] dmask:0x1 unorm glc a16 da
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: atomic_add_i32_1darray:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX10-NEXT: s_mov_b32 s0, s2
; GFX10-NEXT: s_mov_b32 s1, s3
; GFX10-NEXT: s_mov_b32 s2, s4
; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX10-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: s_mov_b32 s4, s6
; GFX10-NEXT: s_mov_b32 s5, s7
; GFX10-NEXT: s_mov_b32 s6, s8
@@ -1699,10 +1699,10 @@ main_body:
define amdgpu_ps float @atomic_add_i32_2darray(<8 x i32> inreg %rsrc, i32 %data, i16 %s, i16 %t, i16 %slice) {
; GFX9-LABEL: atomic_add_i32_2darray:
; GFX9: ; %bb.0: ; %main_body
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v3
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e64 v2, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -1710,21 +1710,21 @@ define amdgpu_ps float @atomic_add_i32_2darray(<8 x i32> inreg %rsrc, i32 %data,
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 unorm glc a16 da
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: atomic_add_i32_2darray:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_and_b32_e32 v3, 0xffff, v3
; GFX10-NEXT: s_mov_b32 s0, s2
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX10-NEXT: v_lshlrev_b32_e64 v4, 16, s0
; GFX10-NEXT: s_mov_b32 s1, s3
; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX10-NEXT: v_lshl_or_b32 v2, s0, 16, v3
; GFX10-NEXT: s_mov_b32 s3, s5
+; GFX10-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v2, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: s_mov_b32 s4, s6
; GFX10-NEXT: s_mov_b32 s5, s7
; GFX10-NEXT: s_mov_b32 s6, s8
@@ -1807,10 +1807,10 @@ main_body:
define amdgpu_ps float @atomic_add_i32_2dmsaa(<8 x i32> inreg %rsrc, i32 %data, i16 %s, i16 %t, i16 %fragid) {
; GFX9-LABEL: atomic_add_i32_2dmsaa:
; GFX9: ; %bb.0: ; %main_body
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v3
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e64 v2, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -1818,21 +1818,21 @@ define amdgpu_ps float @atomic_add_i32_2dmsaa(<8 x i32> inreg %rsrc, i32 %data,
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 unorm glc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: atomic_add_i32_2dmsaa:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_and_b32_e32 v3, 0xffff, v3
; GFX10-NEXT: s_mov_b32 s0, s2
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX10-NEXT: v_lshlrev_b32_e64 v4, 16, s0
; GFX10-NEXT: s_mov_b32 s1, s3
; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX10-NEXT: v_lshl_or_b32 v2, s0, 16, v3
; GFX10-NEXT: s_mov_b32 s3, s5
+; GFX10-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v2, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: s_mov_b32 s4, s6
; GFX10-NEXT: s_mov_b32 s5, s7
; GFX10-NEXT: s_mov_b32 s6, s8
@@ -1915,9 +1915,9 @@ main_body:
define amdgpu_ps float @atomic_add_i32_2darraymsaa(<8 x i32> inreg %rsrc, i32 %data, i16 %s, i16 %t, i16 %slice, i16 %fragid) {
; GFX9-LABEL: atomic_add_i32_2darraymsaa:
; GFX9: ; %bb.0: ; %main_body
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v3
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v4
; GFX9-NEXT: s_mov_b32 s0, s2
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
@@ -1926,20 +1926,20 @@ define amdgpu_ps float @atomic_add_i32_2darraymsaa(<8 x i32> inreg %rsrc, i32 %d
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v2, v4, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 unorm glc a16 da
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: atomic_add_i32_2darraymsaa:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX10-NEXT: s_mov_b32 s0, s2
; GFX10-NEXT: s_mov_b32 s1, s3
; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX10-NEXT: v_lshl_or_b32 v2, v4, 16, v3
+; GFX10-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v2, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: s_mov_b32 s3, s5
; GFX10-NEXT: s_mov_b32 s4, s6
; GFX10-NEXT: s_mov_b32 s5, s7
@@ -2027,7 +2027,7 @@ define amdgpu_ps float @atomic_add_i32_1d_slc(<8 x i32> inreg %rsrc, i32 %data,
; GFX9-LABEL: atomic_add_i32_1d_slc:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT: v_lshlrev_b32_e64 v2, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -2035,21 +2035,21 @@ define amdgpu_ps float @atomic_add_i32_1d_slc(<8 x i32> inreg %rsrc, i32 %data,
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_add v0, v1, s[0:7] dmask:0x1 unorm glc slc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: atomic_add_i32_1d_slc:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX10-NEXT: s_mov_b32 s0, s2
; GFX10-NEXT: s_mov_b32 s1, s3
+; GFX10-NEXT: v_lshlrev_b32_e64 v2, 16, s0
; GFX10-NEXT: s_mov_b32 s2, s4
; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v1, s0, 16, v1
; GFX10-NEXT: s_mov_b32 s4, s6
; GFX10-NEXT: s_mov_b32 s5, s7
+; GFX10-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: s_mov_b32 s6, s8
; GFX10-NEXT: s_mov_b32 s7, s9
; GFX10-NEXT: image_atomic_add v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc slc a16
@@ -2125,7 +2125,7 @@ define amdgpu_ps <2 x float> @atomic_swap_i64_1d(<8 x i32> inreg %rsrc, i64 %dat
; GFX9-LABEL: atomic_swap_i64_1d:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX9-NEXT: v_lshlrev_b32_e64 v3, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -2133,21 +2133,21 @@ define amdgpu_ps <2 x float> @atomic_swap_i64_1d(<8 x i32> inreg %rsrc, i64 %dat
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_swap v[0:1], v2, s[0:7] dmask:0x3 unorm glc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: atomic_swap_i64_1d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX10-NEXT: s_mov_b32 s0, s2
; GFX10-NEXT: s_mov_b32 s1, s3
+; GFX10-NEXT: v_lshlrev_b32_e64 v3, 16, s0
; GFX10-NEXT: s_mov_b32 s2, s4
; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v2, s0, 16, v2
; GFX10-NEXT: s_mov_b32 s4, s6
; GFX10-NEXT: s_mov_b32 s5, s7
+; GFX10-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: s_mov_b32 s6, s8
; GFX10-NEXT: s_mov_b32 s7, s9
; GFX10-NEXT: image_atomic_swap v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
@@ -2223,7 +2223,7 @@ define amdgpu_ps <2 x float> @atomic_add_i64_1d(<8 x i32> inreg %rsrc, i64 %data
; GFX9-LABEL: atomic_add_i64_1d:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX9-NEXT: v_lshlrev_b32_e64 v3, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -2231,21 +2231,21 @@ define amdgpu_ps <2 x float> @atomic_add_i64_1d(<8 x i32> inreg %rsrc, i64 %data
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_add v[0:1], v2, s[0:7] dmask:0x3 unorm glc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: atomic_add_i64_1d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX10-NEXT: s_mov_b32 s0, s2
; GFX10-NEXT: s_mov_b32 s1, s3
+; GFX10-NEXT: v_lshlrev_b32_e64 v3, 16, s0
; GFX10-NEXT: s_mov_b32 s2, s4
; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v2, s0, 16, v2
; GFX10-NEXT: s_mov_b32 s4, s6
; GFX10-NEXT: s_mov_b32 s5, s7
+; GFX10-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: s_mov_b32 s6, s8
; GFX10-NEXT: s_mov_b32 s7, s9
; GFX10-NEXT: image_atomic_add v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
@@ -2321,7 +2321,7 @@ define amdgpu_ps <2 x float> @atomic_sub_i64_1d(<8 x i32> inreg %rsrc, i64 %data
; GFX9-LABEL: atomic_sub_i64_1d:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX9-NEXT: v_lshlrev_b32_e64 v3, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -2329,21 +2329,21 @@ define amdgpu_ps <2 x float> @atomic_sub_i64_1d(<8 x i32> inreg %rsrc, i64 %data
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_sub v[0:1], v2, s[0:7] dmask:0x3 unorm glc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: atomic_sub_i64_1d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX10-NEXT: s_mov_b32 s0, s2
; GFX10-NEXT: s_mov_b32 s1, s3
+; GFX10-NEXT: v_lshlrev_b32_e64 v3, 16, s0
; GFX10-NEXT: s_mov_b32 s2, s4
; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v2, s0, 16, v2
; GFX10-NEXT: s_mov_b32 s4, s6
; GFX10-NEXT: s_mov_b32 s5, s7
+; GFX10-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: s_mov_b32 s6, s8
; GFX10-NEXT: s_mov_b32 s7, s9
; GFX10-NEXT: image_atomic_sub v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
@@ -2419,7 +2419,7 @@ define amdgpu_ps <2 x float> @atomic_smin_i64_1d(<8 x i32> inreg %rsrc, i64 %dat
; GFX9-LABEL: atomic_smin_i64_1d:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX9-NEXT: v_lshlrev_b32_e64 v3, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -2427,21 +2427,21 @@ define amdgpu_ps <2 x float> @atomic_smin_i64_1d(<8 x i32> inreg %rsrc, i64 %dat
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_smin v[0:1], v2, s[0:7] dmask:0x3 unorm glc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: atomic_smin_i64_1d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX10-NEXT: s_mov_b32 s0, s2
; GFX10-NEXT: s_mov_b32 s1, s3
+; GFX10-NEXT: v_lshlrev_b32_e64 v3, 16, s0
; GFX10-NEXT: s_mov_b32 s2, s4
; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v2, s0, 16, v2
; GFX10-NEXT: s_mov_b32 s4, s6
; GFX10-NEXT: s_mov_b32 s5, s7
+; GFX10-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: s_mov_b32 s6, s8
; GFX10-NEXT: s_mov_b32 s7, s9
; GFX10-NEXT: image_atomic_smin v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
@@ -2517,7 +2517,7 @@ define amdgpu_ps <2 x float> @atomic_umin_i64_1d(<8 x i32> inreg %rsrc, i64 %dat
; GFX9-LABEL: atomic_umin_i64_1d:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX9-NEXT: v_lshlrev_b32_e64 v3, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -2525,21 +2525,21 @@ define amdgpu_ps <2 x float> @atomic_umin_i64_1d(<8 x i32> inreg %rsrc, i64 %dat
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_umin v[0:1], v2, s[0:7] dmask:0x3 unorm glc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: atomic_umin_i64_1d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX10-NEXT: s_mov_b32 s0, s2
; GFX10-NEXT: s_mov_b32 s1, s3
+; GFX10-NEXT: v_lshlrev_b32_e64 v3, 16, s0
; GFX10-NEXT: s_mov_b32 s2, s4
; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v2, s0, 16, v2
; GFX10-NEXT: s_mov_b32 s4, s6
; GFX10-NEXT: s_mov_b32 s5, s7
+; GFX10-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: s_mov_b32 s6, s8
; GFX10-NEXT: s_mov_b32 s7, s9
; GFX10-NEXT: image_atomic_umin v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
@@ -2615,7 +2615,7 @@ define amdgpu_ps <2 x float> @atomic_smax_i64_1d(<8 x i32> inreg %rsrc, i64 %dat
; GFX9-LABEL: atomic_smax_i64_1d:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX9-NEXT: v_lshlrev_b32_e64 v3, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -2623,21 +2623,21 @@ define amdgpu_ps <2 x float> @atomic_smax_i64_1d(<8 x i32> inreg %rsrc, i64 %dat
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_smax v[0:1], v2, s[0:7] dmask:0x3 unorm glc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: atomic_smax_i64_1d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX10-NEXT: s_mov_b32 s0, s2
; GFX10-NEXT: s_mov_b32 s1, s3
+; GFX10-NEXT: v_lshlrev_b32_e64 v3, 16, s0
; GFX10-NEXT: s_mov_b32 s2, s4
; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v2, s0, 16, v2
; GFX10-NEXT: s_mov_b32 s4, s6
; GFX10-NEXT: s_mov_b32 s5, s7
+; GFX10-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: s_mov_b32 s6, s8
; GFX10-NEXT: s_mov_b32 s7, s9
; GFX10-NEXT: image_atomic_smax v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
@@ -2713,7 +2713,7 @@ define amdgpu_ps <2 x float> @atomic_umax_i64_1d(<8 x i32> inreg %rsrc, i64 %dat
; GFX9-LABEL: atomic_umax_i64_1d:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX9-NEXT: v_lshlrev_b32_e64 v3, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -2721,21 +2721,21 @@ define amdgpu_ps <2 x float> @atomic_umax_i64_1d(<8 x i32> inreg %rsrc, i64 %dat
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_umax v[0:1], v2, s[0:7] dmask:0x3 unorm glc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: atomic_umax_i64_1d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX10-NEXT: s_mov_b32 s0, s2
; GFX10-NEXT: s_mov_b32 s1, s3
+; GFX10-NEXT: v_lshlrev_b32_e64 v3, 16, s0
; GFX10-NEXT: s_mov_b32 s2, s4
; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v2, s0, 16, v2
; GFX10-NEXT: s_mov_b32 s4, s6
; GFX10-NEXT: s_mov_b32 s5, s7
+; GFX10-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: s_mov_b32 s6, s8
; GFX10-NEXT: s_mov_b32 s7, s9
; GFX10-NEXT: image_atomic_umax v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
@@ -2811,7 +2811,7 @@ define amdgpu_ps <2 x float> @atomic_and_i64_1d(<8 x i32> inreg %rsrc, i64 %data
; GFX9-LABEL: atomic_and_i64_1d:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX9-NEXT: v_lshlrev_b32_e64 v3, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -2819,21 +2819,21 @@ define amdgpu_ps <2 x float> @atomic_and_i64_1d(<8 x i32> inreg %rsrc, i64 %data
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_and v[0:1], v2, s[0:7] dmask:0x3 unorm glc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: atomic_and_i64_1d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX10-NEXT: s_mov_b32 s0, s2
; GFX10-NEXT: s_mov_b32 s1, s3
+; GFX10-NEXT: v_lshlrev_b32_e64 v3, 16, s0
; GFX10-NEXT: s_mov_b32 s2, s4
; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v2, s0, 16, v2
; GFX10-NEXT: s_mov_b32 s4, s6
; GFX10-NEXT: s_mov_b32 s5, s7
+; GFX10-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: s_mov_b32 s6, s8
; GFX10-NEXT: s_mov_b32 s7, s9
; GFX10-NEXT: image_atomic_and v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
@@ -2909,7 +2909,7 @@ define amdgpu_ps <2 x float> @atomic_or_i64_1d(<8 x i32> inreg %rsrc, i64 %data,
; GFX9-LABEL: atomic_or_i64_1d:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX9-NEXT: v_lshlrev_b32_e64 v3, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -2917,21 +2917,21 @@ define amdgpu_ps <2 x float> @atomic_or_i64_1d(<8 x i32> inreg %rsrc, i64 %data,
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_or v[0:1], v2, s[0:7] dmask:0x3 unorm glc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: atomic_or_i64_1d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX10-NEXT: s_mov_b32 s0, s2
; GFX10-NEXT: s_mov_b32 s1, s3
+; GFX10-NEXT: v_lshlrev_b32_e64 v3, 16, s0
; GFX10-NEXT: s_mov_b32 s2, s4
; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v2, s0, 16, v2
; GFX10-NEXT: s_mov_b32 s4, s6
; GFX10-NEXT: s_mov_b32 s5, s7
+; GFX10-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: s_mov_b32 s6, s8
; GFX10-NEXT: s_mov_b32 s7, s9
; GFX10-NEXT: image_atomic_or v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
@@ -3007,7 +3007,7 @@ define amdgpu_ps <2 x float> @atomic_xor_i64_1d(<8 x i32> inreg %rsrc, i64 %data
; GFX9-LABEL: atomic_xor_i64_1d:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX9-NEXT: v_lshlrev_b32_e64 v3, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -3015,21 +3015,21 @@ define amdgpu_ps <2 x float> @atomic_xor_i64_1d(<8 x i32> inreg %rsrc, i64 %data
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_xor v[0:1], v2, s[0:7] dmask:0x3 unorm glc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: atomic_xor_i64_1d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX10-NEXT: s_mov_b32 s0, s2
; GFX10-NEXT: s_mov_b32 s1, s3
+; GFX10-NEXT: v_lshlrev_b32_e64 v3, 16, s0
; GFX10-NEXT: s_mov_b32 s2, s4
; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v2, s0, 16, v2
; GFX10-NEXT: s_mov_b32 s4, s6
; GFX10-NEXT: s_mov_b32 s5, s7
+; GFX10-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: s_mov_b32 s6, s8
; GFX10-NEXT: s_mov_b32 s7, s9
; GFX10-NEXT: image_atomic_xor v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
@@ -3105,7 +3105,7 @@ define amdgpu_ps <2 x float> @atomic_inc_i64_1d(<8 x i32> inreg %rsrc, i64 %data
; GFX9-LABEL: atomic_inc_i64_1d:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX9-NEXT: v_lshlrev_b32_e64 v3, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -3113,21 +3113,21 @@ define amdgpu_ps <2 x float> @atomic_inc_i64_1d(<8 x i32> inreg %rsrc, i64 %data
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_inc v[0:1], v2, s[0:7] dmask:0x3 unorm glc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: atomic_inc_i64_1d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX10-NEXT: s_mov_b32 s0, s2
; GFX10-NEXT: s_mov_b32 s1, s3
+; GFX10-NEXT: v_lshlrev_b32_e64 v3, 16, s0
; GFX10-NEXT: s_mov_b32 s2, s4
; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v2, s0, 16, v2
; GFX10-NEXT: s_mov_b32 s4, s6
; GFX10-NEXT: s_mov_b32 s5, s7
+; GFX10-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: s_mov_b32 s6, s8
; GFX10-NEXT: s_mov_b32 s7, s9
; GFX10-NEXT: image_atomic_inc v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
@@ -3203,7 +3203,7 @@ define amdgpu_ps <2 x float> @atomic_dec_i64_1d(<8 x i32> inreg %rsrc, i64 %data
; GFX9-LABEL: atomic_dec_i64_1d:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX9-NEXT: v_lshlrev_b32_e64 v3, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -3211,21 +3211,21 @@ define amdgpu_ps <2 x float> @atomic_dec_i64_1d(<8 x i32> inreg %rsrc, i64 %data
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_dec v[0:1], v2, s[0:7] dmask:0x3 unorm glc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: atomic_dec_i64_1d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX10-NEXT: s_mov_b32 s0, s2
; GFX10-NEXT: s_mov_b32 s1, s3
+; GFX10-NEXT: v_lshlrev_b32_e64 v3, 16, s0
; GFX10-NEXT: s_mov_b32 s2, s4
; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v2, s0, 16, v2
; GFX10-NEXT: s_mov_b32 s4, s6
; GFX10-NEXT: s_mov_b32 s5, s7
+; GFX10-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: s_mov_b32 s6, s8
; GFX10-NEXT: s_mov_b32 s7, s9
; GFX10-NEXT: image_atomic_dec v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
@@ -3301,7 +3301,7 @@ define amdgpu_ps <2 x float> @atomic_cmpswap_i64_1d(<8 x i32> inreg %rsrc, i64 %
; GFX9-LABEL: atomic_cmpswap_i64_1d:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX9-NEXT: v_lshlrev_b32_e64 v5, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -3309,21 +3309,21 @@ define amdgpu_ps <2 x float> @atomic_cmpswap_i64_1d(<8 x i32> inreg %rsrc, i64 %
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v4, s0, 16, v4
+; GFX9-NEXT: v_or_b32_sdwa v4, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_cmpswap v[0:3], v4, s[0:7] dmask:0xf unorm glc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: atomic_cmpswap_i64_1d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v4, 0xffff, v4
; GFX10-NEXT: s_mov_b32 s0, s2
; GFX10-NEXT: s_mov_b32 s1, s3
+; GFX10-NEXT: v_lshlrev_b32_e64 v5, 16, s0
; GFX10-NEXT: s_mov_b32 s2, s4
; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v4, s0, 16, v4
; GFX10-NEXT: s_mov_b32 s4, s6
; GFX10-NEXT: s_mov_b32 s5, s7
+; GFX10-NEXT: v_or_b32_sdwa v4, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: s_mov_b32 s6, s8
; GFX10-NEXT: s_mov_b32 s7, s9
; GFX10-NEXT: image_atomic_cmpswap v[0:3], v4, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm glc a16
@@ -3398,7 +3398,7 @@ main_body:
define amdgpu_ps <2 x float> @atomic_add_i64_2d(<8 x i32> inreg %rsrc, i64 %data, i16 %s, i16 %t) {
; GFX9-LABEL: atomic_add_i64_2d:
; GFX9: ; %bb.0: ; %main_body
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: s_mov_b32 s0, s2
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
@@ -3407,19 +3407,19 @@ define amdgpu_ps <2 x float> @atomic_add_i64_2d(<8 x i32> inreg %rsrc, i64 %data
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_add v[0:1], v2, s[0:7] dmask:0x3 unorm glc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: atomic_add_i64_2d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX10-NEXT: s_mov_b32 s0, s2
; GFX10-NEXT: s_mov_b32 s1, s3
; GFX10-NEXT: s_mov_b32 s2, s4
; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX10-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: s_mov_b32 s4, s6
; GFX10-NEXT: s_mov_b32 s5, s7
; GFX10-NEXT: s_mov_b32 s6, s8
@@ -3498,10 +3498,10 @@ main_body:
define amdgpu_ps <2 x float> @atomic_add_i64_3d(<8 x i32> inreg %rsrc, i64 %data, i16 %s, i16 %t, i16 %r) {
; GFX9-LABEL: atomic_add_i64_3d:
; GFX9: ; %bb.0: ; %main_body
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_lshl_or_b32 v2, v3, 16, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v4
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e64 v3, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -3509,21 +3509,21 @@ define amdgpu_ps <2 x float> @atomic_add_i64_3d(<8 x i32> inreg %rsrc, i64 %data
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v3, s0, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_add v[0:1], v[2:3], s[0:7] dmask:0x3 unorm glc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: atomic_add_i64_3d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10-NEXT: v_and_b32_e32 v4, 0xffff, v4
; GFX10-NEXT: s_mov_b32 s0, s2
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX10-NEXT: v_lshlrev_b32_e64 v5, 16, s0
; GFX10-NEXT: s_mov_b32 s1, s3
; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: v_lshl_or_b32 v2, v3, 16, v2
-; GFX10-NEXT: v_lshl_or_b32 v3, s0, 16, v4
; GFX10-NEXT: s_mov_b32 s3, s5
+; GFX10-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v3, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: s_mov_b32 s4, s6
; GFX10-NEXT: s_mov_b32 s5, s7
; GFX10-NEXT: s_mov_b32 s6, s8
@@ -3606,10 +3606,10 @@ main_body:
define amdgpu_ps <2 x float> @atomic_add_i64_cube(<8 x i32> inreg %rsrc, i64 %data, i16 %s, i16 %t, i16 %face) {
; GFX9-LABEL: atomic_add_i64_cube:
; GFX9: ; %bb.0: ; %main_body
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_lshl_or_b32 v2, v3, 16, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v4
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e64 v3, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -3617,21 +3617,21 @@ define amdgpu_ps <2 x float> @atomic_add_i64_cube(<8 x i32> inreg %rsrc, i64 %da
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v3, s0, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_add v[0:1], v[2:3], s[0:7] dmask:0x3 unorm glc a16 da
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: atomic_add_i64_cube:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10-NEXT: v_and_b32_e32 v4, 0xffff, v4
; GFX10-NEXT: s_mov_b32 s0, s2
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX10-NEXT: v_lshlrev_b32_e64 v5, 16, s0
; GFX10-NEXT: s_mov_b32 s1, s3
; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: v_lshl_or_b32 v2, v3, 16, v2
-; GFX10-NEXT: v_lshl_or_b32 v3, s0, 16, v4
; GFX10-NEXT: s_mov_b32 s3, s5
+; GFX10-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v3, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: s_mov_b32 s4, s6
; GFX10-NEXT: s_mov_b32 s5, s7
; GFX10-NEXT: s_mov_b32 s6, s8
@@ -3714,7 +3714,7 @@ main_body:
define amdgpu_ps <2 x float> @atomic_add_i64_1darray(<8 x i32> inreg %rsrc, i64 %data, i16 %s, i16 %slice) {
; GFX9-LABEL: atomic_add_i64_1darray:
; GFX9: ; %bb.0: ; %main_body
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: s_mov_b32 s0, s2
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
@@ -3723,19 +3723,19 @@ define amdgpu_ps <2 x float> @atomic_add_i64_1darray(<8 x i32> inreg %rsrc, i64
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_add v[0:1], v2, s[0:7] dmask:0x3 unorm glc a16 da
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: atomic_add_i64_1darray:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX10-NEXT: s_mov_b32 s0, s2
; GFX10-NEXT: s_mov_b32 s1, s3
; GFX10-NEXT: s_mov_b32 s2, s4
; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX10-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: s_mov_b32 s4, s6
; GFX10-NEXT: s_mov_b32 s5, s7
; GFX10-NEXT: s_mov_b32 s6, s8
@@ -3814,10 +3814,10 @@ main_body:
define amdgpu_ps <2 x float> @atomic_add_i64_2darray(<8 x i32> inreg %rsrc, i64 %data, i16 %s, i16 %t, i16 %slice) {
; GFX9-LABEL: atomic_add_i64_2darray:
; GFX9: ; %bb.0: ; %main_body
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_lshl_or_b32 v2, v3, 16, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v4
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e64 v3, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -3825,21 +3825,21 @@ define amdgpu_ps <2 x float> @atomic_add_i64_2darray(<8 x i32> inreg %rsrc, i64
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v3, s0, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_add v[0:1], v[2:3], s[0:7] dmask:0x3 unorm glc a16 da
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: atomic_add_i64_2darray:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10-NEXT: v_and_b32_e32 v4, 0xffff, v4
; GFX10-NEXT: s_mov_b32 s0, s2
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX10-NEXT: v_lshlrev_b32_e64 v5, 16, s0
; GFX10-NEXT: s_mov_b32 s1, s3
; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: v_lshl_or_b32 v2, v3, 16, v2
-; GFX10-NEXT: v_lshl_or_b32 v3, s0, 16, v4
; GFX10-NEXT: s_mov_b32 s3, s5
+; GFX10-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v3, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: s_mov_b32 s4, s6
; GFX10-NEXT: s_mov_b32 s5, s7
; GFX10-NEXT: s_mov_b32 s6, s8
@@ -3922,10 +3922,10 @@ main_body:
define amdgpu_ps <2 x float> @atomic_add_i64_2dmsaa(<8 x i32> inreg %rsrc, i64 %data, i16 %s, i16 %t, i16 %fragid) {
; GFX9-LABEL: atomic_add_i64_2dmsaa:
; GFX9: ; %bb.0: ; %main_body
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_lshl_or_b32 v2, v3, 16, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v4
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e64 v3, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -3933,21 +3933,21 @@ define amdgpu_ps <2 x float> @atomic_add_i64_2dmsaa(<8 x i32> inreg %rsrc, i64 %
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v3, s0, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_add v[0:1], v[2:3], s[0:7] dmask:0x3 unorm glc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: atomic_add_i64_2dmsaa:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10-NEXT: v_and_b32_e32 v4, 0xffff, v4
; GFX10-NEXT: s_mov_b32 s0, s2
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX10-NEXT: v_lshlrev_b32_e64 v5, 16, s0
; GFX10-NEXT: s_mov_b32 s1, s3
; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: v_lshl_or_b32 v2, v3, 16, v2
-; GFX10-NEXT: v_lshl_or_b32 v3, s0, 16, v4
; GFX10-NEXT: s_mov_b32 s3, s5
+; GFX10-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v3, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: s_mov_b32 s4, s6
; GFX10-NEXT: s_mov_b32 s5, s7
; GFX10-NEXT: s_mov_b32 s6, s8
@@ -4030,9 +4030,9 @@ main_body:
define amdgpu_ps <2 x float> @atomic_add_i64_2darraymsaa(<8 x i32> inreg %rsrc, i64 %data, i16 %s, i16 %t, i16 %slice, i16 %fragid) {
; GFX9-LABEL: atomic_add_i64_2darraymsaa:
; GFX9: ; %bb.0: ; %main_body
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_lshl_or_b32 v2, v3, 16, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v4
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v5
; GFX9-NEXT: s_mov_b32 s0, s2
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
@@ -4041,20 +4041,20 @@ define amdgpu_ps <2 x float> @atomic_add_i64_2darraymsaa(<8 x i32> inreg %rsrc,
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v3, v5, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_add v[0:1], v[2:3], s[0:7] dmask:0x3 unorm glc a16 da
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: atomic_add_i64_2darraymsaa:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX10-NEXT: s_mov_b32 s0, s2
; GFX10-NEXT: s_mov_b32 s1, s3
; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: v_lshl_or_b32 v2, v3, 16, v2
-; GFX10-NEXT: v_lshl_or_b32 v3, v5, 16, v4
+; GFX10-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v3, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: s_mov_b32 s3, s5
; GFX10-NEXT: s_mov_b32 s4, s6
; GFX10-NEXT: s_mov_b32 s5, s7
@@ -4142,7 +4142,7 @@ define amdgpu_ps <2 x float> @atomic_add_i64_1d_slc(<8 x i32> inreg %rsrc, i64 %
; GFX9-LABEL: atomic_add_i64_1d_slc:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX9-NEXT: v_lshlrev_b32_e64 v3, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -4150,21 +4150,21 @@ define amdgpu_ps <2 x float> @atomic_add_i64_1d_slc(<8 x i32> inreg %rsrc, i64 %
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_atomic_add v[0:1], v2, s[0:7] dmask:0x3 unorm glc slc a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: atomic_add_i64_1d_slc:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX10-NEXT: s_mov_b32 s0, s2
; GFX10-NEXT: s_mov_b32 s1, s3
+; GFX10-NEXT: v_lshlrev_b32_e64 v3, 16, s0
; GFX10-NEXT: s_mov_b32 s2, s4
; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v2, s0, 16, v2
; GFX10-NEXT: s_mov_b32 s4, s6
; GFX10-NEXT: s_mov_b32 s5, s7
+; GFX10-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: s_mov_b32 s6, s8
; GFX10-NEXT: s_mov_b32 s7, s9
; GFX10-NEXT: image_atomic_add v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc slc a16
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.gather4.a16.dim.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.gather4.a16.dim.ll
index 4a3a0e234162e..0cd3a5d235ea6 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.gather4.a16.dim.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.gather4.a16.dim.ll
@@ -12,7 +12,7 @@ define amdgpu_ps <4 x float> @gather4_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX9-NEXT: s_mov_b64 s[14:15], exec
; GFX9-NEXT: s_mov_b32 s0, s2
; GFX9-NEXT: s_wqm_b64 exec, exec
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -24,7 +24,7 @@ define amdgpu_ps <4 x float> @gather4_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX9-NEXT: s_mov_b32 s9, s11
; GFX9-NEXT: s_mov_b32 s10, s12
; GFX9-NEXT: s_mov_b32 s11, s13
-; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_and_b64 exec, exec, s[14:15]
; GFX9-NEXT: image_gather4 v[0:3], v0, s[0:7], s[8:11] dmask:0x1 a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -35,7 +35,7 @@ define amdgpu_ps <4 x float> @gather4_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX10NSA-NEXT: s_mov_b32 s14, exec_lo
; GFX10NSA-NEXT: s_mov_b32 s0, s2
; GFX10NSA-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX10NSA-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX10NSA-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX10NSA-NEXT: s_mov_b32 s1, s3
; GFX10NSA-NEXT: s_mov_b32 s2, s4
; GFX10NSA-NEXT: s_mov_b32 s3, s5
@@ -47,7 +47,7 @@ define amdgpu_ps <4 x float> @gather4_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX10NSA-NEXT: s_mov_b32 s9, s11
; GFX10NSA-NEXT: s_mov_b32 s10, s12
; GFX10NSA-NEXT: s_mov_b32 s11, s13
-; GFX10NSA-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10NSA-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10NSA-NEXT: s_and_b32 exec_lo, exec_lo, s14
; GFX10NSA-NEXT: image_gather4 v[0:3], v0, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX10NSA-NEXT: s_waitcnt vmcnt(0)
@@ -153,9 +153,9 @@ define amdgpu_ps <4 x float> @gather4_cube(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX9-NEXT: s_mov_b64 s[14:15], exec
; GFX9-NEXT: s_mov_b32 s0, s2
; GFX9-NEXT: s_wqm_b64 exec, exec
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e64 v1, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -167,7 +167,7 @@ define amdgpu_ps <4 x float> @gather4_cube(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX9-NEXT: s_mov_b32 s9, s11
; GFX9-NEXT: s_mov_b32 s10, s12
; GFX9-NEXT: s_mov_b32 s11, s13
-; GFX9-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_and_b64 exec, exec, s[14:15]
; GFX9-NEXT: image_gather4 v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 a16 da
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -178,8 +178,8 @@ define amdgpu_ps <4 x float> @gather4_cube(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX10NSA-NEXT: s_mov_b32 s14, exec_lo
; GFX10NSA-NEXT: s_mov_b32 s0, s2
; GFX10NSA-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX10NSA-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10NSA-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX10NSA-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX10NSA-NEXT: v_lshlrev_b32_e64 v3, 16, s0
; GFX10NSA-NEXT: s_mov_b32 s1, s3
; GFX10NSA-NEXT: s_mov_b32 s2, s4
; GFX10NSA-NEXT: s_mov_b32 s3, s5
@@ -191,8 +191,8 @@ define amdgpu_ps <4 x float> @gather4_cube(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX10NSA-NEXT: s_mov_b32 s9, s11
; GFX10NSA-NEXT: s_mov_b32 s10, s12
; GFX10NSA-NEXT: s_mov_b32 s11, s13
-; GFX10NSA-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX10NSA-NEXT: v_lshl_or_b32 v1, s0, 16, v2
+; GFX10NSA-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10NSA-NEXT: v_or_b32_sdwa v1, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10NSA-NEXT: s_and_b32 exec_lo, exec_lo, s14
; GFX10NSA-NEXT: image_gather4 v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_CUBE a16
; GFX10NSA-NEXT: s_waitcnt vmcnt(0)
@@ -302,9 +302,9 @@ define amdgpu_ps <4 x float> @gather4_2darray(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX9-NEXT: s_mov_b64 s[14:15], exec
; GFX9-NEXT: s_mov_b32 s0, s2
; GFX9-NEXT: s_wqm_b64 exec, exec
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e64 v1, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -316,7 +316,7 @@ define amdgpu_ps <4 x float> @gather4_2darray(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX9-NEXT: s_mov_b32 s9, s11
; GFX9-NEXT: s_mov_b32 s10, s12
; GFX9-NEXT: s_mov_b32 s11, s13
-; GFX9-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_and_b64 exec, exec, s[14:15]
; GFX9-NEXT: image_gather4 v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 a16 da
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -327,8 +327,8 @@ define amdgpu_ps <4 x float> @gather4_2darray(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX10NSA-NEXT: s_mov_b32 s14, exec_lo
; GFX10NSA-NEXT: s_mov_b32 s0, s2
; GFX10NSA-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX10NSA-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10NSA-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX10NSA-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX10NSA-NEXT: v_lshlrev_b32_e64 v3, 16, s0
; GFX10NSA-NEXT: s_mov_b32 s1, s3
; GFX10NSA-NEXT: s_mov_b32 s2, s4
; GFX10NSA-NEXT: s_mov_b32 s3, s5
@@ -340,8 +340,8 @@ define amdgpu_ps <4 x float> @gather4_2darray(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX10NSA-NEXT: s_mov_b32 s9, s11
; GFX10NSA-NEXT: s_mov_b32 s10, s12
; GFX10NSA-NEXT: s_mov_b32 s11, s13
-; GFX10NSA-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX10NSA-NEXT: v_lshl_or_b32 v1, s0, 16, v2
+; GFX10NSA-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10NSA-NEXT: v_or_b32_sdwa v1, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10NSA-NEXT: s_and_b32 exec_lo, exec_lo, s14
; GFX10NSA-NEXT: image_gather4 v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY a16
; GFX10NSA-NEXT: s_waitcnt vmcnt(0)
@@ -451,7 +451,7 @@ define amdgpu_ps <4 x float> @gather4_c_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX9-NEXT: s_mov_b64 s[14:15], exec
; GFX9-NEXT: s_mov_b32 s0, s2
; GFX9-NEXT: s_wqm_b64 exec, exec
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -463,7 +463,7 @@ define amdgpu_ps <4 x float> @gather4_c_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX9-NEXT: s_mov_b32 s9, s11
; GFX9-NEXT: s_mov_b32 s10, s12
; GFX9-NEXT: s_mov_b32 s11, s13
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_and_b64 exec, exec, s[14:15]
; GFX9-NEXT: image_gather4_c v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -474,7 +474,7 @@ define amdgpu_ps <4 x float> @gather4_c_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX10NSA-NEXT: s_mov_b32 s14, exec_lo
; GFX10NSA-NEXT: s_mov_b32 s0, s2
; GFX10NSA-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX10NSA-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX10NSA-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX10NSA-NEXT: s_mov_b32 s1, s3
; GFX10NSA-NEXT: s_mov_b32 s2, s4
; GFX10NSA-NEXT: s_mov_b32 s3, s5
@@ -486,7 +486,7 @@ define amdgpu_ps <4 x float> @gather4_c_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX10NSA-NEXT: s_mov_b32 s9, s11
; GFX10NSA-NEXT: s_mov_b32 s10, s12
; GFX10NSA-NEXT: s_mov_b32 s11, s13
-; GFX10NSA-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX10NSA-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10NSA-NEXT: s_and_b32 exec_lo, exec_lo, s14
; GFX10NSA-NEXT: image_gather4_c v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX10NSA-NEXT: s_waitcnt vmcnt(0)
@@ -592,9 +592,9 @@ define amdgpu_ps <4 x float> @gather4_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX9-NEXT: s_mov_b64 s[14:15], exec
; GFX9-NEXT: s_mov_b32 s0, s2
; GFX9-NEXT: s_wqm_b64 exec, exec
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e64 v1, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -606,7 +606,7 @@ define amdgpu_ps <4 x float> @gather4_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX9-NEXT: s_mov_b32 s9, s11
; GFX9-NEXT: s_mov_b32 s10, s12
; GFX9-NEXT: s_mov_b32 s11, s13
-; GFX9-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_and_b64 exec, exec, s[14:15]
; GFX9-NEXT: image_gather4_cl v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -617,8 +617,8 @@ define amdgpu_ps <4 x float> @gather4_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX10NSA-NEXT: s_mov_b32 s14, exec_lo
; GFX10NSA-NEXT: s_mov_b32 s0, s2
; GFX10NSA-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX10NSA-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10NSA-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX10NSA-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX10NSA-NEXT: v_lshlrev_b32_e64 v3, 16, s0
; GFX10NSA-NEXT: s_mov_b32 s1, s3
; GFX10NSA-NEXT: s_mov_b32 s2, s4
; GFX10NSA-NEXT: s_mov_b32 s3, s5
@@ -630,8 +630,8 @@ define amdgpu_ps <4 x float> @gather4_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX10NSA-NEXT: s_mov_b32 s9, s11
; GFX10NSA-NEXT: s_mov_b32 s10, s12
; GFX10NSA-NEXT: s_mov_b32 s11, s13
-; GFX10NSA-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX10NSA-NEXT: v_lshl_or_b32 v1, s0, 16, v2
+; GFX10NSA-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10NSA-NEXT: v_or_b32_sdwa v1, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10NSA-NEXT: s_and_b32 exec_lo, exec_lo, s14
; GFX10NSA-NEXT: image_gather4_cl v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX10NSA-NEXT: s_waitcnt vmcnt(0)
@@ -741,9 +741,9 @@ define amdgpu_ps <4 x float> @gather4_c_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX9-NEXT: s_mov_b64 s[14:15], exec
; GFX9-NEXT: s_mov_b32 s0, s2
; GFX9-NEXT: s_wqm_b64 exec, exec
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v3
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e64 v2, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -755,7 +755,7 @@ define amdgpu_ps <4 x float> @gather4_c_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX9-NEXT: s_mov_b32 s9, s11
; GFX9-NEXT: s_mov_b32 s10, s12
; GFX9-NEXT: s_mov_b32 s11, s13
-; GFX9-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_and_b64 exec, exec, s[14:15]
; GFX9-NEXT: image_gather4_c_cl v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1 a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -766,8 +766,8 @@ define amdgpu_ps <4 x float> @gather4_c_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX10NSA-NEXT: s_mov_b32 s14, exec_lo
; GFX10NSA-NEXT: s_mov_b32 s0, s2
; GFX10NSA-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX10NSA-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10NSA-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX10NSA-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX10NSA-NEXT: v_lshlrev_b32_e64 v4, 16, s0
; GFX10NSA-NEXT: s_mov_b32 s1, s3
; GFX10NSA-NEXT: s_mov_b32 s2, s4
; GFX10NSA-NEXT: s_mov_b32 s3, s5
@@ -779,8 +779,8 @@ define amdgpu_ps <4 x float> @gather4_c_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX10NSA-NEXT: s_mov_b32 s9, s11
; GFX10NSA-NEXT: s_mov_b32 s10, s12
; GFX10NSA-NEXT: s_mov_b32 s11, s13
-; GFX10NSA-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX10NSA-NEXT: v_lshl_or_b32 v2, s0, 16, v3
+; GFX10NSA-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10NSA-NEXT: v_or_b32_sdwa v2, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10NSA-NEXT: s_and_b32 exec_lo, exec_lo, s14
; GFX10NSA-NEXT: image_gather4_c_cl v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX10NSA-NEXT: s_waitcnt vmcnt(0)
@@ -890,8 +890,8 @@ define amdgpu_ps <4 x float> @gather4_b_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX9-NEXT: s_mov_b64 s[14:15], exec
; GFX9-NEXT: s_mov_b32 s0, s2
; GFX9-NEXT: s_wqm_b64 exec, exec
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT: v_lshlrev_b32_e64 v3, 16, s0
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -903,8 +903,8 @@ define amdgpu_ps <4 x float> @gather4_b_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX9-NEXT: s_mov_b32 s9, s11
; GFX9-NEXT: s_mov_b32 s10, s12
; GFX9-NEXT: s_mov_b32 s11, s13
-; GFX9-NEXT: v_lshl_or_b32 v0, s0, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_and_b64 exec, exec, s[14:15]
; GFX9-NEXT: image_gather4_b v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -915,8 +915,8 @@ define amdgpu_ps <4 x float> @gather4_b_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX10NSA-NEXT: s_mov_b32 s14, exec_lo
; GFX10NSA-NEXT: s_mov_b32 s0, s2
; GFX10NSA-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX10NSA-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10NSA-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX10NSA-NEXT: v_lshlrev_b32_e64 v3, 16, s0
+; GFX10NSA-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX10NSA-NEXT: s_mov_b32 s1, s3
; GFX10NSA-NEXT: s_mov_b32 s2, s4
; GFX10NSA-NEXT: s_mov_b32 s3, s5
@@ -928,8 +928,8 @@ define amdgpu_ps <4 x float> @gather4_b_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX10NSA-NEXT: s_mov_b32 s9, s11
; GFX10NSA-NEXT: s_mov_b32 s10, s12
; GFX10NSA-NEXT: s_mov_b32 s11, s13
-; GFX10NSA-NEXT: v_lshl_or_b32 v0, s0, 16, v0
-; GFX10NSA-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX10NSA-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10NSA-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10NSA-NEXT: s_and_b32 exec_lo, exec_lo, s14
; GFX10NSA-NEXT: image_gather4_b v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX10NSA-NEXT: s_waitcnt vmcnt(0)
@@ -1039,8 +1039,8 @@ define amdgpu_ps <4 x float> @gather4_c_b_2d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX9-NEXT: s_mov_b64 s[14:15], exec
; GFX9-NEXT: s_mov_b32 s0, s2
; GFX9-NEXT: s_wqm_b64 exec, exec
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX9-NEXT: v_lshlrev_b32_e64 v4, 16, s0
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -1052,8 +1052,8 @@ define amdgpu_ps <4 x float> @gather4_c_b_2d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX9-NEXT: s_mov_b32 s9, s11
; GFX9-NEXT: s_mov_b32 s10, s12
; GFX9-NEXT: s_mov_b32 s11, s13
-; GFX9-NEXT: v_lshl_or_b32 v0, s0, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_and_b64 exec, exec, s[14:15]
; GFX9-NEXT: image_gather4_c_b v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1 a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -1064,8 +1064,8 @@ define amdgpu_ps <4 x float> @gather4_c_b_2d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX10NSA-NEXT: s_mov_b32 s14, exec_lo
; GFX10NSA-NEXT: s_mov_b32 s0, s2
; GFX10NSA-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX10NSA-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10NSA-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX10NSA-NEXT: v_lshlrev_b32_e64 v4, 16, s0
+; GFX10NSA-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX10NSA-NEXT: s_mov_b32 s1, s3
; GFX10NSA-NEXT: s_mov_b32 s2, s4
; GFX10NSA-NEXT: s_mov_b32 s3, s5
@@ -1077,8 +1077,8 @@ define amdgpu_ps <4 x float> @gather4_c_b_2d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX10NSA-NEXT: s_mov_b32 s9, s11
; GFX10NSA-NEXT: s_mov_b32 s10, s12
; GFX10NSA-NEXT: s_mov_b32 s11, s13
-; GFX10NSA-NEXT: v_lshl_or_b32 v0, s0, 16, v0
-; GFX10NSA-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX10NSA-NEXT: v_or_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10NSA-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10NSA-NEXT: s_and_b32 exec_lo, exec_lo, s14
; GFX10NSA-NEXT: image_gather4_c_b v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX10NSA-NEXT: s_waitcnt vmcnt(0)
@@ -1188,10 +1188,8 @@ define amdgpu_ps <4 x float> @gather4_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX9-NEXT: s_mov_b64 s[14:15], exec
; GFX9-NEXT: s_mov_b32 s0, s2
; GFX9-NEXT: s_wqm_b64 exec, exec
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v3
+; GFX9-NEXT: v_lshlrev_b32_e64 v4, 16, s0
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -1203,8 +1201,9 @@ define amdgpu_ps <4 x float> @gather4_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX9-NEXT: s_mov_b32 s9, s11
; GFX9-NEXT: s_mov_b32 s10, s12
; GFX9-NEXT: s_mov_b32 s11, s13
-; GFX9-NEXT: v_lshl_or_b32 v0, s0, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_and_b64 exec, exec, s[14:15]
; GFX9-NEXT: image_gather4_b_cl v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1 a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -1215,9 +1214,8 @@ define amdgpu_ps <4 x float> @gather4_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX10NSA-NEXT: s_mov_b32 s14, exec_lo
; GFX10NSA-NEXT: s_mov_b32 s0, s2
; GFX10NSA-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX10NSA-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10NSA-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10NSA-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX10NSA-NEXT: v_lshlrev_b32_e64 v4, 16, s0
+; GFX10NSA-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX10NSA-NEXT: s_mov_b32 s1, s3
; GFX10NSA-NEXT: s_mov_b32 s2, s4
; GFX10NSA-NEXT: s_mov_b32 s3, s5
@@ -1229,9 +1227,9 @@ define amdgpu_ps <4 x float> @gather4_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX10NSA-NEXT: s_mov_b32 s9, s11
; GFX10NSA-NEXT: s_mov_b32 s10, s12
; GFX10NSA-NEXT: s_mov_b32 s11, s13
-; GFX10NSA-NEXT: v_lshl_or_b32 v0, s0, 16, v0
-; GFX10NSA-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX10NSA-NEXT: v_lshl_or_b32 v2, s0, 16, v3
+; GFX10NSA-NEXT: v_or_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10NSA-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10NSA-NEXT: v_or_b32_sdwa v2, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10NSA-NEXT: s_and_b32 exec_lo, exec_lo, s14
; GFX10NSA-NEXT: image_gather4_b_cl v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX10NSA-NEXT: s_waitcnt vmcnt(0)
@@ -1345,10 +1343,8 @@ define amdgpu_ps <4 x float> @gather4_c_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX9-NEXT: s_mov_b64 s[14:15], exec
; GFX9-NEXT: s_mov_b32 s0, s2
; GFX9-NEXT: s_wqm_b64 exec, exec
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v2, v3, 16, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v4
+; GFX9-NEXT: v_lshlrev_b32_e64 v5, 16, s0
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -1360,8 +1356,9 @@ define amdgpu_ps <4 x float> @gather4_c_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX9-NEXT: s_mov_b32 s9, s11
; GFX9-NEXT: s_mov_b32 s10, s12
; GFX9-NEXT: s_mov_b32 s11, s13
-; GFX9-NEXT: v_lshl_or_b32 v0, s0, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v3, s0, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v0, v5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_and_b64 exec, exec, s[14:15]
; GFX9-NEXT: image_gather4_c_b_cl v[0:3], v[0:3], s[0:7], s[8:11] dmask:0x1 a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -1372,9 +1369,8 @@ define amdgpu_ps <4 x float> @gather4_c_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX10NSA-NEXT: s_mov_b32 s14, exec_lo
; GFX10NSA-NEXT: s_mov_b32 s0, s2
; GFX10NSA-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX10NSA-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10NSA-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10NSA-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX10NSA-NEXT: v_lshlrev_b32_e64 v5, 16, s0
+; GFX10NSA-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX10NSA-NEXT: s_mov_b32 s1, s3
; GFX10NSA-NEXT: s_mov_b32 s2, s4
; GFX10NSA-NEXT: s_mov_b32 s3, s5
@@ -1386,9 +1382,9 @@ define amdgpu_ps <4 x float> @gather4_c_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX10NSA-NEXT: s_mov_b32 s9, s11
; GFX10NSA-NEXT: s_mov_b32 s10, s12
; GFX10NSA-NEXT: s_mov_b32 s11, s13
-; GFX10NSA-NEXT: v_lshl_or_b32 v0, s0, 16, v0
-; GFX10NSA-NEXT: v_lshl_or_b32 v2, v3, 16, v2
-; GFX10NSA-NEXT: v_lshl_or_b32 v3, s0, 16, v4
+; GFX10NSA-NEXT: v_or_b32_sdwa v0, v5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10NSA-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10NSA-NEXT: v_or_b32_sdwa v3, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10NSA-NEXT: s_and_b32 exec_lo, exec_lo, s14
; GFX10NSA-NEXT: image_gather4_c_b_cl v[0:3], v[0:3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX10NSA-NEXT: s_waitcnt vmcnt(0)
@@ -1499,10 +1495,10 @@ main_body:
define amdgpu_ps <4 x float> @gather4_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %s, half %t, half %lod) {
; GFX9-LABEL: gather4_l_2d:
; GFX9: ; %bb.0: ; %main_body
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e64 v1, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -1514,21 +1510,21 @@ define amdgpu_ps <4 x float> @gather4_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX9-NEXT: s_mov_b32 s9, s11
; GFX9-NEXT: s_mov_b32 s10, s12
; GFX9-NEXT: s_mov_b32 s11, s13
-; GFX9-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_gather4_l v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10NSA-LABEL: gather4_l_2d:
; GFX10NSA: ; %bb.0: ; %main_body
-; GFX10NSA-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10NSA-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX10NSA-NEXT: s_mov_b32 s0, s2
+; GFX10NSA-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX10NSA-NEXT: v_lshlrev_b32_e64 v3, 16, s0
; GFX10NSA-NEXT: s_mov_b32 s1, s3
; GFX10NSA-NEXT: s_mov_b32 s2, s4
-; GFX10NSA-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX10NSA-NEXT: v_lshl_or_b32 v1, s0, 16, v2
; GFX10NSA-NEXT: s_mov_b32 s3, s5
+; GFX10NSA-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10NSA-NEXT: v_or_b32_sdwa v1, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10NSA-NEXT: s_mov_b32 s4, s6
; GFX10NSA-NEXT: s_mov_b32 s5, s7
; GFX10NSA-NEXT: s_mov_b32 s6, s8
@@ -1630,10 +1626,10 @@ main_body:
define amdgpu_ps <4 x float> @gather4_c_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, half %s, half %t, half %lod) {
; GFX9-LABEL: gather4_c_l_2d:
; GFX9: ; %bb.0: ; %main_body
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v3
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e64 v2, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -1645,21 +1641,21 @@ define amdgpu_ps <4 x float> @gather4_c_l_2d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX9-NEXT: s_mov_b32 s9, s11
; GFX9-NEXT: s_mov_b32 s10, s12
; GFX9-NEXT: s_mov_b32 s11, s13
-; GFX9-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_gather4_c_l v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1 a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10NSA-LABEL: gather4_c_l_2d:
; GFX10NSA: ; %bb.0: ; %main_body
-; GFX10NSA-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10NSA-NEXT: v_and_b32_e32 v3, 0xffff, v3
; GFX10NSA-NEXT: s_mov_b32 s0, s2
+; GFX10NSA-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX10NSA-NEXT: v_lshlrev_b32_e64 v4, 16, s0
; GFX10NSA-NEXT: s_mov_b32 s1, s3
; GFX10NSA-NEXT: s_mov_b32 s2, s4
-; GFX10NSA-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX10NSA-NEXT: v_lshl_or_b32 v2, s0, 16, v3
; GFX10NSA-NEXT: s_mov_b32 s3, s5
+; GFX10NSA-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10NSA-NEXT: v_or_b32_sdwa v2, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10NSA-NEXT: s_mov_b32 s4, s6
; GFX10NSA-NEXT: s_mov_b32 s5, s7
; GFX10NSA-NEXT: s_mov_b32 s6, s8
@@ -1761,7 +1757,7 @@ main_body:
define amdgpu_ps <4 x float> @gather4_lz_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %s, half %t) {
; GFX9-LABEL: gather4_lz_2d:
; GFX9: ; %bb.0: ; %main_body
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_mov_b32 s0, s2
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
@@ -1774,19 +1770,19 @@ define amdgpu_ps <4 x float> @gather4_lz_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX9-NEXT: s_mov_b32 s9, s11
; GFX9-NEXT: s_mov_b32 s10, s12
; GFX9-NEXT: s_mov_b32 s11, s13
-; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_gather4_lz v[0:3], v0, s[0:7], s[8:11] dmask:0x1 a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10NSA-LABEL: gather4_lz_2d:
; GFX10NSA: ; %bb.0: ; %main_body
-; GFX10NSA-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX10NSA-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX10NSA-NEXT: s_mov_b32 s0, s2
; GFX10NSA-NEXT: s_mov_b32 s1, s3
; GFX10NSA-NEXT: s_mov_b32 s2, s4
; GFX10NSA-NEXT: s_mov_b32 s3, s5
-; GFX10NSA-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10NSA-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10NSA-NEXT: s_mov_b32 s4, s6
; GFX10NSA-NEXT: s_mov_b32 s5, s7
; GFX10NSA-NEXT: s_mov_b32 s6, s8
@@ -1884,7 +1880,7 @@ main_body:
define amdgpu_ps <4 x float> @gather4_c_lz_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, half %s, half %t) {
; GFX9-LABEL: gather4_c_lz_2d:
; GFX9: ; %bb.0: ; %main_body
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX9-NEXT: s_mov_b32 s0, s2
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
@@ -1897,19 +1893,19 @@ define amdgpu_ps <4 x float> @gather4_c_lz_2d(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX9-NEXT: s_mov_b32 s9, s11
; GFX9-NEXT: s_mov_b32 s10, s12
; GFX9-NEXT: s_mov_b32 s11, s13
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_gather4_c_lz v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10NSA-LABEL: gather4_c_lz_2d:
; GFX10NSA: ; %bb.0: ; %main_body
-; GFX10NSA-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX10NSA-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX10NSA-NEXT: s_mov_b32 s0, s2
; GFX10NSA-NEXT: s_mov_b32 s1, s3
; GFX10NSA-NEXT: s_mov_b32 s2, s4
; GFX10NSA-NEXT: s_mov_b32 s3, s5
-; GFX10NSA-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX10NSA-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10NSA-NEXT: s_mov_b32 s4, s6
; GFX10NSA-NEXT: s_mov_b32 s5, s7
; GFX10NSA-NEXT: s_mov_b32 s6, s8
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.getresinfo.a16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.getresinfo.a16.ll
index 0b24be357ac5a..6cf7e721605a3 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.getresinfo.a16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.getresinfo.a16.ll
@@ -8,7 +8,7 @@ define amdgpu_ps <4 x float> @getresinfo_1d(<8 x i32> inreg %rsrc, i16 %mip) {
; GFX9-LABEL: getresinfo_1d:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: v_lshlrev_b32_e64 v1, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -16,27 +16,11 @@ define amdgpu_ps <4 x float> @getresinfo_1d(<8 x i32> inreg %rsrc, i16 %mip) {
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v0, s0, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf unorm a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: getresinfo_1d:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: s_mov_b32 s0, s2
-; GFX10-NEXT: s_mov_b32 s1, s3
-; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v0, s0, 16, v0
-; GFX10-NEXT: s_mov_b32 s4, s6
-; GFX10-NEXT: s_mov_b32 s5, s7
-; GFX10-NEXT: s_mov_b32 s6, s8
-; GFX10-NEXT: s_mov_b32 s7, s9
-; GFX10-NEXT: image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm a16
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: getresinfo_1d:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v0, 0xffff, v0
@@ -61,7 +45,7 @@ define amdgpu_ps <4 x float> @getresinfo_2d(<8 x i32> inreg %rsrc, i16 %mip) {
; GFX9-LABEL: getresinfo_2d:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: v_lshlrev_b32_e64 v1, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -69,27 +53,11 @@ define amdgpu_ps <4 x float> @getresinfo_2d(<8 x i32> inreg %rsrc, i16 %mip) {
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v0, s0, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf unorm a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: getresinfo_2d:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: s_mov_b32 s0, s2
-; GFX10-NEXT: s_mov_b32 s1, s3
-; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v0, s0, 16, v0
-; GFX10-NEXT: s_mov_b32 s4, s6
-; GFX10-NEXT: s_mov_b32 s5, s7
-; GFX10-NEXT: s_mov_b32 s6, s8
-; GFX10-NEXT: s_mov_b32 s7, s9
-; GFX10-NEXT: image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D unorm a16
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: getresinfo_2d:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v0, 0xffff, v0
@@ -114,7 +82,7 @@ define amdgpu_ps <4 x float> @getresinfo_3d(<8 x i32> inreg %rsrc, i16 %mip) {
; GFX9-LABEL: getresinfo_3d:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: v_lshlrev_b32_e64 v1, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -122,27 +90,11 @@ define amdgpu_ps <4 x float> @getresinfo_3d(<8 x i32> inreg %rsrc, i16 %mip) {
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v0, s0, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf unorm a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: getresinfo_3d:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: s_mov_b32 s0, s2
-; GFX10-NEXT: s_mov_b32 s1, s3
-; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v0, s0, 16, v0
-; GFX10-NEXT: s_mov_b32 s4, s6
-; GFX10-NEXT: s_mov_b32 s5, s7
-; GFX10-NEXT: s_mov_b32 s6, s8
-; GFX10-NEXT: s_mov_b32 s7, s9
-; GFX10-NEXT: image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D unorm a16
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: getresinfo_3d:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v0, 0xffff, v0
@@ -167,7 +119,7 @@ define amdgpu_ps <4 x float> @getresinfo_cube(<8 x i32> inreg %rsrc, i16 %mip) {
; GFX9-LABEL: getresinfo_cube:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: v_lshlrev_b32_e64 v1, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -175,27 +127,11 @@ define amdgpu_ps <4 x float> @getresinfo_cube(<8 x i32> inreg %rsrc, i16 %mip) {
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v0, s0, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf unorm a16 da
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: getresinfo_cube:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: s_mov_b32 s0, s2
-; GFX10-NEXT: s_mov_b32 s1, s3
-; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v0, s0, 16, v0
-; GFX10-NEXT: s_mov_b32 s4, s6
-; GFX10-NEXT: s_mov_b32 s5, s7
-; GFX10-NEXT: s_mov_b32 s6, s8
-; GFX10-NEXT: s_mov_b32 s7, s9
-; GFX10-NEXT: image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_CUBE unorm a16
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: getresinfo_cube:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v0, 0xffff, v0
@@ -220,7 +156,7 @@ define amdgpu_ps <4 x float> @getresinfo_1darray(<8 x i32> inreg %rsrc, i16 %mip
; GFX9-LABEL: getresinfo_1darray:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: v_lshlrev_b32_e64 v1, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -228,27 +164,11 @@ define amdgpu_ps <4 x float> @getresinfo_1darray(<8 x i32> inreg %rsrc, i16 %mip
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v0, s0, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf unorm a16 da
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: getresinfo_1darray:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: s_mov_b32 s0, s2
-; GFX10-NEXT: s_mov_b32 s1, s3
-; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v0, s0, 16, v0
-; GFX10-NEXT: s_mov_b32 s4, s6
-; GFX10-NEXT: s_mov_b32 s5, s7
-; GFX10-NEXT: s_mov_b32 s6, s8
-; GFX10-NEXT: s_mov_b32 s7, s9
-; GFX10-NEXT: image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D_ARRAY unorm a16
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: getresinfo_1darray:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v0, 0xffff, v0
@@ -273,7 +193,7 @@ define amdgpu_ps <4 x float> @getresinfo_2darray(<8 x i32> inreg %rsrc, i16 %mip
; GFX9-LABEL: getresinfo_2darray:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: v_lshlrev_b32_e64 v1, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -281,27 +201,11 @@ define amdgpu_ps <4 x float> @getresinfo_2darray(<8 x i32> inreg %rsrc, i16 %mip
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v0, s0, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf unorm a16 da
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: getresinfo_2darray:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: s_mov_b32 s0, s2
-; GFX10-NEXT: s_mov_b32 s1, s3
-; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v0, s0, 16, v0
-; GFX10-NEXT: s_mov_b32 s4, s6
-; GFX10-NEXT: s_mov_b32 s5, s7
-; GFX10-NEXT: s_mov_b32 s6, s8
-; GFX10-NEXT: s_mov_b32 s7, s9
-; GFX10-NEXT: image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_ARRAY unorm a16
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: getresinfo_2darray:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v0, 0xffff, v0
@@ -326,7 +230,7 @@ define amdgpu_ps <4 x float> @getresinfo_2dmsaa(<8 x i32> inreg %rsrc, i16 %mip)
; GFX9-LABEL: getresinfo_2dmsaa:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: v_lshlrev_b32_e64 v1, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -334,27 +238,11 @@ define amdgpu_ps <4 x float> @getresinfo_2dmsaa(<8 x i32> inreg %rsrc, i16 %mip)
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v0, s0, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf unorm a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: getresinfo_2dmsaa:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: s_mov_b32 s0, s2
-; GFX10-NEXT: s_mov_b32 s1, s3
-; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v0, s0, 16, v0
-; GFX10-NEXT: s_mov_b32 s4, s6
-; GFX10-NEXT: s_mov_b32 s5, s7
-; GFX10-NEXT: s_mov_b32 s6, s8
-; GFX10-NEXT: s_mov_b32 s7, s9
-; GFX10-NEXT: image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA unorm a16
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: getresinfo_2dmsaa:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v0, 0xffff, v0
@@ -379,7 +267,7 @@ define amdgpu_ps <4 x float> @getresinfo_2darraymsaa(<8 x i32> inreg %rsrc, i16
; GFX9-LABEL: getresinfo_2darraymsaa:
; GFX9: ; %bb.0: ; %main_body
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: v_lshlrev_b32_e64 v1, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -387,27 +275,11 @@ define amdgpu_ps <4 x float> @getresinfo_2darraymsaa(<8 x i32> inreg %rsrc, i16
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v0, s0, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf unorm a16 da
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10-LABEL: getresinfo_2darraymsaa:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: s_mov_b32 s0, s2
-; GFX10-NEXT: s_mov_b32 s1, s3
-; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: s_mov_b32 s3, s5
-; GFX10-NEXT: v_lshl_or_b32 v0, s0, 16, v0
-; GFX10-NEXT: s_mov_b32 s4, s6
-; GFX10-NEXT: s_mov_b32 s5, s7
-; GFX10-NEXT: s_mov_b32 s6, s8
-; GFX10-NEXT: s_mov_b32 s7, s9
-; GFX10-NEXT: image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm a16
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: ; return to shader part epilog
-;
; GFX12-LABEL: getresinfo_2darraymsaa:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v0, 0xffff, v0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.2darraymsaa.a16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.2darraymsaa.a16.ll
index 26a7a9106b3b6..7ca3ae1561b2f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.2darraymsaa.a16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.2darraymsaa.a16.ll
@@ -9,9 +9,9 @@
define amdgpu_ps <4 x float> @load_2darraymsaa_v4f32_xyzw(<8 x i32> inreg %rsrc, i16 %s, i16 %t, i16 %slice, i16 %fragid) {
; GFX9-LABEL: load_2darraymsaa_v4f32_xyzw:
; GFX9: ; %bb.0:
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v3
; GFX9-NEXT: s_mov_b32 s0, s2
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
@@ -20,20 +20,20 @@ define amdgpu_ps <4 x float> @load_2darraymsaa_v4f32_xyzw(<8 x i32> inreg %rsrc,
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v1, v3, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_load v[0:3], v[0:1], s[0:7] dmask:0xf unorm a16 da
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: load_2darraymsaa_v4f32_xyzw:
; GFX10: ; %bb.0:
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX10-NEXT: s_mov_b32 s0, s2
; GFX10-NEXT: s_mov_b32 s1, s3
; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX10-NEXT: v_lshl_or_b32 v1, v3, 16, v2
+; GFX10-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v1, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: s_mov_b32 s3, s5
; GFX10-NEXT: s_mov_b32 s4, s6
; GFX10-NEXT: s_mov_b32 s5, s7
@@ -119,10 +119,10 @@ define amdgpu_ps <4 x float> @load_2darraymsaa_v4f32_xyzw(<8 x i32> inreg %rsrc,
define amdgpu_ps <4 x float> @load_2darraymsaa_v4f32_xyzw_tfe(<8 x i32> inreg %rsrc, ptr addrspace(1) inreg %out, i16 %s, i16 %t, i16 %slice, i16 %fragid) {
; GFX9-LABEL: load_2darraymsaa_v4f32_xyzw_tfe:
; GFX9: ; %bb.0:
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_mov_b32_e32 v5, 0
-; GFX9-NEXT: v_lshl_or_b32 v10, v1, 16, v0
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v2
+; GFX9-NEXT: v_or_b32_sdwa v10, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX9-NEXT: v_mov_b32_e32 v6, v5
; GFX9-NEXT: v_mov_b32_e32 v7, v5
; GFX9-NEXT: v_mov_b32_e32 v8, v5
@@ -135,7 +135,7 @@ define amdgpu_ps <4 x float> @load_2darraymsaa_v4f32_xyzw_tfe(<8 x i32> inreg %r
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v11, v3, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v11, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v0, v5
; GFX9-NEXT: v_mov_b32_e32 v1, v6
; GFX9-NEXT: v_mov_b32_e32 v2, v7
@@ -150,16 +150,16 @@ define amdgpu_ps <4 x float> @load_2darraymsaa_v4f32_xyzw_tfe(<8 x i32> inreg %r
; GFX10-LABEL: load_2darraymsaa_v4f32_xyzw_tfe:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_mov_b32_e32 v5, 0
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX10-NEXT: s_mov_b32 s0, s2
; GFX10-NEXT: s_mov_b32 s1, s3
; GFX10-NEXT: v_mov_b32_e32 v6, v5
; GFX10-NEXT: v_mov_b32_e32 v7, v5
; GFX10-NEXT: v_mov_b32_e32 v8, v5
; GFX10-NEXT: v_mov_b32_e32 v9, v5
-; GFX10-NEXT: v_lshl_or_b32 v10, v1, 16, v0
-; GFX10-NEXT: v_lshl_or_b32 v11, v3, 16, v2
+; GFX10-NEXT: v_or_b32_sdwa v10, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v11, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: s_mov_b32 s2, s4
; GFX10-NEXT: s_mov_b32 s3, s5
; GFX10-NEXT: s_mov_b32 s4, s6
@@ -301,10 +301,10 @@ define amdgpu_ps <4 x float> @load_2darraymsaa_v4f32_xyzw_tfe(<8 x i32> inreg %r
define amdgpu_ps <4 x float> @load_2darraymsaa_v4f32_xyzw_tfe_lwe(<8 x i32> inreg %rsrc, ptr addrspace(1) inreg %out, i16 %s, i16 %t, i16 %slice, i16 %fragid) {
; GFX9-LABEL: load_2darraymsaa_v4f32_xyzw_tfe_lwe:
; GFX9: ; %bb.0:
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_mov_b32_e32 v5, 0
-; GFX9-NEXT: v_lshl_or_b32 v10, v1, 16, v0
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v2
+; GFX9-NEXT: v_or_b32_sdwa v10, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX9-NEXT: v_mov_b32_e32 v6, v5
; GFX9-NEXT: v_mov_b32_e32 v7, v5
; GFX9-NEXT: v_mov_b32_e32 v8, v5
@@ -317,7 +317,7 @@ define amdgpu_ps <4 x float> @load_2darraymsaa_v4f32_xyzw_tfe_lwe(<8 x i32> inre
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v11, v3, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v11, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v0, v5
; GFX9-NEXT: v_mov_b32_e32 v1, v6
; GFX9-NEXT: v_mov_b32_e32 v2, v7
@@ -332,16 +332,16 @@ define amdgpu_ps <4 x float> @load_2darraymsaa_v4f32_xyzw_tfe_lwe(<8 x i32> inre
; GFX10-LABEL: load_2darraymsaa_v4f32_xyzw_tfe_lwe:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_mov_b32_e32 v5, 0
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX10-NEXT: s_mov_b32 s0, s2
; GFX10-NEXT: s_mov_b32 s1, s3
; GFX10-NEXT: v_mov_b32_e32 v6, v5
; GFX10-NEXT: v_mov_b32_e32 v7, v5
; GFX10-NEXT: v_mov_b32_e32 v8, v5
; GFX10-NEXT: v_mov_b32_e32 v9, v5
-; GFX10-NEXT: v_lshl_or_b32 v10, v1, 16, v0
-; GFX10-NEXT: v_lshl_or_b32 v11, v3, 16, v2
+; GFX10-NEXT: v_or_b32_sdwa v10, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v11, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: s_mov_b32 s2, s4
; GFX10-NEXT: s_mov_b32 s3, s5
; GFX10-NEXT: s_mov_b32 s4, s6
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.3d.a16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.3d.a16.ll
index 2c0ee1740375a..cd1b7227e427a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.3d.a16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.3d.a16.ll
@@ -9,10 +9,10 @@
define amdgpu_ps <4 x float> @load_3d_v4f32_xyzw(<8 x i32> inreg %rsrc, i16 %s, i16 %t, i16 %r) {
; GFX9-LABEL: load_3d_v4f32_xyzw:
; GFX9: ; %bb.0:
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e64 v1, 16, s0
; GFX9-NEXT: s_mov_b32 s1, s3
; GFX9-NEXT: s_mov_b32 s2, s4
; GFX9-NEXT: s_mov_b32 s3, s5
@@ -20,26 +20,26 @@ define amdgpu_ps <4 x float> @load_3d_v4f32_xyzw(<8 x i32> inreg %rsrc, i16 %s,
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: image_load v[0:3], v[0:1], s[0:7] dmask:0xf unorm a16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: load_3d_v4f32_xyzw:
; GFX10: ; %bb.0:
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX10-NEXT: s_mov_b32 s0, s2
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-NEXT: v_lshlrev_b32_e64 v3, 16, s0
; GFX10-NEXT: s_mov_b32 s1, s3
; GFX10-NEXT: s_mov_b32 s2, s4
-; GFX10-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX10-NEXT: v_lshl_or_b32 v1, s0, 16, v2
; GFX10-NEXT: s_mov_b32 s3, s5
+; GFX10-NEXT: v_or_b32_sdwa v1, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: s_mov_b32 s4, s6
; GFX10-NEXT: s_mov_b32 s5, s7
; GFX10-NEXT: s_mov_b32 s6, s8
; GFX10-NEXT: s_mov_b32 s7, s9
-; GFX10-NEXT: image_load v[0:3], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D unorm a16
+; GFX10-NEXT: image_load v[0:3], v[1:2], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D unorm a16
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
@@ -115,11 +115,11 @@ define amdgpu_ps <4 x float> @load_3d_v4f32_xyzw(<8 x i32> inreg %rsrc, i16 %s,
define amdgpu_ps <4 x float> @load_3d_v4f32_xyzw_tfe(<8 x i32> inreg %rsrc, ptr addrspace(1) inreg %out, i16 %s, i16 %t, i16 %r) {
; GFX9-LABEL: load_3d_v4f32_xyzw_tfe:
; GFX9: ; %bb.0:
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_mov_b32_e32 v5, 0
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_lshl_or_b32 v10, v1, 16, v0
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_mov_b32_e32 v5, 0
+; GFX9-NEXT: v_or_b32_sdwa v10, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e64 v0, 16, s0
; GFX9-NEXT: v_mov_b32_e32 v6, v5
; GFX9-NEXT: v_mov_b32_e32 v7, v5
; GFX9-NEXT: v_mov_b32_e32 v8, v5
@@ -131,7 +131,7 @@ define amdgpu_ps <4 x float> @load_3d_v4f32_xyzw_tfe(<8 x i32> inreg %rsrc, ptr
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v11, s0, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v11, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v0, v5
; GFX9-NEXT: v_mov_b32_e32 v1, v6
; GFX9-NEXT: v_mov_b32_e32 v2, v7
@@ -145,18 +145,18 @@ define amdgpu_ps <4 x float> @load_3d_v4f32_xyzw_tfe(<8 x i32> inreg %rsrc, ptr
;
; GFX10-LABEL: load_3d_v4f32_xyzw_tfe:
; GFX10: ; %bb.0:
-; GFX10-NEXT: v_mov_b32_e32 v5, 0
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX10-NEXT: s_mov_b32 s0, s2
+; GFX10-NEXT: v_mov_b32_e32 v5, 0
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-NEXT: v_lshlrev_b32_e64 v3, 16, s0
; GFX10-NEXT: s_mov_b32 s1, s3
+; GFX10-NEXT: s_mov_b32 s2, s4
; GFX10-NEXT: v_mov_b32_e32 v6, v5
; GFX10-NEXT: v_mov_b32_e32 v7, v5
; GFX10-NEXT: v_mov_b32_e32 v8, v5
; GFX10-NEXT: v_mov_b32_e32 v9, v5
-; GFX10-NEXT: v_lshl_or_b32 v10, v1, 16, v0
-; GFX10-NEXT: v_lshl_or_b32 v11, s0, 16, v2
-; GFX10-NEXT: s_mov_b32 s2, s4
+; GFX10-NEXT: v_or_b32_sdwa v10, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v11, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: s_mov_b32 s3, s5
; GFX10-NEXT: s_mov_b32 s4, s6
; GFX10-NEXT: s_mov_b32 s5, s7
@@ -283,11 +283,11 @@ define amdgpu_ps <4 x float> @load_3d_v4f32_xyzw_tfe(<8 x i32> inreg %rsrc, ptr
define amdgpu_ps <4 x float> @load_3d_v4f32_xyzw_tfe_lwe(<8 x i32> inreg %rsrc, ptr addrspace(1) inreg %out, i16 %s, i16 %t, i16 %r) {
; GFX9-LABEL: load_3d_v4f32_xyzw_tfe_lwe:
; GFX9: ; %bb.0:
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_mov_b32_e32 v5, 0
; GFX9-NEXT: s_mov_b32 s0, s2
-; GFX9-NEXT: v_lshl_or_b32 v10, v1, 16, v0
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_mov_b32_e32 v5, 0
+; GFX9-NEXT: v_or_b32_sdwa v10, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e64 v0, 16, s0
; GFX9-NEXT: v_mov_b32_e32 v6, v5
; GFX9-NEXT: v_mov_b32_e32 v7, v5
; GFX9-NEXT: v_mov_b32_e32 v8, v5
@@ -299,7 +299,7 @@ define amdgpu_ps <4 x float> @load_3d_v4f32_xyzw_tfe_lwe(<8 x i32> inreg %rsrc,
; GFX9-NEXT: s_mov_b32 s5, s7
; GFX9-NEXT: s_mov_b32 s6, s8
; GFX9-NEXT: s_mov_b32 s7, s9
-; GFX9-NEXT: v_lshl_or_b32 v11, s0, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v11, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v0, v5
; GFX9-NEXT: v_mov_b32_e32 v1, v6
; GFX9-NEXT: v_mov_b32_e32 v2, v7
@@ -313,18 +313,18 @@ define amdgpu_ps <4 x float> @load_3d_v4f32_xyzw_tfe_lwe(<8 x i32> inreg %rsrc,
;
; GFX10-LABEL: load_3d_v4f32_xyzw_tfe_lwe:
; GFX10: ; %bb.0:
-; GFX10-NEXT: v_mov_b32_e32 v5, 0
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX10-NEXT: s_mov_b32 s0, s2
+; GFX10-NEXT: v_mov_b32_e32 v5, 0
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-NEXT: v_lshlrev_b32_e64 v3, 16, s0
; GFX10-NEXT: s_mov_b32 s1, s3
+; GFX10-NEXT: s_mov_b32 s2, s4
; GFX10-NEXT: v_mov_b32_e32 v6, v5
; GFX10-NEXT: v_mov_b32_e32 v7, v5
; GFX10-NEXT: v_mov_b32_e32 v8, v5
; GFX10-NEXT: v_mov_b32_e32 v9, v5
-; GFX10-NEXT: v_lshl_or_b32 v10, v1, 16, v0
-; GFX10-NEXT: v_lshl_or_b32 v11, s0, 16, v2
-; GFX10-NEXT: s_mov_b32 s2, s4
+; GFX10-NEXT: v_or_b32_sdwa v10, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v11, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: s_mov_b32 s3, s5
; GFX10-NEXT: s_mov_b32 s4, s6
; GFX10-NEXT: s_mov_b32 s5, s7
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.sample.cd.g16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.sample.cd.g16.ll
index c417d090efdbe..5657cc175db68 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.sample.cd.g16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.sample.cd.g16.ll
@@ -4,10 +4,9 @@
define amdgpu_ps <4 x float> @sample_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %dsdh, half %dsdv, float %s) {
; GFX10-LABEL: sample_cd_1d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_lshl_or_b32 v0, s0, 16, v0
-; GFX10-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX10-NEXT: v_lshlrev_b32_e64 v3, 16, s0
+; GFX10-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: image_sample_cd_g16 v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
@@ -19,10 +18,10 @@ main_body:
define amdgpu_ps <4 x float> @sample_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %dsdh, half %dtdh, half %dsdv, half %dtdv, float %s, float %t) {
; GFX10-LABEL: sample_cd_2d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX10-NEXT: v_lshl_or_b32 v1, v3, 16, v2
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX10-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v1, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: image_sample_cd_g16 v[0:3], [v0, v1, v4, v5], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
@@ -34,10 +33,9 @@ main_body:
define amdgpu_ps <4 x float> @sample_c_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, half %dsdh, half %dsdv, float %s) {
; GFX10-LABEL: sample_c_cd_1d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10-NEXT: v_lshl_or_b32 v1, s0, 16, v1
-; GFX10-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX10-NEXT: v_lshlrev_b32_e64 v4, 16, s0
+; GFX10-NEXT: v_or_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: image_sample_c_cd_g16 v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
@@ -49,10 +47,10 @@ main_body:
define amdgpu_ps <4 x float> @sample_c_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, half %dsdh, half %dtdh, half %dsdv, half %dtdv, float %s, float %t) {
; GFX10-LABEL: sample_c_cd_2d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX10-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX10-NEXT: v_lshl_or_b32 v2, v4, 16, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX10-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v2, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: image_sample_c_cd_g16 v[0:3], [v0, v1, v2, v5, v6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
@@ -64,10 +62,9 @@ main_body:
define amdgpu_ps <4 x float> @sample_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %dsdh, half %dsdv, float %s, float %clamp) {
; GFX10-LABEL: sample_cd_cl_1d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_lshl_or_b32 v0, s0, 16, v0
-; GFX10-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX10-NEXT: v_lshlrev_b32_e64 v4, 16, s0
+; GFX10-NEXT: v_or_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: image_sample_cd_cl_g16 v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
@@ -79,10 +76,10 @@ main_body:
define amdgpu_ps <4 x float> @sample_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %dsdh, half %dtdh, half %dsdv, half %dtdv, float %s, float %t, float %clamp) {
; GFX10-LABEL: sample_cd_cl_2d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX10-NEXT: v_lshl_or_b32 v1, v3, 16, v2
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX10-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v1, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: image_sample_cd_cl_g16 v[0:3], [v0, v1, v4, v5, v6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
@@ -94,10 +91,9 @@ main_body:
define amdgpu_ps <4 x float> @sample_c_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, half %dsdh, half %dsdv, float %s, float %clamp) {
; GFX10-LABEL: sample_c_cd_cl_1d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10-NEXT: v_lshl_or_b32 v1, s0, 16, v1
-; GFX10-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX10-NEXT: v_lshlrev_b32_e64 v5, 16, s0
+; GFX10-NEXT: v_or_b32_sdwa v1, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v2, v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: image_sample_c_cd_cl_g16 v[0:3], v[0:4], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
@@ -110,11 +106,12 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX10-LABEL: sample_c_cd_cl_2d:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: v_mov_b32_e32 v8, v2
+; GFX10-NEXT: v_mov_b32_e32 v9, v3
; GFX10-NEXT: v_mov_b32_e32 v2, v0
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v1
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v3
-; GFX10-NEXT: v_lshl_or_b32 v3, v8, 16, v0
-; GFX10-NEXT: v_lshl_or_b32 v4, v4, 16, v1
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v8
+; GFX10-NEXT: v_or_b32_sdwa v4, v4, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v3, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: image_sample_c_cd_cl_g16 v[0:3], v[2:7], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.sample.g16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.sample.g16.ll
index 8f99ace9dd94c..096cb9ff625da 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.sample.g16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.sample.g16.ll
@@ -8,10 +8,9 @@
define amdgpu_ps <4 x float> @sample_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %dsdh, half %dsdv, float %s) {
; GFX10-LABEL: sample_d_1d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_lshl_or_b32 v0, s0, 16, v0
-; GFX10-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX10-NEXT: v_lshlrev_b32_e64 v3, 16, s0
+; GFX10-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: image_sample_d_g16 v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
@@ -55,10 +54,10 @@ main_body:
define amdgpu_ps <4 x float> @sample_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %dsdh, half %dtdh, half %dsdv, half %dtdv, float %s, float %t) {
; GFX10-LABEL: sample_d_2d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX10-NEXT: v_lshl_or_b32 v1, v3, 16, v2
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX10-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v1, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: image_sample_d_g16 v[0:3], [v0, v1, v4, v5], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
@@ -106,14 +105,15 @@ main_body:
define amdgpu_ps <4 x float> @sample_d_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %dsdh, half %dtdh, half %drdh, half %dsdv, half %dtdv, half %drdv, float %s, float %t, float %r) {
; GFX10-LABEL: sample_d_3d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: v_and_b32_e32 v9, 0xffff, v2
-; GFX10-NEXT: v_and_b32_e32 v10, 0xffff, v3
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX10-NEXT: v_lshl_or_b32 v2, v1, 16, v0
-; GFX10-NEXT: v_lshl_or_b32 v3, s0, 16, v9
-; GFX10-NEXT: v_lshl_or_b32 v4, v4, 16, v10
-; GFX10-NEXT: v_lshl_or_b32 v5, s0, 16, v5
+; GFX10-NEXT: v_mov_b32_e32 v9, v2
+; GFX10-NEXT: v_mov_b32_e32 v10, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-NEXT: v_lshlrev_b32_e64 v11, 16, s0
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX10-NEXT: v_or_b32_sdwa v2, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v3, v11, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v4, v4, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v5, v11, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: image_sample_d_g16 v[0:3], v[2:8], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_3D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
@@ -169,10 +169,9 @@ main_body:
define amdgpu_ps <4 x float> @sample_c_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, half %dsdh, half %dsdv, float %s) {
; GFX10-LABEL: sample_c_d_1d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10-NEXT: v_lshl_or_b32 v1, s0, 16, v1
-; GFX10-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX10-NEXT: v_lshlrev_b32_e64 v4, 16, s0
+; GFX10-NEXT: v_or_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: image_sample_c_d_g16 v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
@@ -216,10 +215,10 @@ main_body:
define amdgpu_ps <4 x float> @sample_c_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, half %dsdh, half %dtdh, half %dsdv, half %dtdv, float %s, float %t) {
; GFX10-LABEL: sample_c_d_2d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX10-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX10-NEXT: v_lshl_or_b32 v2, v4, 16, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX10-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v2, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: image_sample_c_d_g16 v[0:3], [v0, v1, v2, v5, v6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
@@ -267,10 +266,9 @@ main_body:
define amdgpu_ps <4 x float> @sample_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %dsdh, half %dsdv, float %s, float %clamp) {
; GFX10-LABEL: sample_d_cl_1d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_lshl_or_b32 v0, s0, 16, v0
-; GFX10-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX10-NEXT: v_lshlrev_b32_e64 v4, 16, s0
+; GFX10-NEXT: v_or_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: image_sample_d_cl_g16 v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
@@ -314,10 +312,10 @@ main_body:
define amdgpu_ps <4 x float> @sample_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %dsdh, half %dtdh, half %dsdv, half %dtdv, float %s, float %t, float %clamp) {
; GFX10-LABEL: sample_d_cl_2d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX10-NEXT: v_lshl_or_b32 v1, v3, 16, v2
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX10-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v1, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: image_sample_d_cl_g16 v[0:3], [v0, v1, v4, v5, v6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
@@ -365,10 +363,9 @@ main_body:
define amdgpu_ps <4 x float> @sample_c_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, half %dsdh, half %dsdv, float %s, float %clamp) {
; GFX10-LABEL: sample_c_d_cl_1d:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10-NEXT: v_lshl_or_b32 v1, s0, 16, v1
-; GFX10-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX10-NEXT: v_lshlrev_b32_e64 v5, 16, s0
+; GFX10-NEXT: v_or_b32_sdwa v1, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v2, v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: image_sample_c_d_cl_g16 v[0:3], v[0:4], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
@@ -413,11 +410,12 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX10-LABEL: sample_c_d_cl_2d:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: v_mov_b32_e32 v8, v2
+; GFX10-NEXT: v_mov_b32_e32 v9, v3
; GFX10-NEXT: v_mov_b32_e32 v2, v0
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v1
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v3
-; GFX10-NEXT: v_lshl_or_b32 v3, v8, 16, v0
-; GFX10-NEXT: v_lshl_or_b32 v4, v4, 16, v1
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v8
+; GFX10-NEXT: v_or_b32_sdwa v4, v4, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v3, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: image_sample_c_d_cl_g16 v[0:3], v[2:7], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
@@ -465,14 +463,15 @@ main_body:
define amdgpu_ps float @sample_c_d_o_2darray_V1(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, i32 %offset, float %zcompare, half %dsdh, half %dtdh, half %dsdv, half %dtdv, float %s, float %t, float %slice) {
; GFX10-LABEL: sample_c_d_o_2darray_V1:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_mov_b32_e32 v9, v2
-; GFX10-NEXT: v_mov_b32_e32 v10, v3
+; GFX10-NEXT: v_mov_b32_e32 v9, v3
+; GFX10-NEXT: v_mov_b32_e32 v10, v2
+; GFX10-NEXT: v_mov_b32_e32 v11, v4
; GFX10-NEXT: v_mov_b32_e32 v2, v0
; GFX10-NEXT: v_mov_b32_e32 v3, v1
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v4
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v9
-; GFX10-NEXT: v_lshl_or_b32 v5, v5, 16, v1
-; GFX10-NEXT: v_lshl_or_b32 v4, v10, 16, v0
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v9
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v5
+; GFX10-NEXT: v_or_b32_sdwa v4, v0, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v5, v1, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: image_sample_c_d_o_g16 v0, v[2:8], s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_2D_ARRAY
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
@@ -522,14 +521,15 @@ main_body:
define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, i32 %offset, float %zcompare, half %dsdh, half %dtdh, half %dsdv, half %dtdv, float %s, float %t, float %slice) {
; GFX10-LABEL: sample_c_d_o_2darray_V2:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: v_mov_b32_e32 v9, v2
-; GFX10-NEXT: v_mov_b32_e32 v10, v3
+; GFX10-NEXT: v_mov_b32_e32 v9, v3
+; GFX10-NEXT: v_mov_b32_e32 v10, v2
+; GFX10-NEXT: v_mov_b32_e32 v11, v4
; GFX10-NEXT: v_mov_b32_e32 v2, v0
; GFX10-NEXT: v_mov_b32_e32 v3, v1
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v4
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v9
-; GFX10-NEXT: v_lshl_or_b32 v5, v5, 16, v1
-; GFX10-NEXT: v_lshl_or_b32 v4, v10, 16, v0
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v9
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v5
+; GFX10-NEXT: v_or_b32_sdwa v4, v0, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v5, v1, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: image_sample_c_d_o_g16 v[0:1], v[2:8], s[0:7], s[8:11] dmask:0x6 dim:SQ_RSRC_IMG_2D_ARRAY
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/saddsat.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/saddsat.ll
index 72d1e056bade2..9a43ae888f4e5 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/saddsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/saddsat.ll
@@ -370,12 +370,11 @@ define i16 @v_saddsat_v2i8(i16 %lhs.arg, i16 %rhs.arg) {
; GFX9-LABEL: v_saddsat_v2i8:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_lshrrev_b32_e32 v2, 8, v0
-; GFX9-NEXT: v_lshrrev_b32_e32 v3, 8, v1
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_lshl_or_b32 v0, v2, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v3, 16, v1
+; GFX9-NEXT: v_mov_b32_e32 v2, 8
+; GFX9-NEXT: v_lshrrev_b32_sdwa v3, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_lshrrev_b32_sdwa v2, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
; GFX9-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
; GFX9-NEXT: v_pk_add_i16 v0, v0, v1 clamp
@@ -388,12 +387,11 @@ define i16 @v_saddsat_v2i8(i16 %lhs.arg, i16 %rhs.arg) {
; GFX10-LABEL: v_saddsat_v2i8:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_e32 v2, 8, v0
-; GFX10-NEXT: v_lshrrev_b32_e32 v3, 8, v1
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_lshl_or_b32 v0, v2, 16, v0
-; GFX10-NEXT: v_lshl_or_b32 v1, v3, 16, v1
+; GFX10-NEXT: v_mov_b32_e32 v2, 8
+; GFX10-NEXT: v_lshrrev_b32_sdwa v3, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_lshrrev_b32_sdwa v2, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
; GFX10-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
; GFX10-NEXT: v_pk_add_i16 v0, v0, v1 clamp
@@ -709,28 +707,26 @@ define i32 @v_saddsat_v4i8(i32 %lhs.arg, i32 %rhs.arg) {
; GFX9-LABEL: v_saddsat_v4i8:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_lshrrev_b32_e32 v3, 24, v0
-; GFX9-NEXT: v_lshrrev_b32_e32 v2, 8, v0
-; GFX9-NEXT: v_lshrrev_b32_e32 v4, 8, v1
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v0
-; GFX9-NEXT: v_alignbit_b32 v0, v3, v0, 16
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v1
-; GFX9-NEXT: v_lshrrev_b32_e32 v5, 24, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v2, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v3, v4, 16, v3
-; GFX9-NEXT: v_alignbit_b32 v1, v5, v1, 16
-; GFX9-NEXT: v_pk_lshlrev_b16 v2, 8, v2 op_sel_hi:[0,1]
+; GFX9-NEXT: v_mov_b32_e32 v2, 8
+; GFX9-NEXT: v_lshrrev_b32_sdwa v3, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 24, v0
+; GFX9-NEXT: v_lshrrev_b32_sdwa v5, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_lshrrev_b32_e32 v6, 24, v1
+; GFX9-NEXT: v_or_b32_sdwa v3, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_alignbit_b32 v0, v4, v0, 16
+; GFX9-NEXT: v_or_b32_sdwa v4, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_alignbit_b32 v1, v6, v1, 16
; GFX9-NEXT: v_pk_lshlrev_b16 v3, 8, v3 op_sel_hi:[0,1]
+; GFX9-NEXT: v_pk_lshlrev_b16 v4, 8, v4 op_sel_hi:[0,1]
; GFX9-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
; GFX9-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
-; GFX9-NEXT: v_pk_add_i16 v2, v2, v3 clamp
+; GFX9-NEXT: v_pk_add_i16 v3, v3, v4 clamp
; GFX9-NEXT: v_pk_add_i16 v0, v0, v1 clamp
-; GFX9-NEXT: v_pk_ashrrev_i16 v1, 8, v2 op_sel_hi:[0,1]
-; GFX9-NEXT: v_mov_b32_e32 v3, 8
+; GFX9-NEXT: v_pk_ashrrev_i16 v1, 8, v3 op_sel_hi:[0,1]
; GFX9-NEXT: v_pk_ashrrev_i16 v0, 8, v0 op_sel_hi:[0,1]
-; GFX9-NEXT: v_mov_b32_e32 v2, 0xff
-; GFX9-NEXT: v_lshlrev_b32_sdwa v3, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
-; GFX9-NEXT: v_and_or_b32 v1, v1, v2, v3
+; GFX9-NEXT: v_mov_b32_e32 v3, 0xff
+; GFX9-NEXT: v_lshlrev_b32_sdwa v2, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
+; GFX9-NEXT: v_and_or_b32 v1, v1, v3, v2
; GFX9-NEXT: v_and_b32_e32 v2, 0xff, v0
; GFX9-NEXT: v_mov_b32_e32 v3, 24
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
@@ -741,30 +737,28 @@ define i32 @v_saddsat_v4i8(i32 %lhs.arg, i32 %rhs.arg) {
; GFX10-LABEL: v_saddsat_v4i8:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_e32 v2, 8, v0
-; GFX10-NEXT: v_lshrrev_b32_e32 v3, 24, v0
-; GFX10-NEXT: v_and_b32_e32 v4, 0xffff, v0
-; GFX10-NEXT: v_lshrrev_b32_e32 v5, 8, v1
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffff, v1
-; GFX10-NEXT: v_lshrrev_b32_e32 v7, 24, v1
-; GFX10-NEXT: v_alignbit_b32 v0, v3, v0, 16
-; GFX10-NEXT: v_lshl_or_b32 v2, v2, 16, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, 24
-; GFX10-NEXT: v_lshl_or_b32 v3, v5, 16, v6
-; GFX10-NEXT: v_alignbit_b32 v1, v7, v1, 16
-; GFX10-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
-; GFX10-NEXT: v_pk_lshlrev_b16 v2, 8, v2 op_sel_hi:[0,1]
+; GFX10-NEXT: v_mov_b32_e32 v2, 8
+; GFX10-NEXT: v_lshrrev_b32_e32 v4, 24, v0
+; GFX10-NEXT: v_lshrrev_b32_e32 v6, 24, v1
+; GFX10-NEXT: v_lshrrev_b32_sdwa v3, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_lshrrev_b32_sdwa v5, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v3, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_alignbit_b32 v0, v4, v0, 16
+; GFX10-NEXT: v_or_b32_sdwa v4, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_alignbit_b32 v1, v6, v1, 16
; GFX10-NEXT: v_pk_lshlrev_b16 v3, 8, v3 op_sel_hi:[0,1]
+; GFX10-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX10-NEXT: v_pk_lshlrev_b16 v4, 8, v4 op_sel_hi:[0,1]
; GFX10-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
-; GFX10-NEXT: v_pk_add_i16 v2, v2, v3 clamp
+; GFX10-NEXT: v_pk_add_i16 v3, v3, v4 clamp
; GFX10-NEXT: v_pk_add_i16 v0, v0, v1 clamp
-; GFX10-NEXT: v_mov_b32_e32 v1, 8
-; GFX10-NEXT: v_pk_ashrrev_i16 v2, 8, v2 op_sel_hi:[0,1]
+; GFX10-NEXT: v_mov_b32_e32 v4, 24
+; GFX10-NEXT: v_pk_ashrrev_i16 v1, 8, v3 op_sel_hi:[0,1]
; GFX10-NEXT: v_pk_ashrrev_i16 v0, 8, v0 op_sel_hi:[0,1]
-; GFX10-NEXT: v_lshlrev_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
+; GFX10-NEXT: v_lshlrev_b32_sdwa v2, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
; GFX10-NEXT: v_and_b32_e32 v3, 0xff, v0
; GFX10-NEXT: v_lshlrev_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
-; GFX10-NEXT: v_and_or_b32 v1, 0xff, v2, v1
+; GFX10-NEXT: v_and_or_b32 v1, 0xff, v1, v2
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
; GFX10-NEXT: v_or3_b32 v0, v1, v2, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/select-to-fmin-fmax.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/select-to-fmin-fmax.ll
index 66e6e1d75015f..ec149479868b5 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/select-to-fmin-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/select-to-fmin-fmax.ll
@@ -46,20 +46,20 @@ define <4 x half> @test_v4s16(<4 x half> %a) #0 {
; GCN: ; %bb.0: ; %entry
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mov_b32_e32 v4, 0
-; GCN-NEXT: v_cmp_gt_f16_e32 vcc, 0, v0
; GCN-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GCN-NEXT: v_cndmask_b32_e64 v5, v0, 0, vcc
+; GCN-NEXT: v_cmp_gt_f16_e32 vcc, 0, v0
; GCN-NEXT: v_cmp_lt_f16_sdwa s[4:5], v0, v4 src0_sel:WORD_1 src1_sel:DWORD
-; GCN-NEXT: v_cmp_gt_f16_e32 vcc, 0, v1
; GCN-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GCN-NEXT: v_cndmask_b32_e64 v5, v0, 0, vcc
; GCN-NEXT: v_cndmask_b32_e64 v0, v2, 0, s[4:5]
-; GCN-NEXT: v_cndmask_b32_e64 v2, v1, 0, vcc
+; GCN-NEXT: v_cmp_gt_f16_e32 vcc, 0, v1
; GCN-NEXT: v_cmp_lt_f16_sdwa s[4:5], v1, v4 src0_sel:WORD_1 src1_sel:DWORD
+; GCN-NEXT: v_cndmask_b32_e64 v2, v1, 0, vcc
; GCN-NEXT: v_cndmask_b32_e64 v1, v3, 0, s[4:5]
-; GCN-NEXT: v_and_b32_e32 v3, 0xffff, v5
-; GCN-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GCN-NEXT: v_lshl_or_b32 v0, v0, 16, v3
-; GCN-NEXT: v_lshl_or_b32 v1, v1, 16, v2
+; GCN-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GCN-NEXT: v_or_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GCN-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GCN-NEXT: s_setpc_b64 s[30:31]
entry:
%fcmp = fcmp olt <4 x half> %a, zeroinitializer
@@ -72,34 +72,34 @@ define <8 x half> @test_v8s16(<8 x half> %a) #0 {
; GCN: ; %bb.0: ; %entry
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mov_b32_e32 v8, 0
-; GCN-NEXT: v_cmp_gt_f16_e32 vcc, 0, v0
; GCN-NEXT: v_lshrrev_b32_e32 v4, 16, v0
-; GCN-NEXT: v_cndmask_b32_e64 v9, v0, 0, vcc
+; GCN-NEXT: v_cmp_gt_f16_e32 vcc, 0, v0
; GCN-NEXT: v_cmp_lt_f16_sdwa s[4:5], v0, v8 src0_sel:WORD_1 src1_sel:DWORD
-; GCN-NEXT: v_cmp_gt_f16_e32 vcc, 0, v1
; GCN-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GCN-NEXT: v_cndmask_b32_e64 v9, v0, 0, vcc
; GCN-NEXT: v_cndmask_b32_e64 v0, v4, 0, s[4:5]
-; GCN-NEXT: v_cndmask_b32_e64 v4, v1, 0, vcc
+; GCN-NEXT: v_cmp_gt_f16_e32 vcc, 0, v1
; GCN-NEXT: v_cmp_lt_f16_sdwa s[4:5], v1, v8 src0_sel:WORD_1 src1_sel:DWORD
-; GCN-NEXT: v_cmp_gt_f16_e32 vcc, 0, v2
; GCN-NEXT: v_lshrrev_b32_e32 v6, 16, v2
+; GCN-NEXT: v_cndmask_b32_e64 v4, v1, 0, vcc
; GCN-NEXT: v_cndmask_b32_e64 v1, v5, 0, s[4:5]
-; GCN-NEXT: v_cndmask_b32_e64 v5, v2, 0, vcc
+; GCN-NEXT: v_cmp_gt_f16_e32 vcc, 0, v2
; GCN-NEXT: v_cmp_lt_f16_sdwa s[4:5], v2, v8 src0_sel:WORD_1 src1_sel:DWORD
-; GCN-NEXT: v_cmp_gt_f16_e32 vcc, 0, v3
-; GCN-NEXT: v_and_b32_e32 v4, 0xffff, v4
; GCN-NEXT: v_lshrrev_b32_e32 v7, 16, v3
+; GCN-NEXT: v_cndmask_b32_e64 v5, v2, 0, vcc
; GCN-NEXT: v_cndmask_b32_e64 v2, v6, 0, s[4:5]
-; GCN-NEXT: v_cndmask_b32_e64 v6, v3, 0, vcc
+; GCN-NEXT: v_cmp_gt_f16_e32 vcc, 0, v3
; GCN-NEXT: v_cmp_lt_f16_sdwa s[4:5], v3, v8 src0_sel:WORD_1 src1_sel:DWORD
-; GCN-NEXT: v_lshl_or_b32 v1, v1, 16, v4
-; GCN-NEXT: v_and_b32_e32 v4, 0xffff, v5
+; GCN-NEXT: v_cndmask_b32_e64 v6, v3, 0, vcc
; GCN-NEXT: v_cndmask_b32_e64 v3, v7, 0, s[4:5]
-; GCN-NEXT: v_and_b32_e32 v7, 0xffff, v9
-; GCN-NEXT: v_lshl_or_b32 v2, v2, 16, v4
-; GCN-NEXT: v_and_b32_e32 v4, 0xffff, v6
-; GCN-NEXT: v_lshl_or_b32 v0, v0, 16, v7
-; GCN-NEXT: v_lshl_or_b32 v3, v3, 16, v4
+; GCN-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GCN-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GCN-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GCN-NEXT: v_or_b32_sdwa v0, v0, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GCN-NEXT: v_or_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GCN-NEXT: v_or_b32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GCN-NEXT: v_or_b32_sdwa v3, v3, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GCN-NEXT: s_setpc_b64 s[30:31]
entry:
%fcmp = fcmp olt <8 x half> %a, zeroinitializer
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll
index 83d3cde9879f4..f5056d6247023 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll
@@ -370,12 +370,11 @@ define i16 @v_ssubsat_v2i8(i16 %lhs.arg, i16 %rhs.arg) {
; GFX9-LABEL: v_ssubsat_v2i8:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_lshrrev_b32_e32 v2, 8, v0
-; GFX9-NEXT: v_lshrrev_b32_e32 v3, 8, v1
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_lshl_or_b32 v0, v2, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v3, 16, v1
+; GFX9-NEXT: v_mov_b32_e32 v2, 8
+; GFX9-NEXT: v_lshrrev_b32_sdwa v3, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_lshrrev_b32_sdwa v2, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
; GFX9-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
; GFX9-NEXT: v_pk_sub_i16 v0, v0, v1 clamp
@@ -388,12 +387,11 @@ define i16 @v_ssubsat_v2i8(i16 %lhs.arg, i16 %rhs.arg) {
; GFX10-LABEL: v_ssubsat_v2i8:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_e32 v2, 8, v0
-; GFX10-NEXT: v_lshrrev_b32_e32 v3, 8, v1
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_lshl_or_b32 v0, v2, 16, v0
-; GFX10-NEXT: v_lshl_or_b32 v1, v3, 16, v1
+; GFX10-NEXT: v_mov_b32_e32 v2, 8
+; GFX10-NEXT: v_lshrrev_b32_sdwa v3, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_lshrrev_b32_sdwa v2, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
; GFX10-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
; GFX10-NEXT: v_pk_sub_i16 v0, v0, v1 clamp
@@ -709,28 +707,26 @@ define i32 @v_ssubsat_v4i8(i32 %lhs.arg, i32 %rhs.arg) {
; GFX9-LABEL: v_ssubsat_v4i8:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_lshrrev_b32_e32 v3, 24, v0
-; GFX9-NEXT: v_lshrrev_b32_e32 v2, 8, v0
-; GFX9-NEXT: v_lshrrev_b32_e32 v4, 8, v1
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v0
-; GFX9-NEXT: v_alignbit_b32 v0, v3, v0, 16
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v1
-; GFX9-NEXT: v_lshrrev_b32_e32 v5, 24, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v2, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v3, v4, 16, v3
-; GFX9-NEXT: v_alignbit_b32 v1, v5, v1, 16
-; GFX9-NEXT: v_pk_lshlrev_b16 v2, 8, v2 op_sel_hi:[0,1]
+; GFX9-NEXT: v_mov_b32_e32 v2, 8
+; GFX9-NEXT: v_lshrrev_b32_sdwa v3, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 24, v0
+; GFX9-NEXT: v_lshrrev_b32_sdwa v5, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_lshrrev_b32_e32 v6, 24, v1
+; GFX9-NEXT: v_or_b32_sdwa v3, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_alignbit_b32 v0, v4, v0, 16
+; GFX9-NEXT: v_or_b32_sdwa v4, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_alignbit_b32 v1, v6, v1, 16
; GFX9-NEXT: v_pk_lshlrev_b16 v3, 8, v3 op_sel_hi:[0,1]
+; GFX9-NEXT: v_pk_lshlrev_b16 v4, 8, v4 op_sel_hi:[0,1]
; GFX9-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
; GFX9-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
-; GFX9-NEXT: v_pk_sub_i16 v2, v2, v3 clamp
+; GFX9-NEXT: v_pk_sub_i16 v3, v3, v4 clamp
; GFX9-NEXT: v_pk_sub_i16 v0, v0, v1 clamp
-; GFX9-NEXT: v_pk_ashrrev_i16 v1, 8, v2 op_sel_hi:[0,1]
-; GFX9-NEXT: v_mov_b32_e32 v3, 8
+; GFX9-NEXT: v_pk_ashrrev_i16 v1, 8, v3 op_sel_hi:[0,1]
; GFX9-NEXT: v_pk_ashrrev_i16 v0, 8, v0 op_sel_hi:[0,1]
-; GFX9-NEXT: v_mov_b32_e32 v2, 0xff
-; GFX9-NEXT: v_lshlrev_b32_sdwa v3, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
-; GFX9-NEXT: v_and_or_b32 v1, v1, v2, v3
+; GFX9-NEXT: v_mov_b32_e32 v3, 0xff
+; GFX9-NEXT: v_lshlrev_b32_sdwa v2, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
+; GFX9-NEXT: v_and_or_b32 v1, v1, v3, v2
; GFX9-NEXT: v_and_b32_e32 v2, 0xff, v0
; GFX9-NEXT: v_mov_b32_e32 v3, 24
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
@@ -741,30 +737,28 @@ define i32 @v_ssubsat_v4i8(i32 %lhs.arg, i32 %rhs.arg) {
; GFX10-LABEL: v_ssubsat_v4i8:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_e32 v2, 8, v0
-; GFX10-NEXT: v_lshrrev_b32_e32 v3, 24, v0
-; GFX10-NEXT: v_and_b32_e32 v4, 0xffff, v0
-; GFX10-NEXT: v_lshrrev_b32_e32 v5, 8, v1
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffff, v1
-; GFX10-NEXT: v_lshrrev_b32_e32 v7, 24, v1
-; GFX10-NEXT: v_alignbit_b32 v0, v3, v0, 16
-; GFX10-NEXT: v_lshl_or_b32 v2, v2, 16, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, 24
-; GFX10-NEXT: v_lshl_or_b32 v3, v5, 16, v6
-; GFX10-NEXT: v_alignbit_b32 v1, v7, v1, 16
-; GFX10-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
-; GFX10-NEXT: v_pk_lshlrev_b16 v2, 8, v2 op_sel_hi:[0,1]
+; GFX10-NEXT: v_mov_b32_e32 v2, 8
+; GFX10-NEXT: v_lshrrev_b32_e32 v4, 24, v0
+; GFX10-NEXT: v_lshrrev_b32_e32 v6, 24, v1
+; GFX10-NEXT: v_lshrrev_b32_sdwa v3, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_lshrrev_b32_sdwa v5, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v3, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_alignbit_b32 v0, v4, v0, 16
+; GFX10-NEXT: v_or_b32_sdwa v4, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_alignbit_b32 v1, v6, v1, 16
; GFX10-NEXT: v_pk_lshlrev_b16 v3, 8, v3 op_sel_hi:[0,1]
+; GFX10-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX10-NEXT: v_pk_lshlrev_b16 v4, 8, v4 op_sel_hi:[0,1]
; GFX10-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
-; GFX10-NEXT: v_pk_sub_i16 v2, v2, v3 clamp
+; GFX10-NEXT: v_pk_sub_i16 v3, v3, v4 clamp
; GFX10-NEXT: v_pk_sub_i16 v0, v0, v1 clamp
-; GFX10-NEXT: v_mov_b32_e32 v1, 8
-; GFX10-NEXT: v_pk_ashrrev_i16 v2, 8, v2 op_sel_hi:[0,1]
+; GFX10-NEXT: v_mov_b32_e32 v4, 24
+; GFX10-NEXT: v_pk_ashrrev_i16 v1, 8, v3 op_sel_hi:[0,1]
; GFX10-NEXT: v_pk_ashrrev_i16 v0, 8, v0 op_sel_hi:[0,1]
-; GFX10-NEXT: v_lshlrev_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
+; GFX10-NEXT: v_lshlrev_b32_sdwa v2, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
; GFX10-NEXT: v_and_b32_e32 v3, 0xff, v0
; GFX10-NEXT: v_lshlrev_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
-; GFX10-NEXT: v_and_or_b32 v1, 0xff, v2, v1
+; GFX10-NEXT: v_and_or_b32 v1, 0xff, v1, v2
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
; GFX10-NEXT: v_or3_b32 v0, v1, v2, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll
index 54f7339ab4c32..749fe014ff706 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll
@@ -296,12 +296,11 @@ define i16 @v_uaddsat_v2i8(i16 %lhs.arg, i16 %rhs.arg) {
; GFX9-LABEL: v_uaddsat_v2i8:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_lshrrev_b32_e32 v2, 8, v0
-; GFX9-NEXT: v_lshrrev_b32_e32 v3, 8, v1
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_lshl_or_b32 v0, v2, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v3, 16, v1
+; GFX9-NEXT: v_mov_b32_e32 v2, 8
+; GFX9-NEXT: v_lshrrev_b32_sdwa v3, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_lshrrev_b32_sdwa v2, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
; GFX9-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
; GFX9-NEXT: v_pk_add_u16 v0, v0, v1 clamp
@@ -314,12 +313,11 @@ define i16 @v_uaddsat_v2i8(i16 %lhs.arg, i16 %rhs.arg) {
; GFX10-LABEL: v_uaddsat_v2i8:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_e32 v2, 8, v0
-; GFX10-NEXT: v_lshrrev_b32_e32 v3, 8, v1
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_lshl_or_b32 v0, v2, 16, v0
-; GFX10-NEXT: v_lshl_or_b32 v1, v3, 16, v1
+; GFX10-NEXT: v_mov_b32_e32 v2, 8
+; GFX10-NEXT: v_lshrrev_b32_sdwa v3, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_lshrrev_b32_sdwa v2, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
; GFX10-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
; GFX10-NEXT: v_pk_add_u16 v0, v0, v1 clamp
@@ -604,28 +602,26 @@ define i32 @v_uaddsat_v4i8(i32 %lhs.arg, i32 %rhs.arg) {
; GFX9-LABEL: v_uaddsat_v4i8:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_lshrrev_b32_e32 v3, 24, v0
-; GFX9-NEXT: v_lshrrev_b32_e32 v2, 8, v0
-; GFX9-NEXT: v_lshrrev_b32_e32 v4, 8, v1
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v0
-; GFX9-NEXT: v_alignbit_b32 v0, v3, v0, 16
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v1
-; GFX9-NEXT: v_lshrrev_b32_e32 v5, 24, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v2, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v3, v4, 16, v3
-; GFX9-NEXT: v_alignbit_b32 v1, v5, v1, 16
-; GFX9-NEXT: v_pk_lshlrev_b16 v2, 8, v2 op_sel_hi:[0,1]
+; GFX9-NEXT: v_mov_b32_e32 v2, 8
+; GFX9-NEXT: v_lshrrev_b32_sdwa v3, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 24, v0
+; GFX9-NEXT: v_lshrrev_b32_sdwa v5, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_lshrrev_b32_e32 v6, 24, v1
+; GFX9-NEXT: v_or_b32_sdwa v3, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_alignbit_b32 v0, v4, v0, 16
+; GFX9-NEXT: v_or_b32_sdwa v4, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_alignbit_b32 v1, v6, v1, 16
; GFX9-NEXT: v_pk_lshlrev_b16 v3, 8, v3 op_sel_hi:[0,1]
+; GFX9-NEXT: v_pk_lshlrev_b16 v4, 8, v4 op_sel_hi:[0,1]
; GFX9-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
; GFX9-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
-; GFX9-NEXT: v_pk_add_u16 v2, v2, v3 clamp
+; GFX9-NEXT: v_pk_add_u16 v3, v3, v4 clamp
; GFX9-NEXT: v_pk_add_u16 v0, v0, v1 clamp
-; GFX9-NEXT: v_pk_lshrrev_b16 v1, 8, v2 op_sel_hi:[0,1]
-; GFX9-NEXT: v_mov_b32_e32 v3, 8
+; GFX9-NEXT: v_pk_lshrrev_b16 v1, 8, v3 op_sel_hi:[0,1]
; GFX9-NEXT: v_pk_lshrrev_b16 v0, 8, v0 op_sel_hi:[0,1]
-; GFX9-NEXT: v_mov_b32_e32 v2, 0xff
-; GFX9-NEXT: v_lshlrev_b32_sdwa v3, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
-; GFX9-NEXT: v_and_or_b32 v1, v1, v2, v3
+; GFX9-NEXT: v_mov_b32_e32 v3, 0xff
+; GFX9-NEXT: v_lshlrev_b32_sdwa v2, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
+; GFX9-NEXT: v_and_or_b32 v1, v1, v3, v2
; GFX9-NEXT: v_and_b32_e32 v2, 0xff, v0
; GFX9-NEXT: v_mov_b32_e32 v3, 24
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
@@ -636,30 +632,28 @@ define i32 @v_uaddsat_v4i8(i32 %lhs.arg, i32 %rhs.arg) {
; GFX10-LABEL: v_uaddsat_v4i8:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_e32 v2, 8, v0
-; GFX10-NEXT: v_lshrrev_b32_e32 v3, 24, v0
-; GFX10-NEXT: v_and_b32_e32 v4, 0xffff, v0
-; GFX10-NEXT: v_lshrrev_b32_e32 v5, 8, v1
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffff, v1
-; GFX10-NEXT: v_lshrrev_b32_e32 v7, 24, v1
-; GFX10-NEXT: v_alignbit_b32 v0, v3, v0, 16
-; GFX10-NEXT: v_lshl_or_b32 v2, v2, 16, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, 24
-; GFX10-NEXT: v_lshl_or_b32 v3, v5, 16, v6
-; GFX10-NEXT: v_alignbit_b32 v1, v7, v1, 16
-; GFX10-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
-; GFX10-NEXT: v_pk_lshlrev_b16 v2, 8, v2 op_sel_hi:[0,1]
+; GFX10-NEXT: v_mov_b32_e32 v2, 8
+; GFX10-NEXT: v_lshrrev_b32_e32 v4, 24, v0
+; GFX10-NEXT: v_lshrrev_b32_e32 v6, 24, v1
+; GFX10-NEXT: v_lshrrev_b32_sdwa v3, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_lshrrev_b32_sdwa v5, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v3, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_alignbit_b32 v0, v4, v0, 16
+; GFX10-NEXT: v_or_b32_sdwa v4, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_alignbit_b32 v1, v6, v1, 16
; GFX10-NEXT: v_pk_lshlrev_b16 v3, 8, v3 op_sel_hi:[0,1]
+; GFX10-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX10-NEXT: v_pk_lshlrev_b16 v4, 8, v4 op_sel_hi:[0,1]
; GFX10-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
-; GFX10-NEXT: v_pk_add_u16 v2, v2, v3 clamp
+; GFX10-NEXT: v_pk_add_u16 v3, v3, v4 clamp
; GFX10-NEXT: v_pk_add_u16 v0, v0, v1 clamp
-; GFX10-NEXT: v_mov_b32_e32 v1, 8
-; GFX10-NEXT: v_pk_lshrrev_b16 v2, 8, v2 op_sel_hi:[0,1]
+; GFX10-NEXT: v_mov_b32_e32 v4, 24
+; GFX10-NEXT: v_pk_lshrrev_b16 v1, 8, v3 op_sel_hi:[0,1]
; GFX10-NEXT: v_pk_lshrrev_b16 v0, 8, v0 op_sel_hi:[0,1]
-; GFX10-NEXT: v_lshlrev_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
+; GFX10-NEXT: v_lshlrev_b32_sdwa v2, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
; GFX10-NEXT: v_and_b32_e32 v3, 0xff, v0
; GFX10-NEXT: v_lshlrev_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
-; GFX10-NEXT: v_and_or_b32 v1, 0xff, v2, v1
+; GFX10-NEXT: v_and_or_b32 v1, 0xff, v1, v2
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
; GFX10-NEXT: v_or3_b32 v0, v1, v2, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/usubsat.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/usubsat.ll
index fc4b7bd4c9cfd..2e76abae2ff4f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/usubsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/usubsat.ll
@@ -290,12 +290,11 @@ define i16 @v_usubsat_v2i8(i16 %lhs.arg, i16 %rhs.arg) {
; GFX9-LABEL: v_usubsat_v2i8:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_lshrrev_b32_e32 v2, 8, v0
-; GFX9-NEXT: v_lshrrev_b32_e32 v3, 8, v1
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_lshl_or_b32 v0, v2, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v3, 16, v1
+; GFX9-NEXT: v_mov_b32_e32 v2, 8
+; GFX9-NEXT: v_lshrrev_b32_sdwa v3, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_lshrrev_b32_sdwa v2, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
; GFX9-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
; GFX9-NEXT: v_pk_sub_u16 v0, v0, v1 clamp
@@ -308,12 +307,11 @@ define i16 @v_usubsat_v2i8(i16 %lhs.arg, i16 %rhs.arg) {
; GFX10-LABEL: v_usubsat_v2i8:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_e32 v2, 8, v0
-; GFX10-NEXT: v_lshrrev_b32_e32 v3, 8, v1
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_lshl_or_b32 v0, v2, 16, v0
-; GFX10-NEXT: v_lshl_or_b32 v1, v3, 16, v1
+; GFX10-NEXT: v_mov_b32_e32 v2, 8
+; GFX10-NEXT: v_lshrrev_b32_sdwa v3, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_lshrrev_b32_sdwa v2, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
; GFX10-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
; GFX10-NEXT: v_pk_sub_u16 v0, v0, v1 clamp
@@ -592,28 +590,26 @@ define i32 @v_usubsat_v4i8(i32 %lhs.arg, i32 %rhs.arg) {
; GFX9-LABEL: v_usubsat_v4i8:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_lshrrev_b32_e32 v3, 24, v0
-; GFX9-NEXT: v_lshrrev_b32_e32 v2, 8, v0
-; GFX9-NEXT: v_lshrrev_b32_e32 v4, 8, v1
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v0
-; GFX9-NEXT: v_alignbit_b32 v0, v3, v0, 16
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v1
-; GFX9-NEXT: v_lshrrev_b32_e32 v5, 24, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v2, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v3, v4, 16, v3
-; GFX9-NEXT: v_alignbit_b32 v1, v5, v1, 16
-; GFX9-NEXT: v_pk_lshlrev_b16 v2, 8, v2 op_sel_hi:[0,1]
+; GFX9-NEXT: v_mov_b32_e32 v2, 8
+; GFX9-NEXT: v_lshrrev_b32_sdwa v3, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 24, v0
+; GFX9-NEXT: v_lshrrev_b32_sdwa v5, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_lshrrev_b32_e32 v6, 24, v1
+; GFX9-NEXT: v_or_b32_sdwa v3, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_alignbit_b32 v0, v4, v0, 16
+; GFX9-NEXT: v_or_b32_sdwa v4, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_alignbit_b32 v1, v6, v1, 16
; GFX9-NEXT: v_pk_lshlrev_b16 v3, 8, v3 op_sel_hi:[0,1]
+; GFX9-NEXT: v_pk_lshlrev_b16 v4, 8, v4 op_sel_hi:[0,1]
; GFX9-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
; GFX9-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
-; GFX9-NEXT: v_pk_sub_u16 v2, v2, v3 clamp
+; GFX9-NEXT: v_pk_sub_u16 v3, v3, v4 clamp
; GFX9-NEXT: v_pk_sub_u16 v0, v0, v1 clamp
-; GFX9-NEXT: v_pk_lshrrev_b16 v1, 8, v2 op_sel_hi:[0,1]
-; GFX9-NEXT: v_mov_b32_e32 v3, 8
+; GFX9-NEXT: v_pk_lshrrev_b16 v1, 8, v3 op_sel_hi:[0,1]
; GFX9-NEXT: v_pk_lshrrev_b16 v0, 8, v0 op_sel_hi:[0,1]
-; GFX9-NEXT: v_mov_b32_e32 v2, 0xff
-; GFX9-NEXT: v_lshlrev_b32_sdwa v3, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
-; GFX9-NEXT: v_and_or_b32 v1, v1, v2, v3
+; GFX9-NEXT: v_mov_b32_e32 v3, 0xff
+; GFX9-NEXT: v_lshlrev_b32_sdwa v2, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
+; GFX9-NEXT: v_and_or_b32 v1, v1, v3, v2
; GFX9-NEXT: v_and_b32_e32 v2, 0xff, v0
; GFX9-NEXT: v_mov_b32_e32 v3, 24
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
@@ -624,30 +620,28 @@ define i32 @v_usubsat_v4i8(i32 %lhs.arg, i32 %rhs.arg) {
; GFX10-LABEL: v_usubsat_v4i8:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_e32 v2, 8, v0
-; GFX10-NEXT: v_lshrrev_b32_e32 v3, 24, v0
-; GFX10-NEXT: v_and_b32_e32 v4, 0xffff, v0
-; GFX10-NEXT: v_lshrrev_b32_e32 v5, 8, v1
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffff, v1
-; GFX10-NEXT: v_lshrrev_b32_e32 v7, 24, v1
-; GFX10-NEXT: v_alignbit_b32 v0, v3, v0, 16
-; GFX10-NEXT: v_lshl_or_b32 v2, v2, 16, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, 24
-; GFX10-NEXT: v_lshl_or_b32 v3, v5, 16, v6
-; GFX10-NEXT: v_alignbit_b32 v1, v7, v1, 16
-; GFX10-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
-; GFX10-NEXT: v_pk_lshlrev_b16 v2, 8, v2 op_sel_hi:[0,1]
+; GFX10-NEXT: v_mov_b32_e32 v2, 8
+; GFX10-NEXT: v_lshrrev_b32_e32 v4, 24, v0
+; GFX10-NEXT: v_lshrrev_b32_e32 v6, 24, v1
+; GFX10-NEXT: v_lshrrev_b32_sdwa v3, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_lshrrev_b32_sdwa v5, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v3, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_alignbit_b32 v0, v4, v0, 16
+; GFX10-NEXT: v_or_b32_sdwa v4, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_alignbit_b32 v1, v6, v1, 16
; GFX10-NEXT: v_pk_lshlrev_b16 v3, 8, v3 op_sel_hi:[0,1]
+; GFX10-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX10-NEXT: v_pk_lshlrev_b16 v4, 8, v4 op_sel_hi:[0,1]
; GFX10-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
-; GFX10-NEXT: v_pk_sub_u16 v2, v2, v3 clamp
+; GFX10-NEXT: v_pk_sub_u16 v3, v3, v4 clamp
; GFX10-NEXT: v_pk_sub_u16 v0, v0, v1 clamp
-; GFX10-NEXT: v_mov_b32_e32 v1, 8
-; GFX10-NEXT: v_pk_lshrrev_b16 v2, 8, v2 op_sel_hi:[0,1]
+; GFX10-NEXT: v_mov_b32_e32 v4, 24
+; GFX10-NEXT: v_pk_lshrrev_b16 v1, 8, v3 op_sel_hi:[0,1]
; GFX10-NEXT: v_pk_lshrrev_b16 v0, 8, v0 op_sel_hi:[0,1]
-; GFX10-NEXT: v_lshlrev_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
+; GFX10-NEXT: v_lshlrev_b32_sdwa v2, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
; GFX10-NEXT: v_and_b32_e32 v3, 0xff, v0
; GFX10-NEXT: v_lshlrev_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
-; GFX10-NEXT: v_and_or_b32 v1, 0xff, v2, v1
+; GFX10-NEXT: v_and_or_b32 v1, 0xff, v1, v2
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
; GFX10-NEXT: v_or3_b32 v0, v1, v2, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/vni8-across-blocks.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/vni8-across-blocks.ll
index c819ef7a7f6dd..2530e82e6e015 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/vni8-across-blocks.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/vni8-across-blocks.ll
@@ -31,12 +31,12 @@ define amdgpu_kernel void @v3i8_liveout(ptr addrspace(1) %src1, ptr addrspace(1)
; GFX906-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX906-NEXT: v_lshrrev_b32_e32 v0, 8, v4
; GFX906-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX906-NEXT: v_lshlrev_b16_e32 v0, 8, v0
-; GFX906-NEXT: v_or_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX906-NEXT: v_and_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX906-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX906-NEXT: v_lshlrev_b16_e32 v0, 8, v0
; GFX906-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX906-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX906-NEXT: v_or_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX906-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX906-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX906-NEXT: v_mov_b32_e32 v1, 0
; GFX906-NEXT: global_store_short v1, v0, s[6:7]
; GFX906-NEXT: global_store_byte_d16_hi v1, v0, s[6:7] offset:2
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
index 3e0fd263fcc5c..2b1650e429b86 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
@@ -27799,24 +27799,24 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
-; GFX9-NEXT: buffer_store_dword v36, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v35, off, s[0:3], s32 ; 4-byte Folded Spill
; GFX9-NEXT: s_mov_b64 exec, s[4:5]
-; GFX9-NEXT: v_writelane_b32 v36, s36, 0
-; GFX9-NEXT: v_writelane_b32 v36, s37, 1
-; GFX9-NEXT: v_writelane_b32 v36, s38, 2
-; GFX9-NEXT: v_writelane_b32 v36, s39, 3
-; GFX9-NEXT: v_writelane_b32 v36, s48, 4
-; GFX9-NEXT: v_writelane_b32 v36, s49, 5
-; GFX9-NEXT: v_writelane_b32 v36, s50, 6
-; GFX9-NEXT: v_writelane_b32 v36, s51, 7
-; GFX9-NEXT: v_writelane_b32 v36, s52, 8
-; GFX9-NEXT: v_writelane_b32 v36, s53, 9
-; GFX9-NEXT: v_writelane_b32 v36, s54, 10
-; GFX9-NEXT: v_writelane_b32 v36, s55, 11
-; GFX9-NEXT: v_writelane_b32 v36, s64, 12
-; GFX9-NEXT: v_writelane_b32 v36, s65, 13
-; GFX9-NEXT: v_writelane_b32 v36, s66, 14
-; GFX9-NEXT: v_writelane_b32 v36, s67, 15
+; GFX9-NEXT: v_writelane_b32 v35, s36, 0
+; GFX9-NEXT: v_writelane_b32 v35, s37, 1
+; GFX9-NEXT: v_writelane_b32 v35, s38, 2
+; GFX9-NEXT: v_writelane_b32 v35, s39, 3
+; GFX9-NEXT: v_writelane_b32 v35, s48, 4
+; GFX9-NEXT: v_writelane_b32 v35, s49, 5
+; GFX9-NEXT: v_writelane_b32 v35, s50, 6
+; GFX9-NEXT: v_writelane_b32 v35, s51, 7
+; GFX9-NEXT: v_writelane_b32 v35, s52, 8
+; GFX9-NEXT: v_writelane_b32 v35, s53, 9
+; GFX9-NEXT: v_writelane_b32 v35, s54, 10
+; GFX9-NEXT: v_writelane_b32 v35, s55, 11
+; GFX9-NEXT: v_writelane_b32 v35, s64, 12
+; GFX9-NEXT: v_writelane_b32 v35, s65, 13
+; GFX9-NEXT: v_writelane_b32 v35, s66, 14
+; GFX9-NEXT: v_writelane_b32 v35, s67, 15
; GFX9-NEXT: v_readfirstlane_b32 s4, v18
; GFX9-NEXT: s_mov_b32 s49, s29
; GFX9-NEXT: s_mov_b32 s48, s28
@@ -27888,25 +27888,24 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: s_lshl_b32 s4, s50, 16
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_add_f32_e32 v3, s4, v16
-; GFX9-NEXT: v_mov_b32_e32 v32, 0xffff
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_bfe_u32 v4, v3, 16, 1
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_add_u32_e32 v4, v4, v3
; GFX9-NEXT: s_and_b32 s4, s49, 0xffff0000
+; GFX9-NEXT: v_add_u32_e32 v4, v4, v3
+; GFX9-NEXT: v_or_b32_sdwa v15, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v3
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX9-NEXT: v_lshl_or_b32 v15, v0, 16, v1
-; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX9-NEXT: v_and_b32_sdwa v3, v32, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
-; GFX9-NEXT: v_lshl_or_b32 v14, v2, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v14, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_u32_e32 v1, 0x7fff, v1
; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
@@ -27918,11 +27917,11 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s48, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v13, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v13, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -27937,11 +27936,11 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s47, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v12, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v12, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -27956,11 +27955,11 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s46, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v11, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v11, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -27975,11 +27974,11 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s45, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v10, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v10, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -27994,11 +27993,11 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s44, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v9, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v9, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -28013,11 +28012,11 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s43, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v8, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v8, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -28032,11 +28031,11 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s42, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v7, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v7, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -28051,11 +28050,11 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s41, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v6, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v6, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -28070,11 +28069,11 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s40, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v5, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v5, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -28089,11 +28088,11 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s39, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v4, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v4, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -28108,11 +28107,11 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s38, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v3, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v3, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -28127,11 +28126,11 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v17, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v17, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s37, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v2, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v2, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -28146,11 +28145,11 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v17, 0x7fff, v17
; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v17, v18, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v17, v18, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s36, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v1, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v17, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v17, v17, v0
@@ -28165,11 +28164,11 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v18, 0x7fff, v18
; GFX9-NEXT: v_or_b32_e32 v19, 0x400000, v17
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
-; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v19, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v17, v32, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v19, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s67, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v17
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -28184,11 +28183,11 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s66, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v31, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v31, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -28203,11 +28202,11 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s65, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v30, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v30, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -28222,11 +28221,11 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s64, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v29, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v29, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -28241,11 +28240,11 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s63, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v28, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v28, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -28260,11 +28259,11 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s62, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v27, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v27, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -28279,11 +28278,11 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s61, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v26, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v26, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -28298,11 +28297,11 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s60, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v25, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v25, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -28317,11 +28316,11 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s59, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v24, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v24, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -28336,11 +28335,11 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s58, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v23, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v23, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -28355,11 +28354,11 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s57, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v22, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v22, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -28374,11 +28373,11 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s56, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v21, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v21, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -28393,11 +28392,11 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s55, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v20, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v20, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -28410,69 +28409,69 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_bfe_u32 v19, v18, 16, 1
; GFX9-NEXT: v_add_u32_e32 v19, v19, v18
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
-; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v18
+; GFX9-NEXT: v_or_b32_e32 v32, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v33, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v32, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s54, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v19, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v19, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
; GFX9-NEXT: v_add_u32_e32 v18, 0x7fff, v18
-; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v17
+; GFX9-NEXT: v_or_b32_e32 v32, 0x400000, v17
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
; GFX9-NEXT: s_lshl_b32 s4, s54, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v33, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v32, vcc
; GFX9-NEXT: v_add_f32_e32 v18, s4, v16
-; GFX9-NEXT: v_bfe_u32 v33, v18, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v33, v33, v18
-; GFX9-NEXT: v_add_u32_e32 v33, 0x7fff, v33
-; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v18
+; GFX9-NEXT: v_bfe_u32 v32, v18, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v32, v32, v18
+; GFX9-NEXT: v_add_u32_e32 v32, 0x7fff, v32
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v33, v34, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v32, v33, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s53, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v18, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v18, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
-; GFX9-NEXT: v_bfe_u32 v33, v17, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v33, v33, v17
-; GFX9-NEXT: v_add_u32_e32 v33, 0x7fff, v33
-; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v17
+; GFX9-NEXT: v_bfe_u32 v32, v17, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v32, v32, v17
+; GFX9-NEXT: v_add_u32_e32 v32, 0x7fff, v32
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v17
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
; GFX9-NEXT: s_lshl_b32 s4, s53, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v17, v33, v34, vcc
-; GFX9-NEXT: v_add_f32_e32 v33, s4, v16
-; GFX9-NEXT: v_bfe_u32 v34, v33, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v34, v34, v33
-; GFX9-NEXT: v_add_u32_e32 v34, 0x7fff, v34
-; GFX9-NEXT: v_or_b32_e32 v35, 0x400000, v33
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v33, v33
-; GFX9-NEXT: v_cndmask_b32_e32 v33, v34, v35, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v17, v32, v33, vcc
+; GFX9-NEXT: v_add_f32_e32 v32, s4, v16
+; GFX9-NEXT: v_bfe_u32 v33, v32, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v33, v33, v32
+; GFX9-NEXT: v_add_u32_e32 v33, 0x7fff, v33
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v32
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v32, v32
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v33, v32, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v32, v33, v34, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s52, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v17, v17, 16, v33
-; GFX9-NEXT: v_add_f32_e32 v33, s4, v16
-; GFX9-NEXT: v_bfe_u32 v34, v33, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v34, v34, v33
+; GFX9-NEXT: v_or_b32_sdwa v17, v17, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f32_e32 v32, s4, v16
+; GFX9-NEXT: v_bfe_u32 v33, v32, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v33, v33, v32
; GFX9-NEXT: s_lshl_b32 s4, s52, 16
-; GFX9-NEXT: v_add_u32_e32 v34, 0x7fff, v34
-; GFX9-NEXT: v_or_b32_e32 v35, 0x400000, v33
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v33, v33
+; GFX9-NEXT: v_add_u32_e32 v33, 0x7fff, v33
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v32
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v32, v32
; GFX9-NEXT: v_add_f32_e32 v16, s4, v16
-; GFX9-NEXT: v_cndmask_b32_e32 v33, v34, v35, vcc
-; GFX9-NEXT: v_bfe_u32 v34, v16, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v34, v34, v16
-; GFX9-NEXT: v_add_u32_e32 v34, 0x7fff, v34
-; GFX9-NEXT: v_or_b32_e32 v35, 0x400000, v16
+; GFX9-NEXT: v_cndmask_b32_e32 v32, v33, v34, vcc
+; GFX9-NEXT: v_bfe_u32 v33, v16, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v33, v33, v16
+; GFX9-NEXT: v_add_u32_e32 v33, 0x7fff, v33
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v16
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v16, v16
-; GFX9-NEXT: v_cndmask_b32_e32 v16, v34, v35, vcc
-; GFX9-NEXT: v_and_b32_sdwa v16, v32, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshrrev_b32_e32 v32, 16, v33
-; GFX9-NEXT: v_lshl_or_b32 v16, v32, 16, v16
+; GFX9-NEXT: v_lshrrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_cndmask_b32_e32 v16, v33, v34, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_or_b32_sdwa v16, v32, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_branch .LBB19_6
; GFX9-NEXT: .LBB19_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
@@ -28508,24 +28507,24 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
; GFX9-NEXT: .LBB19_6: ; %end
-; GFX9-NEXT: v_readlane_b32 s67, v36, 15
-; GFX9-NEXT: v_readlane_b32 s66, v36, 14
-; GFX9-NEXT: v_readlane_b32 s65, v36, 13
-; GFX9-NEXT: v_readlane_b32 s64, v36, 12
-; GFX9-NEXT: v_readlane_b32 s55, v36, 11
-; GFX9-NEXT: v_readlane_b32 s54, v36, 10
-; GFX9-NEXT: v_readlane_b32 s53, v36, 9
-; GFX9-NEXT: v_readlane_b32 s52, v36, 8
-; GFX9-NEXT: v_readlane_b32 s51, v36, 7
-; GFX9-NEXT: v_readlane_b32 s50, v36, 6
-; GFX9-NEXT: v_readlane_b32 s49, v36, 5
-; GFX9-NEXT: v_readlane_b32 s48, v36, 4
-; GFX9-NEXT: v_readlane_b32 s39, v36, 3
-; GFX9-NEXT: v_readlane_b32 s38, v36, 2
-; GFX9-NEXT: v_readlane_b32 s37, v36, 1
-; GFX9-NEXT: v_readlane_b32 s36, v36, 0
+; GFX9-NEXT: v_readlane_b32 s67, v35, 15
+; GFX9-NEXT: v_readlane_b32 s66, v35, 14
+; GFX9-NEXT: v_readlane_b32 s65, v35, 13
+; GFX9-NEXT: v_readlane_b32 s64, v35, 12
+; GFX9-NEXT: v_readlane_b32 s55, v35, 11
+; GFX9-NEXT: v_readlane_b32 s54, v35, 10
+; GFX9-NEXT: v_readlane_b32 s53, v35, 9
+; GFX9-NEXT: v_readlane_b32 s52, v35, 8
+; GFX9-NEXT: v_readlane_b32 s51, v35, 7
+; GFX9-NEXT: v_readlane_b32 s50, v35, 6
+; GFX9-NEXT: v_readlane_b32 s49, v35, 5
+; GFX9-NEXT: v_readlane_b32 s48, v35, 4
+; GFX9-NEXT: v_readlane_b32 s39, v35, 3
+; GFX9-NEXT: v_readlane_b32 s38, v35, 2
+; GFX9-NEXT: v_readlane_b32 s37, v35, 1
+; GFX9-NEXT: v_readlane_b32 s36, v35, 0
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
-; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v35, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-NEXT: s_mov_b64 exec, s[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -64008,24 +64007,24 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
-; GFX9-NEXT: buffer_store_dword v36, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v35, off, s[0:3], s32 ; 4-byte Folded Spill
; GFX9-NEXT: s_mov_b64 exec, s[4:5]
-; GFX9-NEXT: v_writelane_b32 v36, s36, 0
-; GFX9-NEXT: v_writelane_b32 v36, s37, 1
-; GFX9-NEXT: v_writelane_b32 v36, s38, 2
-; GFX9-NEXT: v_writelane_b32 v36, s39, 3
-; GFX9-NEXT: v_writelane_b32 v36, s48, 4
-; GFX9-NEXT: v_writelane_b32 v36, s49, 5
-; GFX9-NEXT: v_writelane_b32 v36, s50, 6
-; GFX9-NEXT: v_writelane_b32 v36, s51, 7
-; GFX9-NEXT: v_writelane_b32 v36, s52, 8
-; GFX9-NEXT: v_writelane_b32 v36, s53, 9
-; GFX9-NEXT: v_writelane_b32 v36, s54, 10
-; GFX9-NEXT: v_writelane_b32 v36, s55, 11
-; GFX9-NEXT: v_writelane_b32 v36, s64, 12
-; GFX9-NEXT: v_writelane_b32 v36, s65, 13
-; GFX9-NEXT: v_writelane_b32 v36, s66, 14
-; GFX9-NEXT: v_writelane_b32 v36, s67, 15
+; GFX9-NEXT: v_writelane_b32 v35, s36, 0
+; GFX9-NEXT: v_writelane_b32 v35, s37, 1
+; GFX9-NEXT: v_writelane_b32 v35, s38, 2
+; GFX9-NEXT: v_writelane_b32 v35, s39, 3
+; GFX9-NEXT: v_writelane_b32 v35, s48, 4
+; GFX9-NEXT: v_writelane_b32 v35, s49, 5
+; GFX9-NEXT: v_writelane_b32 v35, s50, 6
+; GFX9-NEXT: v_writelane_b32 v35, s51, 7
+; GFX9-NEXT: v_writelane_b32 v35, s52, 8
+; GFX9-NEXT: v_writelane_b32 v35, s53, 9
+; GFX9-NEXT: v_writelane_b32 v35, s54, 10
+; GFX9-NEXT: v_writelane_b32 v35, s55, 11
+; GFX9-NEXT: v_writelane_b32 v35, s64, 12
+; GFX9-NEXT: v_writelane_b32 v35, s65, 13
+; GFX9-NEXT: v_writelane_b32 v35, s66, 14
+; GFX9-NEXT: v_writelane_b32 v35, s67, 15
; GFX9-NEXT: v_readfirstlane_b32 s4, v18
; GFX9-NEXT: s_mov_b32 s49, s29
; GFX9-NEXT: s_mov_b32 s48, s28
@@ -64097,25 +64096,24 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: s_lshl_b32 s4, s50, 16
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_add_f32_e32 v3, s4, v16
-; GFX9-NEXT: v_mov_b32_e32 v32, 0xffff
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_bfe_u32 v4, v3, 16, 1
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_add_u32_e32 v4, v4, v3
; GFX9-NEXT: s_and_b32 s4, s49, 0xffff0000
+; GFX9-NEXT: v_add_u32_e32 v4, v4, v3
+; GFX9-NEXT: v_or_b32_sdwa v15, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v3
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX9-NEXT: v_lshl_or_b32 v15, v0, 16, v1
-; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX9-NEXT: v_and_b32_sdwa v3, v32, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
-; GFX9-NEXT: v_lshl_or_b32 v14, v2, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v14, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_u32_e32 v1, 0x7fff, v1
; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
@@ -64127,11 +64125,11 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s48, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v13, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v13, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -64146,11 +64144,11 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s47, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v12, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v12, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -64165,11 +64163,11 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s46, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v11, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v11, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -64184,11 +64182,11 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s45, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v10, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v10, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -64203,11 +64201,11 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s44, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v9, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v9, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -64222,11 +64220,11 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s43, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v8, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v8, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -64241,11 +64239,11 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s42, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v7, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v7, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -64260,11 +64258,11 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s41, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v6, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v6, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -64279,11 +64277,11 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s40, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v5, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v5, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -64298,11 +64296,11 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s39, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v4, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v4, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -64317,11 +64315,11 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s38, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v3, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v3, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -64336,11 +64334,11 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v17, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v17, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s37, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v2, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v2, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -64355,11 +64353,11 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v17, 0x7fff, v17
; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v17, v18, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v17, v18, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s36, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v1, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v17, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v17, v17, v0
@@ -64374,11 +64372,11 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v18, 0x7fff, v18
; GFX9-NEXT: v_or_b32_e32 v19, 0x400000, v17
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
-; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v19, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v17, v32, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v19, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s67, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v17
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -64393,11 +64391,11 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s66, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v31, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v31, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -64412,11 +64410,11 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s65, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v30, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v30, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -64431,11 +64429,11 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s64, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v29, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v29, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -64450,11 +64448,11 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s63, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v28, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v28, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -64469,11 +64467,11 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s62, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v27, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v27, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -64488,11 +64486,11 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s61, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v26, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v26, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -64507,11 +64505,11 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s60, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v25, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v25, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -64526,11 +64524,11 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s59, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v24, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v24, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -64545,11 +64543,11 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s58, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v23, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v23, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -64564,11 +64562,11 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s57, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v22, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v22, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -64583,11 +64581,11 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s56, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v21, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v21, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -64602,11 +64600,11 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s55, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v20, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v20, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -64619,69 +64617,69 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_bfe_u32 v19, v18, 16, 1
; GFX9-NEXT: v_add_u32_e32 v19, v19, v18
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
-; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v18
+; GFX9-NEXT: v_or_b32_e32 v32, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v33, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v32, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s54, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v19, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v19, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
; GFX9-NEXT: v_add_u32_e32 v18, 0x7fff, v18
-; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v17
+; GFX9-NEXT: v_or_b32_e32 v32, 0x400000, v17
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
; GFX9-NEXT: s_lshl_b32 s4, s54, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v33, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v32, vcc
; GFX9-NEXT: v_add_f32_e32 v18, s4, v16
-; GFX9-NEXT: v_bfe_u32 v33, v18, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v33, v33, v18
-; GFX9-NEXT: v_add_u32_e32 v33, 0x7fff, v33
-; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v18
+; GFX9-NEXT: v_bfe_u32 v32, v18, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v32, v32, v18
+; GFX9-NEXT: v_add_u32_e32 v32, 0x7fff, v32
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v33, v34, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v32, v33, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s53, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v18, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v18, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
-; GFX9-NEXT: v_bfe_u32 v33, v17, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v33, v33, v17
-; GFX9-NEXT: v_add_u32_e32 v33, 0x7fff, v33
-; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v17
+; GFX9-NEXT: v_bfe_u32 v32, v17, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v32, v32, v17
+; GFX9-NEXT: v_add_u32_e32 v32, 0x7fff, v32
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v17
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
; GFX9-NEXT: s_lshl_b32 s4, s53, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v17, v33, v34, vcc
-; GFX9-NEXT: v_add_f32_e32 v33, s4, v16
-; GFX9-NEXT: v_bfe_u32 v34, v33, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v34, v34, v33
-; GFX9-NEXT: v_add_u32_e32 v34, 0x7fff, v34
-; GFX9-NEXT: v_or_b32_e32 v35, 0x400000, v33
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v33, v33
-; GFX9-NEXT: v_cndmask_b32_e32 v33, v34, v35, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v17, v32, v33, vcc
+; GFX9-NEXT: v_add_f32_e32 v32, s4, v16
+; GFX9-NEXT: v_bfe_u32 v33, v32, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v33, v33, v32
+; GFX9-NEXT: v_add_u32_e32 v33, 0x7fff, v33
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v32
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v32, v32
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v33, v32, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v32, v33, v34, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s52, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v17, v17, 16, v33
-; GFX9-NEXT: v_add_f32_e32 v33, s4, v16
-; GFX9-NEXT: v_bfe_u32 v34, v33, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v34, v34, v33
+; GFX9-NEXT: v_or_b32_sdwa v17, v17, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f32_e32 v32, s4, v16
+; GFX9-NEXT: v_bfe_u32 v33, v32, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v33, v33, v32
; GFX9-NEXT: s_lshl_b32 s4, s52, 16
-; GFX9-NEXT: v_add_u32_e32 v34, 0x7fff, v34
-; GFX9-NEXT: v_or_b32_e32 v35, 0x400000, v33
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v33, v33
+; GFX9-NEXT: v_add_u32_e32 v33, 0x7fff, v33
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v32
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v32, v32
; GFX9-NEXT: v_add_f32_e32 v16, s4, v16
-; GFX9-NEXT: v_cndmask_b32_e32 v33, v34, v35, vcc
-; GFX9-NEXT: v_bfe_u32 v34, v16, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v34, v34, v16
-; GFX9-NEXT: v_add_u32_e32 v34, 0x7fff, v34
-; GFX9-NEXT: v_or_b32_e32 v35, 0x400000, v16
+; GFX9-NEXT: v_cndmask_b32_e32 v32, v33, v34, vcc
+; GFX9-NEXT: v_bfe_u32 v33, v16, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v33, v33, v16
+; GFX9-NEXT: v_add_u32_e32 v33, 0x7fff, v33
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v16
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v16, v16
-; GFX9-NEXT: v_cndmask_b32_e32 v16, v34, v35, vcc
-; GFX9-NEXT: v_and_b32_sdwa v16, v32, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshrrev_b32_e32 v32, 16, v33
-; GFX9-NEXT: v_lshl_or_b32 v16, v32, 16, v16
+; GFX9-NEXT: v_lshrrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_cndmask_b32_e32 v16, v33, v34, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_or_b32_sdwa v16, v32, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_branch .LBB43_6
; GFX9-NEXT: .LBB43_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
@@ -64717,24 +64715,24 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
; GFX9-NEXT: .LBB43_6: ; %end
-; GFX9-NEXT: v_readlane_b32 s67, v36, 15
-; GFX9-NEXT: v_readlane_b32 s66, v36, 14
-; GFX9-NEXT: v_readlane_b32 s65, v36, 13
-; GFX9-NEXT: v_readlane_b32 s64, v36, 12
-; GFX9-NEXT: v_readlane_b32 s55, v36, 11
-; GFX9-NEXT: v_readlane_b32 s54, v36, 10
-; GFX9-NEXT: v_readlane_b32 s53, v36, 9
-; GFX9-NEXT: v_readlane_b32 s52, v36, 8
-; GFX9-NEXT: v_readlane_b32 s51, v36, 7
-; GFX9-NEXT: v_readlane_b32 s50, v36, 6
-; GFX9-NEXT: v_readlane_b32 s49, v36, 5
-; GFX9-NEXT: v_readlane_b32 s48, v36, 4
-; GFX9-NEXT: v_readlane_b32 s39, v36, 3
-; GFX9-NEXT: v_readlane_b32 s38, v36, 2
-; GFX9-NEXT: v_readlane_b32 s37, v36, 1
-; GFX9-NEXT: v_readlane_b32 s36, v36, 0
+; GFX9-NEXT: v_readlane_b32 s67, v35, 15
+; GFX9-NEXT: v_readlane_b32 s66, v35, 14
+; GFX9-NEXT: v_readlane_b32 s65, v35, 13
+; GFX9-NEXT: v_readlane_b32 s64, v35, 12
+; GFX9-NEXT: v_readlane_b32 s55, v35, 11
+; GFX9-NEXT: v_readlane_b32 s54, v35, 10
+; GFX9-NEXT: v_readlane_b32 s53, v35, 9
+; GFX9-NEXT: v_readlane_b32 s52, v35, 8
+; GFX9-NEXT: v_readlane_b32 s51, v35, 7
+; GFX9-NEXT: v_readlane_b32 s50, v35, 6
+; GFX9-NEXT: v_readlane_b32 s49, v35, 5
+; GFX9-NEXT: v_readlane_b32 s48, v35, 4
+; GFX9-NEXT: v_readlane_b32 s39, v35, 3
+; GFX9-NEXT: v_readlane_b32 s38, v35, 2
+; GFX9-NEXT: v_readlane_b32 s37, v35, 1
+; GFX9-NEXT: v_readlane_b32 s36, v35, 0
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
-; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v35, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-NEXT: s_mov_b64 exec, s[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -97807,24 +97805,24 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
-; GFX9-NEXT: buffer_store_dword v36, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v35, off, s[0:3], s32 ; 4-byte Folded Spill
; GFX9-NEXT: s_mov_b64 exec, s[4:5]
-; GFX9-NEXT: v_writelane_b32 v36, s36, 0
-; GFX9-NEXT: v_writelane_b32 v36, s37, 1
-; GFX9-NEXT: v_writelane_b32 v36, s38, 2
-; GFX9-NEXT: v_writelane_b32 v36, s39, 3
-; GFX9-NEXT: v_writelane_b32 v36, s48, 4
-; GFX9-NEXT: v_writelane_b32 v36, s49, 5
-; GFX9-NEXT: v_writelane_b32 v36, s50, 6
-; GFX9-NEXT: v_writelane_b32 v36, s51, 7
-; GFX9-NEXT: v_writelane_b32 v36, s52, 8
-; GFX9-NEXT: v_writelane_b32 v36, s53, 9
-; GFX9-NEXT: v_writelane_b32 v36, s54, 10
-; GFX9-NEXT: v_writelane_b32 v36, s55, 11
-; GFX9-NEXT: v_writelane_b32 v36, s64, 12
-; GFX9-NEXT: v_writelane_b32 v36, s65, 13
-; GFX9-NEXT: v_writelane_b32 v36, s66, 14
-; GFX9-NEXT: v_writelane_b32 v36, s67, 15
+; GFX9-NEXT: v_writelane_b32 v35, s36, 0
+; GFX9-NEXT: v_writelane_b32 v35, s37, 1
+; GFX9-NEXT: v_writelane_b32 v35, s38, 2
+; GFX9-NEXT: v_writelane_b32 v35, s39, 3
+; GFX9-NEXT: v_writelane_b32 v35, s48, 4
+; GFX9-NEXT: v_writelane_b32 v35, s49, 5
+; GFX9-NEXT: v_writelane_b32 v35, s50, 6
+; GFX9-NEXT: v_writelane_b32 v35, s51, 7
+; GFX9-NEXT: v_writelane_b32 v35, s52, 8
+; GFX9-NEXT: v_writelane_b32 v35, s53, 9
+; GFX9-NEXT: v_writelane_b32 v35, s54, 10
+; GFX9-NEXT: v_writelane_b32 v35, s55, 11
+; GFX9-NEXT: v_writelane_b32 v35, s64, 12
+; GFX9-NEXT: v_writelane_b32 v35, s65, 13
+; GFX9-NEXT: v_writelane_b32 v35, s66, 14
+; GFX9-NEXT: v_writelane_b32 v35, s67, 15
; GFX9-NEXT: v_readfirstlane_b32 s4, v18
; GFX9-NEXT: s_mov_b32 s49, s29
; GFX9-NEXT: s_mov_b32 s48, s28
@@ -97896,25 +97894,24 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: s_lshl_b32 s4, s50, 16
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_add_f32_e32 v3, s4, v16
-; GFX9-NEXT: v_mov_b32_e32 v32, 0xffff
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_bfe_u32 v4, v3, 16, 1
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_add_u32_e32 v4, v4, v3
; GFX9-NEXT: s_and_b32 s4, s49, 0xffff0000
+; GFX9-NEXT: v_add_u32_e32 v4, v4, v3
+; GFX9-NEXT: v_or_b32_sdwa v15, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v3
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX9-NEXT: v_lshl_or_b32 v15, v0, 16, v1
-; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX9-NEXT: v_and_b32_sdwa v3, v32, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
-; GFX9-NEXT: v_lshl_or_b32 v14, v2, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v14, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_u32_e32 v1, 0x7fff, v1
; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
@@ -97926,11 +97923,11 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s48, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v13, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v13, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -97945,11 +97942,11 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s47, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v12, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v12, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -97964,11 +97961,11 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s46, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v11, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v11, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -97983,11 +97980,11 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s45, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v10, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v10, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -98002,11 +97999,11 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s44, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v9, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v9, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -98021,11 +98018,11 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s43, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v8, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v8, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -98040,11 +98037,11 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s42, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v7, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v7, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -98059,11 +98056,11 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s41, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v6, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v6, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -98078,11 +98075,11 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s40, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v5, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v5, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -98097,11 +98094,11 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s39, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v4, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v4, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -98116,11 +98113,11 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s38, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v3, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v3, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -98135,11 +98132,11 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v17, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v17, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s37, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v2, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v2, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -98154,11 +98151,11 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v17, 0x7fff, v17
; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v17, v18, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v17, v18, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s36, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v1, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v17, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v17, v17, v0
@@ -98173,11 +98170,11 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v18, 0x7fff, v18
; GFX9-NEXT: v_or_b32_e32 v19, 0x400000, v17
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
-; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v19, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v17, v32, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v19, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s67, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v17
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -98192,11 +98189,11 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s66, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v31, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v31, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -98211,11 +98208,11 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s65, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v30, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v30, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -98230,11 +98227,11 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s64, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v29, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v29, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -98249,11 +98246,11 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s63, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v28, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v28, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -98268,11 +98265,11 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s62, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v27, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v27, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -98287,11 +98284,11 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s61, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v26, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v26, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -98306,11 +98303,11 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s60, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v25, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v25, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -98325,11 +98322,11 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s59, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v24, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v24, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -98344,11 +98341,11 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s58, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v23, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v23, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -98363,11 +98360,11 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s57, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v22, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v22, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -98382,11 +98379,11 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s56, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v21, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v21, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -98401,11 +98398,11 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s55, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v20, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v20, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -98418,69 +98415,69 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_bfe_u32 v19, v18, 16, 1
; GFX9-NEXT: v_add_u32_e32 v19, v19, v18
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
-; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v18
+; GFX9-NEXT: v_or_b32_e32 v32, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v33, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v32, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s54, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v19, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v19, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
; GFX9-NEXT: v_add_u32_e32 v18, 0x7fff, v18
-; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v17
+; GFX9-NEXT: v_or_b32_e32 v32, 0x400000, v17
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
; GFX9-NEXT: s_lshl_b32 s4, s54, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v33, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v32, vcc
; GFX9-NEXT: v_add_f32_e32 v18, s4, v16
-; GFX9-NEXT: v_bfe_u32 v33, v18, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v33, v33, v18
-; GFX9-NEXT: v_add_u32_e32 v33, 0x7fff, v33
-; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v18
+; GFX9-NEXT: v_bfe_u32 v32, v18, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v32, v32, v18
+; GFX9-NEXT: v_add_u32_e32 v32, 0x7fff, v32
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v33, v34, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v32, v33, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s53, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v18, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v18, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
-; GFX9-NEXT: v_bfe_u32 v33, v17, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v33, v33, v17
-; GFX9-NEXT: v_add_u32_e32 v33, 0x7fff, v33
-; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v17
+; GFX9-NEXT: v_bfe_u32 v32, v17, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v32, v32, v17
+; GFX9-NEXT: v_add_u32_e32 v32, 0x7fff, v32
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v17
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
; GFX9-NEXT: s_lshl_b32 s4, s53, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v17, v33, v34, vcc
-; GFX9-NEXT: v_add_f32_e32 v33, s4, v16
-; GFX9-NEXT: v_bfe_u32 v34, v33, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v34, v34, v33
-; GFX9-NEXT: v_add_u32_e32 v34, 0x7fff, v34
-; GFX9-NEXT: v_or_b32_e32 v35, 0x400000, v33
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v33, v33
-; GFX9-NEXT: v_cndmask_b32_e32 v33, v34, v35, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v17, v32, v33, vcc
+; GFX9-NEXT: v_add_f32_e32 v32, s4, v16
+; GFX9-NEXT: v_bfe_u32 v33, v32, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v33, v33, v32
+; GFX9-NEXT: v_add_u32_e32 v33, 0x7fff, v33
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v32
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v32, v32
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v33, v32, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v32, v33, v34, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s52, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v17, v17, 16, v33
-; GFX9-NEXT: v_add_f32_e32 v33, s4, v16
-; GFX9-NEXT: v_bfe_u32 v34, v33, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v34, v34, v33
+; GFX9-NEXT: v_or_b32_sdwa v17, v17, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f32_e32 v32, s4, v16
+; GFX9-NEXT: v_bfe_u32 v33, v32, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v33, v33, v32
; GFX9-NEXT: s_lshl_b32 s4, s52, 16
-; GFX9-NEXT: v_add_u32_e32 v34, 0x7fff, v34
-; GFX9-NEXT: v_or_b32_e32 v35, 0x400000, v33
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v33, v33
+; GFX9-NEXT: v_add_u32_e32 v33, 0x7fff, v33
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v32
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v32, v32
; GFX9-NEXT: v_add_f32_e32 v16, s4, v16
-; GFX9-NEXT: v_cndmask_b32_e32 v33, v34, v35, vcc
-; GFX9-NEXT: v_bfe_u32 v34, v16, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v34, v34, v16
-; GFX9-NEXT: v_add_u32_e32 v34, 0x7fff, v34
-; GFX9-NEXT: v_or_b32_e32 v35, 0x400000, v16
+; GFX9-NEXT: v_cndmask_b32_e32 v32, v33, v34, vcc
+; GFX9-NEXT: v_bfe_u32 v33, v16, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v33, v33, v16
+; GFX9-NEXT: v_add_u32_e32 v33, 0x7fff, v33
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v16
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v16, v16
-; GFX9-NEXT: v_cndmask_b32_e32 v16, v34, v35, vcc
-; GFX9-NEXT: v_and_b32_sdwa v16, v32, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshrrev_b32_e32 v32, 16, v33
-; GFX9-NEXT: v_lshl_or_b32 v16, v32, 16, v16
+; GFX9-NEXT: v_lshrrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_cndmask_b32_e32 v16, v33, v34, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_or_b32_sdwa v16, v32, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_branch .LBB63_6
; GFX9-NEXT: .LBB63_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
@@ -98516,24 +98513,24 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
; GFX9-NEXT: .LBB63_6: ; %end
-; GFX9-NEXT: v_readlane_b32 s67, v36, 15
-; GFX9-NEXT: v_readlane_b32 s66, v36, 14
-; GFX9-NEXT: v_readlane_b32 s65, v36, 13
-; GFX9-NEXT: v_readlane_b32 s64, v36, 12
-; GFX9-NEXT: v_readlane_b32 s55, v36, 11
-; GFX9-NEXT: v_readlane_b32 s54, v36, 10
-; GFX9-NEXT: v_readlane_b32 s53, v36, 9
-; GFX9-NEXT: v_readlane_b32 s52, v36, 8
-; GFX9-NEXT: v_readlane_b32 s51, v36, 7
-; GFX9-NEXT: v_readlane_b32 s50, v36, 6
-; GFX9-NEXT: v_readlane_b32 s49, v36, 5
-; GFX9-NEXT: v_readlane_b32 s48, v36, 4
-; GFX9-NEXT: v_readlane_b32 s39, v36, 3
-; GFX9-NEXT: v_readlane_b32 s38, v36, 2
-; GFX9-NEXT: v_readlane_b32 s37, v36, 1
-; GFX9-NEXT: v_readlane_b32 s36, v36, 0
+; GFX9-NEXT: v_readlane_b32 s67, v35, 15
+; GFX9-NEXT: v_readlane_b32 s66, v35, 14
+; GFX9-NEXT: v_readlane_b32 s65, v35, 13
+; GFX9-NEXT: v_readlane_b32 s64, v35, 12
+; GFX9-NEXT: v_readlane_b32 s55, v35, 11
+; GFX9-NEXT: v_readlane_b32 s54, v35, 10
+; GFX9-NEXT: v_readlane_b32 s53, v35, 9
+; GFX9-NEXT: v_readlane_b32 s52, v35, 8
+; GFX9-NEXT: v_readlane_b32 s51, v35, 7
+; GFX9-NEXT: v_readlane_b32 s50, v35, 6
+; GFX9-NEXT: v_readlane_b32 s49, v35, 5
+; GFX9-NEXT: v_readlane_b32 s48, v35, 4
+; GFX9-NEXT: v_readlane_b32 s39, v35, 3
+; GFX9-NEXT: v_readlane_b32 s38, v35, 2
+; GFX9-NEXT: v_readlane_b32 s37, v35, 1
+; GFX9-NEXT: v_readlane_b32 s36, v35, 0
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
-; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v35, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-NEXT: s_mov_b64 exec, s[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -130920,24 +130917,24 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
-; GFX9-NEXT: buffer_store_dword v36, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v35, off, s[0:3], s32 ; 4-byte Folded Spill
; GFX9-NEXT: s_mov_b64 exec, s[4:5]
-; GFX9-NEXT: v_writelane_b32 v36, s36, 0
-; GFX9-NEXT: v_writelane_b32 v36, s37, 1
-; GFX9-NEXT: v_writelane_b32 v36, s38, 2
-; GFX9-NEXT: v_writelane_b32 v36, s39, 3
-; GFX9-NEXT: v_writelane_b32 v36, s48, 4
-; GFX9-NEXT: v_writelane_b32 v36, s49, 5
-; GFX9-NEXT: v_writelane_b32 v36, s50, 6
-; GFX9-NEXT: v_writelane_b32 v36, s51, 7
-; GFX9-NEXT: v_writelane_b32 v36, s52, 8
-; GFX9-NEXT: v_writelane_b32 v36, s53, 9
-; GFX9-NEXT: v_writelane_b32 v36, s54, 10
-; GFX9-NEXT: v_writelane_b32 v36, s55, 11
-; GFX9-NEXT: v_writelane_b32 v36, s64, 12
-; GFX9-NEXT: v_writelane_b32 v36, s65, 13
-; GFX9-NEXT: v_writelane_b32 v36, s66, 14
-; GFX9-NEXT: v_writelane_b32 v36, s67, 15
+; GFX9-NEXT: v_writelane_b32 v35, s36, 0
+; GFX9-NEXT: v_writelane_b32 v35, s37, 1
+; GFX9-NEXT: v_writelane_b32 v35, s38, 2
+; GFX9-NEXT: v_writelane_b32 v35, s39, 3
+; GFX9-NEXT: v_writelane_b32 v35, s48, 4
+; GFX9-NEXT: v_writelane_b32 v35, s49, 5
+; GFX9-NEXT: v_writelane_b32 v35, s50, 6
+; GFX9-NEXT: v_writelane_b32 v35, s51, 7
+; GFX9-NEXT: v_writelane_b32 v35, s52, 8
+; GFX9-NEXT: v_writelane_b32 v35, s53, 9
+; GFX9-NEXT: v_writelane_b32 v35, s54, 10
+; GFX9-NEXT: v_writelane_b32 v35, s55, 11
+; GFX9-NEXT: v_writelane_b32 v35, s64, 12
+; GFX9-NEXT: v_writelane_b32 v35, s65, 13
+; GFX9-NEXT: v_writelane_b32 v35, s66, 14
+; GFX9-NEXT: v_writelane_b32 v35, s67, 15
; GFX9-NEXT: v_readfirstlane_b32 s4, v18
; GFX9-NEXT: s_mov_b32 s49, s29
; GFX9-NEXT: s_mov_b32 s48, s28
@@ -131009,25 +131006,24 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: s_lshl_b32 s4, s50, 16
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_add_f32_e32 v3, s4, v16
-; GFX9-NEXT: v_mov_b32_e32 v32, 0xffff
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_bfe_u32 v4, v3, 16, 1
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_add_u32_e32 v4, v4, v3
; GFX9-NEXT: s_and_b32 s4, s49, 0xffff0000
+; GFX9-NEXT: v_add_u32_e32 v4, v4, v3
+; GFX9-NEXT: v_or_b32_sdwa v15, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v3
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX9-NEXT: v_lshl_or_b32 v15, v0, 16, v1
-; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX9-NEXT: v_and_b32_sdwa v3, v32, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
-; GFX9-NEXT: v_lshl_or_b32 v14, v2, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v14, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_u32_e32 v1, 0x7fff, v1
; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
@@ -131039,11 +131035,11 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s48, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v13, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v13, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -131058,11 +131054,11 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s47, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v12, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v12, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -131077,11 +131073,11 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s46, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v11, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v11, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -131096,11 +131092,11 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s45, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v10, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v10, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -131115,11 +131111,11 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s44, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v9, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v9, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -131134,11 +131130,11 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s43, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v8, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v8, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -131153,11 +131149,11 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s42, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v7, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v7, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -131172,11 +131168,11 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s41, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v6, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v6, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -131191,11 +131187,11 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s40, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v5, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v5, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -131210,11 +131206,11 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s39, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v4, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v4, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -131229,11 +131225,11 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s38, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v3, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v3, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -131248,11 +131244,11 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v17, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v17, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s37, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v2, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v2, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -131267,11 +131263,11 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v17, 0x7fff, v17
; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v17, v18, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v17, v18, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s36, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v1, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v17, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v17, v17, v0
@@ -131286,11 +131282,11 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v18, 0x7fff, v18
; GFX9-NEXT: v_or_b32_e32 v19, 0x400000, v17
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
-; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v19, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v17, v32, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v19, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_and_b32 s4, s67, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v17
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -131305,11 +131301,11 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s66, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v31, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v31, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -131324,11 +131320,11 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s65, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v30, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v30, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -131343,11 +131339,11 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s64, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v29, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v29, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -131362,11 +131358,11 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s63, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v28, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v28, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -131381,11 +131377,11 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s62, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v27, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v27, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -131400,11 +131396,11 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s61, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v26, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v26, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -131419,11 +131415,11 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s60, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v25, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v25, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -131438,11 +131434,11 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s59, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v24, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v24, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -131457,11 +131453,11 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s58, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v23, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v23, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -131476,11 +131472,11 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s57, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v22, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v22, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -131495,11 +131491,11 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s56, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v21, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v21, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -131514,11 +131510,11 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s55, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v20, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v20, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -131531,69 +131527,69 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_bfe_u32 v19, v18, 16, 1
; GFX9-NEXT: v_add_u32_e32 v19, v19, v18
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
-; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v18
+; GFX9-NEXT: v_or_b32_e32 v32, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v33, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v32, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s54, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v19, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v19, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
; GFX9-NEXT: v_add_u32_e32 v18, 0x7fff, v18
-; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v17
+; GFX9-NEXT: v_or_b32_e32 v32, 0x400000, v17
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
; GFX9-NEXT: s_lshl_b32 s4, s54, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v33, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v32, vcc
; GFX9-NEXT: v_add_f32_e32 v18, s4, v16
-; GFX9-NEXT: v_bfe_u32 v33, v18, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v33, v33, v18
-; GFX9-NEXT: v_add_u32_e32 v33, 0x7fff, v33
-; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v18
+; GFX9-NEXT: v_bfe_u32 v32, v18, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v32, v32, v18
+; GFX9-NEXT: v_add_u32_e32 v32, 0x7fff, v32
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v33, v34, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v32, v33, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s53, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v18, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v18, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
-; GFX9-NEXT: v_bfe_u32 v33, v17, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v33, v33, v17
-; GFX9-NEXT: v_add_u32_e32 v33, 0x7fff, v33
-; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v17
+; GFX9-NEXT: v_bfe_u32 v32, v17, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v32, v32, v17
+; GFX9-NEXT: v_add_u32_e32 v32, 0x7fff, v32
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v17
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
; GFX9-NEXT: s_lshl_b32 s4, s53, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v17, v33, v34, vcc
-; GFX9-NEXT: v_add_f32_e32 v33, s4, v16
-; GFX9-NEXT: v_bfe_u32 v34, v33, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v34, v34, v33
-; GFX9-NEXT: v_add_u32_e32 v34, 0x7fff, v34
-; GFX9-NEXT: v_or_b32_e32 v35, 0x400000, v33
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v33, v33
-; GFX9-NEXT: v_cndmask_b32_e32 v33, v34, v35, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v17, v32, v33, vcc
+; GFX9-NEXT: v_add_f32_e32 v32, s4, v16
+; GFX9-NEXT: v_bfe_u32 v33, v32, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v33, v33, v32
+; GFX9-NEXT: v_add_u32_e32 v33, 0x7fff, v33
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v32
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v32, v32
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v33, v32, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v32, v33, v34, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s52, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v17, v17, 16, v33
-; GFX9-NEXT: v_add_f32_e32 v33, s4, v16
-; GFX9-NEXT: v_bfe_u32 v34, v33, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v34, v34, v33
+; GFX9-NEXT: v_or_b32_sdwa v17, v17, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f32_e32 v32, s4, v16
+; GFX9-NEXT: v_bfe_u32 v33, v32, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v33, v33, v32
; GFX9-NEXT: s_lshl_b32 s4, s52, 16
-; GFX9-NEXT: v_add_u32_e32 v34, 0x7fff, v34
-; GFX9-NEXT: v_or_b32_e32 v35, 0x400000, v33
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v33, v33
+; GFX9-NEXT: v_add_u32_e32 v33, 0x7fff, v33
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v32
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v32, v32
; GFX9-NEXT: v_add_f32_e32 v16, s4, v16
-; GFX9-NEXT: v_cndmask_b32_e32 v33, v34, v35, vcc
-; GFX9-NEXT: v_bfe_u32 v34, v16, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v34, v34, v16
-; GFX9-NEXT: v_add_u32_e32 v34, 0x7fff, v34
-; GFX9-NEXT: v_or_b32_e32 v35, 0x400000, v16
+; GFX9-NEXT: v_cndmask_b32_e32 v32, v33, v34, vcc
+; GFX9-NEXT: v_bfe_u32 v33, v16, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v33, v33, v16
+; GFX9-NEXT: v_add_u32_e32 v33, 0x7fff, v33
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v16
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v16, v16
-; GFX9-NEXT: v_cndmask_b32_e32 v16, v34, v35, vcc
-; GFX9-NEXT: v_and_b32_sdwa v16, v32, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshrrev_b32_e32 v32, 16, v33
-; GFX9-NEXT: v_lshl_or_b32 v16, v32, 16, v16
+; GFX9-NEXT: v_lshrrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_cndmask_b32_e32 v16, v33, v34, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_or_b32_sdwa v16, v32, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_branch .LBB79_6
; GFX9-NEXT: .LBB79_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
@@ -131629,24 +131625,24 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
; GFX9-NEXT: .LBB79_6: ; %end
-; GFX9-NEXT: v_readlane_b32 s67, v36, 15
-; GFX9-NEXT: v_readlane_b32 s66, v36, 14
-; GFX9-NEXT: v_readlane_b32 s65, v36, 13
-; GFX9-NEXT: v_readlane_b32 s64, v36, 12
-; GFX9-NEXT: v_readlane_b32 s55, v36, 11
-; GFX9-NEXT: v_readlane_b32 s54, v36, 10
-; GFX9-NEXT: v_readlane_b32 s53, v36, 9
-; GFX9-NEXT: v_readlane_b32 s52, v36, 8
-; GFX9-NEXT: v_readlane_b32 s51, v36, 7
-; GFX9-NEXT: v_readlane_b32 s50, v36, 6
-; GFX9-NEXT: v_readlane_b32 s49, v36, 5
-; GFX9-NEXT: v_readlane_b32 s48, v36, 4
-; GFX9-NEXT: v_readlane_b32 s39, v36, 3
-; GFX9-NEXT: v_readlane_b32 s38, v36, 2
-; GFX9-NEXT: v_readlane_b32 s37, v36, 1
-; GFX9-NEXT: v_readlane_b32 s36, v36, 0
+; GFX9-NEXT: v_readlane_b32 s67, v35, 15
+; GFX9-NEXT: v_readlane_b32 s66, v35, 14
+; GFX9-NEXT: v_readlane_b32 s65, v35, 13
+; GFX9-NEXT: v_readlane_b32 s64, v35, 12
+; GFX9-NEXT: v_readlane_b32 s55, v35, 11
+; GFX9-NEXT: v_readlane_b32 s54, v35, 10
+; GFX9-NEXT: v_readlane_b32 s53, v35, 9
+; GFX9-NEXT: v_readlane_b32 s52, v35, 8
+; GFX9-NEXT: v_readlane_b32 s51, v35, 7
+; GFX9-NEXT: v_readlane_b32 s50, v35, 6
+; GFX9-NEXT: v_readlane_b32 s49, v35, 5
+; GFX9-NEXT: v_readlane_b32 s48, v35, 4
+; GFX9-NEXT: v_readlane_b32 s39, v35, 3
+; GFX9-NEXT: v_readlane_b32 s38, v35, 2
+; GFX9-NEXT: v_readlane_b32 s37, v35, 1
+; GFX9-NEXT: v_readlane_b32 s36, v35, 0
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
-; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v35, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-NEXT: s_mov_b64 exec, s[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -148166,25 +148162,25 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX9-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
+; GFX9-NEXT: v_readfirstlane_b32 s42, v13
; GFX9-NEXT: v_readfirstlane_b32 s46, v12
; GFX9-NEXT: v_readfirstlane_b32 s61, v11
-; GFX9-NEXT: v_readfirstlane_b32 s63, v10
+; GFX9-NEXT: v_readfirstlane_b32 s15, v3
; GFX9-NEXT: v_readfirstlane_b32 s7, v0
-; GFX9-NEXT: buffer_load_ushort v10, off, s[0:3], s32 offset:8
-; GFX9-NEXT: buffer_load_ushort v11, off, s[0:3], s32 offset:4
-; GFX9-NEXT: buffer_load_ushort v12, off, s[0:3], s32
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:328
-; GFX9-NEXT: v_readfirstlane_b32 s43, v2
-; GFX9-NEXT: v_readfirstlane_b32 s6, v1
+; GFX9-NEXT: buffer_load_ushort v11, off, s[0:3], s32 offset:8
+; GFX9-NEXT: buffer_load_ushort v12, off, s[0:3], s32 offset:4
+; GFX9-NEXT: buffer_load_ushort v13, off, s[0:3], s32
+; GFX9-NEXT: buffer_load_ushort v3, off, s[0:3], s32 offset:328
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:324
; GFX9-NEXT: v_readfirstlane_b32 s8, v5
; GFX9-NEXT: v_readfirstlane_b32 s10, v4
-; GFX9-NEXT: v_readfirstlane_b32 s15, v3
+; GFX9-NEXT: v_readfirstlane_b32 s43, v2
+; GFX9-NEXT: v_readfirstlane_b32 s6, v1
+; GFX9-NEXT: v_readfirstlane_b32 s63, v10
; GFX9-NEXT: v_readfirstlane_b32 s12, v9
; GFX9-NEXT: v_readfirstlane_b32 s14, v8
; GFX9-NEXT: v_readfirstlane_b32 s47, v7
-; GFX9-NEXT: v_readfirstlane_b32 s59, v6
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:332
-; GFX9-NEXT: v_readfirstlane_b32 s42, v13
; GFX9-NEXT: v_readfirstlane_b32 s74, v30
; GFX9-NEXT: v_readfirstlane_b32 s44, v29
; GFX9-NEXT: v_readfirstlane_b32 s56, v28
@@ -148202,401 +148198,403 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX9-NEXT: v_readfirstlane_b32 s58, v16
; GFX9-NEXT: v_readfirstlane_b32 s73, v15
; GFX9-NEXT: v_readfirstlane_b32 s75, v14
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:472 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:324
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:320
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:484 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:316
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:480 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v2, off, s[0:3], s32 offset:312
-; GFX9-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:308
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:304
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:496 ; 4-byte Folded Spill
+; GFX9-NEXT: v_readfirstlane_b32 s59, v6
+; GFX9-NEXT: buffer_load_ushort v39, off, s[0:3], s32 offset:296
+; GFX9-NEXT: buffer_load_ushort v50, off, s[0:3], s32 offset:292
+; GFX9-NEXT: buffer_load_ushort v62, off, s[0:3], s32 offset:288
+; GFX9-NEXT: buffer_load_ushort v6, off, s[0:3], s32 offset:284
+; GFX9-NEXT: s_waitcnt vmcnt(5)
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:320
+; GFX9-NEXT: buffer_load_ushort v2, off, s[0:3], s32 offset:316
+; GFX9-NEXT: buffer_load_ushort v4, off, s[0:3], s32 offset:312
+; GFX9-NEXT: buffer_load_ushort v5, off, s[0:3], s32 offset:308
+; GFX9-NEXT: buffer_load_ushort v56, off, s[0:3], s32 offset:304
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:300
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:492 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v4, off, s[0:3], s32 offset:296
-; GFX9-NEXT: buffer_load_ushort v5, off, s[0:3], s32 offset:292
-; GFX9-NEXT: buffer_load_ushort v3, off, s[0:3], s32 offset:288
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:284
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:488 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v8, off, s[0:3], s32 offset:280
-; GFX9-NEXT: buffer_load_ushort v7, off, s[0:3], s32 offset:276
-; GFX9-NEXT: buffer_load_ushort v6, off, s[0:3], s32 offset:272
-; GFX9-NEXT: buffer_load_ushort v63, off, s[0:3], s32 offset:268
-; GFX9-NEXT: buffer_load_ushort v48, off, s[0:3], s32 offset:264
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v63, off, s[0:3], s32 offset:280
+; GFX9-NEXT: buffer_load_ushort v38, off, s[0:3], s32 offset:276
+; GFX9-NEXT: buffer_load_ushort v52, off, s[0:3], s32 offset:272
+; GFX9-NEXT: buffer_load_ushort v7, off, s[0:3], s32 offset:268
+; GFX9-NEXT: buffer_load_ushort v8, off, s[0:3], s32 offset:264
; GFX9-NEXT: buffer_load_ushort v54, off, s[0:3], s32 offset:260
-; GFX9-NEXT: buffer_load_ushort v61, off, s[0:3], s32 offset:256
-; GFX9-NEXT: buffer_load_ushort v33, off, s[0:3], s32 offset:252
-; GFX9-NEXT: buffer_load_ushort v9, off, s[0:3], s32 offset:248
-; GFX9-NEXT: buffer_load_ushort v62, off, s[0:3], s32 offset:244
+; GFX9-NEXT: buffer_load_ushort v9, off, s[0:3], s32 offset:256
+; GFX9-NEXT: buffer_load_ushort v37, off, s[0:3], s32 offset:252
+; GFX9-NEXT: buffer_load_ushort v10, off, s[0:3], s32 offset:248
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:244
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:428 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:240
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:464 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v43, off, s[0:3], s32 offset:236
-; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:452 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:236
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:432 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:232
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:500 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:464 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:228
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:436 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:400 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:224
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:436 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:220
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:468 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:440 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:216
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:504 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:472 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:212
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:468 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:208
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:440 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:204
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:400 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:408 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v45, off, s[0:3], s32 offset:204
+; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:200
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:512 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:488 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:196
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:484 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:192
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v51, off, s[0:3], s32 offset:188
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:184
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:492 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:180
; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_ushort v61, off, s[0:3], s32 offset:140
+; GFX9-NEXT: buffer_load_ushort v36, off, s[0:3], s32 offset:136
+; GFX9-NEXT: buffer_load_ushort v60, off, s[0:3], s32 offset:132
+; GFX9-NEXT: buffer_load_ushort v43, off, s[0:3], s32 offset:128
+; GFX9-NEXT: buffer_load_ushort v58, off, s[0:3], s32 offset:124
+; GFX9-NEXT: buffer_load_ushort v34, off, s[0:3], s32 offset:120
+; GFX9-NEXT: buffer_load_ushort v55, off, s[0:3], s32 offset:116
+; GFX9-NEXT: buffer_load_ushort v42, off, s[0:3], s32 offset:112
+; GFX9-NEXT: buffer_load_ushort v47, off, s[0:3], s32 offset:108
+; GFX9-NEXT: buffer_load_ushort v28, off, s[0:3], s32 offset:104
+; GFX9-NEXT: buffer_load_ushort v35, off, s[0:3], s32 offset:100
+; GFX9-NEXT: buffer_load_ushort v48, off, s[0:3], s32 offset:96
+; GFX9-NEXT: buffer_load_ushort v44, off, s[0:3], s32 offset:92
+; GFX9-NEXT: buffer_load_ushort v24, off, s[0:3], s32 offset:88
+; GFX9-NEXT: buffer_load_ushort v29, off, s[0:3], s32 offset:84
+; GFX9-NEXT: buffer_load_ushort v53, off, s[0:3], s32 offset:80
+; GFX9-NEXT: buffer_load_ushort v49, off, s[0:3], s32 offset:76
+; GFX9-NEXT: buffer_load_ushort v20, off, s[0:3], s32 offset:72
+; GFX9-NEXT: buffer_load_ushort v25, off, s[0:3], s32 offset:68
+; GFX9-NEXT: buffer_load_ushort v30, off, s[0:3], s32 offset:64
+; GFX9-NEXT: buffer_load_ushort v33, off, s[0:3], s32 offset:60
+; GFX9-NEXT: buffer_load_ushort v17, off, s[0:3], s32 offset:56
+; GFX9-NEXT: buffer_load_ushort v21, off, s[0:3], s32 offset:52
+; GFX9-NEXT: buffer_load_ushort v26, off, s[0:3], s32 offset:48
+; GFX9-NEXT: buffer_load_ushort v31, off, s[0:3], s32 offset:44
+; GFX9-NEXT: buffer_load_ushort v15, off, s[0:3], s32 offset:40
+; GFX9-NEXT: buffer_load_ushort v18, off, s[0:3], s32 offset:36
+; GFX9-NEXT: buffer_load_ushort v22, off, s[0:3], s32 offset:32
+; GFX9-NEXT: buffer_load_ushort v27, off, s[0:3], s32 offset:28
+; GFX9-NEXT: buffer_load_ushort v14, off, s[0:3], s32 offset:24
+; GFX9-NEXT: buffer_load_ushort v16, off, s[0:3], s32 offset:20
+; GFX9-NEXT: buffer_load_ushort v19, off, s[0:3], s32 offset:16
+; GFX9-NEXT: buffer_load_ushort v23, off, s[0:3], s32 offset:12
; GFX9-NEXT: s_and_b64 s[4:5], vcc, exec
+; GFX9-NEXT: s_waitcnt vmcnt(33)
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:496 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v32, off, s[0:3], s32 offset:176
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:172
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:508 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:192
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:480 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v46, off, s[0:3], s32 offset:168
+; GFX9-NEXT: buffer_load_ushort v57, off, s[0:3], s32 offset:164
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_ushort v13, off, s[0:3], s32 offset:188
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:444 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v47, off, s[0:3], s32 offset:184
-; GFX9-NEXT: buffer_load_ushort v59, off, s[0:3], s32 offset:180
-; GFX9-NEXT: buffer_load_ushort v42, off, s[0:3], s32 offset:176
-; GFX9-NEXT: buffer_load_ushort v58, off, s[0:3], s32 offset:172
-; GFX9-NEXT: buffer_load_ushort v44, off, s[0:3], s32 offset:168
-; GFX9-NEXT: buffer_load_ushort v56, off, s[0:3], s32 offset:164
-; GFX9-NEXT: buffer_load_ushort v60, off, s[0:3], s32 offset:160
-; GFX9-NEXT: buffer_load_ushort v53, off, s[0:3], s32 offset:156
-; GFX9-NEXT: buffer_load_ushort v49, off, s[0:3], s32 offset:152
-; GFX9-NEXT: buffer_load_ushort v45, off, s[0:3], s32 offset:148
-; GFX9-NEXT: buffer_load_ushort v57, off, s[0:3], s32 offset:144
-; GFX9-NEXT: buffer_load_ushort v55, off, s[0:3], s32 offset:140
-; GFX9-NEXT: buffer_load_ushort v38, off, s[0:3], s32 offset:136
-; GFX9-NEXT: buffer_load_ushort v32, off, s[0:3], s32 offset:132
-; GFX9-NEXT: buffer_load_ushort v46, off, s[0:3], s32 offset:128
-; GFX9-NEXT: buffer_load_ushort v40, off, s[0:3], s32 offset:124
-; GFX9-NEXT: buffer_load_ushort v31, off, s[0:3], s32 offset:120
-; GFX9-NEXT: buffer_load_ushort v52, off, s[0:3], s32 offset:116
-; GFX9-NEXT: buffer_load_ushort v50, off, s[0:3], s32 offset:112
-; GFX9-NEXT: buffer_load_ushort v41, off, s[0:3], s32 offset:108
-; GFX9-NEXT: buffer_load_ushort v27, off, s[0:3], s32 offset:104
-; GFX9-NEXT: buffer_load_ushort v37, off, s[0:3], s32 offset:100
-; GFX9-NEXT: buffer_load_ushort v36, off, s[0:3], s32 offset:96
-; GFX9-NEXT: buffer_load_ushort v51, off, s[0:3], s32 offset:92
-; GFX9-NEXT: buffer_load_ushort v23, off, s[0:3], s32 offset:88
-; GFX9-NEXT: buffer_load_ushort v28, off, s[0:3], s32 offset:84
-; GFX9-NEXT: buffer_load_ushort v34, off, s[0:3], s32 offset:80
-; GFX9-NEXT: buffer_load_ushort v39, off, s[0:3], s32 offset:76
-; GFX9-NEXT: buffer_load_ushort v19, off, s[0:3], s32 offset:72
-; GFX9-NEXT: buffer_load_ushort v24, off, s[0:3], s32 offset:68
-; GFX9-NEXT: buffer_load_ushort v29, off, s[0:3], s32 offset:64
-; GFX9-NEXT: buffer_load_ushort v35, off, s[0:3], s32 offset:60
-; GFX9-NEXT: buffer_load_ushort v16, off, s[0:3], s32 offset:56
-; GFX9-NEXT: buffer_load_ushort v20, off, s[0:3], s32 offset:52
-; GFX9-NEXT: buffer_load_ushort v25, off, s[0:3], s32 offset:48
-; GFX9-NEXT: buffer_load_ushort v30, off, s[0:3], s32 offset:44
-; GFX9-NEXT: buffer_load_ushort v14, off, s[0:3], s32 offset:40
-; GFX9-NEXT: buffer_load_ushort v17, off, s[0:3], s32 offset:36
-; GFX9-NEXT: buffer_load_ushort v21, off, s[0:3], s32 offset:32
-; GFX9-NEXT: buffer_load_ushort v26, off, s[0:3], s32 offset:28
-; GFX9-NEXT: buffer_load_ushort v13, off, s[0:3], s32 offset:24
-; GFX9-NEXT: buffer_load_ushort v15, off, s[0:3], s32 offset:20
-; GFX9-NEXT: buffer_load_ushort v18, off, s[0:3], s32 offset:16
-; GFX9-NEXT: buffer_load_ushort v22, off, s[0:3], s32 offset:12
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:516 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:520 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(6)
-; GFX9-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:528 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:532 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(7)
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(9)
-; GFX9-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(12)
-; GFX9-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:160
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:500 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v40, off, s[0:3], s32 offset:156
+; GFX9-NEXT: buffer_load_ushort v59, off, s[0:3], s32 offset:152
+; GFX9-NEXT: buffer_load_ushort v41, off, s[0:3], s32 offset:148
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:144
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:512 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:516 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:520 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:528 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:532 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:408 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v49, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
+; GFX9-NEXT: s_waitcnt vmcnt(27)
+; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:412 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:420 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:424 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:428 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:432 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:420 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:424 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:628 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:636 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v52, off, s[0:3], s32 offset:640 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:644 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v38, off, s[0:3], s32 offset:648 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:444 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:448 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v48, off, s[0:3], s32 offset:628 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:452 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:636 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:640 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:644 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:648 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:656 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:660 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v48, off, s[0:3], s32 offset:664 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:668 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:672 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:508 ; 4-byte Folded Spill
+; GFX9-NEXT: s_waitcnt vmcnt(49)
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:504 ; 4-byte Folded Spill
; GFX9-NEXT: s_cbranch_scc0 .LBB89_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v8, 0xc0c0004
-; GFX9-NEXT: v_mov_b32_e32 v2, s21
-; GFX9-NEXT: v_perm_b32 v0, s16, v0, v8
+; GFX9-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:676 ; 4-byte Folded Spill
; GFX9-NEXT: v_mov_b32_e32 v1, s19
-; GFX9-NEXT: v_perm_b32 v2, s20, v2, v8
; GFX9-NEXT: v_mov_b32_e32 v3, s23
+; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_perm_b32 v1, s18, v1, v8
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:680 ; 4-byte Folded Spill
+; GFX9-NEXT: v_mov_b32_e32 v2, s21
; GFX9-NEXT: v_perm_b32 v3, s22, v3, v8
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v3, 16, v2
-; GFX9-NEXT: v_mov_b32_e32 v2, s25
-; GFX9-NEXT: v_perm_b32 v2, s24, v2, v8
+; GFX9-NEXT: v_perm_b32 v0, s16, v0, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_perm_b32 v2, s20, v2, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v3, s27
+; GFX9-NEXT: v_mov_b32_e32 v2, s25
; GFX9-NEXT: v_perm_b32 v3, s26, v3, v8
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_lshl_or_b32 v2, v3, 16, v2
-; GFX9-NEXT: v_mov_b32_e32 v3, s29
-; GFX9-NEXT: v_perm_b32 v3, s28, v3, v8
+; GFX9-NEXT: v_perm_b32 v2, s24, v2, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_mov_b32_e32 v4, s6
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v3, s29
; GFX9-NEXT: v_perm_b32 v4, s7, v4, v8
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_lshl_or_b32 v3, v4, 16, v3
-; GFX9-NEXT: v_mov_b32_e32 v4, s15
-; GFX9-NEXT: v_perm_b32 v4, s43, v4, v8
+; GFX9-NEXT: v_perm_b32 v3, s28, v3, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX9-NEXT: v_mov_b32_e32 v5, s8
+; GFX9-NEXT: v_or_b32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v4, s15
; GFX9-NEXT: v_perm_b32 v5, s10, v5, v8
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_lshl_or_b32 v4, v5, 16, v4
-; GFX9-NEXT: v_mov_b32_e32 v5, s47
-; GFX9-NEXT: v_perm_b32 v5, s59, v5, v8
+; GFX9-NEXT: v_perm_b32 v4, s43, v4, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX9-NEXT: v_mov_b32_e32 v6, s12
+; GFX9-NEXT: v_or_b32_sdwa v4, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v5, s47
; GFX9-NEXT: v_perm_b32 v6, s14, v6, v8
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_lshl_or_b32 v5, v6, 16, v5
-; GFX9-NEXT: v_mov_b32_e32 v6, s61
-; GFX9-NEXT: v_perm_b32 v6, s63, v6, v8
+; GFX9-NEXT: v_perm_b32 v5, s59, v5, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX9-NEXT: v_mov_b32_e32 v50, v54
+; GFX9-NEXT: v_mov_b32_e32 v54, v7
; GFX9-NEXT: v_mov_b32_e32 v7, s42
+; GFX9-NEXT: v_or_b32_sdwa v5, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v6, s61
; GFX9-NEXT: v_perm_b32 v7, s46, v7, v8
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_lshl_or_b32 v6, v7, 16, v6
-; GFX9-NEXT: v_mov_b32_e32 v7, s73
-; GFX9-NEXT: v_perm_b32 v7, s75, v7, v8
-; GFX9-NEXT: v_mov_b32_e32 v63, v33
-; GFX9-NEXT: v_mov_b32_e32 v33, v9
+; GFX9-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:684 ; 4-byte Folded Spill
+; GFX9-NEXT: v_perm_b32 v6, s63, v6, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX9-NEXT: v_mov_b32_e32 v39, v9
; GFX9-NEXT: v_mov_b32_e32 v9, s45
+; GFX9-NEXT: v_or_b32_sdwa v6, v7, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v7, s73
; GFX9-NEXT: v_perm_b32 v9, s58, v9, v8
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_lshl_or_b32 v7, v9, 16, v7
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v10
-; GFX9-NEXT: v_perm_b32 v10, s74, v12, v8
-; GFX9-NEXT: v_or_b32_sdwa v9, v11, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v9, 16, v10
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v18
-; GFX9-NEXT: v_or_b32_sdwa v9, v22, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v13
-; GFX9-NEXT: v_or_b32_sdwa v10, v15, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
+; GFX9-NEXT: v_perm_b32 v7, s75, v7, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX9-NEXT: s_and_b32 s4, s40, 0xff
+; GFX9-NEXT: s_lshl_b32 s5, s13, 8
+; GFX9-NEXT: v_or_b32_sdwa v7, v9, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v9, s57
+; GFX9-NEXT: s_or_b32 s4, s4, s5
+; GFX9-NEXT: v_mov_b32_e32 v56, v38
+; GFX9-NEXT: v_perm_b32 v9, s60, v9, v8
+; GFX9-NEXT: v_mov_b32_e32 v38, v37
+; GFX9-NEXT: v_mov_b32_e32 v37, v10
+; GFX9-NEXT: v_lshlrev_b32_e64 v10, 16, s4
+; GFX9-NEXT: v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v11
+; GFX9-NEXT: v_perm_b32 v8, s74, v13, v8
+; GFX9-NEXT: v_or_b32_sdwa v10, v12, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v11, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v19
+; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v14
+; GFX9-NEXT: v_or_b32_sdwa v8, v23, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v10, v16, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v12, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_mov_b32 s4, 0xc0c0004
-; GFX9-NEXT: v_lshl_or_b32 v12, v10, 16, v9
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v14
-; GFX9-NEXT: v_perm_b32 v10, v26, v21, s4
-; GFX9-NEXT: v_or_b32_sdwa v9, v17, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v13, v9, 16, v10
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v25
-; GFX9-NEXT: v_or_b32_sdwa v9, v30, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v16
-; GFX9-NEXT: v_or_b32_sdwa v10, v20, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_lshl_or_b32 v14, v10, 16, v9
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v19
-; GFX9-NEXT: v_perm_b32 v10, v35, v29, s4
-; GFX9-NEXT: v_or_b32_sdwa v9, v24, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v15, v9, 16, v10
-; GFX9-NEXT: v_perm_b32 v9, v39, v34, s4
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_perm_b32 v10, v51, v36, s4
-; GFX9-NEXT: v_perm_b32 v16, v28, v23, s4
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_perm_b32 v17, v37, v27, s4
-; GFX9-NEXT: v_lshl_or_b32 v16, v16, 16, v9
-; GFX9-NEXT: v_perm_b32 v9, v41, v50, s4
-; GFX9-NEXT: v_lshl_or_b32 v17, v17, 16, v10
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_perm_b32 v10, v52, v31, s4
-; GFX9-NEXT: v_lshl_or_b32 v18, v10, 16, v9
-; GFX9-NEXT: v_perm_b32 v9, v40, v46, s4
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_perm_b32 v10, v32, v38, s4
-; GFX9-NEXT: v_lshl_or_b32 v19, v10, 16, v9
-; GFX9-NEXT: v_perm_b32 v9, v55, v57, s4
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_perm_b32 v10, v45, v49, s4
-; GFX9-NEXT: v_lshl_or_b32 v20, v10, 16, v9
-; GFX9-NEXT: v_perm_b32 v9, v53, v60, s4
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_perm_b32 v10, v56, v44, s4
-; GFX9-NEXT: v_lshl_or_b32 v21, v10, 16, v9
-; GFX9-NEXT: v_perm_b32 v9, v58, v42, s4
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_perm_b32 v10, v59, v47, s4
-; GFX9-NEXT: v_lshl_or_b32 v22, v10, 16, v9
-; GFX9-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
-; GFX9-NEXT: v_mov_b32_e32 v50, v32
-; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
-; GFX9-NEXT: v_mov_b32_e32 v51, v49
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v15
+; GFX9-NEXT: v_or_b32_sdwa v8, v18, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_perm_b32 v10, v27, v22, s4
+; GFX9-NEXT: v_perm_b32 v16, v49, v53, s4
; GFX9-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:404 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
-; GFX9-NEXT: v_mov_b32_e32 v41, v46
-; GFX9-NEXT: v_mov_b32_e32 v39, v52
-; GFX9-NEXT: v_mov_b32_e32 v52, v38
-; GFX9-NEXT: v_mov_b32_e32 v34, v37
-; GFX9-NEXT: v_mov_b32_e32 v37, v31
-; GFX9-NEXT: s_lshl_b32 s5, s13, 8
+; GFX9-NEXT: v_or_b32_sdwa v13, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v17
+; GFX9-NEXT: v_perm_b32 v17, v44, v48, s4
+; GFX9-NEXT: v_mov_b32_e32 v44, v55
+; GFX9-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v26
+; GFX9-NEXT: v_or_b32_sdwa v8, v31, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v10, v21, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v14, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v20
+; GFX9-NEXT: v_or_b32_sdwa v8, v25, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_perm_b32 v10, v33, v30, s4
+; GFX9-NEXT: v_or_b32_sdwa v15, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_perm_b32 v8, v29, v24, s4
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX9-NEXT: v_perm_b32 v10, v35, v28, s4
+; GFX9-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX9-NEXT: v_or_b32_sdwa v16, v8, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_perm_b32 v8, v44, v34, s4
+; GFX9-NEXT: v_or_b32_sdwa v17, v10, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX9-NEXT: v_perm_b32 v10, v47, v42, s4
+; GFX9-NEXT: v_mov_b32_e32 v34, v36
+; GFX9-NEXT: v_mov_b32_e32 v35, v60
+; GFX9-NEXT: v_or_b32_sdwa v18, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_perm_b32 v8, v35, v34, s4
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX9-NEXT: v_perm_b32 v10, v58, v43, s4
+; GFX9-NEXT: v_mov_b32_e32 v53, v59
+; GFX9-NEXT: v_or_b32_sdwa v19, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_perm_b32 v8, v41, v53, s4
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX9-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:484 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
+; GFX9-NEXT: s_lshl_b32 s5, s41, 8
; GFX9-NEXT: s_lshl_b32 s77, s9, 8
; GFX9-NEXT: s_lshl_b32 s78, s44, 8
-; GFX9-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:452 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:428 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(9)
-; GFX9-NEXT: v_perm_b32 v9, v9, v10, s4
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: s_waitcnt vmcnt(7)
-; GFX9-NEXT: v_perm_b32 v10, v32, v58, s4
-; GFX9-NEXT: v_lshl_or_b32 v23, v10, 16, v9
-; GFX9-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:440 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:400 ; 4-byte Folded Reload
+; GFX9-NEXT: v_perm_b32 v10, v61, v49, s4
+; GFX9-NEXT: v_or_b32_sdwa v20, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_perm_b32 v8, v57, v46, s4
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX9-NEXT: s_waitcnt vmcnt(8)
+; GFX9-NEXT: v_perm_b32 v10, v40, v55, s4
+; GFX9-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
+; GFX9-NEXT: v_or_b32_sdwa v21, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: s_waitcnt vmcnt(9)
+; GFX9-NEXT: v_perm_b32 v10, v33, v32, s4
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_perm_b32 v8, v60, v40, s4
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX9-NEXT: v_or_b32_sdwa v22, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_perm_b32 v8, v42, v59, s4
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX9-NEXT: v_perm_b32 v10, v51, v36, s4
+; GFX9-NEXT: v_or_b32_sdwa v23, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v8
+; GFX9-NEXT: v_or_b32_sdwa v8, v45, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v9
+; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v51
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v49
-; GFX9-NEXT: v_or_b32_sdwa v10, v24, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_lshl_or_b32 v24, v10, 16, v9
-; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:436 ; 4-byte Folded Reload
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v35
+; GFX9-NEXT: v_or_b32_sdwa v10, v45, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v24, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:400 ; 4-byte Folded Reload
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v32
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
+; GFX9-NEXT: v_or_b32_sdwa v8, v10, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:436 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:440 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v10, v25, v10, s4
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v25, v9, 16, v10
-; GFX9-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
-; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v33
-; GFX9-NEXT: v_or_b32_sdwa v10, v62, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
-; GFX9-NEXT: v_mov_b32_e32 v33, v46
+; GFX9-NEXT: v_or_b32_sdwa v25, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:452 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:432 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v9
-; GFX9-NEXT: v_or_b32_sdwa v9, v43, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_lshl_or_b32 v26, v10, 16, v9
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v48
-; GFX9-NEXT: v_perm_b32 v10, v63, v61, s4
-; GFX9-NEXT: v_or_b32_sdwa v9, v54, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v27, v9, 16, v10
-; GFX9-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
-; GFX9-NEXT: v_mov_b32_e32 v54, v61
-; GFX9-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:484 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(4)
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v9
-; GFX9-NEXT: s_waitcnt vmcnt(3)
-; GFX9-NEXT: v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_mov_b32_e32 v48, v63
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v10
-; GFX9-NEXT: v_or_b32_sdwa v10, v28, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_lshl_or_b32 v28, v10, 16, v9
-; GFX9-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:428 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v8
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_or_b32_sdwa v8, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v37
+; GFX9-NEXT: v_or_b32_sdwa v10, v26, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v26, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:424 ; 4-byte Folded Reload
+; GFX9-NEXT: v_perm_b32 v10, v38, v39, s4
+; GFX9-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
+; GFX9-NEXT: v_mov_b32_e32 v38, v34
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v8
+; GFX9-NEXT: v_or_b32_sdwa v8, v50, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v27, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v52
+; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v63
+; GFX9-NEXT: v_or_b32_sdwa v8, v54, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v10, v56, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v28, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
+; GFX9-NEXT: v_mov_b32_e32 v54, v31
+; GFX9-NEXT: v_mov_b32_e32 v50, v53
+; GFX9-NEXT: v_mov_b32_e32 v56, v40
+; GFX9-NEXT: v_mov_b32_e32 v52, v32
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v9
+; GFX9-NEXT: v_mov_b32_e32 v63, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v8
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:432 ; 4-byte Folded Reload
+; GFX9-NEXT: v_or_b32_sdwa v8, v10, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v10, v36, v10, s4
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v29, v9, 16, v10
-; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:424 ; 4-byte Folded Reload
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v43
-; GFX9-NEXT: v_or_b32_sdwa v9, v62, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
+; GFX9-NEXT: v_perm_b32 v10, v10, v62, s4
+; GFX9-NEXT: v_or_b32_sdwa v29, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:404 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v8
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v10
-; GFX9-NEXT: v_or_b32_sdwa v10, v30, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_lshl_or_b32 v30, v10, 16, v9
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v38
-; GFX9-NEXT: v_perm_b32 v10, v63, v61, s4
-; GFX9-NEXT: v_or_b32_sdwa v9, v46, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v31, v9, 16, v10
-; GFX9-NEXT: v_mov_b32_e32 v9, s57
-; GFX9-NEXT: v_perm_b32 v8, s60, v9, v8
-; GFX9-NEXT: s_and_b32 s4, s40, 0xff
-; GFX9-NEXT: s_or_b32 s4, s4, s5
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, s4, 16, v8
+; GFX9-NEXT: v_or_b32_sdwa v8, v39, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v10, v30, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v30, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v8
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v62, v10
+; GFX9-NEXT: v_perm_b32 v10, v31, v10, s4
; GFX9-NEXT: s_and_b32 s4, s76, 0xff
-; GFX9-NEXT: s_lshl_b32 s5, s41, 8
; GFX9-NEXT: s_or_b32 s4, s4, s5
; GFX9-NEXT: s_and_b32 s5, s11, 0xff
; GFX9-NEXT: s_or_b32 s5, s5, s77
@@ -148606,60 +148604,54 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX9-NEXT: s_or_b32 s5, s5, s77
; GFX9-NEXT: s_and_b32 s77, s56, 0xff
; GFX9-NEXT: s_or_b32 s77, s77, s78
+; GFX9-NEXT: v_or_b32_sdwa v8, v48, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: s_pack_ll_b32_b16 s5, s5, s77
+; GFX9-NEXT: v_or_b32_sdwa v31, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v8, s4
; GFX9-NEXT: v_mov_b32_e32 v10, s5
; GFX9-NEXT: s_mov_b64 s[4:5], 0
-; GFX9-NEXT: v_mov_b32_e32 v63, v36
; GFX9-NEXT: s_branch .LBB89_3
; GFX9-NEXT: .LBB89_2:
-; GFX9-NEXT: v_mov_b32_e32 v54, v61
-; GFX9-NEXT: v_mov_b32_e32 v50, v32
-; GFX9-NEXT: v_mov_b32_e32 v51, v49
-; GFX9-NEXT: v_mov_b32_e32 v39, v52
-; GFX9-NEXT: v_mov_b32_e32 v52, v38
-; GFX9-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:484 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
+; GFX9-NEXT: v_mov_b32_e32 v63, v39
+; GFX9-NEXT: v_mov_b32_e32 v38, v36
+; GFX9-NEXT: v_mov_b32_e32 v44, v55
+; GFX9-NEXT: v_mov_b32_e32 v50, v59
+; GFX9-NEXT: v_mov_b32_e32 v35, v60
+; GFX9-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
-; GFX9-NEXT: v_mov_b32_e32 v41, v46
-; GFX9-NEXT: v_mov_b32_e32 v34, v37
-; GFX9-NEXT: v_mov_b32_e32 v37, v31
+; GFX9-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:484 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
+; GFX9-NEXT: v_mov_b32_e32 v54, v2
+; GFX9-NEXT: v_mov_b32_e32 v62, v1
; GFX9-NEXT: s_mov_b64 s[4:5], -1
; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
; GFX9-NEXT: ; implicit-def: $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
; GFX9-NEXT: .LBB89_3: ; %Flow
-; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:400 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
; GFX9-NEXT: s_cselect_b32 s4, 1, 0
; GFX9-NEXT: s_cmp_lg_u32 s4, 1
; GFX9-NEXT: s_cbranch_scc1 .LBB89_5
; GFX9-NEXT: ; %bb.4: ; %cmp.true
-; GFX9-NEXT: s_waitcnt vmcnt(5)
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v48
-; GFX9-NEXT: s_mov_b32 s4, 0xc0c0004
-; GFX9-NEXT: v_perm_b32 v0, v0, v61, s4
-; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:472 ; 4-byte Folded Spill
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v33
-; GFX9-NEXT: s_waitcnt vmcnt(4)
-; GFX9-NEXT: v_perm_b32 v0, v0, v38, s4
-; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v62
-; GFX9-NEXT: v_perm_b32 v0, v0, v43, s4
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v54
+; GFX9-NEXT: s_mov_b32 s5, 0xc0c0004
+; GFX9-NEXT: v_perm_b32 v0, v0, v62, s5
; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:480 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
+; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v48
+; GFX9-NEXT: s_movk_i32 s4, 0x300
; GFX9-NEXT: s_add_i32 s74, s74, 3
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:424 ; 4-byte Folded Reload
; GFX9-NEXT: s_add_i32 s72, s72, 3
; GFX9-NEXT: s_add_i32 s56, s56, 3
; GFX9-NEXT: s_add_i32 s60, s60, 3
@@ -148680,373 +148672,358 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX9-NEXT: s_add_i32 s26, s26, 3
; GFX9-NEXT: s_add_i32 s20, s20, 3
; GFX9-NEXT: s_add_i32 s22, s22, 3
+; GFX9-NEXT: v_mov_b32_e32 v32, s23
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: v_mov_b32_e32 v33, s17
; GFX9-NEXT: s_add_i32 s18, s18, 3
-; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
+; GFX9-NEXT: v_mov_b32_e32 v34, s19
+; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:432 ; 4-byte Folded Reload
-; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:408 ; 4-byte Folded Spill
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v63
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:404 ; 4-byte Folded Reload
+; GFX9-NEXT: v_add_u32_sdwa v0, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:444 ; 4-byte Folded Spill
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v39
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:424 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:428 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_sdwa v0, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:636 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:428 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:412 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT: v_perm_b32 v0, v0, v63, s5
+; GFX9-NEXT: v_add_u32_sdwa v0, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:448 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: v_add_u32_e32 v38, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:452 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_e32 v39, 0x300, v0
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: v_add_u32_e32 v62, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_sdwa v63, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:628 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT: v_perm_b32 v0, v0, v54, s4
-; GFX9-NEXT: v_add_u32_e32 v63, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:424 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_sdwa v48, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:432 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:452 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:412 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:632 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:420 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:428 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: v_add_u32_e32 v31, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_sdwa v31, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:440 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:436 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v30, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:436 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:440 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:400 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT: v_perm_b32 v0, v0, v35, s4
-; GFX9-NEXT: v_add_u32_e32 v29, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v36
+; GFX9-NEXT: v_perm_b32 v0, v0, v52, s5
+; GFX9-NEXT: v_add_u32_sdwa v29, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:632 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v28, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:404 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v45
+; GFX9-NEXT: v_perm_b32 v0, v0, v51, s5
+; GFX9-NEXT: v_add_u32_sdwa v27, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT: v_perm_b32 v0, v0, v49, s4
-; GFX9-NEXT: v_add_u32_e32 v27, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v36, s5
; GFX9-NEXT: v_add_u32_e32 v26, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v32
-; GFX9-NEXT: v_perm_b32 v0, v0, v58, s4
-; GFX9-NEXT: v_add_u32_e32 v25, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
-; GFX9-NEXT: v_mov_b32_e32 v32, s23
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v42
+; GFX9-NEXT: v_perm_b32 v0, v0, v59, s5
+; GFX9-NEXT: v_add_u32_sdwa v25, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v33
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_e32 v24, 0x300, v0
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v60
+; GFX9-NEXT: v_perm_b32 v0, v0, v56, s5
+; GFX9-NEXT: v_add_u32_sdwa v23, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
+; GFX9-NEXT: v_mov_b32_e32 v33, s17
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT: v_perm_b32 v0, v0, v42, s4
-; GFX9-NEXT: v_add_u32_e32 v24, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v59
-; GFX9-NEXT: v_perm_b32 v0, v0, v47, s4
-; GFX9-NEXT: v_add_u32_e32 v23, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v53
-; GFX9-NEXT: v_perm_b32 v0, v0, v60, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v55, s5
; GFX9-NEXT: v_add_u32_e32 v22, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v56
-; GFX9-NEXT: v_perm_b32 v0, v0, v44, s4
-; GFX9-NEXT: v_add_u32_e32 v21, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v55
-; GFX9-NEXT: v_perm_b32 v0, v0, v57, s4
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v57
+; GFX9-NEXT: v_perm_b32 v0, v0, v46, s5
+; GFX9-NEXT: v_add_u32_sdwa v21, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v61
+; GFX9-NEXT: v_perm_b32 v0, v0, v49, s5
; GFX9-NEXT: v_add_u32_e32 v20, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v45
-; GFX9-NEXT: v_perm_b32 v0, v0, v51, s4
-; GFX9-NEXT: v_add_u32_e32 v49, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v40
-; GFX9-NEXT: v_perm_b32 v0, v0, v41, s4
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v41
+; GFX9-NEXT: v_perm_b32 v0, v0, v50, s5
+; GFX9-NEXT: v_add_u32_sdwa v51, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v58
+; GFX9-NEXT: v_perm_b32 v0, v0, v43, s5
; GFX9-NEXT: v_add_u32_e32 v19, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v50
-; GFX9-NEXT: v_perm_b32 v0, v0, v52, s4
-; GFX9-NEXT: v_add_u32_e32 v52, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v22, 0xffff, v22
-; GFX9-NEXT: v_lshl_or_b32 v21, v21, 16, v22
-; GFX9-NEXT: v_and_b32_e32 v22, 0xffff, v24
-; GFX9-NEXT: v_lshl_or_b32 v22, v23, 16, v22
-; GFX9-NEXT: v_and_b32_e32 v23, 0xffff, v26
-; GFX9-NEXT: v_lshl_or_b32 v23, v25, 16, v23
-; GFX9-NEXT: v_and_b32_e32 v25, 0xffff, v30
-; GFX9-NEXT: v_lshl_or_b32 v25, v29, 16, v25
-; GFX9-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:424 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v24, 0xffff, v28
-; GFX9-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:428 ; 4-byte Folded Reload
-; GFX9-NEXT: v_lshl_or_b32 v24, v27, 16, v24
-; GFX9-NEXT: v_and_b32_e32 v27, 0xffff, v63
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_lshl_or_b32 v19, v52, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v49, 16, v20
-; GFX9-NEXT: s_waitcnt vmcnt(4)
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v35
+; GFX9-NEXT: v_perm_b32 v0, v0, v38, s5
+; GFX9-NEXT: v_add_u32_sdwa v38, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v47
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
; GFX9-NEXT: v_add_u32_e32 v18, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v39
-; GFX9-NEXT: v_perm_b32 v0, v0, v37, s4
-; GFX9-NEXT: v_add_u32_e32 v37, 0x300, v0
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v44
+; GFX9-NEXT: v_or_b32_sdwa v21, v21, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v22, v23, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v23, v25, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v25, v29, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
+; GFX9-NEXT: v_or_b32_sdwa v24, v27, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v38, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v20, v51, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v28, v63, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(4)
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_sdwa v53, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_lshl_or_b32 v18, v37, 16, v18
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
+; GFX9-NEXT: v_or_b32_sdwa v18, v53, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: s_waitcnt vmcnt(5)
+; GFX9-NEXT: v_or_b32_sdwa v26, v31, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: v_or_b32_sdwa v29, v30, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:404 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(4)
+; GFX9-NEXT: v_or_b32_sdwa v27, v48, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: s_waitcnt vmcnt(3)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
-; GFX9-NEXT: v_add_u32_e32 v54, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v34
-; GFX9-NEXT: v_mov_b32_e32 v34, s19
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v17, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: v_or_b32_sdwa v30, v31, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: v_add_u32_e32 v16, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:636 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_sdwa v54, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
+; GFX9-NEXT: v_or_b32_sdwa v17, v54, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT: v_perm_b32 v0, v0, v37, s5
+; GFX9-NEXT: v_add_u32_e32 v16, 0x300, v0
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: v_add_u32_e32 v55, 0x300, v0
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_sdwa v55, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
-; GFX9-NEXT: v_lshl_or_b32 v16, v55, 16, v16
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
+; GFX9-NEXT: v_or_b32_sdwa v16, v55, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v15, 0x300, v0
; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: v_add_u32_e32 v40, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_sdwa v40, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
-; GFX9-NEXT: v_lshl_or_b32 v15, v40, 16, v15
+; GFX9-NEXT: v_or_b32_sdwa v15, v40, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v14, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: v_add_u32_e32 v41, 0x300, v0
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_sdwa v41, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
-; GFX9-NEXT: v_lshl_or_b32 v14, v41, 16, v14
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
+; GFX9-NEXT: v_or_b32_sdwa v14, v41, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v13, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: v_add_u32_e32 v42, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
-; GFX9-NEXT: v_lshl_or_b32 v13, v42, 16, v13
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_sdwa v42, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
+; GFX9-NEXT: v_or_b32_sdwa v13, v42, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v12, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
-; GFX9-NEXT: v_add_u32_e32 v43, 0x300, v0
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
+; GFX9-NEXT: v_add_u32_sdwa v43, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX9-NEXT: v_perm_b32 v32, s22, v32, v0
-; GFX9-NEXT: v_add_u32_e32 v32, 0x300, v32
+; GFX9-NEXT: v_add_u32_sdwa v32, v32, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_perm_b32 v33, s16, v33, v0
; GFX9-NEXT: v_add_u32_e32 v33, 0x300, v33
-; GFX9-NEXT: v_and_b32_e32 v33, 0xffff, v33
-; GFX9-NEXT: v_lshl_or_b32 v12, v43, 16, v12
+; GFX9-NEXT: v_or_b32_sdwa v12, v43, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v1, s74, v1, v0
; GFX9-NEXT: v_add_u32_e32 v11, 0x300, v1
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_add_u32_e32 v1, 3, v1
-; GFX9-NEXT: v_perm_b32 v1, v1, v2, s4
-; GFX9-NEXT: v_add_u32_e32 v44, 0x300, v1
+; GFX9-NEXT: v_perm_b32 v1, v1, v2, s5
+; GFX9-NEXT: v_add_u32_sdwa v44, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v1, s62
; GFX9-NEXT: v_perm_b32 v1, s72, v1, v0
; GFX9-NEXT: v_add_u32_e32 v10, 0x300, v1
; GFX9-NEXT: v_mov_b32_e32 v1, s44
; GFX9-NEXT: v_perm_b32 v1, s56, v1, v0
-; GFX9-NEXT: v_add_u32_e32 v45, 0x300, v1
+; GFX9-NEXT: v_add_u32_sdwa v45, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v1, s57
; GFX9-NEXT: v_perm_b32 v1, s60, v1, v0
; GFX9-NEXT: v_add_u32_e32 v9, 0x300, v1
; GFX9-NEXT: v_mov_b32_e32 v1, s13
; GFX9-NEXT: v_perm_b32 v1, s40, v1, v0
-; GFX9-NEXT: v_add_u32_e32 v46, 0x300, v1
+; GFX9-NEXT: v_add_u32_sdwa v46, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v1, s41
; GFX9-NEXT: v_perm_b32 v1, s76, v1, v0
; GFX9-NEXT: v_add_u32_e32 v8, 0x300, v1
; GFX9-NEXT: v_mov_b32_e32 v1, s9
; GFX9-NEXT: v_perm_b32 v1, s11, v1, v0
-; GFX9-NEXT: v_add_u32_e32 v47, 0x300, v1
+; GFX9-NEXT: v_add_u32_sdwa v47, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v1, s73
; GFX9-NEXT: v_perm_b32 v1, s75, v1, v0
; GFX9-NEXT: v_add_u32_e32 v7, 0x300, v1
; GFX9-NEXT: v_mov_b32_e32 v1, s45
; GFX9-NEXT: v_perm_b32 v1, s58, v1, v0
-; GFX9-NEXT: v_add_u32_e32 v56, 0x300, v1
+; GFX9-NEXT: v_add_u32_sdwa v56, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v1, s61
; GFX9-NEXT: v_perm_b32 v1, s63, v1, v0
; GFX9-NEXT: v_add_u32_e32 v6, 0x300, v1
; GFX9-NEXT: v_mov_b32_e32 v1, s42
; GFX9-NEXT: v_perm_b32 v1, s46, v1, v0
-; GFX9-NEXT: v_add_u32_e32 v57, 0x300, v1
+; GFX9-NEXT: v_add_u32_sdwa v57, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v1, s47
; GFX9-NEXT: v_perm_b32 v1, s59, v1, v0
; GFX9-NEXT: v_add_u32_e32 v5, 0x300, v1
; GFX9-NEXT: v_mov_b32_e32 v1, s12
; GFX9-NEXT: v_perm_b32 v1, s14, v1, v0
-; GFX9-NEXT: v_add_u32_e32 v58, 0x300, v1
+; GFX9-NEXT: v_add_u32_sdwa v58, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v1, s15
; GFX9-NEXT: v_perm_b32 v1, s43, v1, v0
; GFX9-NEXT: v_add_u32_e32 v4, 0x300, v1
; GFX9-NEXT: v_mov_b32_e32 v1, s8
; GFX9-NEXT: v_perm_b32 v1, s10, v1, v0
-; GFX9-NEXT: v_add_u32_e32 v59, 0x300, v1
+; GFX9-NEXT: v_add_u32_sdwa v59, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v1, s29
; GFX9-NEXT: v_perm_b32 v1, s28, v1, v0
; GFX9-NEXT: v_add_u32_e32 v3, 0x300, v1
; GFX9-NEXT: v_mov_b32_e32 v1, s6
; GFX9-NEXT: v_perm_b32 v1, s7, v1, v0
-; GFX9-NEXT: v_add_u32_e32 v60, 0x300, v1
+; GFX9-NEXT: v_add_u32_sdwa v60, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v1, s25
; GFX9-NEXT: v_perm_b32 v1, s24, v1, v0
; GFX9-NEXT: v_add_u32_e32 v2, 0x300, v1
; GFX9-NEXT: v_mov_b32_e32 v1, s27
; GFX9-NEXT: v_perm_b32 v1, s26, v1, v0
-; GFX9-NEXT: v_add_u32_e32 v61, 0x300, v1
+; GFX9-NEXT: v_add_u32_sdwa v61, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v1, s21
; GFX9-NEXT: v_perm_b32 v1, s20, v1, v0
; GFX9-NEXT: v_add_u32_e32 v1, 0x300, v1
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_lshl_or_b32 v1, v32, 16, v1
-; GFX9-NEXT: v_and_b32_e32 v32, 0xffff, v54
-; GFX9-NEXT: v_lshl_or_b32 v17, v17, 16, v32
-; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v29, 0xffff, v29
-; GFX9-NEXT: v_and_b32_e32 v26, 0xffff, v26
-; GFX9-NEXT: v_lshl_or_b32 v29, v30, 16, v29
-; GFX9-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
-; GFX9-NEXT: v_lshl_or_b32 v26, v31, 16, v26
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
+; GFX9-NEXT: v_or_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
; GFX9-NEXT: v_perm_b32 v0, s18, v34, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v27, v28, 16, v27
-; GFX9-NEXT: v_and_b32_e32 v28, 0xffff, v38
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v33
-; GFX9-NEXT: v_lshl_or_b32 v2, v61, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v60, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v59, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v58, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v57, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v56, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v47, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v46, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v45, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v44, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v28, v62, 16, v28
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_and_b32_e32 v30, 0xffff, v30
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_lshl_or_b32 v30, v31, 16, v30
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_and_b32_e32 v31, 0xffff, v31
-; GFX9-NEXT: v_lshl_or_b32 v31, v32, 16, v31
+; GFX9-NEXT: v_add_u32_sdwa v0, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v61, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v60, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v4, v59, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v58, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v57, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v56, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v47, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v46, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v45, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v44, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_or_b32_sdwa v31, v32, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: .LBB89_5: ; %end
; GFX9-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
@@ -161454,11 +161431,11 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v4
; GFX9-NEXT: s_and_b32 s45, s44, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v2, v4, 16, v1
+; GFX9-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GFX9-NEXT: v_or_b32_sdwa v2, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v1, s45, v13
; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v1
@@ -161475,16 +161452,16 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: s_and_b32 s44, s43, 0xffff0000
; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
+; GFX9-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v5
+; GFX9-NEXT: s_and_b32 s44, s43, 0xffff0000
+; GFX9-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v3, s44, v13
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_bfe_u32 v4, v3, 16, 1
-; GFX9-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: v_add_u32_e32 v4, v4, v3
; GFX9-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshl_or_b32 v1, v5, 16, v1
; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v3
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
@@ -161498,11 +161475,11 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v3
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX9-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v6
; GFX9-NEXT: s_and_b32 s43, s42, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v4, v6, 16, v3
+; GFX9-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GFX9-NEXT: v_or_b32_sdwa v4, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v3, s43, v13
; GFX9-NEXT: v_bfe_u32 v5, v3, 16, 1
; GFX9-NEXT: v_add_u32_e32 v5, v5, v3
@@ -161519,16 +161496,16 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v3
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX9-NEXT: s_and_b32 s42, s41, 0xffff0000
; GFX9-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc
+; GFX9-NEXT: v_lshrrev_b32_e32 v5, 16, v3
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v7
+; GFX9-NEXT: s_and_b32 s42, s41, 0xffff0000
+; GFX9-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GFX9-NEXT: v_or_b32_sdwa v3, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v5, s42, v13
-; GFX9-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX9-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
; GFX9-NEXT: v_add_u32_e32 v6, v6, v5
; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshl_or_b32 v3, v7, 16, v3
; GFX9-NEXT: v_add_u32_e32 v6, 0x7fff, v6
; GFX9-NEXT: v_or_b32_e32 v7, 0x400000, v5
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
@@ -161542,11 +161519,11 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_or_b32_e32 v7, 0x400000, v5
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
; GFX9-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX9-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX9-NEXT: v_lshrrev_b32_e32 v6, 16, v5
+; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v8
; GFX9-NEXT: s_and_b32 s41, s40, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v6, v8, 16, v5
+; GFX9-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX9-NEXT: v_or_b32_sdwa v6, v5, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v5, s41, v13
; GFX9-NEXT: v_bfe_u32 v7, v5, 16, 1
; GFX9-NEXT: v_add_u32_e32 v7, v7, v5
@@ -161563,16 +161540,16 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v7, 0x7fff, v7
; GFX9-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX9-NEXT: s_and_b32 s40, s13, 0xffff0000
; GFX9-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc
+; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v5
+; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v9
+; GFX9-NEXT: s_and_b32 s40, s13, 0xffff0000
+; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GFX9-NEXT: v_or_b32_sdwa v5, v5, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v7, s40, v13
-; GFX9-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX9-NEXT: v_bfe_u32 v8, v7, 16, 1
-; GFX9-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
; GFX9-NEXT: v_add_u32_e32 v8, v8, v7
; GFX9-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshl_or_b32 v5, v9, 16, v5
; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v7
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
@@ -161586,12 +161563,12 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v7
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX9-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v10
; GFX9-NEXT: s_and_b32 s13, s12, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v11, v10, 16, v7
+; GFX9-NEXT: v_or_b32_sdwa v11, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v7, s13, v13
+; GFX9-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
; GFX9-NEXT: v_bfe_u32 v8, v7, 16, 1
; GFX9-NEXT: v_add_u32_e32 v8, v8, v7
; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
@@ -161608,13 +161585,13 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v7
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX9-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v10
; GFX9-NEXT: s_and_b32 s12, s15, 0xffff0000
; GFX9-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshl_or_b32 v10, v10, 16, v7
+; GFX9-NEXT: v_or_b32_sdwa v10, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v7, s12, v13
+; GFX9-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
; GFX9-NEXT: v_bfe_u32 v8, v7, 16, 1
; GFX9-NEXT: v_add_u32_e32 v8, v8, v7
; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
@@ -161630,12 +161607,12 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v7
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX9-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v12
; GFX9-NEXT: s_and_b32 s12, s14, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v17, v12, 16, v7
+; GFX9-NEXT: v_or_b32_sdwa v17, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v7, s12, v13
+; GFX9-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
; GFX9-NEXT: v_bfe_u32 v8, v7, 16, 1
; GFX9-NEXT: v_add_u32_e32 v8, v8, v7
; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
@@ -161652,12 +161629,12 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v7
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX9-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v12
; GFX9-NEXT: s_and_b32 s12, s11, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v16, v12, 16, v7
+; GFX9-NEXT: v_or_b32_sdwa v16, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v7, s12, v13
+; GFX9-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
; GFX9-NEXT: v_bfe_u32 v8, v7, 16, 1
; GFX9-NEXT: v_add_u32_e32 v8, v8, v7
; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
@@ -161674,12 +161651,12 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v7
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX9-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v12
; GFX9-NEXT: s_and_b32 s11, s10, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v19, v12, 16, v7
+; GFX9-NEXT: v_or_b32_sdwa v19, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v7, s11, v13
+; GFX9-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
; GFX9-NEXT: v_bfe_u32 v8, v7, 16, 1
; GFX9-NEXT: v_add_u32_e32 v8, v8, v7
; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
@@ -161696,12 +161673,12 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v7
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX9-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v12
; GFX9-NEXT: s_and_b32 s10, s9, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v18, v12, 16, v7
+; GFX9-NEXT: v_or_b32_sdwa v18, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v7, s10, v13
+; GFX9-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
; GFX9-NEXT: v_bfe_u32 v8, v7, 16, 1
; GFX9-NEXT: v_add_u32_e32 v8, v8, v7
; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
@@ -161718,12 +161695,12 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v7
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX9-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v12
; GFX9-NEXT: s_and_b32 s9, s8, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v23, v12, 16, v7
+; GFX9-NEXT: v_or_b32_sdwa v23, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v7, s9, v13
+; GFX9-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
; GFX9-NEXT: v_bfe_u32 v8, v7, 16, 1
; GFX9-NEXT: v_add_u32_e32 v8, v8, v7
; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
@@ -161740,12 +161717,12 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v7
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX9-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v12
; GFX9-NEXT: s_and_b32 s8, s7, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v22, v12, 16, v7
+; GFX9-NEXT: v_or_b32_sdwa v22, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v7, s8, v13
+; GFX9-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
; GFX9-NEXT: v_bfe_u32 v8, v7, 16, 1
; GFX9-NEXT: v_add_u32_e32 v8, v8, v7
; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
@@ -161762,12 +161739,12 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v7
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX9-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v12
; GFX9-NEXT: s_and_b32 s7, s6, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v25, v12, 16, v7
+; GFX9-NEXT: v_or_b32_sdwa v25, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v7, s7, v13
+; GFX9-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
; GFX9-NEXT: v_bfe_u32 v8, v7, 16, 1
; GFX9-NEXT: v_add_u32_e32 v8, v8, v7
; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
@@ -161784,12 +161761,12 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v7
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX9-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v12
; GFX9-NEXT: s_and_b32 s6, s17, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v24, v12, 16, v7
+; GFX9-NEXT: v_or_b32_sdwa v24, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v7, s6, v13
+; GFX9-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
; GFX9-NEXT: v_bfe_u32 v8, v7, 16, 1
; GFX9-NEXT: v_add_u32_e32 v8, v8, v7
; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
@@ -161806,12 +161783,12 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v7
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX9-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v12
; GFX9-NEXT: s_and_b32 s6, s16, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v27, v12, 16, v7
+; GFX9-NEXT: v_or_b32_sdwa v27, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v7, s6, v13
+; GFX9-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
; GFX9-NEXT: v_bfe_u32 v8, v7, 16, 1
; GFX9-NEXT: v_add_u32_e32 v8, v8, v7
; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
@@ -161828,12 +161805,12 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v7
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX9-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v12
; GFX9-NEXT: s_and_b32 s6, s19, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v26, v12, 16, v7
+; GFX9-NEXT: v_or_b32_sdwa v26, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v7, s6, v13
+; GFX9-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
; GFX9-NEXT: v_bfe_u32 v8, v7, 16, 1
; GFX9-NEXT: v_add_u32_e32 v8, v8, v7
; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
@@ -161850,12 +161827,12 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v7
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX9-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v12
; GFX9-NEXT: s_and_b32 s6, s18, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v29, v12, 16, v7
+; GFX9-NEXT: v_or_b32_sdwa v29, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v7, s6, v13
+; GFX9-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
; GFX9-NEXT: v_bfe_u32 v8, v7, 16, 1
; GFX9-NEXT: v_add_u32_e32 v8, v8, v7
; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
@@ -161872,12 +161849,12 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v7
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX9-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v12
; GFX9-NEXT: s_and_b32 s6, s21, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v28, v12, 16, v7
+; GFX9-NEXT: v_or_b32_sdwa v28, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v7, s6, v13
+; GFX9-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
; GFX9-NEXT: v_bfe_u32 v8, v7, 16, 1
; GFX9-NEXT: v_add_u32_e32 v8, v8, v7
; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
@@ -161894,12 +161871,12 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v7
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX9-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v12
; GFX9-NEXT: s_and_b32 s6, s20, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v53, v12, 16, v7
+; GFX9-NEXT: v_or_b32_sdwa v53, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v7, s6, v13
+; GFX9-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
; GFX9-NEXT: v_bfe_u32 v8, v7, 16, 1
; GFX9-NEXT: v_add_u32_e32 v8, v8, v7
; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
@@ -161916,12 +161893,12 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v7
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX9-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v12
; GFX9-NEXT: s_and_b32 s6, s23, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v52, v12, 16, v7
+; GFX9-NEXT: v_or_b32_sdwa v52, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v7, s6, v13
+; GFX9-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
; GFX9-NEXT: v_bfe_u32 v8, v7, 16, 1
; GFX9-NEXT: v_add_u32_e32 v8, v8, v7
; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
@@ -161938,12 +161915,12 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v7
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX9-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v12
; GFX9-NEXT: s_and_b32 s6, s22, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v55, v12, 16, v7
+; GFX9-NEXT: v_or_b32_sdwa v55, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v7, s6, v13
+; GFX9-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
; GFX9-NEXT: v_bfe_u32 v8, v7, 16, 1
; GFX9-NEXT: v_add_u32_e32 v8, v8, v7
; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
@@ -161960,12 +161937,12 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v7
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX9-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v12
; GFX9-NEXT: s_and_b32 s6, s25, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v54, v12, 16, v7
+; GFX9-NEXT: v_or_b32_sdwa v54, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v7, s6, v13
+; GFX9-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
; GFX9-NEXT: v_bfe_u32 v8, v7, 16, 1
; GFX9-NEXT: v_add_u32_e32 v8, v8, v7
; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
@@ -161982,12 +161959,12 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v7
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX9-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v12
; GFX9-NEXT: s_and_b32 s6, s24, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v41, v12, 16, v7
+; GFX9-NEXT: v_or_b32_sdwa v41, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v7, s6, v13
+; GFX9-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
; GFX9-NEXT: v_bfe_u32 v8, v7, 16, 1
; GFX9-NEXT: v_add_u32_e32 v8, v8, v7
; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
@@ -162004,12 +161981,12 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v7
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX9-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v12
; GFX9-NEXT: s_and_b32 s6, s27, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v40, v12, 16, v7
+; GFX9-NEXT: v_or_b32_sdwa v40, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v7, s6, v13
+; GFX9-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
; GFX9-NEXT: v_bfe_u32 v8, v7, 16, 1
; GFX9-NEXT: v_add_u32_e32 v8, v8, v7
; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
@@ -162025,12 +162002,12 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v7
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX9-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v14
; GFX9-NEXT: s_and_b32 s6, s26, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v43, v14, 16, v7
+; GFX9-NEXT: v_or_b32_sdwa v43, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v7, s6, v13
+; GFX9-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
; GFX9-NEXT: v_bfe_u32 v8, v7, 16, 1
; GFX9-NEXT: v_add_u32_e32 v8, v8, v7
; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
@@ -162046,12 +162023,12 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v7
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX9-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v57
; GFX9-NEXT: s_and_b32 s6, s29, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v42, v57, 16, v7
+; GFX9-NEXT: v_or_b32_sdwa v42, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v7, s6, v13
+; GFX9-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
; GFX9-NEXT: v_bfe_u32 v8, v7, 16, 1
; GFX9-NEXT: v_add_u32_e32 v8, v8, v7
; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
@@ -162067,12 +162044,12 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v7
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX9-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v56
; GFX9-NEXT: s_and_b32 s6, s28, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v45, v56, 16, v7
+; GFX9-NEXT: v_or_b32_sdwa v45, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v7, s6, v13
+; GFX9-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
; GFX9-NEXT: v_bfe_u32 v8, v7, 16, 1
; GFX9-NEXT: v_add_u32_e32 v8, v8, v7
; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
@@ -162089,12 +162066,12 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v7
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX9-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v14
; GFX9-NEXT: s_and_b32 s6, s5, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v44, v14, 16, v7
+; GFX9-NEXT: v_or_b32_sdwa v44, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v7, s6, v13
+; GFX9-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
; GFX9-NEXT: v_bfe_u32 v8, v7, 16, 1
; GFX9-NEXT: v_add_u32_e32 v8, v8, v7
; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
@@ -162111,9 +162088,9 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX9-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v61, 16, v7
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v61
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v58
; GFX9-NEXT: s_and_b32 s5, s4, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v47, v58, 16, v7
+; GFX9-NEXT: v_or_b32_sdwa v47, v7, v61 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v7, s5, v13
; GFX9-NEXT: v_bfe_u32 v8, v7, 16, 1
; GFX9-NEXT: v_add_u32_e32 v8, v8, v7
@@ -162131,8 +162108,8 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX9-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v13, 16, v7
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v13
-; GFX9-NEXT: v_lshl_or_b32 v46, v59, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v59
+; GFX9-NEXT: v_or_b32_sdwa v46, v7, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_lshrrev_b64 v[7:8], 24, v[24:25]
; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
@@ -174602,25 +174579,25 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX9-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
+; GFX9-NEXT: v_readfirstlane_b32 s42, v13
; GFX9-NEXT: v_readfirstlane_b32 s46, v12
; GFX9-NEXT: v_readfirstlane_b32 s61, v11
-; GFX9-NEXT: v_readfirstlane_b32 s63, v10
+; GFX9-NEXT: v_readfirstlane_b32 s15, v3
; GFX9-NEXT: v_readfirstlane_b32 s7, v0
-; GFX9-NEXT: buffer_load_ushort v10, off, s[0:3], s32 offset:8
-; GFX9-NEXT: buffer_load_ushort v11, off, s[0:3], s32 offset:4
-; GFX9-NEXT: buffer_load_ushort v12, off, s[0:3], s32
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:328
-; GFX9-NEXT: v_readfirstlane_b32 s43, v2
-; GFX9-NEXT: v_readfirstlane_b32 s6, v1
+; GFX9-NEXT: buffer_load_ushort v11, off, s[0:3], s32 offset:8
+; GFX9-NEXT: buffer_load_ushort v12, off, s[0:3], s32 offset:4
+; GFX9-NEXT: buffer_load_ushort v13, off, s[0:3], s32
+; GFX9-NEXT: buffer_load_ushort v3, off, s[0:3], s32 offset:328
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:324
; GFX9-NEXT: v_readfirstlane_b32 s8, v5
; GFX9-NEXT: v_readfirstlane_b32 s10, v4
-; GFX9-NEXT: v_readfirstlane_b32 s15, v3
+; GFX9-NEXT: v_readfirstlane_b32 s43, v2
+; GFX9-NEXT: v_readfirstlane_b32 s6, v1
+; GFX9-NEXT: v_readfirstlane_b32 s63, v10
; GFX9-NEXT: v_readfirstlane_b32 s12, v9
; GFX9-NEXT: v_readfirstlane_b32 s14, v8
; GFX9-NEXT: v_readfirstlane_b32 s47, v7
-; GFX9-NEXT: v_readfirstlane_b32 s59, v6
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:332
-; GFX9-NEXT: v_readfirstlane_b32 s42, v13
; GFX9-NEXT: v_readfirstlane_b32 s74, v30
; GFX9-NEXT: v_readfirstlane_b32 s44, v29
; GFX9-NEXT: v_readfirstlane_b32 s56, v28
@@ -174638,401 +174615,403 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX9-NEXT: v_readfirstlane_b32 s58, v16
; GFX9-NEXT: v_readfirstlane_b32 s73, v15
; GFX9-NEXT: v_readfirstlane_b32 s75, v14
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:472 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:324
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:320
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:484 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:316
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:480 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v2, off, s[0:3], s32 offset:312
-; GFX9-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:308
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:304
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:496 ; 4-byte Folded Spill
+; GFX9-NEXT: v_readfirstlane_b32 s59, v6
+; GFX9-NEXT: buffer_load_ushort v39, off, s[0:3], s32 offset:296
+; GFX9-NEXT: buffer_load_ushort v50, off, s[0:3], s32 offset:292
+; GFX9-NEXT: buffer_load_ushort v62, off, s[0:3], s32 offset:288
+; GFX9-NEXT: buffer_load_ushort v6, off, s[0:3], s32 offset:284
+; GFX9-NEXT: s_waitcnt vmcnt(5)
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:320
+; GFX9-NEXT: buffer_load_ushort v2, off, s[0:3], s32 offset:316
+; GFX9-NEXT: buffer_load_ushort v4, off, s[0:3], s32 offset:312
+; GFX9-NEXT: buffer_load_ushort v5, off, s[0:3], s32 offset:308
+; GFX9-NEXT: buffer_load_ushort v56, off, s[0:3], s32 offset:304
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:300
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:492 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v4, off, s[0:3], s32 offset:296
-; GFX9-NEXT: buffer_load_ushort v5, off, s[0:3], s32 offset:292
-; GFX9-NEXT: buffer_load_ushort v3, off, s[0:3], s32 offset:288
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:284
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:488 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v8, off, s[0:3], s32 offset:280
-; GFX9-NEXT: buffer_load_ushort v7, off, s[0:3], s32 offset:276
-; GFX9-NEXT: buffer_load_ushort v6, off, s[0:3], s32 offset:272
-; GFX9-NEXT: buffer_load_ushort v63, off, s[0:3], s32 offset:268
-; GFX9-NEXT: buffer_load_ushort v48, off, s[0:3], s32 offset:264
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v63, off, s[0:3], s32 offset:280
+; GFX9-NEXT: buffer_load_ushort v38, off, s[0:3], s32 offset:276
+; GFX9-NEXT: buffer_load_ushort v52, off, s[0:3], s32 offset:272
+; GFX9-NEXT: buffer_load_ushort v7, off, s[0:3], s32 offset:268
+; GFX9-NEXT: buffer_load_ushort v8, off, s[0:3], s32 offset:264
; GFX9-NEXT: buffer_load_ushort v54, off, s[0:3], s32 offset:260
-; GFX9-NEXT: buffer_load_ushort v61, off, s[0:3], s32 offset:256
-; GFX9-NEXT: buffer_load_ushort v33, off, s[0:3], s32 offset:252
-; GFX9-NEXT: buffer_load_ushort v9, off, s[0:3], s32 offset:248
-; GFX9-NEXT: buffer_load_ushort v62, off, s[0:3], s32 offset:244
+; GFX9-NEXT: buffer_load_ushort v9, off, s[0:3], s32 offset:256
+; GFX9-NEXT: buffer_load_ushort v37, off, s[0:3], s32 offset:252
+; GFX9-NEXT: buffer_load_ushort v10, off, s[0:3], s32 offset:248
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:244
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:428 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:240
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:464 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v43, off, s[0:3], s32 offset:236
-; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:452 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:236
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:432 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:232
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:500 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:464 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:228
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:436 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:400 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:224
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:436 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:220
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:468 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:440 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:216
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:504 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:472 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:212
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:468 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:208
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:440 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:204
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:400 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:408 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v45, off, s[0:3], s32 offset:204
+; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:200
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:512 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:488 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:196
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:484 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:192
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v51, off, s[0:3], s32 offset:188
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:184
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:492 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:180
; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_ushort v61, off, s[0:3], s32 offset:140
+; GFX9-NEXT: buffer_load_ushort v36, off, s[0:3], s32 offset:136
+; GFX9-NEXT: buffer_load_ushort v60, off, s[0:3], s32 offset:132
+; GFX9-NEXT: buffer_load_ushort v43, off, s[0:3], s32 offset:128
+; GFX9-NEXT: buffer_load_ushort v58, off, s[0:3], s32 offset:124
+; GFX9-NEXT: buffer_load_ushort v34, off, s[0:3], s32 offset:120
+; GFX9-NEXT: buffer_load_ushort v55, off, s[0:3], s32 offset:116
+; GFX9-NEXT: buffer_load_ushort v42, off, s[0:3], s32 offset:112
+; GFX9-NEXT: buffer_load_ushort v47, off, s[0:3], s32 offset:108
+; GFX9-NEXT: buffer_load_ushort v28, off, s[0:3], s32 offset:104
+; GFX9-NEXT: buffer_load_ushort v35, off, s[0:3], s32 offset:100
+; GFX9-NEXT: buffer_load_ushort v48, off, s[0:3], s32 offset:96
+; GFX9-NEXT: buffer_load_ushort v44, off, s[0:3], s32 offset:92
+; GFX9-NEXT: buffer_load_ushort v24, off, s[0:3], s32 offset:88
+; GFX9-NEXT: buffer_load_ushort v29, off, s[0:3], s32 offset:84
+; GFX9-NEXT: buffer_load_ushort v53, off, s[0:3], s32 offset:80
+; GFX9-NEXT: buffer_load_ushort v49, off, s[0:3], s32 offset:76
+; GFX9-NEXT: buffer_load_ushort v20, off, s[0:3], s32 offset:72
+; GFX9-NEXT: buffer_load_ushort v25, off, s[0:3], s32 offset:68
+; GFX9-NEXT: buffer_load_ushort v30, off, s[0:3], s32 offset:64
+; GFX9-NEXT: buffer_load_ushort v33, off, s[0:3], s32 offset:60
+; GFX9-NEXT: buffer_load_ushort v17, off, s[0:3], s32 offset:56
+; GFX9-NEXT: buffer_load_ushort v21, off, s[0:3], s32 offset:52
+; GFX9-NEXT: buffer_load_ushort v26, off, s[0:3], s32 offset:48
+; GFX9-NEXT: buffer_load_ushort v31, off, s[0:3], s32 offset:44
+; GFX9-NEXT: buffer_load_ushort v15, off, s[0:3], s32 offset:40
+; GFX9-NEXT: buffer_load_ushort v18, off, s[0:3], s32 offset:36
+; GFX9-NEXT: buffer_load_ushort v22, off, s[0:3], s32 offset:32
+; GFX9-NEXT: buffer_load_ushort v27, off, s[0:3], s32 offset:28
+; GFX9-NEXT: buffer_load_ushort v14, off, s[0:3], s32 offset:24
+; GFX9-NEXT: buffer_load_ushort v16, off, s[0:3], s32 offset:20
+; GFX9-NEXT: buffer_load_ushort v19, off, s[0:3], s32 offset:16
+; GFX9-NEXT: buffer_load_ushort v23, off, s[0:3], s32 offset:12
; GFX9-NEXT: s_and_b64 s[4:5], vcc, exec
+; GFX9-NEXT: s_waitcnt vmcnt(33)
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:496 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v32, off, s[0:3], s32 offset:176
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:172
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:508 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:192
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:480 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v46, off, s[0:3], s32 offset:168
+; GFX9-NEXT: buffer_load_ushort v57, off, s[0:3], s32 offset:164
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_ushort v13, off, s[0:3], s32 offset:188
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:444 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v47, off, s[0:3], s32 offset:184
-; GFX9-NEXT: buffer_load_ushort v59, off, s[0:3], s32 offset:180
-; GFX9-NEXT: buffer_load_ushort v42, off, s[0:3], s32 offset:176
-; GFX9-NEXT: buffer_load_ushort v58, off, s[0:3], s32 offset:172
-; GFX9-NEXT: buffer_load_ushort v44, off, s[0:3], s32 offset:168
-; GFX9-NEXT: buffer_load_ushort v56, off, s[0:3], s32 offset:164
-; GFX9-NEXT: buffer_load_ushort v60, off, s[0:3], s32 offset:160
-; GFX9-NEXT: buffer_load_ushort v53, off, s[0:3], s32 offset:156
-; GFX9-NEXT: buffer_load_ushort v49, off, s[0:3], s32 offset:152
-; GFX9-NEXT: buffer_load_ushort v45, off, s[0:3], s32 offset:148
-; GFX9-NEXT: buffer_load_ushort v57, off, s[0:3], s32 offset:144
-; GFX9-NEXT: buffer_load_ushort v55, off, s[0:3], s32 offset:140
-; GFX9-NEXT: buffer_load_ushort v38, off, s[0:3], s32 offset:136
-; GFX9-NEXT: buffer_load_ushort v32, off, s[0:3], s32 offset:132
-; GFX9-NEXT: buffer_load_ushort v46, off, s[0:3], s32 offset:128
-; GFX9-NEXT: buffer_load_ushort v40, off, s[0:3], s32 offset:124
-; GFX9-NEXT: buffer_load_ushort v31, off, s[0:3], s32 offset:120
-; GFX9-NEXT: buffer_load_ushort v52, off, s[0:3], s32 offset:116
-; GFX9-NEXT: buffer_load_ushort v50, off, s[0:3], s32 offset:112
-; GFX9-NEXT: buffer_load_ushort v41, off, s[0:3], s32 offset:108
-; GFX9-NEXT: buffer_load_ushort v27, off, s[0:3], s32 offset:104
-; GFX9-NEXT: buffer_load_ushort v37, off, s[0:3], s32 offset:100
-; GFX9-NEXT: buffer_load_ushort v36, off, s[0:3], s32 offset:96
-; GFX9-NEXT: buffer_load_ushort v51, off, s[0:3], s32 offset:92
-; GFX9-NEXT: buffer_load_ushort v23, off, s[0:3], s32 offset:88
-; GFX9-NEXT: buffer_load_ushort v28, off, s[0:3], s32 offset:84
-; GFX9-NEXT: buffer_load_ushort v34, off, s[0:3], s32 offset:80
-; GFX9-NEXT: buffer_load_ushort v39, off, s[0:3], s32 offset:76
-; GFX9-NEXT: buffer_load_ushort v19, off, s[0:3], s32 offset:72
-; GFX9-NEXT: buffer_load_ushort v24, off, s[0:3], s32 offset:68
-; GFX9-NEXT: buffer_load_ushort v29, off, s[0:3], s32 offset:64
-; GFX9-NEXT: buffer_load_ushort v35, off, s[0:3], s32 offset:60
-; GFX9-NEXT: buffer_load_ushort v16, off, s[0:3], s32 offset:56
-; GFX9-NEXT: buffer_load_ushort v20, off, s[0:3], s32 offset:52
-; GFX9-NEXT: buffer_load_ushort v25, off, s[0:3], s32 offset:48
-; GFX9-NEXT: buffer_load_ushort v30, off, s[0:3], s32 offset:44
-; GFX9-NEXT: buffer_load_ushort v14, off, s[0:3], s32 offset:40
-; GFX9-NEXT: buffer_load_ushort v17, off, s[0:3], s32 offset:36
-; GFX9-NEXT: buffer_load_ushort v21, off, s[0:3], s32 offset:32
-; GFX9-NEXT: buffer_load_ushort v26, off, s[0:3], s32 offset:28
-; GFX9-NEXT: buffer_load_ushort v13, off, s[0:3], s32 offset:24
-; GFX9-NEXT: buffer_load_ushort v15, off, s[0:3], s32 offset:20
-; GFX9-NEXT: buffer_load_ushort v18, off, s[0:3], s32 offset:16
-; GFX9-NEXT: buffer_load_ushort v22, off, s[0:3], s32 offset:12
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:516 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:520 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(6)
-; GFX9-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:528 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:532 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(7)
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(9)
-; GFX9-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(12)
-; GFX9-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:160
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:500 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v40, off, s[0:3], s32 offset:156
+; GFX9-NEXT: buffer_load_ushort v59, off, s[0:3], s32 offset:152
+; GFX9-NEXT: buffer_load_ushort v41, off, s[0:3], s32 offset:148
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:144
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:512 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:516 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:520 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:528 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:532 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:408 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v49, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
+; GFX9-NEXT: s_waitcnt vmcnt(27)
+; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:412 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:420 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:424 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:428 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:432 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:420 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:424 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:628 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:636 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v52, off, s[0:3], s32 offset:640 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:644 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v38, off, s[0:3], s32 offset:648 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:444 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:448 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v48, off, s[0:3], s32 offset:628 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:452 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:636 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:640 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:644 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:648 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:656 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:660 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v48, off, s[0:3], s32 offset:664 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:668 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:672 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:508 ; 4-byte Folded Spill
+; GFX9-NEXT: s_waitcnt vmcnt(49)
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:504 ; 4-byte Folded Spill
; GFX9-NEXT: s_cbranch_scc0 .LBB93_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v8, 0xc0c0004
-; GFX9-NEXT: v_mov_b32_e32 v2, s21
-; GFX9-NEXT: v_perm_b32 v0, s16, v0, v8
+; GFX9-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:676 ; 4-byte Folded Spill
; GFX9-NEXT: v_mov_b32_e32 v1, s19
-; GFX9-NEXT: v_perm_b32 v2, s20, v2, v8
; GFX9-NEXT: v_mov_b32_e32 v3, s23
+; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_perm_b32 v1, s18, v1, v8
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:680 ; 4-byte Folded Spill
+; GFX9-NEXT: v_mov_b32_e32 v2, s21
; GFX9-NEXT: v_perm_b32 v3, s22, v3, v8
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v3, 16, v2
-; GFX9-NEXT: v_mov_b32_e32 v2, s25
-; GFX9-NEXT: v_perm_b32 v2, s24, v2, v8
+; GFX9-NEXT: v_perm_b32 v0, s16, v0, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_perm_b32 v2, s20, v2, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v3, s27
+; GFX9-NEXT: v_mov_b32_e32 v2, s25
; GFX9-NEXT: v_perm_b32 v3, s26, v3, v8
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_lshl_or_b32 v2, v3, 16, v2
-; GFX9-NEXT: v_mov_b32_e32 v3, s29
-; GFX9-NEXT: v_perm_b32 v3, s28, v3, v8
+; GFX9-NEXT: v_perm_b32 v2, s24, v2, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_mov_b32_e32 v4, s6
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v3, s29
; GFX9-NEXT: v_perm_b32 v4, s7, v4, v8
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_lshl_or_b32 v3, v4, 16, v3
-; GFX9-NEXT: v_mov_b32_e32 v4, s15
-; GFX9-NEXT: v_perm_b32 v4, s43, v4, v8
+; GFX9-NEXT: v_perm_b32 v3, s28, v3, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX9-NEXT: v_mov_b32_e32 v5, s8
+; GFX9-NEXT: v_or_b32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v4, s15
; GFX9-NEXT: v_perm_b32 v5, s10, v5, v8
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_lshl_or_b32 v4, v5, 16, v4
-; GFX9-NEXT: v_mov_b32_e32 v5, s47
-; GFX9-NEXT: v_perm_b32 v5, s59, v5, v8
+; GFX9-NEXT: v_perm_b32 v4, s43, v4, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX9-NEXT: v_mov_b32_e32 v6, s12
+; GFX9-NEXT: v_or_b32_sdwa v4, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v5, s47
; GFX9-NEXT: v_perm_b32 v6, s14, v6, v8
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_lshl_or_b32 v5, v6, 16, v5
-; GFX9-NEXT: v_mov_b32_e32 v6, s61
-; GFX9-NEXT: v_perm_b32 v6, s63, v6, v8
+; GFX9-NEXT: v_perm_b32 v5, s59, v5, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX9-NEXT: v_mov_b32_e32 v50, v54
+; GFX9-NEXT: v_mov_b32_e32 v54, v7
; GFX9-NEXT: v_mov_b32_e32 v7, s42
+; GFX9-NEXT: v_or_b32_sdwa v5, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v6, s61
; GFX9-NEXT: v_perm_b32 v7, s46, v7, v8
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_lshl_or_b32 v6, v7, 16, v6
-; GFX9-NEXT: v_mov_b32_e32 v7, s73
-; GFX9-NEXT: v_perm_b32 v7, s75, v7, v8
-; GFX9-NEXT: v_mov_b32_e32 v63, v33
-; GFX9-NEXT: v_mov_b32_e32 v33, v9
+; GFX9-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:684 ; 4-byte Folded Spill
+; GFX9-NEXT: v_perm_b32 v6, s63, v6, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX9-NEXT: v_mov_b32_e32 v39, v9
; GFX9-NEXT: v_mov_b32_e32 v9, s45
+; GFX9-NEXT: v_or_b32_sdwa v6, v7, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v7, s73
; GFX9-NEXT: v_perm_b32 v9, s58, v9, v8
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_lshl_or_b32 v7, v9, 16, v7
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v10
-; GFX9-NEXT: v_perm_b32 v10, s74, v12, v8
-; GFX9-NEXT: v_or_b32_sdwa v9, v11, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v9, 16, v10
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v18
-; GFX9-NEXT: v_or_b32_sdwa v9, v22, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v13
-; GFX9-NEXT: v_or_b32_sdwa v10, v15, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
+; GFX9-NEXT: v_perm_b32 v7, s75, v7, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX9-NEXT: s_and_b32 s4, s40, 0xff
+; GFX9-NEXT: s_lshl_b32 s5, s13, 8
+; GFX9-NEXT: v_or_b32_sdwa v7, v9, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v9, s57
+; GFX9-NEXT: s_or_b32 s4, s4, s5
+; GFX9-NEXT: v_mov_b32_e32 v56, v38
+; GFX9-NEXT: v_perm_b32 v9, s60, v9, v8
+; GFX9-NEXT: v_mov_b32_e32 v38, v37
+; GFX9-NEXT: v_mov_b32_e32 v37, v10
+; GFX9-NEXT: v_lshlrev_b32_e64 v10, 16, s4
+; GFX9-NEXT: v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v11
+; GFX9-NEXT: v_perm_b32 v8, s74, v13, v8
+; GFX9-NEXT: v_or_b32_sdwa v10, v12, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v11, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v19
+; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v14
+; GFX9-NEXT: v_or_b32_sdwa v8, v23, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v10, v16, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v12, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_mov_b32 s4, 0xc0c0004
-; GFX9-NEXT: v_lshl_or_b32 v12, v10, 16, v9
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v14
-; GFX9-NEXT: v_perm_b32 v10, v26, v21, s4
-; GFX9-NEXT: v_or_b32_sdwa v9, v17, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v13, v9, 16, v10
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v25
-; GFX9-NEXT: v_or_b32_sdwa v9, v30, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v16
-; GFX9-NEXT: v_or_b32_sdwa v10, v20, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_lshl_or_b32 v14, v10, 16, v9
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v19
-; GFX9-NEXT: v_perm_b32 v10, v35, v29, s4
-; GFX9-NEXT: v_or_b32_sdwa v9, v24, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v15, v9, 16, v10
-; GFX9-NEXT: v_perm_b32 v9, v39, v34, s4
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_perm_b32 v10, v51, v36, s4
-; GFX9-NEXT: v_perm_b32 v16, v28, v23, s4
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_perm_b32 v17, v37, v27, s4
-; GFX9-NEXT: v_lshl_or_b32 v16, v16, 16, v9
-; GFX9-NEXT: v_perm_b32 v9, v41, v50, s4
-; GFX9-NEXT: v_lshl_or_b32 v17, v17, 16, v10
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_perm_b32 v10, v52, v31, s4
-; GFX9-NEXT: v_lshl_or_b32 v18, v10, 16, v9
-; GFX9-NEXT: v_perm_b32 v9, v40, v46, s4
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_perm_b32 v10, v32, v38, s4
-; GFX9-NEXT: v_lshl_or_b32 v19, v10, 16, v9
-; GFX9-NEXT: v_perm_b32 v9, v55, v57, s4
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_perm_b32 v10, v45, v49, s4
-; GFX9-NEXT: v_lshl_or_b32 v20, v10, 16, v9
-; GFX9-NEXT: v_perm_b32 v9, v53, v60, s4
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_perm_b32 v10, v56, v44, s4
-; GFX9-NEXT: v_lshl_or_b32 v21, v10, 16, v9
-; GFX9-NEXT: v_perm_b32 v9, v58, v42, s4
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_perm_b32 v10, v59, v47, s4
-; GFX9-NEXT: v_lshl_or_b32 v22, v10, 16, v9
-; GFX9-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
-; GFX9-NEXT: v_mov_b32_e32 v50, v32
-; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
-; GFX9-NEXT: v_mov_b32_e32 v51, v49
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v15
+; GFX9-NEXT: v_or_b32_sdwa v8, v18, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_perm_b32 v10, v27, v22, s4
+; GFX9-NEXT: v_perm_b32 v16, v49, v53, s4
; GFX9-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:404 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
-; GFX9-NEXT: v_mov_b32_e32 v41, v46
-; GFX9-NEXT: v_mov_b32_e32 v39, v52
-; GFX9-NEXT: v_mov_b32_e32 v52, v38
-; GFX9-NEXT: v_mov_b32_e32 v34, v37
-; GFX9-NEXT: v_mov_b32_e32 v37, v31
-; GFX9-NEXT: s_lshl_b32 s5, s13, 8
+; GFX9-NEXT: v_or_b32_sdwa v13, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v17
+; GFX9-NEXT: v_perm_b32 v17, v44, v48, s4
+; GFX9-NEXT: v_mov_b32_e32 v44, v55
+; GFX9-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v26
+; GFX9-NEXT: v_or_b32_sdwa v8, v31, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v10, v21, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v14, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v20
+; GFX9-NEXT: v_or_b32_sdwa v8, v25, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_perm_b32 v10, v33, v30, s4
+; GFX9-NEXT: v_or_b32_sdwa v15, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_perm_b32 v8, v29, v24, s4
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX9-NEXT: v_perm_b32 v10, v35, v28, s4
+; GFX9-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX9-NEXT: v_or_b32_sdwa v16, v8, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_perm_b32 v8, v44, v34, s4
+; GFX9-NEXT: v_or_b32_sdwa v17, v10, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX9-NEXT: v_perm_b32 v10, v47, v42, s4
+; GFX9-NEXT: v_mov_b32_e32 v34, v36
+; GFX9-NEXT: v_mov_b32_e32 v35, v60
+; GFX9-NEXT: v_or_b32_sdwa v18, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_perm_b32 v8, v35, v34, s4
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX9-NEXT: v_perm_b32 v10, v58, v43, s4
+; GFX9-NEXT: v_mov_b32_e32 v53, v59
+; GFX9-NEXT: v_or_b32_sdwa v19, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_perm_b32 v8, v41, v53, s4
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX9-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:484 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
+; GFX9-NEXT: s_lshl_b32 s5, s41, 8
; GFX9-NEXT: s_lshl_b32 s77, s9, 8
; GFX9-NEXT: s_lshl_b32 s78, s44, 8
-; GFX9-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:452 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:428 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(9)
-; GFX9-NEXT: v_perm_b32 v9, v9, v10, s4
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: s_waitcnt vmcnt(7)
-; GFX9-NEXT: v_perm_b32 v10, v32, v58, s4
-; GFX9-NEXT: v_lshl_or_b32 v23, v10, 16, v9
-; GFX9-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:440 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:400 ; 4-byte Folded Reload
+; GFX9-NEXT: v_perm_b32 v10, v61, v49, s4
+; GFX9-NEXT: v_or_b32_sdwa v20, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_perm_b32 v8, v57, v46, s4
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX9-NEXT: s_waitcnt vmcnt(8)
+; GFX9-NEXT: v_perm_b32 v10, v40, v55, s4
+; GFX9-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
+; GFX9-NEXT: v_or_b32_sdwa v21, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: s_waitcnt vmcnt(9)
+; GFX9-NEXT: v_perm_b32 v10, v33, v32, s4
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_perm_b32 v8, v60, v40, s4
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX9-NEXT: v_or_b32_sdwa v22, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_perm_b32 v8, v42, v59, s4
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX9-NEXT: v_perm_b32 v10, v51, v36, s4
+; GFX9-NEXT: v_or_b32_sdwa v23, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v8
+; GFX9-NEXT: v_or_b32_sdwa v8, v45, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v9
+; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v51
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v49
-; GFX9-NEXT: v_or_b32_sdwa v10, v24, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_lshl_or_b32 v24, v10, 16, v9
-; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:436 ; 4-byte Folded Reload
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v35
+; GFX9-NEXT: v_or_b32_sdwa v10, v45, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v24, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:400 ; 4-byte Folded Reload
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v32
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
+; GFX9-NEXT: v_or_b32_sdwa v8, v10, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:436 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:440 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v10, v25, v10, s4
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v25, v9, 16, v10
-; GFX9-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
-; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v33
-; GFX9-NEXT: v_or_b32_sdwa v10, v62, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
-; GFX9-NEXT: v_mov_b32_e32 v33, v46
+; GFX9-NEXT: v_or_b32_sdwa v25, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:452 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:432 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v9
-; GFX9-NEXT: v_or_b32_sdwa v9, v43, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_lshl_or_b32 v26, v10, 16, v9
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v48
-; GFX9-NEXT: v_perm_b32 v10, v63, v61, s4
-; GFX9-NEXT: v_or_b32_sdwa v9, v54, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v27, v9, 16, v10
-; GFX9-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
-; GFX9-NEXT: v_mov_b32_e32 v54, v61
-; GFX9-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:484 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(4)
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v9
-; GFX9-NEXT: s_waitcnt vmcnt(3)
-; GFX9-NEXT: v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_mov_b32_e32 v48, v63
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v10
-; GFX9-NEXT: v_or_b32_sdwa v10, v28, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_lshl_or_b32 v28, v10, 16, v9
-; GFX9-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:428 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v8
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_or_b32_sdwa v8, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v37
+; GFX9-NEXT: v_or_b32_sdwa v10, v26, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v26, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:424 ; 4-byte Folded Reload
+; GFX9-NEXT: v_perm_b32 v10, v38, v39, s4
+; GFX9-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
+; GFX9-NEXT: v_mov_b32_e32 v38, v34
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v8
+; GFX9-NEXT: v_or_b32_sdwa v8, v50, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v27, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v52
+; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v63
+; GFX9-NEXT: v_or_b32_sdwa v8, v54, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v10, v56, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v28, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
+; GFX9-NEXT: v_mov_b32_e32 v54, v31
+; GFX9-NEXT: v_mov_b32_e32 v50, v53
+; GFX9-NEXT: v_mov_b32_e32 v56, v40
+; GFX9-NEXT: v_mov_b32_e32 v52, v32
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v9
+; GFX9-NEXT: v_mov_b32_e32 v63, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v8
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:432 ; 4-byte Folded Reload
+; GFX9-NEXT: v_or_b32_sdwa v8, v10, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v10, v36, v10, s4
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v29, v9, 16, v10
-; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:424 ; 4-byte Folded Reload
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v43
-; GFX9-NEXT: v_or_b32_sdwa v9, v62, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
+; GFX9-NEXT: v_perm_b32 v10, v10, v62, s4
+; GFX9-NEXT: v_or_b32_sdwa v29, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:404 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v8
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v10
-; GFX9-NEXT: v_or_b32_sdwa v10, v30, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_lshl_or_b32 v30, v10, 16, v9
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v38
-; GFX9-NEXT: v_perm_b32 v10, v63, v61, s4
-; GFX9-NEXT: v_or_b32_sdwa v9, v46, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v31, v9, 16, v10
-; GFX9-NEXT: v_mov_b32_e32 v9, s57
-; GFX9-NEXT: v_perm_b32 v8, s60, v9, v8
-; GFX9-NEXT: s_and_b32 s4, s40, 0xff
-; GFX9-NEXT: s_or_b32 s4, s4, s5
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, s4, 16, v8
+; GFX9-NEXT: v_or_b32_sdwa v8, v39, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v10, v30, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v30, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v8
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v62, v10
+; GFX9-NEXT: v_perm_b32 v10, v31, v10, s4
; GFX9-NEXT: s_and_b32 s4, s76, 0xff
-; GFX9-NEXT: s_lshl_b32 s5, s41, 8
; GFX9-NEXT: s_or_b32 s4, s4, s5
; GFX9-NEXT: s_and_b32 s5, s11, 0xff
; GFX9-NEXT: s_or_b32 s5, s5, s77
@@ -175042,60 +175021,54 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX9-NEXT: s_or_b32 s5, s5, s77
; GFX9-NEXT: s_and_b32 s77, s56, 0xff
; GFX9-NEXT: s_or_b32 s77, s77, s78
+; GFX9-NEXT: v_or_b32_sdwa v8, v48, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: s_pack_ll_b32_b16 s5, s5, s77
+; GFX9-NEXT: v_or_b32_sdwa v31, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v8, s4
; GFX9-NEXT: v_mov_b32_e32 v10, s5
; GFX9-NEXT: s_mov_b64 s[4:5], 0
-; GFX9-NEXT: v_mov_b32_e32 v63, v36
; GFX9-NEXT: s_branch .LBB93_3
; GFX9-NEXT: .LBB93_2:
-; GFX9-NEXT: v_mov_b32_e32 v54, v61
-; GFX9-NEXT: v_mov_b32_e32 v50, v32
-; GFX9-NEXT: v_mov_b32_e32 v51, v49
-; GFX9-NEXT: v_mov_b32_e32 v39, v52
-; GFX9-NEXT: v_mov_b32_e32 v52, v38
-; GFX9-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:484 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
+; GFX9-NEXT: v_mov_b32_e32 v63, v39
+; GFX9-NEXT: v_mov_b32_e32 v38, v36
+; GFX9-NEXT: v_mov_b32_e32 v44, v55
+; GFX9-NEXT: v_mov_b32_e32 v50, v59
+; GFX9-NEXT: v_mov_b32_e32 v35, v60
+; GFX9-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
-; GFX9-NEXT: v_mov_b32_e32 v41, v46
-; GFX9-NEXT: v_mov_b32_e32 v34, v37
-; GFX9-NEXT: v_mov_b32_e32 v37, v31
+; GFX9-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:484 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
+; GFX9-NEXT: v_mov_b32_e32 v54, v2
+; GFX9-NEXT: v_mov_b32_e32 v62, v1
; GFX9-NEXT: s_mov_b64 s[4:5], -1
; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
; GFX9-NEXT: ; implicit-def: $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
; GFX9-NEXT: .LBB93_3: ; %Flow
-; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:400 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
; GFX9-NEXT: s_cselect_b32 s4, 1, 0
; GFX9-NEXT: s_cmp_lg_u32 s4, 1
; GFX9-NEXT: s_cbranch_scc1 .LBB93_5
; GFX9-NEXT: ; %bb.4: ; %cmp.true
-; GFX9-NEXT: s_waitcnt vmcnt(5)
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v48
-; GFX9-NEXT: s_mov_b32 s4, 0xc0c0004
-; GFX9-NEXT: v_perm_b32 v0, v0, v61, s4
-; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:472 ; 4-byte Folded Spill
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v33
-; GFX9-NEXT: s_waitcnt vmcnt(4)
-; GFX9-NEXT: v_perm_b32 v0, v0, v38, s4
-; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v62
-; GFX9-NEXT: v_perm_b32 v0, v0, v43, s4
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v54
+; GFX9-NEXT: s_mov_b32 s5, 0xc0c0004
+; GFX9-NEXT: v_perm_b32 v0, v0, v62, s5
; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:480 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
+; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v48
+; GFX9-NEXT: s_movk_i32 s4, 0x300
; GFX9-NEXT: s_add_i32 s74, s74, 3
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:424 ; 4-byte Folded Reload
; GFX9-NEXT: s_add_i32 s72, s72, 3
; GFX9-NEXT: s_add_i32 s56, s56, 3
; GFX9-NEXT: s_add_i32 s60, s60, 3
@@ -175116,373 +175089,358 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX9-NEXT: s_add_i32 s26, s26, 3
; GFX9-NEXT: s_add_i32 s20, s20, 3
; GFX9-NEXT: s_add_i32 s22, s22, 3
+; GFX9-NEXT: v_mov_b32_e32 v32, s23
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: v_mov_b32_e32 v33, s17
; GFX9-NEXT: s_add_i32 s18, s18, 3
-; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
+; GFX9-NEXT: v_mov_b32_e32 v34, s19
+; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:432 ; 4-byte Folded Reload
-; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:408 ; 4-byte Folded Spill
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v63
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:404 ; 4-byte Folded Reload
+; GFX9-NEXT: v_add_u32_sdwa v0, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:444 ; 4-byte Folded Spill
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v39
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:424 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:428 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_sdwa v0, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:636 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:428 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:412 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT: v_perm_b32 v0, v0, v63, s5
+; GFX9-NEXT: v_add_u32_sdwa v0, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:448 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: v_add_u32_e32 v38, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:452 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_e32 v39, 0x300, v0
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: v_add_u32_e32 v62, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_sdwa v63, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:628 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT: v_perm_b32 v0, v0, v54, s4
-; GFX9-NEXT: v_add_u32_e32 v63, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:424 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_sdwa v48, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:432 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:452 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:412 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:632 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:420 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:428 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: v_add_u32_e32 v31, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_sdwa v31, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:440 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:436 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v30, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:436 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:440 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:400 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT: v_perm_b32 v0, v0, v35, s4
-; GFX9-NEXT: v_add_u32_e32 v29, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v36
+; GFX9-NEXT: v_perm_b32 v0, v0, v52, s5
+; GFX9-NEXT: v_add_u32_sdwa v29, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:632 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v28, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:404 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v45
+; GFX9-NEXT: v_perm_b32 v0, v0, v51, s5
+; GFX9-NEXT: v_add_u32_sdwa v27, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT: v_perm_b32 v0, v0, v49, s4
-; GFX9-NEXT: v_add_u32_e32 v27, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v36, s5
; GFX9-NEXT: v_add_u32_e32 v26, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v32
-; GFX9-NEXT: v_perm_b32 v0, v0, v58, s4
-; GFX9-NEXT: v_add_u32_e32 v25, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
-; GFX9-NEXT: v_mov_b32_e32 v32, s23
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v42
+; GFX9-NEXT: v_perm_b32 v0, v0, v59, s5
+; GFX9-NEXT: v_add_u32_sdwa v25, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v33
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_e32 v24, 0x300, v0
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v60
+; GFX9-NEXT: v_perm_b32 v0, v0, v56, s5
+; GFX9-NEXT: v_add_u32_sdwa v23, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
+; GFX9-NEXT: v_mov_b32_e32 v33, s17
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT: v_perm_b32 v0, v0, v42, s4
-; GFX9-NEXT: v_add_u32_e32 v24, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v59
-; GFX9-NEXT: v_perm_b32 v0, v0, v47, s4
-; GFX9-NEXT: v_add_u32_e32 v23, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v53
-; GFX9-NEXT: v_perm_b32 v0, v0, v60, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v55, s5
; GFX9-NEXT: v_add_u32_e32 v22, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v56
-; GFX9-NEXT: v_perm_b32 v0, v0, v44, s4
-; GFX9-NEXT: v_add_u32_e32 v21, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v55
-; GFX9-NEXT: v_perm_b32 v0, v0, v57, s4
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v57
+; GFX9-NEXT: v_perm_b32 v0, v0, v46, s5
+; GFX9-NEXT: v_add_u32_sdwa v21, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v61
+; GFX9-NEXT: v_perm_b32 v0, v0, v49, s5
; GFX9-NEXT: v_add_u32_e32 v20, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v45
-; GFX9-NEXT: v_perm_b32 v0, v0, v51, s4
-; GFX9-NEXT: v_add_u32_e32 v49, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v40
-; GFX9-NEXT: v_perm_b32 v0, v0, v41, s4
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v41
+; GFX9-NEXT: v_perm_b32 v0, v0, v50, s5
+; GFX9-NEXT: v_add_u32_sdwa v51, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v58
+; GFX9-NEXT: v_perm_b32 v0, v0, v43, s5
; GFX9-NEXT: v_add_u32_e32 v19, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v50
-; GFX9-NEXT: v_perm_b32 v0, v0, v52, s4
-; GFX9-NEXT: v_add_u32_e32 v52, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v22, 0xffff, v22
-; GFX9-NEXT: v_lshl_or_b32 v21, v21, 16, v22
-; GFX9-NEXT: v_and_b32_e32 v22, 0xffff, v24
-; GFX9-NEXT: v_lshl_or_b32 v22, v23, 16, v22
-; GFX9-NEXT: v_and_b32_e32 v23, 0xffff, v26
-; GFX9-NEXT: v_lshl_or_b32 v23, v25, 16, v23
-; GFX9-NEXT: v_and_b32_e32 v25, 0xffff, v30
-; GFX9-NEXT: v_lshl_or_b32 v25, v29, 16, v25
-; GFX9-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:424 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v24, 0xffff, v28
-; GFX9-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:428 ; 4-byte Folded Reload
-; GFX9-NEXT: v_lshl_or_b32 v24, v27, 16, v24
-; GFX9-NEXT: v_and_b32_e32 v27, 0xffff, v63
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_lshl_or_b32 v19, v52, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v49, 16, v20
-; GFX9-NEXT: s_waitcnt vmcnt(4)
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v35
+; GFX9-NEXT: v_perm_b32 v0, v0, v38, s5
+; GFX9-NEXT: v_add_u32_sdwa v38, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v47
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
; GFX9-NEXT: v_add_u32_e32 v18, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v39
-; GFX9-NEXT: v_perm_b32 v0, v0, v37, s4
-; GFX9-NEXT: v_add_u32_e32 v37, 0x300, v0
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v44
+; GFX9-NEXT: v_or_b32_sdwa v21, v21, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v22, v23, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v23, v25, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v25, v29, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
+; GFX9-NEXT: v_or_b32_sdwa v24, v27, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v38, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v20, v51, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v28, v63, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(4)
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_sdwa v53, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_lshl_or_b32 v18, v37, 16, v18
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
+; GFX9-NEXT: v_or_b32_sdwa v18, v53, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: s_waitcnt vmcnt(5)
+; GFX9-NEXT: v_or_b32_sdwa v26, v31, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: v_or_b32_sdwa v29, v30, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:404 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(4)
+; GFX9-NEXT: v_or_b32_sdwa v27, v48, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: s_waitcnt vmcnt(3)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
-; GFX9-NEXT: v_add_u32_e32 v54, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v34
-; GFX9-NEXT: v_mov_b32_e32 v34, s19
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v17, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: v_or_b32_sdwa v30, v31, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: v_add_u32_e32 v16, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:636 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_sdwa v54, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
+; GFX9-NEXT: v_or_b32_sdwa v17, v54, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT: v_perm_b32 v0, v0, v37, s5
+; GFX9-NEXT: v_add_u32_e32 v16, 0x300, v0
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: v_add_u32_e32 v55, 0x300, v0
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_sdwa v55, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
-; GFX9-NEXT: v_lshl_or_b32 v16, v55, 16, v16
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
+; GFX9-NEXT: v_or_b32_sdwa v16, v55, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v15, 0x300, v0
; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: v_add_u32_e32 v40, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_sdwa v40, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
-; GFX9-NEXT: v_lshl_or_b32 v15, v40, 16, v15
+; GFX9-NEXT: v_or_b32_sdwa v15, v40, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v14, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: v_add_u32_e32 v41, 0x300, v0
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_sdwa v41, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
-; GFX9-NEXT: v_lshl_or_b32 v14, v41, 16, v14
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
+; GFX9-NEXT: v_or_b32_sdwa v14, v41, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v13, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: v_add_u32_e32 v42, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
-; GFX9-NEXT: v_lshl_or_b32 v13, v42, 16, v13
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_sdwa v42, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
+; GFX9-NEXT: v_or_b32_sdwa v13, v42, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v12, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
-; GFX9-NEXT: v_add_u32_e32 v43, 0x300, v0
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
+; GFX9-NEXT: v_add_u32_sdwa v43, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX9-NEXT: v_perm_b32 v32, s22, v32, v0
-; GFX9-NEXT: v_add_u32_e32 v32, 0x300, v32
+; GFX9-NEXT: v_add_u32_sdwa v32, v32, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_perm_b32 v33, s16, v33, v0
; GFX9-NEXT: v_add_u32_e32 v33, 0x300, v33
-; GFX9-NEXT: v_and_b32_e32 v33, 0xffff, v33
-; GFX9-NEXT: v_lshl_or_b32 v12, v43, 16, v12
+; GFX9-NEXT: v_or_b32_sdwa v12, v43, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v1, s74, v1, v0
; GFX9-NEXT: v_add_u32_e32 v11, 0x300, v1
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_add_u32_e32 v1, 3, v1
-; GFX9-NEXT: v_perm_b32 v1, v1, v2, s4
-; GFX9-NEXT: v_add_u32_e32 v44, 0x300, v1
+; GFX9-NEXT: v_perm_b32 v1, v1, v2, s5
+; GFX9-NEXT: v_add_u32_sdwa v44, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v1, s62
; GFX9-NEXT: v_perm_b32 v1, s72, v1, v0
; GFX9-NEXT: v_add_u32_e32 v10, 0x300, v1
; GFX9-NEXT: v_mov_b32_e32 v1, s44
; GFX9-NEXT: v_perm_b32 v1, s56, v1, v0
-; GFX9-NEXT: v_add_u32_e32 v45, 0x300, v1
+; GFX9-NEXT: v_add_u32_sdwa v45, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v1, s57
; GFX9-NEXT: v_perm_b32 v1, s60, v1, v0
; GFX9-NEXT: v_add_u32_e32 v9, 0x300, v1
; GFX9-NEXT: v_mov_b32_e32 v1, s13
; GFX9-NEXT: v_perm_b32 v1, s40, v1, v0
-; GFX9-NEXT: v_add_u32_e32 v46, 0x300, v1
+; GFX9-NEXT: v_add_u32_sdwa v46, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v1, s41
; GFX9-NEXT: v_perm_b32 v1, s76, v1, v0
; GFX9-NEXT: v_add_u32_e32 v8, 0x300, v1
; GFX9-NEXT: v_mov_b32_e32 v1, s9
; GFX9-NEXT: v_perm_b32 v1, s11, v1, v0
-; GFX9-NEXT: v_add_u32_e32 v47, 0x300, v1
+; GFX9-NEXT: v_add_u32_sdwa v47, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v1, s73
; GFX9-NEXT: v_perm_b32 v1, s75, v1, v0
; GFX9-NEXT: v_add_u32_e32 v7, 0x300, v1
; GFX9-NEXT: v_mov_b32_e32 v1, s45
; GFX9-NEXT: v_perm_b32 v1, s58, v1, v0
-; GFX9-NEXT: v_add_u32_e32 v56, 0x300, v1
+; GFX9-NEXT: v_add_u32_sdwa v56, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v1, s61
; GFX9-NEXT: v_perm_b32 v1, s63, v1, v0
; GFX9-NEXT: v_add_u32_e32 v6, 0x300, v1
; GFX9-NEXT: v_mov_b32_e32 v1, s42
; GFX9-NEXT: v_perm_b32 v1, s46, v1, v0
-; GFX9-NEXT: v_add_u32_e32 v57, 0x300, v1
+; GFX9-NEXT: v_add_u32_sdwa v57, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v1, s47
; GFX9-NEXT: v_perm_b32 v1, s59, v1, v0
; GFX9-NEXT: v_add_u32_e32 v5, 0x300, v1
; GFX9-NEXT: v_mov_b32_e32 v1, s12
; GFX9-NEXT: v_perm_b32 v1, s14, v1, v0
-; GFX9-NEXT: v_add_u32_e32 v58, 0x300, v1
+; GFX9-NEXT: v_add_u32_sdwa v58, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v1, s15
; GFX9-NEXT: v_perm_b32 v1, s43, v1, v0
; GFX9-NEXT: v_add_u32_e32 v4, 0x300, v1
; GFX9-NEXT: v_mov_b32_e32 v1, s8
; GFX9-NEXT: v_perm_b32 v1, s10, v1, v0
-; GFX9-NEXT: v_add_u32_e32 v59, 0x300, v1
+; GFX9-NEXT: v_add_u32_sdwa v59, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v1, s29
; GFX9-NEXT: v_perm_b32 v1, s28, v1, v0
; GFX9-NEXT: v_add_u32_e32 v3, 0x300, v1
; GFX9-NEXT: v_mov_b32_e32 v1, s6
; GFX9-NEXT: v_perm_b32 v1, s7, v1, v0
-; GFX9-NEXT: v_add_u32_e32 v60, 0x300, v1
+; GFX9-NEXT: v_add_u32_sdwa v60, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v1, s25
; GFX9-NEXT: v_perm_b32 v1, s24, v1, v0
; GFX9-NEXT: v_add_u32_e32 v2, 0x300, v1
; GFX9-NEXT: v_mov_b32_e32 v1, s27
; GFX9-NEXT: v_perm_b32 v1, s26, v1, v0
-; GFX9-NEXT: v_add_u32_e32 v61, 0x300, v1
+; GFX9-NEXT: v_add_u32_sdwa v61, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v1, s21
; GFX9-NEXT: v_perm_b32 v1, s20, v1, v0
; GFX9-NEXT: v_add_u32_e32 v1, 0x300, v1
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_lshl_or_b32 v1, v32, 16, v1
-; GFX9-NEXT: v_and_b32_e32 v32, 0xffff, v54
-; GFX9-NEXT: v_lshl_or_b32 v17, v17, 16, v32
-; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v29, 0xffff, v29
-; GFX9-NEXT: v_and_b32_e32 v26, 0xffff, v26
-; GFX9-NEXT: v_lshl_or_b32 v29, v30, 16, v29
-; GFX9-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
-; GFX9-NEXT: v_lshl_or_b32 v26, v31, 16, v26
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
+; GFX9-NEXT: v_or_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
; GFX9-NEXT: v_perm_b32 v0, s18, v34, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v27, v28, 16, v27
-; GFX9-NEXT: v_and_b32_e32 v28, 0xffff, v38
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v33
-; GFX9-NEXT: v_lshl_or_b32 v2, v61, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v60, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v59, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v58, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v57, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v56, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v47, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v46, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v45, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v44, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v28, v62, 16, v28
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_and_b32_e32 v30, 0xffff, v30
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_lshl_or_b32 v30, v31, 16, v30
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_and_b32_e32 v31, 0xffff, v31
-; GFX9-NEXT: v_lshl_or_b32 v31, v32, 16, v31
+; GFX9-NEXT: v_add_u32_sdwa v0, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v61, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v60, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v4, v59, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v58, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v57, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v56, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v47, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v46, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v45, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v44, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_or_b32_sdwa v31, v32, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: .LBB93_5: ; %end
; GFX9-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
@@ -195449,25 +195407,25 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX9-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
+; GFX9-NEXT: v_readfirstlane_b32 s42, v13
; GFX9-NEXT: v_readfirstlane_b32 s46, v12
; GFX9-NEXT: v_readfirstlane_b32 s61, v11
-; GFX9-NEXT: v_readfirstlane_b32 s63, v10
+; GFX9-NEXT: v_readfirstlane_b32 s15, v3
; GFX9-NEXT: v_readfirstlane_b32 s7, v0
-; GFX9-NEXT: buffer_load_ushort v10, off, s[0:3], s32 offset:8
-; GFX9-NEXT: buffer_load_ushort v11, off, s[0:3], s32 offset:4
-; GFX9-NEXT: buffer_load_ushort v12, off, s[0:3], s32
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:328
-; GFX9-NEXT: v_readfirstlane_b32 s43, v2
-; GFX9-NEXT: v_readfirstlane_b32 s6, v1
+; GFX9-NEXT: buffer_load_ushort v11, off, s[0:3], s32 offset:8
+; GFX9-NEXT: buffer_load_ushort v12, off, s[0:3], s32 offset:4
+; GFX9-NEXT: buffer_load_ushort v13, off, s[0:3], s32
+; GFX9-NEXT: buffer_load_ushort v3, off, s[0:3], s32 offset:328
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:324
; GFX9-NEXT: v_readfirstlane_b32 s8, v5
; GFX9-NEXT: v_readfirstlane_b32 s10, v4
-; GFX9-NEXT: v_readfirstlane_b32 s15, v3
+; GFX9-NEXT: v_readfirstlane_b32 s43, v2
+; GFX9-NEXT: v_readfirstlane_b32 s6, v1
+; GFX9-NEXT: v_readfirstlane_b32 s63, v10
; GFX9-NEXT: v_readfirstlane_b32 s12, v9
; GFX9-NEXT: v_readfirstlane_b32 s14, v8
; GFX9-NEXT: v_readfirstlane_b32 s47, v7
-; GFX9-NEXT: v_readfirstlane_b32 s59, v6
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:332
-; GFX9-NEXT: v_readfirstlane_b32 s42, v13
; GFX9-NEXT: v_readfirstlane_b32 s74, v30
; GFX9-NEXT: v_readfirstlane_b32 s44, v29
; GFX9-NEXT: v_readfirstlane_b32 s56, v28
@@ -195485,401 +195443,403 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX9-NEXT: v_readfirstlane_b32 s58, v16
; GFX9-NEXT: v_readfirstlane_b32 s73, v15
; GFX9-NEXT: v_readfirstlane_b32 s75, v14
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:472 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:324
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:320
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:484 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:316
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:480 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v2, off, s[0:3], s32 offset:312
-; GFX9-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:308
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:304
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:496 ; 4-byte Folded Spill
+; GFX9-NEXT: v_readfirstlane_b32 s59, v6
+; GFX9-NEXT: buffer_load_ushort v39, off, s[0:3], s32 offset:296
+; GFX9-NEXT: buffer_load_ushort v50, off, s[0:3], s32 offset:292
+; GFX9-NEXT: buffer_load_ushort v62, off, s[0:3], s32 offset:288
+; GFX9-NEXT: buffer_load_ushort v6, off, s[0:3], s32 offset:284
+; GFX9-NEXT: s_waitcnt vmcnt(5)
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:320
+; GFX9-NEXT: buffer_load_ushort v2, off, s[0:3], s32 offset:316
+; GFX9-NEXT: buffer_load_ushort v4, off, s[0:3], s32 offset:312
+; GFX9-NEXT: buffer_load_ushort v5, off, s[0:3], s32 offset:308
+; GFX9-NEXT: buffer_load_ushort v56, off, s[0:3], s32 offset:304
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:300
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:492 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v4, off, s[0:3], s32 offset:296
-; GFX9-NEXT: buffer_load_ushort v5, off, s[0:3], s32 offset:292
-; GFX9-NEXT: buffer_load_ushort v3, off, s[0:3], s32 offset:288
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:284
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:488 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v8, off, s[0:3], s32 offset:280
-; GFX9-NEXT: buffer_load_ushort v7, off, s[0:3], s32 offset:276
-; GFX9-NEXT: buffer_load_ushort v6, off, s[0:3], s32 offset:272
-; GFX9-NEXT: buffer_load_ushort v63, off, s[0:3], s32 offset:268
-; GFX9-NEXT: buffer_load_ushort v48, off, s[0:3], s32 offset:264
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v63, off, s[0:3], s32 offset:280
+; GFX9-NEXT: buffer_load_ushort v38, off, s[0:3], s32 offset:276
+; GFX9-NEXT: buffer_load_ushort v52, off, s[0:3], s32 offset:272
+; GFX9-NEXT: buffer_load_ushort v7, off, s[0:3], s32 offset:268
+; GFX9-NEXT: buffer_load_ushort v8, off, s[0:3], s32 offset:264
; GFX9-NEXT: buffer_load_ushort v54, off, s[0:3], s32 offset:260
-; GFX9-NEXT: buffer_load_ushort v61, off, s[0:3], s32 offset:256
-; GFX9-NEXT: buffer_load_ushort v33, off, s[0:3], s32 offset:252
-; GFX9-NEXT: buffer_load_ushort v9, off, s[0:3], s32 offset:248
-; GFX9-NEXT: buffer_load_ushort v62, off, s[0:3], s32 offset:244
+; GFX9-NEXT: buffer_load_ushort v9, off, s[0:3], s32 offset:256
+; GFX9-NEXT: buffer_load_ushort v37, off, s[0:3], s32 offset:252
+; GFX9-NEXT: buffer_load_ushort v10, off, s[0:3], s32 offset:248
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:244
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:428 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:240
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:464 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v43, off, s[0:3], s32 offset:236
-; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:452 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:236
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:432 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:232
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:500 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:464 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:228
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:436 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:400 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:224
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:436 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:220
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:468 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:440 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:216
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:504 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:472 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:212
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:468 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:208
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:440 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:204
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:400 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:408 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v45, off, s[0:3], s32 offset:204
+; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:200
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:512 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:488 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:196
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:484 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:192
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v51, off, s[0:3], s32 offset:188
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:184
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:492 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:180
; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_ushort v61, off, s[0:3], s32 offset:140
+; GFX9-NEXT: buffer_load_ushort v36, off, s[0:3], s32 offset:136
+; GFX9-NEXT: buffer_load_ushort v60, off, s[0:3], s32 offset:132
+; GFX9-NEXT: buffer_load_ushort v43, off, s[0:3], s32 offset:128
+; GFX9-NEXT: buffer_load_ushort v58, off, s[0:3], s32 offset:124
+; GFX9-NEXT: buffer_load_ushort v34, off, s[0:3], s32 offset:120
+; GFX9-NEXT: buffer_load_ushort v55, off, s[0:3], s32 offset:116
+; GFX9-NEXT: buffer_load_ushort v42, off, s[0:3], s32 offset:112
+; GFX9-NEXT: buffer_load_ushort v47, off, s[0:3], s32 offset:108
+; GFX9-NEXT: buffer_load_ushort v28, off, s[0:3], s32 offset:104
+; GFX9-NEXT: buffer_load_ushort v35, off, s[0:3], s32 offset:100
+; GFX9-NEXT: buffer_load_ushort v48, off, s[0:3], s32 offset:96
+; GFX9-NEXT: buffer_load_ushort v44, off, s[0:3], s32 offset:92
+; GFX9-NEXT: buffer_load_ushort v24, off, s[0:3], s32 offset:88
+; GFX9-NEXT: buffer_load_ushort v29, off, s[0:3], s32 offset:84
+; GFX9-NEXT: buffer_load_ushort v53, off, s[0:3], s32 offset:80
+; GFX9-NEXT: buffer_load_ushort v49, off, s[0:3], s32 offset:76
+; GFX9-NEXT: buffer_load_ushort v20, off, s[0:3], s32 offset:72
+; GFX9-NEXT: buffer_load_ushort v25, off, s[0:3], s32 offset:68
+; GFX9-NEXT: buffer_load_ushort v30, off, s[0:3], s32 offset:64
+; GFX9-NEXT: buffer_load_ushort v33, off, s[0:3], s32 offset:60
+; GFX9-NEXT: buffer_load_ushort v17, off, s[0:3], s32 offset:56
+; GFX9-NEXT: buffer_load_ushort v21, off, s[0:3], s32 offset:52
+; GFX9-NEXT: buffer_load_ushort v26, off, s[0:3], s32 offset:48
+; GFX9-NEXT: buffer_load_ushort v31, off, s[0:3], s32 offset:44
+; GFX9-NEXT: buffer_load_ushort v15, off, s[0:3], s32 offset:40
+; GFX9-NEXT: buffer_load_ushort v18, off, s[0:3], s32 offset:36
+; GFX9-NEXT: buffer_load_ushort v22, off, s[0:3], s32 offset:32
+; GFX9-NEXT: buffer_load_ushort v27, off, s[0:3], s32 offset:28
+; GFX9-NEXT: buffer_load_ushort v14, off, s[0:3], s32 offset:24
+; GFX9-NEXT: buffer_load_ushort v16, off, s[0:3], s32 offset:20
+; GFX9-NEXT: buffer_load_ushort v19, off, s[0:3], s32 offset:16
+; GFX9-NEXT: buffer_load_ushort v23, off, s[0:3], s32 offset:12
; GFX9-NEXT: s_and_b64 s[4:5], vcc, exec
+; GFX9-NEXT: s_waitcnt vmcnt(33)
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:496 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v32, off, s[0:3], s32 offset:176
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:172
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:508 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:192
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:480 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v46, off, s[0:3], s32 offset:168
+; GFX9-NEXT: buffer_load_ushort v57, off, s[0:3], s32 offset:164
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_ushort v13, off, s[0:3], s32 offset:188
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:444 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v47, off, s[0:3], s32 offset:184
-; GFX9-NEXT: buffer_load_ushort v59, off, s[0:3], s32 offset:180
-; GFX9-NEXT: buffer_load_ushort v42, off, s[0:3], s32 offset:176
-; GFX9-NEXT: buffer_load_ushort v58, off, s[0:3], s32 offset:172
-; GFX9-NEXT: buffer_load_ushort v44, off, s[0:3], s32 offset:168
-; GFX9-NEXT: buffer_load_ushort v56, off, s[0:3], s32 offset:164
-; GFX9-NEXT: buffer_load_ushort v60, off, s[0:3], s32 offset:160
-; GFX9-NEXT: buffer_load_ushort v53, off, s[0:3], s32 offset:156
-; GFX9-NEXT: buffer_load_ushort v49, off, s[0:3], s32 offset:152
-; GFX9-NEXT: buffer_load_ushort v45, off, s[0:3], s32 offset:148
-; GFX9-NEXT: buffer_load_ushort v57, off, s[0:3], s32 offset:144
-; GFX9-NEXT: buffer_load_ushort v55, off, s[0:3], s32 offset:140
-; GFX9-NEXT: buffer_load_ushort v38, off, s[0:3], s32 offset:136
-; GFX9-NEXT: buffer_load_ushort v32, off, s[0:3], s32 offset:132
-; GFX9-NEXT: buffer_load_ushort v46, off, s[0:3], s32 offset:128
-; GFX9-NEXT: buffer_load_ushort v40, off, s[0:3], s32 offset:124
-; GFX9-NEXT: buffer_load_ushort v31, off, s[0:3], s32 offset:120
-; GFX9-NEXT: buffer_load_ushort v52, off, s[0:3], s32 offset:116
-; GFX9-NEXT: buffer_load_ushort v50, off, s[0:3], s32 offset:112
-; GFX9-NEXT: buffer_load_ushort v41, off, s[0:3], s32 offset:108
-; GFX9-NEXT: buffer_load_ushort v27, off, s[0:3], s32 offset:104
-; GFX9-NEXT: buffer_load_ushort v37, off, s[0:3], s32 offset:100
-; GFX9-NEXT: buffer_load_ushort v36, off, s[0:3], s32 offset:96
-; GFX9-NEXT: buffer_load_ushort v51, off, s[0:3], s32 offset:92
-; GFX9-NEXT: buffer_load_ushort v23, off, s[0:3], s32 offset:88
-; GFX9-NEXT: buffer_load_ushort v28, off, s[0:3], s32 offset:84
-; GFX9-NEXT: buffer_load_ushort v34, off, s[0:3], s32 offset:80
-; GFX9-NEXT: buffer_load_ushort v39, off, s[0:3], s32 offset:76
-; GFX9-NEXT: buffer_load_ushort v19, off, s[0:3], s32 offset:72
-; GFX9-NEXT: buffer_load_ushort v24, off, s[0:3], s32 offset:68
-; GFX9-NEXT: buffer_load_ushort v29, off, s[0:3], s32 offset:64
-; GFX9-NEXT: buffer_load_ushort v35, off, s[0:3], s32 offset:60
-; GFX9-NEXT: buffer_load_ushort v16, off, s[0:3], s32 offset:56
-; GFX9-NEXT: buffer_load_ushort v20, off, s[0:3], s32 offset:52
-; GFX9-NEXT: buffer_load_ushort v25, off, s[0:3], s32 offset:48
-; GFX9-NEXT: buffer_load_ushort v30, off, s[0:3], s32 offset:44
-; GFX9-NEXT: buffer_load_ushort v14, off, s[0:3], s32 offset:40
-; GFX9-NEXT: buffer_load_ushort v17, off, s[0:3], s32 offset:36
-; GFX9-NEXT: buffer_load_ushort v21, off, s[0:3], s32 offset:32
-; GFX9-NEXT: buffer_load_ushort v26, off, s[0:3], s32 offset:28
-; GFX9-NEXT: buffer_load_ushort v13, off, s[0:3], s32 offset:24
-; GFX9-NEXT: buffer_load_ushort v15, off, s[0:3], s32 offset:20
-; GFX9-NEXT: buffer_load_ushort v18, off, s[0:3], s32 offset:16
-; GFX9-NEXT: buffer_load_ushort v22, off, s[0:3], s32 offset:12
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:516 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:520 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(6)
-; GFX9-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:528 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:532 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(7)
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(9)
-; GFX9-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(12)
-; GFX9-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:160
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:500 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v40, off, s[0:3], s32 offset:156
+; GFX9-NEXT: buffer_load_ushort v59, off, s[0:3], s32 offset:152
+; GFX9-NEXT: buffer_load_ushort v41, off, s[0:3], s32 offset:148
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:144
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:512 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:516 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:520 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:528 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:532 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:408 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v49, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
+; GFX9-NEXT: s_waitcnt vmcnt(27)
+; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:412 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:420 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:424 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:428 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:432 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:420 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:424 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:628 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:636 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v52, off, s[0:3], s32 offset:640 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:644 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v38, off, s[0:3], s32 offset:648 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:444 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:448 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v48, off, s[0:3], s32 offset:628 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:452 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:636 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:640 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:644 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:648 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:656 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:660 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v48, off, s[0:3], s32 offset:664 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:668 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:672 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:508 ; 4-byte Folded Spill
+; GFX9-NEXT: s_waitcnt vmcnt(49)
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:504 ; 4-byte Folded Spill
; GFX9-NEXT: s_cbranch_scc0 .LBB97_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v8, 0xc0c0004
-; GFX9-NEXT: v_mov_b32_e32 v2, s21
-; GFX9-NEXT: v_perm_b32 v0, s16, v0, v8
+; GFX9-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:676 ; 4-byte Folded Spill
; GFX9-NEXT: v_mov_b32_e32 v1, s19
-; GFX9-NEXT: v_perm_b32 v2, s20, v2, v8
; GFX9-NEXT: v_mov_b32_e32 v3, s23
+; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_perm_b32 v1, s18, v1, v8
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:680 ; 4-byte Folded Spill
+; GFX9-NEXT: v_mov_b32_e32 v2, s21
; GFX9-NEXT: v_perm_b32 v3, s22, v3, v8
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v3, 16, v2
-; GFX9-NEXT: v_mov_b32_e32 v2, s25
-; GFX9-NEXT: v_perm_b32 v2, s24, v2, v8
+; GFX9-NEXT: v_perm_b32 v0, s16, v0, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_perm_b32 v2, s20, v2, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v3, s27
+; GFX9-NEXT: v_mov_b32_e32 v2, s25
; GFX9-NEXT: v_perm_b32 v3, s26, v3, v8
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_lshl_or_b32 v2, v3, 16, v2
-; GFX9-NEXT: v_mov_b32_e32 v3, s29
-; GFX9-NEXT: v_perm_b32 v3, s28, v3, v8
+; GFX9-NEXT: v_perm_b32 v2, s24, v2, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_mov_b32_e32 v4, s6
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v3, s29
; GFX9-NEXT: v_perm_b32 v4, s7, v4, v8
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_lshl_or_b32 v3, v4, 16, v3
-; GFX9-NEXT: v_mov_b32_e32 v4, s15
-; GFX9-NEXT: v_perm_b32 v4, s43, v4, v8
+; GFX9-NEXT: v_perm_b32 v3, s28, v3, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX9-NEXT: v_mov_b32_e32 v5, s8
+; GFX9-NEXT: v_or_b32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v4, s15
; GFX9-NEXT: v_perm_b32 v5, s10, v5, v8
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_lshl_or_b32 v4, v5, 16, v4
-; GFX9-NEXT: v_mov_b32_e32 v5, s47
-; GFX9-NEXT: v_perm_b32 v5, s59, v5, v8
+; GFX9-NEXT: v_perm_b32 v4, s43, v4, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX9-NEXT: v_mov_b32_e32 v6, s12
+; GFX9-NEXT: v_or_b32_sdwa v4, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v5, s47
; GFX9-NEXT: v_perm_b32 v6, s14, v6, v8
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_lshl_or_b32 v5, v6, 16, v5
-; GFX9-NEXT: v_mov_b32_e32 v6, s61
-; GFX9-NEXT: v_perm_b32 v6, s63, v6, v8
+; GFX9-NEXT: v_perm_b32 v5, s59, v5, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX9-NEXT: v_mov_b32_e32 v50, v54
+; GFX9-NEXT: v_mov_b32_e32 v54, v7
; GFX9-NEXT: v_mov_b32_e32 v7, s42
+; GFX9-NEXT: v_or_b32_sdwa v5, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v6, s61
; GFX9-NEXT: v_perm_b32 v7, s46, v7, v8
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_lshl_or_b32 v6, v7, 16, v6
-; GFX9-NEXT: v_mov_b32_e32 v7, s73
-; GFX9-NEXT: v_perm_b32 v7, s75, v7, v8
-; GFX9-NEXT: v_mov_b32_e32 v63, v33
-; GFX9-NEXT: v_mov_b32_e32 v33, v9
+; GFX9-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:684 ; 4-byte Folded Spill
+; GFX9-NEXT: v_perm_b32 v6, s63, v6, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX9-NEXT: v_mov_b32_e32 v39, v9
; GFX9-NEXT: v_mov_b32_e32 v9, s45
+; GFX9-NEXT: v_or_b32_sdwa v6, v7, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v7, s73
; GFX9-NEXT: v_perm_b32 v9, s58, v9, v8
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_lshl_or_b32 v7, v9, 16, v7
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v10
-; GFX9-NEXT: v_perm_b32 v10, s74, v12, v8
-; GFX9-NEXT: v_or_b32_sdwa v9, v11, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v9, 16, v10
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v18
-; GFX9-NEXT: v_or_b32_sdwa v9, v22, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v13
-; GFX9-NEXT: v_or_b32_sdwa v10, v15, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
+; GFX9-NEXT: v_perm_b32 v7, s75, v7, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX9-NEXT: s_and_b32 s4, s40, 0xff
+; GFX9-NEXT: s_lshl_b32 s5, s13, 8
+; GFX9-NEXT: v_or_b32_sdwa v7, v9, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v9, s57
+; GFX9-NEXT: s_or_b32 s4, s4, s5
+; GFX9-NEXT: v_mov_b32_e32 v56, v38
+; GFX9-NEXT: v_perm_b32 v9, s60, v9, v8
+; GFX9-NEXT: v_mov_b32_e32 v38, v37
+; GFX9-NEXT: v_mov_b32_e32 v37, v10
+; GFX9-NEXT: v_lshlrev_b32_e64 v10, 16, s4
+; GFX9-NEXT: v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v11
+; GFX9-NEXT: v_perm_b32 v8, s74, v13, v8
+; GFX9-NEXT: v_or_b32_sdwa v10, v12, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v11, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v19
+; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v14
+; GFX9-NEXT: v_or_b32_sdwa v8, v23, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v10, v16, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v12, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_mov_b32 s4, 0xc0c0004
-; GFX9-NEXT: v_lshl_or_b32 v12, v10, 16, v9
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v14
-; GFX9-NEXT: v_perm_b32 v10, v26, v21, s4
-; GFX9-NEXT: v_or_b32_sdwa v9, v17, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v13, v9, 16, v10
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v25
-; GFX9-NEXT: v_or_b32_sdwa v9, v30, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v16
-; GFX9-NEXT: v_or_b32_sdwa v10, v20, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_lshl_or_b32 v14, v10, 16, v9
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v19
-; GFX9-NEXT: v_perm_b32 v10, v35, v29, s4
-; GFX9-NEXT: v_or_b32_sdwa v9, v24, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v15, v9, 16, v10
-; GFX9-NEXT: v_perm_b32 v9, v39, v34, s4
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_perm_b32 v10, v51, v36, s4
-; GFX9-NEXT: v_perm_b32 v16, v28, v23, s4
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_perm_b32 v17, v37, v27, s4
-; GFX9-NEXT: v_lshl_or_b32 v16, v16, 16, v9
-; GFX9-NEXT: v_perm_b32 v9, v41, v50, s4
-; GFX9-NEXT: v_lshl_or_b32 v17, v17, 16, v10
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_perm_b32 v10, v52, v31, s4
-; GFX9-NEXT: v_lshl_or_b32 v18, v10, 16, v9
-; GFX9-NEXT: v_perm_b32 v9, v40, v46, s4
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_perm_b32 v10, v32, v38, s4
-; GFX9-NEXT: v_lshl_or_b32 v19, v10, 16, v9
-; GFX9-NEXT: v_perm_b32 v9, v55, v57, s4
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_perm_b32 v10, v45, v49, s4
-; GFX9-NEXT: v_lshl_or_b32 v20, v10, 16, v9
-; GFX9-NEXT: v_perm_b32 v9, v53, v60, s4
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_perm_b32 v10, v56, v44, s4
-; GFX9-NEXT: v_lshl_or_b32 v21, v10, 16, v9
-; GFX9-NEXT: v_perm_b32 v9, v58, v42, s4
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_perm_b32 v10, v59, v47, s4
-; GFX9-NEXT: v_lshl_or_b32 v22, v10, 16, v9
-; GFX9-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
-; GFX9-NEXT: v_mov_b32_e32 v50, v32
-; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
-; GFX9-NEXT: v_mov_b32_e32 v51, v49
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v15
+; GFX9-NEXT: v_or_b32_sdwa v8, v18, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_perm_b32 v10, v27, v22, s4
+; GFX9-NEXT: v_perm_b32 v16, v49, v53, s4
; GFX9-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:404 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
-; GFX9-NEXT: v_mov_b32_e32 v41, v46
-; GFX9-NEXT: v_mov_b32_e32 v39, v52
-; GFX9-NEXT: v_mov_b32_e32 v52, v38
-; GFX9-NEXT: v_mov_b32_e32 v34, v37
-; GFX9-NEXT: v_mov_b32_e32 v37, v31
-; GFX9-NEXT: s_lshl_b32 s5, s13, 8
+; GFX9-NEXT: v_or_b32_sdwa v13, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v17
+; GFX9-NEXT: v_perm_b32 v17, v44, v48, s4
+; GFX9-NEXT: v_mov_b32_e32 v44, v55
+; GFX9-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v26
+; GFX9-NEXT: v_or_b32_sdwa v8, v31, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v10, v21, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v14, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v20
+; GFX9-NEXT: v_or_b32_sdwa v8, v25, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_perm_b32 v10, v33, v30, s4
+; GFX9-NEXT: v_or_b32_sdwa v15, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_perm_b32 v8, v29, v24, s4
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX9-NEXT: v_perm_b32 v10, v35, v28, s4
+; GFX9-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX9-NEXT: v_or_b32_sdwa v16, v8, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_perm_b32 v8, v44, v34, s4
+; GFX9-NEXT: v_or_b32_sdwa v17, v10, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX9-NEXT: v_perm_b32 v10, v47, v42, s4
+; GFX9-NEXT: v_mov_b32_e32 v34, v36
+; GFX9-NEXT: v_mov_b32_e32 v35, v60
+; GFX9-NEXT: v_or_b32_sdwa v18, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_perm_b32 v8, v35, v34, s4
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX9-NEXT: v_perm_b32 v10, v58, v43, s4
+; GFX9-NEXT: v_mov_b32_e32 v53, v59
+; GFX9-NEXT: v_or_b32_sdwa v19, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_perm_b32 v8, v41, v53, s4
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX9-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:484 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
+; GFX9-NEXT: s_lshl_b32 s5, s41, 8
; GFX9-NEXT: s_lshl_b32 s77, s9, 8
; GFX9-NEXT: s_lshl_b32 s78, s44, 8
-; GFX9-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:452 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:428 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(9)
-; GFX9-NEXT: v_perm_b32 v9, v9, v10, s4
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: s_waitcnt vmcnt(7)
-; GFX9-NEXT: v_perm_b32 v10, v32, v58, s4
-; GFX9-NEXT: v_lshl_or_b32 v23, v10, 16, v9
-; GFX9-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:440 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:400 ; 4-byte Folded Reload
+; GFX9-NEXT: v_perm_b32 v10, v61, v49, s4
+; GFX9-NEXT: v_or_b32_sdwa v20, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_perm_b32 v8, v57, v46, s4
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX9-NEXT: s_waitcnt vmcnt(8)
+; GFX9-NEXT: v_perm_b32 v10, v40, v55, s4
+; GFX9-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
+; GFX9-NEXT: v_or_b32_sdwa v21, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: s_waitcnt vmcnt(9)
+; GFX9-NEXT: v_perm_b32 v10, v33, v32, s4
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v9
+; GFX9-NEXT: v_perm_b32 v8, v60, v40, s4
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX9-NEXT: v_or_b32_sdwa v22, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_perm_b32 v8, v42, v59, s4
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX9-NEXT: v_perm_b32 v10, v51, v36, s4
+; GFX9-NEXT: v_or_b32_sdwa v23, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v8
+; GFX9-NEXT: v_or_b32_sdwa v8, v45, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v51
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v49
-; GFX9-NEXT: v_or_b32_sdwa v10, v24, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_lshl_or_b32 v24, v10, 16, v9
-; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:436 ; 4-byte Folded Reload
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v35
+; GFX9-NEXT: v_or_b32_sdwa v10, v45, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v24, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:400 ; 4-byte Folded Reload
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v32
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
+; GFX9-NEXT: v_or_b32_sdwa v8, v10, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:436 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:440 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v10, v25, v10, s4
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v25, v9, 16, v10
-; GFX9-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
-; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v33
-; GFX9-NEXT: v_or_b32_sdwa v10, v62, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
-; GFX9-NEXT: v_mov_b32_e32 v33, v46
+; GFX9-NEXT: v_or_b32_sdwa v25, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:452 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:432 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v9
-; GFX9-NEXT: v_or_b32_sdwa v9, v43, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_lshl_or_b32 v26, v10, 16, v9
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v48
-; GFX9-NEXT: v_perm_b32 v10, v63, v61, s4
-; GFX9-NEXT: v_or_b32_sdwa v9, v54, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v27, v9, 16, v10
-; GFX9-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
-; GFX9-NEXT: v_mov_b32_e32 v54, v61
-; GFX9-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:484 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(4)
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v9
-; GFX9-NEXT: s_waitcnt vmcnt(3)
-; GFX9-NEXT: v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_mov_b32_e32 v48, v63
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v10
-; GFX9-NEXT: v_or_b32_sdwa v10, v28, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_lshl_or_b32 v28, v10, 16, v9
-; GFX9-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:428 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v8
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_or_b32_sdwa v8, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v37
+; GFX9-NEXT: v_or_b32_sdwa v10, v26, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v26, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:424 ; 4-byte Folded Reload
+; GFX9-NEXT: v_perm_b32 v10, v38, v39, s4
+; GFX9-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
+; GFX9-NEXT: v_mov_b32_e32 v38, v34
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v8
+; GFX9-NEXT: v_or_b32_sdwa v8, v50, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v27, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v52
+; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v63
+; GFX9-NEXT: v_or_b32_sdwa v8, v54, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v10, v56, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v28, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
+; GFX9-NEXT: v_mov_b32_e32 v54, v31
+; GFX9-NEXT: v_mov_b32_e32 v50, v53
+; GFX9-NEXT: v_mov_b32_e32 v56, v40
+; GFX9-NEXT: v_mov_b32_e32 v52, v32
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v9
+; GFX9-NEXT: v_mov_b32_e32 v63, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v8
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:432 ; 4-byte Folded Reload
+; GFX9-NEXT: v_or_b32_sdwa v8, v10, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v10, v36, v10, s4
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v29, v9, 16, v10
-; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:424 ; 4-byte Folded Reload
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v43
-; GFX9-NEXT: v_or_b32_sdwa v9, v62, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
+; GFX9-NEXT: v_perm_b32 v10, v10, v62, s4
+; GFX9-NEXT: v_or_b32_sdwa v29, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:404 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v8
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v10
-; GFX9-NEXT: v_or_b32_sdwa v10, v30, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_lshl_or_b32 v30, v10, 16, v9
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v38
-; GFX9-NEXT: v_perm_b32 v10, v63, v61, s4
-; GFX9-NEXT: v_or_b32_sdwa v9, v46, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v31, v9, 16, v10
-; GFX9-NEXT: v_mov_b32_e32 v9, s57
-; GFX9-NEXT: v_perm_b32 v8, s60, v9, v8
-; GFX9-NEXT: s_and_b32 s4, s40, 0xff
-; GFX9-NEXT: s_or_b32 s4, s4, s5
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, s4, 16, v8
+; GFX9-NEXT: v_or_b32_sdwa v8, v39, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v10, v30, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v30, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v8
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v62, v10
+; GFX9-NEXT: v_perm_b32 v10, v31, v10, s4
; GFX9-NEXT: s_and_b32 s4, s76, 0xff
-; GFX9-NEXT: s_lshl_b32 s5, s41, 8
; GFX9-NEXT: s_or_b32 s4, s4, s5
; GFX9-NEXT: s_and_b32 s5, s11, 0xff
; GFX9-NEXT: s_or_b32 s5, s5, s77
@@ -195889,60 +195849,54 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX9-NEXT: s_or_b32 s5, s5, s77
; GFX9-NEXT: s_and_b32 s77, s56, 0xff
; GFX9-NEXT: s_or_b32 s77, s77, s78
+; GFX9-NEXT: v_or_b32_sdwa v8, v48, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: s_pack_ll_b32_b16 s5, s5, s77
+; GFX9-NEXT: v_or_b32_sdwa v31, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v8, s4
; GFX9-NEXT: v_mov_b32_e32 v10, s5
; GFX9-NEXT: s_mov_b64 s[4:5], 0
-; GFX9-NEXT: v_mov_b32_e32 v63, v36
; GFX9-NEXT: s_branch .LBB97_3
; GFX9-NEXT: .LBB97_2:
-; GFX9-NEXT: v_mov_b32_e32 v54, v61
-; GFX9-NEXT: v_mov_b32_e32 v50, v32
-; GFX9-NEXT: v_mov_b32_e32 v51, v49
-; GFX9-NEXT: v_mov_b32_e32 v39, v52
-; GFX9-NEXT: v_mov_b32_e32 v52, v38
-; GFX9-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:484 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
+; GFX9-NEXT: v_mov_b32_e32 v63, v39
+; GFX9-NEXT: v_mov_b32_e32 v38, v36
+; GFX9-NEXT: v_mov_b32_e32 v44, v55
+; GFX9-NEXT: v_mov_b32_e32 v50, v59
+; GFX9-NEXT: v_mov_b32_e32 v35, v60
+; GFX9-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
-; GFX9-NEXT: v_mov_b32_e32 v41, v46
-; GFX9-NEXT: v_mov_b32_e32 v34, v37
-; GFX9-NEXT: v_mov_b32_e32 v37, v31
+; GFX9-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:484 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
+; GFX9-NEXT: v_mov_b32_e32 v54, v2
+; GFX9-NEXT: v_mov_b32_e32 v62, v1
; GFX9-NEXT: s_mov_b64 s[4:5], -1
; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
; GFX9-NEXT: ; implicit-def: $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
; GFX9-NEXT: .LBB97_3: ; %Flow
-; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:400 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
; GFX9-NEXT: s_cselect_b32 s4, 1, 0
; GFX9-NEXT: s_cmp_lg_u32 s4, 1
; GFX9-NEXT: s_cbranch_scc1 .LBB97_5
; GFX9-NEXT: ; %bb.4: ; %cmp.true
-; GFX9-NEXT: s_waitcnt vmcnt(5)
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v48
-; GFX9-NEXT: s_mov_b32 s4, 0xc0c0004
-; GFX9-NEXT: v_perm_b32 v0, v0, v61, s4
-; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:472 ; 4-byte Folded Spill
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v33
-; GFX9-NEXT: s_waitcnt vmcnt(4)
-; GFX9-NEXT: v_perm_b32 v0, v0, v38, s4
-; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v62
-; GFX9-NEXT: v_perm_b32 v0, v0, v43, s4
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v54
+; GFX9-NEXT: s_mov_b32 s5, 0xc0c0004
+; GFX9-NEXT: v_perm_b32 v0, v0, v62, s5
; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:480 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
+; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v48
+; GFX9-NEXT: s_movk_i32 s4, 0x300
; GFX9-NEXT: s_add_i32 s74, s74, 3
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:424 ; 4-byte Folded Reload
; GFX9-NEXT: s_add_i32 s72, s72, 3
; GFX9-NEXT: s_add_i32 s56, s56, 3
; GFX9-NEXT: s_add_i32 s60, s60, 3
@@ -195963,373 +195917,358 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX9-NEXT: s_add_i32 s26, s26, 3
; GFX9-NEXT: s_add_i32 s20, s20, 3
; GFX9-NEXT: s_add_i32 s22, s22, 3
+; GFX9-NEXT: v_mov_b32_e32 v32, s23
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: v_mov_b32_e32 v33, s17
; GFX9-NEXT: s_add_i32 s18, s18, 3
-; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
+; GFX9-NEXT: v_mov_b32_e32 v34, s19
+; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:432 ; 4-byte Folded Reload
-; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:408 ; 4-byte Folded Spill
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v63
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:404 ; 4-byte Folded Reload
+; GFX9-NEXT: v_add_u32_sdwa v0, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:444 ; 4-byte Folded Spill
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v39
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:424 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:428 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_sdwa v0, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:636 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:428 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:412 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT: v_perm_b32 v0, v0, v63, s5
+; GFX9-NEXT: v_add_u32_sdwa v0, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:448 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: v_add_u32_e32 v38, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:452 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_e32 v39, 0x300, v0
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: v_add_u32_e32 v62, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_sdwa v63, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:628 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT: v_perm_b32 v0, v0, v54, s4
-; GFX9-NEXT: v_add_u32_e32 v63, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:424 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_sdwa v48, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:432 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:452 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:412 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:632 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:420 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:428 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: v_add_u32_e32 v31, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_sdwa v31, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:440 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:436 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v30, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:436 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:440 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:400 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT: v_perm_b32 v0, v0, v35, s4
-; GFX9-NEXT: v_add_u32_e32 v29, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v36
+; GFX9-NEXT: v_perm_b32 v0, v0, v52, s5
+; GFX9-NEXT: v_add_u32_sdwa v29, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:632 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v28, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:404 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v45
+; GFX9-NEXT: v_perm_b32 v0, v0, v51, s5
+; GFX9-NEXT: v_add_u32_sdwa v27, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT: v_perm_b32 v0, v0, v49, s4
-; GFX9-NEXT: v_add_u32_e32 v27, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v36, s5
; GFX9-NEXT: v_add_u32_e32 v26, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v32
-; GFX9-NEXT: v_perm_b32 v0, v0, v58, s4
-; GFX9-NEXT: v_add_u32_e32 v25, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
-; GFX9-NEXT: v_mov_b32_e32 v32, s23
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v42
+; GFX9-NEXT: v_perm_b32 v0, v0, v59, s5
+; GFX9-NEXT: v_add_u32_sdwa v25, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v33
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_e32 v24, 0x300, v0
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v60
+; GFX9-NEXT: v_perm_b32 v0, v0, v56, s5
+; GFX9-NEXT: v_add_u32_sdwa v23, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
+; GFX9-NEXT: v_mov_b32_e32 v33, s17
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT: v_perm_b32 v0, v0, v42, s4
-; GFX9-NEXT: v_add_u32_e32 v24, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v59
-; GFX9-NEXT: v_perm_b32 v0, v0, v47, s4
-; GFX9-NEXT: v_add_u32_e32 v23, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v53
-; GFX9-NEXT: v_perm_b32 v0, v0, v60, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v55, s5
; GFX9-NEXT: v_add_u32_e32 v22, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v56
-; GFX9-NEXT: v_perm_b32 v0, v0, v44, s4
-; GFX9-NEXT: v_add_u32_e32 v21, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v55
-; GFX9-NEXT: v_perm_b32 v0, v0, v57, s4
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v57
+; GFX9-NEXT: v_perm_b32 v0, v0, v46, s5
+; GFX9-NEXT: v_add_u32_sdwa v21, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v61
+; GFX9-NEXT: v_perm_b32 v0, v0, v49, s5
; GFX9-NEXT: v_add_u32_e32 v20, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v45
-; GFX9-NEXT: v_perm_b32 v0, v0, v51, s4
-; GFX9-NEXT: v_add_u32_e32 v49, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v40
-; GFX9-NEXT: v_perm_b32 v0, v0, v41, s4
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v41
+; GFX9-NEXT: v_perm_b32 v0, v0, v50, s5
+; GFX9-NEXT: v_add_u32_sdwa v51, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v58
+; GFX9-NEXT: v_perm_b32 v0, v0, v43, s5
; GFX9-NEXT: v_add_u32_e32 v19, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v50
-; GFX9-NEXT: v_perm_b32 v0, v0, v52, s4
-; GFX9-NEXT: v_add_u32_e32 v52, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v22, 0xffff, v22
-; GFX9-NEXT: v_lshl_or_b32 v21, v21, 16, v22
-; GFX9-NEXT: v_and_b32_e32 v22, 0xffff, v24
-; GFX9-NEXT: v_lshl_or_b32 v22, v23, 16, v22
-; GFX9-NEXT: v_and_b32_e32 v23, 0xffff, v26
-; GFX9-NEXT: v_lshl_or_b32 v23, v25, 16, v23
-; GFX9-NEXT: v_and_b32_e32 v25, 0xffff, v30
-; GFX9-NEXT: v_lshl_or_b32 v25, v29, 16, v25
-; GFX9-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:424 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v24, 0xffff, v28
-; GFX9-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:428 ; 4-byte Folded Reload
-; GFX9-NEXT: v_lshl_or_b32 v24, v27, 16, v24
-; GFX9-NEXT: v_and_b32_e32 v27, 0xffff, v63
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_lshl_or_b32 v19, v52, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v49, 16, v20
-; GFX9-NEXT: s_waitcnt vmcnt(4)
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v35
+; GFX9-NEXT: v_perm_b32 v0, v0, v38, s5
+; GFX9-NEXT: v_add_u32_sdwa v38, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v47
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
; GFX9-NEXT: v_add_u32_e32 v18, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v39
-; GFX9-NEXT: v_perm_b32 v0, v0, v37, s4
-; GFX9-NEXT: v_add_u32_e32 v37, 0x300, v0
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v44
+; GFX9-NEXT: v_or_b32_sdwa v21, v21, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v22, v23, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v23, v25, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v25, v29, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
+; GFX9-NEXT: v_or_b32_sdwa v24, v27, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v38, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v20, v51, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v28, v63, v39 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(4)
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_sdwa v53, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_lshl_or_b32 v18, v37, 16, v18
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
+; GFX9-NEXT: v_or_b32_sdwa v18, v53, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: s_waitcnt vmcnt(5)
+; GFX9-NEXT: v_or_b32_sdwa v26, v31, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: v_or_b32_sdwa v29, v30, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:404 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(4)
+; GFX9-NEXT: v_or_b32_sdwa v27, v48, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: s_waitcnt vmcnt(3)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
-; GFX9-NEXT: v_add_u32_e32 v54, 0x300, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v34
-; GFX9-NEXT: v_mov_b32_e32 v34, s19
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v17, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: v_or_b32_sdwa v30, v31, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: v_add_u32_e32 v16, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:636 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_sdwa v54, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
+; GFX9-NEXT: v_or_b32_sdwa v17, v54, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT: v_perm_b32 v0, v0, v37, s5
+; GFX9-NEXT: v_add_u32_e32 v16, 0x300, v0
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: v_add_u32_e32 v55, 0x300, v0
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_sdwa v55, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
-; GFX9-NEXT: v_lshl_or_b32 v16, v55, 16, v16
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
+; GFX9-NEXT: v_or_b32_sdwa v16, v55, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v15, 0x300, v0
; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: v_add_u32_e32 v40, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_sdwa v40, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
-; GFX9-NEXT: v_lshl_or_b32 v15, v40, 16, v15
+; GFX9-NEXT: v_or_b32_sdwa v15, v40, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v14, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: v_add_u32_e32 v41, 0x300, v0
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_sdwa v41, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
-; GFX9-NEXT: v_lshl_or_b32 v14, v41, 16, v14
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
+; GFX9-NEXT: v_or_b32_sdwa v14, v41, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v13, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: v_add_u32_e32 v42, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
-; GFX9-NEXT: v_lshl_or_b32 v13, v42, 16, v13
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: v_add_u32_sdwa v42, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
+; GFX9-NEXT: v_or_b32_sdwa v13, v42, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
; GFX9-NEXT: v_add_u32_e32 v12, 0x300, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
-; GFX9-NEXT: v_add_u32_e32 v43, 0x300, v0
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s5
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
+; GFX9-NEXT: v_add_u32_sdwa v43, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX9-NEXT: v_perm_b32 v32, s22, v32, v0
-; GFX9-NEXT: v_add_u32_e32 v32, 0x300, v32
+; GFX9-NEXT: v_add_u32_sdwa v32, v32, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_perm_b32 v33, s16, v33, v0
; GFX9-NEXT: v_add_u32_e32 v33, 0x300, v33
-; GFX9-NEXT: v_and_b32_e32 v33, 0xffff, v33
-; GFX9-NEXT: v_lshl_or_b32 v12, v43, 16, v12
+; GFX9-NEXT: v_or_b32_sdwa v12, v43, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v1, s74, v1, v0
; GFX9-NEXT: v_add_u32_e32 v11, 0x300, v1
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_add_u32_e32 v1, 3, v1
-; GFX9-NEXT: v_perm_b32 v1, v1, v2, s4
-; GFX9-NEXT: v_add_u32_e32 v44, 0x300, v1
+; GFX9-NEXT: v_perm_b32 v1, v1, v2, s5
+; GFX9-NEXT: v_add_u32_sdwa v44, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v1, s62
; GFX9-NEXT: v_perm_b32 v1, s72, v1, v0
; GFX9-NEXT: v_add_u32_e32 v10, 0x300, v1
; GFX9-NEXT: v_mov_b32_e32 v1, s44
; GFX9-NEXT: v_perm_b32 v1, s56, v1, v0
-; GFX9-NEXT: v_add_u32_e32 v45, 0x300, v1
+; GFX9-NEXT: v_add_u32_sdwa v45, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v1, s57
; GFX9-NEXT: v_perm_b32 v1, s60, v1, v0
; GFX9-NEXT: v_add_u32_e32 v9, 0x300, v1
; GFX9-NEXT: v_mov_b32_e32 v1, s13
; GFX9-NEXT: v_perm_b32 v1, s40, v1, v0
-; GFX9-NEXT: v_add_u32_e32 v46, 0x300, v1
+; GFX9-NEXT: v_add_u32_sdwa v46, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v1, s41
; GFX9-NEXT: v_perm_b32 v1, s76, v1, v0
; GFX9-NEXT: v_add_u32_e32 v8, 0x300, v1
; GFX9-NEXT: v_mov_b32_e32 v1, s9
; GFX9-NEXT: v_perm_b32 v1, s11, v1, v0
-; GFX9-NEXT: v_add_u32_e32 v47, 0x300, v1
+; GFX9-NEXT: v_add_u32_sdwa v47, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v1, s73
; GFX9-NEXT: v_perm_b32 v1, s75, v1, v0
; GFX9-NEXT: v_add_u32_e32 v7, 0x300, v1
; GFX9-NEXT: v_mov_b32_e32 v1, s45
; GFX9-NEXT: v_perm_b32 v1, s58, v1, v0
-; GFX9-NEXT: v_add_u32_e32 v56, 0x300, v1
+; GFX9-NEXT: v_add_u32_sdwa v56, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v1, s61
; GFX9-NEXT: v_perm_b32 v1, s63, v1, v0
; GFX9-NEXT: v_add_u32_e32 v6, 0x300, v1
; GFX9-NEXT: v_mov_b32_e32 v1, s42
; GFX9-NEXT: v_perm_b32 v1, s46, v1, v0
-; GFX9-NEXT: v_add_u32_e32 v57, 0x300, v1
+; GFX9-NEXT: v_add_u32_sdwa v57, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v1, s47
; GFX9-NEXT: v_perm_b32 v1, s59, v1, v0
; GFX9-NEXT: v_add_u32_e32 v5, 0x300, v1
; GFX9-NEXT: v_mov_b32_e32 v1, s12
; GFX9-NEXT: v_perm_b32 v1, s14, v1, v0
-; GFX9-NEXT: v_add_u32_e32 v58, 0x300, v1
+; GFX9-NEXT: v_add_u32_sdwa v58, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v1, s15
; GFX9-NEXT: v_perm_b32 v1, s43, v1, v0
; GFX9-NEXT: v_add_u32_e32 v4, 0x300, v1
; GFX9-NEXT: v_mov_b32_e32 v1, s8
; GFX9-NEXT: v_perm_b32 v1, s10, v1, v0
-; GFX9-NEXT: v_add_u32_e32 v59, 0x300, v1
+; GFX9-NEXT: v_add_u32_sdwa v59, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v1, s29
; GFX9-NEXT: v_perm_b32 v1, s28, v1, v0
; GFX9-NEXT: v_add_u32_e32 v3, 0x300, v1
; GFX9-NEXT: v_mov_b32_e32 v1, s6
; GFX9-NEXT: v_perm_b32 v1, s7, v1, v0
-; GFX9-NEXT: v_add_u32_e32 v60, 0x300, v1
+; GFX9-NEXT: v_add_u32_sdwa v60, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v1, s25
; GFX9-NEXT: v_perm_b32 v1, s24, v1, v0
; GFX9-NEXT: v_add_u32_e32 v2, 0x300, v1
; GFX9-NEXT: v_mov_b32_e32 v1, s27
; GFX9-NEXT: v_perm_b32 v1, s26, v1, v0
-; GFX9-NEXT: v_add_u32_e32 v61, 0x300, v1
+; GFX9-NEXT: v_add_u32_sdwa v61, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mov_b32_e32 v1, s21
; GFX9-NEXT: v_perm_b32 v1, s20, v1, v0
; GFX9-NEXT: v_add_u32_e32 v1, 0x300, v1
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_lshl_or_b32 v1, v32, 16, v1
-; GFX9-NEXT: v_and_b32_e32 v32, 0xffff, v54
-; GFX9-NEXT: v_lshl_or_b32 v17, v17, 16, v32
-; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v29, 0xffff, v29
-; GFX9-NEXT: v_and_b32_e32 v26, 0xffff, v26
-; GFX9-NEXT: v_lshl_or_b32 v29, v30, 16, v29
-; GFX9-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
-; GFX9-NEXT: v_lshl_or_b32 v26, v31, 16, v26
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
+; GFX9-NEXT: v_or_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
; GFX9-NEXT: v_perm_b32 v0, s18, v34, v0
-; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v27, v28, 16, v27
-; GFX9-NEXT: v_and_b32_e32 v28, 0xffff, v38
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v33
-; GFX9-NEXT: v_lshl_or_b32 v2, v61, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v60, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v59, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v58, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v57, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v56, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v47, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v46, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v45, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v44, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v28, v62, 16, v28
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_and_b32_e32 v30, 0xffff, v30
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_lshl_or_b32 v30, v31, 16, v30
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_and_b32_e32 v31, 0xffff, v31
-; GFX9-NEXT: v_lshl_or_b32 v31, v32, 16, v31
+; GFX9-NEXT: v_add_u32_sdwa v0, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v61, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v60, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v4, v59, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v58, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v57, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v56, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v47, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v46, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v45, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v44, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_or_b32_sdwa v31, v32, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: .LBB97_5: ; %end
; GFX9-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
@@ -212917,18 +212856,18 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
-; GFX9-NEXT: buffer_store_dword v36, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v35, off, s[0:3], s32 ; 4-byte Folded Spill
; GFX9-NEXT: s_mov_b64 exec, s[4:5]
-; GFX9-NEXT: v_writelane_b32 v36, s36, 0
-; GFX9-NEXT: v_writelane_b32 v36, s37, 1
-; GFX9-NEXT: v_writelane_b32 v36, s38, 2
-; GFX9-NEXT: v_writelane_b32 v36, s39, 3
-; GFX9-NEXT: v_writelane_b32 v36, s48, 4
-; GFX9-NEXT: v_writelane_b32 v36, s49, 5
-; GFX9-NEXT: v_writelane_b32 v36, s50, 6
-; GFX9-NEXT: v_writelane_b32 v36, s51, 7
-; GFX9-NEXT: v_writelane_b32 v36, s30, 8
-; GFX9-NEXT: v_writelane_b32 v36, s31, 9
+; GFX9-NEXT: v_writelane_b32 v35, s36, 0
+; GFX9-NEXT: v_writelane_b32 v35, s37, 1
+; GFX9-NEXT: v_writelane_b32 v35, s38, 2
+; GFX9-NEXT: v_writelane_b32 v35, s39, 3
+; GFX9-NEXT: v_writelane_b32 v35, s48, 4
+; GFX9-NEXT: v_writelane_b32 v35, s49, 5
+; GFX9-NEXT: v_writelane_b32 v35, s50, 6
+; GFX9-NEXT: v_writelane_b32 v35, s51, 7
+; GFX9-NEXT: v_writelane_b32 v35, s30, 8
+; GFX9-NEXT: v_writelane_b32 v35, s31, 9
; GFX9-NEXT: v_readfirstlane_b32 s4, v18
; GFX9-NEXT: v_readfirstlane_b32 s51, v17
; GFX9-NEXT: v_readfirstlane_b32 s50, v16
@@ -212966,6 +212905,14 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
+; GFX9-NEXT: s_lshl_b32 s4, s50, 16
+; GFX9-NEXT: v_add_u32_e32 v1, 0x7fff, v1
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v1, v2, vcc
+; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
; GFX9-NEXT: s_and_b32 s5, s30, 0xffff0000
; GFX9-NEXT: v_add_u32_e32 v1, 0x7fff, v1
; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
@@ -212994,25 +212941,24 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: s_lshl_b32 s5, s31, 16
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_add_f32_e32 v3, s5, v16
-; GFX9-NEXT: v_bfe_u32 v4, v3, 16, 1
-; GFX9-NEXT: v_mov_b32_e32 v32, 0xffff
; GFX9-NEXT: s_and_b32 s4, s29, 0xffff0000
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_bfe_u32 v4, v3, 16, 1
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_add_u32_e32 v4, v4, v3
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_or_b32_sdwa v14, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v3
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX9-NEXT: v_lshl_or_b32 v14, v0, 16, v1
-; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX9-NEXT: v_and_b32_sdwa v3, v32, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
-; GFX9-NEXT: v_lshl_or_b32 v15, v2, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v15, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_u32_e32 v1, 0x7fff, v1
; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
@@ -213024,11 +212970,11 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: s_and_b32 s4, s28, 0xffff0000
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v13, v0, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v13, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -213043,11 +212989,11 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: s_and_b32 s4, s27, 0xffff0000
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v12, v0, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v12, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -213062,11 +213008,11 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: s_and_b32 s4, s26, 0xffff0000
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v11, v0, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v11, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -213081,11 +213027,11 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: s_and_b32 s4, s25, 0xffff0000
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v10, v0, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v10, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -213100,11 +213046,11 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: s_and_b32 s4, s24, 0xffff0000
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v9, v0, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v9, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -213119,11 +213065,11 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: s_and_b32 s4, s23, 0xffff0000
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v8, v0, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v8, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -213138,11 +213084,11 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: s_and_b32 s4, s22, 0xffff0000
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v7, v0, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v7, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -213157,11 +213103,11 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: s_and_b32 s4, s21, 0xffff0000
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v6, v0, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v6, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -213176,11 +213122,11 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: s_and_b32 s4, s20, 0xffff0000
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v5, v0, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v5, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -213195,11 +213141,11 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: s_and_b32 s4, s19, 0xffff0000
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v4, v0, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v4, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -213214,11 +213160,11 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: s_and_b32 s4, s18, 0xffff0000
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v3, v0, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v3, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
@@ -213231,59 +213177,51 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
-; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v1
+; GFX9-NEXT: v_or_b32_e32 v19, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v18, vcc
-; GFX9-NEXT: s_and_b32 s4, s17, 0xffff0000
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v2, v0, 16, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v19, vcc
+; GFX9-NEXT: s_and_b32 s4, s17, 0xffff0000
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v2, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v0
; GFX9-NEXT: v_add_u32_e32 v1, 0x7fff, v1
-; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v0
+; GFX9-NEXT: v_or_b32_e32 v19, 0x400000, v0
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
; GFX9-NEXT: s_lshl_b32 s4, s17, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v18, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v19, vcc
; GFX9-NEXT: v_add_f32_e32 v1, s4, v16
-; GFX9-NEXT: v_bfe_u32 v18, v1, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v18, v18, v1
-; GFX9-NEXT: v_add_u32_e32 v18, 0x7fff, v18
-; GFX9-NEXT: v_or_b32_e32 v19, 0x400000, v1
+; GFX9-NEXT: v_bfe_u32 v19, v1, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v19, v19, v1
+; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
+; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v18, v19, vcc
-; GFX9-NEXT: s_and_b32 s4, s16, 0xffff0000
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v32, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v1, v0, 16, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v19, v20, vcc
+; GFX9-NEXT: s_and_b32 s4, s16, 0xffff0000
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
-; GFX9-NEXT: v_bfe_u32 v18, v0, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v18, v18, v0
-; GFX9-NEXT: v_add_u32_e32 v18, 0x7fff, v18
-; GFX9-NEXT: v_or_b32_e32 v19, 0x400000, v0
+; GFX9-NEXT: v_bfe_u32 v19, v0, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v19, v19, v0
+; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
+; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v0
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
; GFX9-NEXT: s_lshl_b32 s4, s16, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v18, v19, vcc
-; GFX9-NEXT: v_add_f32_e32 v18, s4, v16
-; GFX9-NEXT: v_bfe_u32 v19, v18, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v19, v19, v18
-; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
-; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
-; GFX9-NEXT: s_lshl_b32 s4, s50, 16
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v19, v20, vcc
+; GFX9-NEXT: v_add_f32_e32 v19, s4, v16
+; GFX9-NEXT: v_bfe_u32 v20, v19, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v20, v20, v19
+; GFX9-NEXT: v_add_u32_e32 v20, 0x7fff, v20
+; GFX9-NEXT: v_or_b32_e32 v21, 0x400000, v19
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v19, v19
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v18
-; GFX9-NEXT: v_add_f32_e32 v18, s4, v16
-; GFX9-NEXT: v_bfe_u32 v19, v18, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v19, v19, v18
-; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
-; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
+; GFX9-NEXT: v_cndmask_b32_e32 v19, v20, v21, vcc
; GFX9-NEXT: s_and_b32 s4, s51, 0xffff0000
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v19, s4, v16
; GFX9-NEXT: v_bfe_u32 v20, v19, 16, 1
; GFX9-NEXT: v_add_u32_e32 v20, v20, v19
@@ -213293,22 +213231,22 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; GFX9-NEXT: s_lshl_b32 s4, s51, 16
; GFX9-NEXT: v_cndmask_b32_e32 v19, v20, v21, vcc
; GFX9-NEXT: v_add_f32_e32 v20, s4, v16
+; GFX9-NEXT: v_lshrrev_b32_e32 v18, 16, v18
; GFX9-NEXT: v_bfe_u32 v21, v20, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v21, v21, v20
-; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v18, 16, v18
; GFX9-NEXT: s_and_b32 s4, s49, 0xffff0000
+; GFX9-NEXT: v_add_u32_e32 v21, v21, v20
+; GFX9-NEXT: v_or_b32_sdwa v30, v18, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_add_u32_e32 v21, 0x7fff, v21
; GFX9-NEXT: v_or_b32_e32 v22, 0x400000, v20
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v20, v20
-; GFX9-NEXT: v_lshl_or_b32 v30, v17, 16, v18
-; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
-; GFX9-NEXT: v_cndmask_b32_e32 v20, v21, v22, vcc
-; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_lshrrev_b32_e32 v19, 16, v19
-; GFX9-NEXT: v_and_b32_sdwa v20, v32, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
+; GFX9-NEXT: v_cndmask_b32_e32 v20, v21, v22, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v19, 16, v19
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
-; GFX9-NEXT: v_lshl_or_b32 v31, v19, 16, v20
+; GFX9-NEXT: v_or_b32_sdwa v31, v19, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_u32_e32 v18, 0x7fff, v18
; GFX9-NEXT: v_or_b32_e32 v19, 0x400000, v17
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
@@ -213320,11 +213258,11 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s48, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v29, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v29, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -213339,11 +213277,11 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s47, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v28, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v28, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -213358,11 +213296,11 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s46, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v27, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v27, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -213377,11 +213315,11 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s45, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v26, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v26, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -213396,11 +213334,11 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s44, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v25, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v25, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -213415,11 +213353,11 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s43, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v24, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v24, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -213434,11 +213372,11 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s42, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v23, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v23, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -213453,11 +213391,11 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s41, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v22, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v22, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -213472,11 +213410,11 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s40, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v21, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v21, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -213491,11 +213429,11 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v20, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s39, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v20, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v20, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
@@ -213508,69 +213446,69 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; GFX9-NEXT: v_bfe_u32 v19, v18, 16, 1
; GFX9-NEXT: v_add_u32_e32 v19, v19, v18
; GFX9-NEXT: v_add_u32_e32 v19, 0x7fff, v19
-; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v18
+; GFX9-NEXT: v_or_b32_e32 v32, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v33, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v19, v32, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s38, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v19, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v19, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
; GFX9-NEXT: v_add_u32_e32 v18, 0x7fff, v18
-; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v17
+; GFX9-NEXT: v_or_b32_e32 v32, 0x400000, v17
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
; GFX9-NEXT: s_lshl_b32 s4, s38, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v33, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v32, vcc
; GFX9-NEXT: v_add_f32_e32 v18, s4, v16
-; GFX9-NEXT: v_bfe_u32 v33, v18, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v33, v33, v18
-; GFX9-NEXT: v_add_u32_e32 v33, 0x7fff, v33
-; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v18
+; GFX9-NEXT: v_bfe_u32 v32, v18, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v32, v32, v18
+; GFX9-NEXT: v_add_u32_e32 v32, 0x7fff, v32
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
-; GFX9-NEXT: v_cndmask_b32_e32 v18, v33, v34, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v18, v32, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v32, v33, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s37, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v18, v17, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v18, v17, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v17, s4, v16
-; GFX9-NEXT: v_bfe_u32 v33, v17, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v33, v33, v17
-; GFX9-NEXT: v_add_u32_e32 v33, 0x7fff, v33
-; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v17
+; GFX9-NEXT: v_bfe_u32 v32, v17, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v32, v32, v17
+; GFX9-NEXT: v_add_u32_e32 v32, 0x7fff, v32
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v17
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
; GFX9-NEXT: s_lshl_b32 s4, s37, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v17, v33, v34, vcc
-; GFX9-NEXT: v_add_f32_e32 v33, s4, v16
-; GFX9-NEXT: v_bfe_u32 v34, v33, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v34, v34, v33
-; GFX9-NEXT: v_add_u32_e32 v34, 0x7fff, v34
-; GFX9-NEXT: v_or_b32_e32 v35, 0x400000, v33
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v33, v33
-; GFX9-NEXT: v_cndmask_b32_e32 v33, v34, v35, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v17, v32, v33, vcc
+; GFX9-NEXT: v_add_f32_e32 v32, s4, v16
+; GFX9-NEXT: v_bfe_u32 v33, v32, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v33, v33, v32
+; GFX9-NEXT: v_add_u32_e32 v33, 0x7fff, v33
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v32
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v32, v32
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX9-NEXT: v_and_b32_sdwa v33, v32, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v32, v33, v34, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX9-NEXT: s_and_b32 s4, s36, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v17, v17, 16, v33
-; GFX9-NEXT: v_add_f32_e32 v33, s4, v16
-; GFX9-NEXT: v_bfe_u32 v34, v33, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v34, v34, v33
+; GFX9-NEXT: v_or_b32_sdwa v17, v17, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f32_e32 v32, s4, v16
+; GFX9-NEXT: v_bfe_u32 v33, v32, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v33, v33, v32
; GFX9-NEXT: s_lshl_b32 s4, s36, 16
-; GFX9-NEXT: v_add_u32_e32 v34, 0x7fff, v34
-; GFX9-NEXT: v_or_b32_e32 v35, 0x400000, v33
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v33, v33
+; GFX9-NEXT: v_add_u32_e32 v33, 0x7fff, v33
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v32
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v32, v32
; GFX9-NEXT: v_add_f32_e32 v16, s4, v16
-; GFX9-NEXT: v_cndmask_b32_e32 v33, v34, v35, vcc
-; GFX9-NEXT: v_bfe_u32 v34, v16, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v34, v34, v16
-; GFX9-NEXT: v_add_u32_e32 v34, 0x7fff, v34
-; GFX9-NEXT: v_or_b32_e32 v35, 0x400000, v16
+; GFX9-NEXT: v_cndmask_b32_e32 v32, v33, v34, vcc
+; GFX9-NEXT: v_bfe_u32 v33, v16, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v33, v33, v16
+; GFX9-NEXT: v_add_u32_e32 v33, 0x7fff, v33
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v16
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v16, v16
-; GFX9-NEXT: v_cndmask_b32_e32 v16, v34, v35, vcc
-; GFX9-NEXT: v_and_b32_sdwa v16, v32, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshrrev_b32_e32 v32, 16, v33
-; GFX9-NEXT: v_lshl_or_b32 v16, v32, 16, v16
+; GFX9-NEXT: v_lshrrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_cndmask_b32_e32 v16, v33, v34, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_or_b32_sdwa v16, v32, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_branch .LBB101_6
; GFX9-NEXT: .LBB101_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
@@ -213606,18 +213544,18 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; GFX9-NEXT: v_mov_b32_e32 v30, s50
; GFX9-NEXT: v_mov_b32_e32 v31, s51
; GFX9-NEXT: .LBB101_6: ; %end
-; GFX9-NEXT: v_readlane_b32 s30, v36, 8
-; GFX9-NEXT: v_readlane_b32 s31, v36, 9
-; GFX9-NEXT: v_readlane_b32 s51, v36, 7
-; GFX9-NEXT: v_readlane_b32 s50, v36, 6
-; GFX9-NEXT: v_readlane_b32 s49, v36, 5
-; GFX9-NEXT: v_readlane_b32 s48, v36, 4
-; GFX9-NEXT: v_readlane_b32 s39, v36, 3
-; GFX9-NEXT: v_readlane_b32 s38, v36, 2
-; GFX9-NEXT: v_readlane_b32 s37, v36, 1
-; GFX9-NEXT: v_readlane_b32 s36, v36, 0
+; GFX9-NEXT: v_readlane_b32 s30, v35, 8
+; GFX9-NEXT: v_readlane_b32 s31, v35, 9
+; GFX9-NEXT: v_readlane_b32 s51, v35, 7
+; GFX9-NEXT: v_readlane_b32 s50, v35, 6
+; GFX9-NEXT: v_readlane_b32 s49, v35, 5
+; GFX9-NEXT: v_readlane_b32 s48, v35, 4
+; GFX9-NEXT: v_readlane_b32 s39, v35, 3
+; GFX9-NEXT: v_readlane_b32 s38, v35, 2
+; GFX9-NEXT: v_readlane_b32 s37, v35, 1
+; GFX9-NEXT: v_readlane_b32 s36, v35, 0
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
-; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v35, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-NEXT: s_mov_b64 exec, s[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll
index d5edd9a96e1a0..17fa6366bbca1 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll
@@ -3310,71 +3310,70 @@ define inreg <4 x i32> @bitcast_v8bf16_to_v4i32_scalar(<8 x bfloat> inreg %a, i3
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX9-NEXT: v_mov_b32_e32 v4, 0xffff
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s18
-; GFX9-NEXT: v_lshl_or_b32 v3, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v3, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
-; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: s_lshl_b32 s4, s18, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v5, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
-; GFX9-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v5, v5, v2
-; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
-; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v2
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc
+; GFX9-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v4, v4, v2
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v2
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s17
-; GFX9-NEXT: v_lshl_or_b32 v2, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
-; GFX9-NEXT: v_bfe_u32 v5, v1, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v5, v5, v1
-; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
-; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v1
+; GFX9-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v4, v4, v1
+; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: s_lshl_b32 s4, s17, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v6, vcc
-; GFX9-NEXT: v_add_f32_e32 v5, s4, v0
-; GFX9-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v6, v6, v5
-; GFX9-NEXT: v_add_u32_e32 v6, 0x7fff, v6
-; GFX9-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX9-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v5, vcc
+; GFX9-NEXT: v_add_f32_e32 v4, s4, v0
+; GFX9-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v5, v5, v4
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v5, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
+; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
+; GFX9-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s16
-; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v5
-; GFX9-NEXT: v_add_f32_e32 v5, s4, v0
-; GFX9-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v6, v6, v5
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f32_e32 v4, s4, v0
+; GFX9-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v5, v5, v4
; GFX9-NEXT: s_lshl_b32 s4, s16, 16
-; GFX9-NEXT: v_add_u32_e32 v6, 0x7fff, v6
-; GFX9-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
+; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
; GFX9-NEXT: v_add_f32_e32 v0, s4, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc
-; GFX9-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v6, v6, v0
-; GFX9-NEXT: v_add_u32_e32 v6, 0x7fff, v6
-; GFX9-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc
+; GFX9-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v5, v5, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
+; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v0
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v7, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX9-NEXT: v_and_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v0, v5, 16, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v5, v6, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX9-NEXT: v_or_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_setpc_b64 s[30:31]
; GFX9-NEXT: .LBB23_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
@@ -8072,71 +8071,70 @@ define inreg <4 x float> @bitcast_v8bf16_to_v4f32_scalar(<8 x bfloat> inreg %a,
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX9-NEXT: v_mov_b32_e32 v4, 0xffff
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s18
-; GFX9-NEXT: v_lshl_or_b32 v3, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v3, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
-; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: s_lshl_b32 s4, s18, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v5, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
-; GFX9-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v5, v5, v2
-; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
-; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v2
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc
+; GFX9-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v4, v4, v2
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v2
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s17
-; GFX9-NEXT: v_lshl_or_b32 v2, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
-; GFX9-NEXT: v_bfe_u32 v5, v1, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v5, v5, v1
-; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
-; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v1
+; GFX9-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v4, v4, v1
+; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: s_lshl_b32 s4, s17, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v6, vcc
-; GFX9-NEXT: v_add_f32_e32 v5, s4, v0
-; GFX9-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v6, v6, v5
-; GFX9-NEXT: v_add_u32_e32 v6, 0x7fff, v6
-; GFX9-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX9-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v5, vcc
+; GFX9-NEXT: v_add_f32_e32 v4, s4, v0
+; GFX9-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v5, v5, v4
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v5, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
+; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
+; GFX9-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s16
-; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v5
-; GFX9-NEXT: v_add_f32_e32 v5, s4, v0
-; GFX9-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v6, v6, v5
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f32_e32 v4, s4, v0
+; GFX9-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v5, v5, v4
; GFX9-NEXT: s_lshl_b32 s4, s16, 16
-; GFX9-NEXT: v_add_u32_e32 v6, 0x7fff, v6
-; GFX9-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
+; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
; GFX9-NEXT: v_add_f32_e32 v0, s4, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc
-; GFX9-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v6, v6, v0
-; GFX9-NEXT: v_add_u32_e32 v6, 0x7fff, v6
-; GFX9-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc
+; GFX9-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v5, v5, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
+; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v0
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v7, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX9-NEXT: v_and_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v0, v5, 16, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v5, v6, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX9-NEXT: v_or_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_setpc_b64 s[30:31]
; GFX9-NEXT: .LBB47_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
@@ -12427,71 +12425,70 @@ define inreg <2 x i64> @bitcast_v8bf16_to_v2i64_scalar(<8 x bfloat> inreg %a, i3
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX9-NEXT: v_mov_b32_e32 v4, 0xffff
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s18
-; GFX9-NEXT: v_lshl_or_b32 v3, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v3, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
-; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: s_lshl_b32 s4, s18, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v5, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
-; GFX9-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v5, v5, v2
-; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
-; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v2
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc
+; GFX9-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v4, v4, v2
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v2
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s17
-; GFX9-NEXT: v_lshl_or_b32 v2, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
-; GFX9-NEXT: v_bfe_u32 v5, v1, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v5, v5, v1
-; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
-; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v1
+; GFX9-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v4, v4, v1
+; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: s_lshl_b32 s4, s17, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v6, vcc
-; GFX9-NEXT: v_add_f32_e32 v5, s4, v0
-; GFX9-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v6, v6, v5
-; GFX9-NEXT: v_add_u32_e32 v6, 0x7fff, v6
-; GFX9-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX9-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v5, vcc
+; GFX9-NEXT: v_add_f32_e32 v4, s4, v0
+; GFX9-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v5, v5, v4
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v5, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
+; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
+; GFX9-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s16
-; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v5
-; GFX9-NEXT: v_add_f32_e32 v5, s4, v0
-; GFX9-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v6, v6, v5
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f32_e32 v4, s4, v0
+; GFX9-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v5, v5, v4
; GFX9-NEXT: s_lshl_b32 s4, s16, 16
-; GFX9-NEXT: v_add_u32_e32 v6, 0x7fff, v6
-; GFX9-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
+; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
; GFX9-NEXT: v_add_f32_e32 v0, s4, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc
-; GFX9-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v6, v6, v0
-; GFX9-NEXT: v_add_u32_e32 v6, 0x7fff, v6
-; GFX9-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc
+; GFX9-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v5, v5, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
+; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v0
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v7, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX9-NEXT: v_and_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v0, v5, 16, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v5, v6, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX9-NEXT: v_or_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_setpc_b64 s[30:31]
; GFX9-NEXT: .LBB67_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
@@ -16341,71 +16338,70 @@ define inreg <2 x double> @bitcast_v8bf16_to_v2f64_scalar(<8 x bfloat> inreg %a,
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX9-NEXT: v_mov_b32_e32 v4, 0xffff
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s18
-; GFX9-NEXT: v_lshl_or_b32 v3, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v3, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
-; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: s_lshl_b32 s4, s18, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v5, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
-; GFX9-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v5, v5, v2
-; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
-; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v2
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc
+; GFX9-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v4, v4, v2
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v2
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s17
-; GFX9-NEXT: v_lshl_or_b32 v2, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
-; GFX9-NEXT: v_bfe_u32 v5, v1, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v5, v5, v1
-; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
-; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v1
+; GFX9-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v4, v4, v1
+; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: s_lshl_b32 s4, s17, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v6, vcc
-; GFX9-NEXT: v_add_f32_e32 v5, s4, v0
-; GFX9-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v6, v6, v5
-; GFX9-NEXT: v_add_u32_e32 v6, 0x7fff, v6
-; GFX9-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX9-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v5, vcc
+; GFX9-NEXT: v_add_f32_e32 v4, s4, v0
+; GFX9-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v5, v5, v4
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v5, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
+; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
+; GFX9-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s16
-; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v5
-; GFX9-NEXT: v_add_f32_e32 v5, s4, v0
-; GFX9-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v6, v6, v5
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f32_e32 v4, s4, v0
+; GFX9-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v5, v5, v4
; GFX9-NEXT: s_lshl_b32 s4, s16, 16
-; GFX9-NEXT: v_add_u32_e32 v6, 0x7fff, v6
-; GFX9-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
+; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
; GFX9-NEXT: v_add_f32_e32 v0, s4, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc
-; GFX9-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v6, v6, v0
-; GFX9-NEXT: v_add_u32_e32 v6, 0x7fff, v6
-; GFX9-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc
+; GFX9-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v5, v5, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
+; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v0
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v7, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX9-NEXT: v_and_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v0, v5, 16, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v5, v6, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX9-NEXT: v_or_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_setpc_b64 s[30:31]
; GFX9-NEXT: .LBB83_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
@@ -22995,22 +22991,21 @@ define inreg <8 x half> @bitcast_v8bf16_to_v8f16_scalar(<8 x bfloat> inreg %a, i
; GFX9-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc
; GFX9-NEXT: v_bfe_u32 v8, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v8, v8, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v0
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v8, v9, vcc
-; GFX9-NEXT: v_mov_b32_e32 v8, 0xffff
-; GFX9-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX9-NEXT: v_and_b32_sdwa v0, v8, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v3, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v0, v8, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v8, v9, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v2, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v0, v8, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v0, v8, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v0, v4, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v3, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v0, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v0, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v4
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_setpc_b64 s[30:31]
; GFX9-NEXT: .LBB103_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
@@ -25955,9 +25950,9 @@ define inreg <16 x i8> @bitcast_v8bf16_to_v16i8_scalar(<8 x bfloat> inreg %a, i3
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v0
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v17
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v6
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s16
-; GFX9-NEXT: v_lshl_or_b32 v19, v6, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v19, v0, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v0, s4, v1
; GFX9-NEXT: v_bfe_u32 v2, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v0
@@ -25975,9 +25970,9 @@ define inreg <16 x i8> @bitcast_v8bf16_to_v16i8_scalar(<8 x bfloat> inreg %a, i3
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s19
-; GFX9-NEXT: v_lshl_or_b32 v18, v2, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v18, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v3, s4, v1
; GFX9-NEXT: v_bfe_u32 v4, v3, 16, 1
; GFX9-NEXT: v_add_u32_e32 v4, v4, v3
@@ -25995,9 +25990,9 @@ define inreg <16 x i8> @bitcast_v8bf16_to_v16i8_scalar(<8 x bfloat> inreg %a, i3
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v16, 16, v3
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v16
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v14
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s18
-; GFX9-NEXT: v_lshl_or_b32 v21, v14, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v21, v3, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v3, s4, v1
; GFX9-NEXT: v_bfe_u32 v4, v3, 16, 1
; GFX9-NEXT: v_add_u32_e32 v4, v4, v3
@@ -26015,8 +26010,8 @@ define inreg <16 x i8> @bitcast_v8bf16_to_v16i8_scalar(<8 x bfloat> inreg %a, i3
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v1
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v8
-; GFX9-NEXT: v_lshl_or_b32 v20, v10, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v10
+; GFX9-NEXT: v_or_b32_sdwa v20, v1, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_lshrrev_b64 v[11:12], 24, v[20:21]
; GFX9-NEXT: v_lshrrev_b64 v[3:4], 24, v[18:19]
; GFX9-NEXT: v_lshrrev_b32_e32 v15, 24, v21
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll
index ff778c4b646d6..24cebc1097205 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll
@@ -4670,15 +4670,14 @@ define inreg <8 x i32> @bitcast_v16bf16_to_v8i32_scalar(<16 x bfloat> inreg %a,
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX9-NEXT: v_mov_b32_e32 v8, 0xffff
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v8, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s22, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v7, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v7, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -4690,14 +4689,14 @@ define inreg <8 x i32> @bitcast_v16bf16_to_v8i32_scalar(<16 x bfloat> inreg %a,
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v8, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s21, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v6, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v6, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -4709,14 +4708,14 @@ define inreg <8 x i32> @bitcast_v16bf16_to_v8i32_scalar(<16 x bfloat> inreg %a,
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v8, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s20, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v5, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -4728,14 +4727,14 @@ define inreg <8 x i32> @bitcast_v16bf16_to_v8i32_scalar(<16 x bfloat> inreg %a,
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v8, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s19, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v4, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v4, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -4747,70 +4746,70 @@ define inreg <8 x i32> @bitcast_v16bf16_to_v8i32_scalar(<16 x bfloat> inreg %a,
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
-; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX9-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v9, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v8, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v8, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s18, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v3, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v3, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
-; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX9-NEXT: v_or_b32_e32 v8, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: s_lshl_b32 s4, s18, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v9, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v8, vcc
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
-; GFX9-NEXT: v_bfe_u32 v9, v2, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v9, v9, v2
-; GFX9-NEXT: v_add_u32_e32 v9, 0x7fff, v9
-; GFX9-NEXT: v_or_b32_e32 v10, 0x400000, v2
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v9, v10, vcc
+; GFX9-NEXT: v_bfe_u32 v8, v2, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v8, v8, v2
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v8, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
+; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v8, v9, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s17, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v2, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
-; GFX9-NEXT: v_bfe_u32 v9, v1, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v9, v9, v1
-; GFX9-NEXT: v_add_u32_e32 v9, 0x7fff, v9
-; GFX9-NEXT: v_or_b32_e32 v10, 0x400000, v1
+; GFX9-NEXT: v_bfe_u32 v8, v1, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v8, v8, v1
+; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
+; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: s_lshl_b32 s4, s17, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v9, v10, vcc
-; GFX9-NEXT: v_add_f32_e32 v9, s4, v0
-; GFX9-NEXT: v_bfe_u32 v10, v9, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v10, v10, v9
-; GFX9-NEXT: v_add_u32_e32 v10, 0x7fff, v10
-; GFX9-NEXT: v_or_b32_e32 v11, 0x400000, v9
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v9, v9
-; GFX9-NEXT: v_cndmask_b32_e32 v9, v10, v11, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v8, v9, vcc
+; GFX9-NEXT: v_add_f32_e32 v8, s4, v0
+; GFX9-NEXT: v_bfe_u32 v9, v8, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v9, v9, v8
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v9, v8, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_u32_e32 v9, 0x7fff, v9
+; GFX9-NEXT: v_or_b32_e32 v10, 0x400000, v8
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v8, v8
+; GFX9-NEXT: v_cndmask_b32_e32 v8, v9, v10, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s16, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v9
-; GFX9-NEXT: v_add_f32_e32 v9, s4, v0
-; GFX9-NEXT: v_bfe_u32 v10, v9, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v10, v10, v9
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f32_e32 v8, s4, v0
+; GFX9-NEXT: v_bfe_u32 v9, v8, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v9, v9, v8
; GFX9-NEXT: s_lshl_b32 s4, s16, 16
-; GFX9-NEXT: v_add_u32_e32 v10, 0x7fff, v10
-; GFX9-NEXT: v_or_b32_e32 v11, 0x400000, v9
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v9, v9
+; GFX9-NEXT: v_add_u32_e32 v9, 0x7fff, v9
+; GFX9-NEXT: v_or_b32_e32 v10, 0x400000, v8
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v8, v8
; GFX9-NEXT: v_add_f32_e32 v0, s4, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v9, v10, v11, vcc
-; GFX9-NEXT: v_bfe_u32 v10, v0, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v10, v10, v0
-; GFX9-NEXT: v_add_u32_e32 v10, 0x7fff, v10
-; GFX9-NEXT: v_or_b32_e32 v11, 0x400000, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v8, v9, v10, vcc
+; GFX9-NEXT: v_bfe_u32 v9, v0, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v9, v9, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; GFX9-NEXT: v_add_u32_e32 v9, 0x7fff, v9
+; GFX9-NEXT: v_or_b32_e32 v10, 0x400000, v0
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v10, v11, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v9, 16, v9
-; GFX9-NEXT: v_and_b32_sdwa v0, v8, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v0, v9, 16, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v9, v10, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX9-NEXT: v_or_b32_sdwa v0, v8, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_setpc_b64 s[30:31]
; GFX9-NEXT: .LBB23_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
@@ -12162,15 +12161,14 @@ define inreg <8 x float> @bitcast_v16bf16_to_v8f32_scalar(<16 x bfloat> inreg %a
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX9-NEXT: v_mov_b32_e32 v8, 0xffff
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v8, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s22, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v7, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v7, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -12182,14 +12180,14 @@ define inreg <8 x float> @bitcast_v16bf16_to_v8f32_scalar(<16 x bfloat> inreg %a
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v8, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s21, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v6, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v6, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -12201,14 +12199,14 @@ define inreg <8 x float> @bitcast_v16bf16_to_v8f32_scalar(<16 x bfloat> inreg %a
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v8, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s20, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v5, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -12220,14 +12218,14 @@ define inreg <8 x float> @bitcast_v16bf16_to_v8f32_scalar(<16 x bfloat> inreg %a
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v8, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s19, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v4, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v4, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -12239,70 +12237,70 @@ define inreg <8 x float> @bitcast_v16bf16_to_v8f32_scalar(<16 x bfloat> inreg %a
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
-; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX9-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v9, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v8, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v8, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s18, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v3, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v3, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
-; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX9-NEXT: v_or_b32_e32 v8, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: s_lshl_b32 s4, s18, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v9, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v8, vcc
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
-; GFX9-NEXT: v_bfe_u32 v9, v2, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v9, v9, v2
-; GFX9-NEXT: v_add_u32_e32 v9, 0x7fff, v9
-; GFX9-NEXT: v_or_b32_e32 v10, 0x400000, v2
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v9, v10, vcc
+; GFX9-NEXT: v_bfe_u32 v8, v2, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v8, v8, v2
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v8, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
+; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v8, v9, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s17, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v2, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
-; GFX9-NEXT: v_bfe_u32 v9, v1, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v9, v9, v1
-; GFX9-NEXT: v_add_u32_e32 v9, 0x7fff, v9
-; GFX9-NEXT: v_or_b32_e32 v10, 0x400000, v1
+; GFX9-NEXT: v_bfe_u32 v8, v1, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v8, v8, v1
+; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
+; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: s_lshl_b32 s4, s17, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v9, v10, vcc
-; GFX9-NEXT: v_add_f32_e32 v9, s4, v0
-; GFX9-NEXT: v_bfe_u32 v10, v9, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v10, v10, v9
-; GFX9-NEXT: v_add_u32_e32 v10, 0x7fff, v10
-; GFX9-NEXT: v_or_b32_e32 v11, 0x400000, v9
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v9, v9
-; GFX9-NEXT: v_cndmask_b32_e32 v9, v10, v11, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v8, v9, vcc
+; GFX9-NEXT: v_add_f32_e32 v8, s4, v0
+; GFX9-NEXT: v_bfe_u32 v9, v8, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v9, v9, v8
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v9, v8, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_u32_e32 v9, 0x7fff, v9
+; GFX9-NEXT: v_or_b32_e32 v10, 0x400000, v8
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v8, v8
+; GFX9-NEXT: v_cndmask_b32_e32 v8, v9, v10, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s16, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v9
-; GFX9-NEXT: v_add_f32_e32 v9, s4, v0
-; GFX9-NEXT: v_bfe_u32 v10, v9, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v10, v10, v9
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f32_e32 v8, s4, v0
+; GFX9-NEXT: v_bfe_u32 v9, v8, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v9, v9, v8
; GFX9-NEXT: s_lshl_b32 s4, s16, 16
-; GFX9-NEXT: v_add_u32_e32 v10, 0x7fff, v10
-; GFX9-NEXT: v_or_b32_e32 v11, 0x400000, v9
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v9, v9
+; GFX9-NEXT: v_add_u32_e32 v9, 0x7fff, v9
+; GFX9-NEXT: v_or_b32_e32 v10, 0x400000, v8
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v8, v8
; GFX9-NEXT: v_add_f32_e32 v0, s4, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v9, v10, v11, vcc
-; GFX9-NEXT: v_bfe_u32 v10, v0, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v10, v10, v0
-; GFX9-NEXT: v_add_u32_e32 v10, 0x7fff, v10
-; GFX9-NEXT: v_or_b32_e32 v11, 0x400000, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v8, v9, v10, vcc
+; GFX9-NEXT: v_bfe_u32 v9, v0, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v9, v9, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; GFX9-NEXT: v_add_u32_e32 v9, 0x7fff, v9
+; GFX9-NEXT: v_or_b32_e32 v10, 0x400000, v0
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v10, v11, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v9, 16, v9
-; GFX9-NEXT: v_and_b32_sdwa v0, v8, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v0, v9, 16, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v9, v10, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX9-NEXT: v_or_b32_sdwa v0, v8, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_setpc_b64 s[30:31]
; GFX9-NEXT: .LBB47_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
@@ -19152,15 +19150,14 @@ define inreg <4 x i64> @bitcast_v16bf16_to_v4i64_scalar(<16 x bfloat> inreg %a,
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX9-NEXT: v_mov_b32_e32 v8, 0xffff
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v8, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s22, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v7, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v7, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -19172,14 +19169,14 @@ define inreg <4 x i64> @bitcast_v16bf16_to_v4i64_scalar(<16 x bfloat> inreg %a,
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v8, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s21, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v6, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v6, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -19191,14 +19188,14 @@ define inreg <4 x i64> @bitcast_v16bf16_to_v4i64_scalar(<16 x bfloat> inreg %a,
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v8, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s20, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v5, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -19210,14 +19207,14 @@ define inreg <4 x i64> @bitcast_v16bf16_to_v4i64_scalar(<16 x bfloat> inreg %a,
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v8, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s19, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v4, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v4, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -19229,70 +19226,70 @@ define inreg <4 x i64> @bitcast_v16bf16_to_v4i64_scalar(<16 x bfloat> inreg %a,
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
-; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX9-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v9, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v8, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v8, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s18, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v3, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v3, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
-; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX9-NEXT: v_or_b32_e32 v8, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: s_lshl_b32 s4, s18, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v9, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v8, vcc
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
-; GFX9-NEXT: v_bfe_u32 v9, v2, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v9, v9, v2
-; GFX9-NEXT: v_add_u32_e32 v9, 0x7fff, v9
-; GFX9-NEXT: v_or_b32_e32 v10, 0x400000, v2
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v9, v10, vcc
+; GFX9-NEXT: v_bfe_u32 v8, v2, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v8, v8, v2
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v8, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
+; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v8, v9, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s17, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v2, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
-; GFX9-NEXT: v_bfe_u32 v9, v1, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v9, v9, v1
-; GFX9-NEXT: v_add_u32_e32 v9, 0x7fff, v9
-; GFX9-NEXT: v_or_b32_e32 v10, 0x400000, v1
+; GFX9-NEXT: v_bfe_u32 v8, v1, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v8, v8, v1
+; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
+; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: s_lshl_b32 s4, s17, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v9, v10, vcc
-; GFX9-NEXT: v_add_f32_e32 v9, s4, v0
-; GFX9-NEXT: v_bfe_u32 v10, v9, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v10, v10, v9
-; GFX9-NEXT: v_add_u32_e32 v10, 0x7fff, v10
-; GFX9-NEXT: v_or_b32_e32 v11, 0x400000, v9
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v9, v9
-; GFX9-NEXT: v_cndmask_b32_e32 v9, v10, v11, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v8, v9, vcc
+; GFX9-NEXT: v_add_f32_e32 v8, s4, v0
+; GFX9-NEXT: v_bfe_u32 v9, v8, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v9, v9, v8
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v9, v8, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_u32_e32 v9, 0x7fff, v9
+; GFX9-NEXT: v_or_b32_e32 v10, 0x400000, v8
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v8, v8
+; GFX9-NEXT: v_cndmask_b32_e32 v8, v9, v10, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s16, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v9
-; GFX9-NEXT: v_add_f32_e32 v9, s4, v0
-; GFX9-NEXT: v_bfe_u32 v10, v9, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v10, v10, v9
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f32_e32 v8, s4, v0
+; GFX9-NEXT: v_bfe_u32 v9, v8, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v9, v9, v8
; GFX9-NEXT: s_lshl_b32 s4, s16, 16
-; GFX9-NEXT: v_add_u32_e32 v10, 0x7fff, v10
-; GFX9-NEXT: v_or_b32_e32 v11, 0x400000, v9
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v9, v9
+; GFX9-NEXT: v_add_u32_e32 v9, 0x7fff, v9
+; GFX9-NEXT: v_or_b32_e32 v10, 0x400000, v8
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v8, v8
; GFX9-NEXT: v_add_f32_e32 v0, s4, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v9, v10, v11, vcc
-; GFX9-NEXT: v_bfe_u32 v10, v0, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v10, v10, v0
-; GFX9-NEXT: v_add_u32_e32 v10, 0x7fff, v10
-; GFX9-NEXT: v_or_b32_e32 v11, 0x400000, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v8, v9, v10, vcc
+; GFX9-NEXT: v_bfe_u32 v9, v0, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v9, v9, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; GFX9-NEXT: v_add_u32_e32 v9, 0x7fff, v9
+; GFX9-NEXT: v_or_b32_e32 v10, 0x400000, v0
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v10, v11, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v9, 16, v9
-; GFX9-NEXT: v_and_b32_sdwa v0, v8, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v0, v9, 16, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v9, v10, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX9-NEXT: v_or_b32_sdwa v0, v8, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_setpc_b64 s[30:31]
; GFX9-NEXT: .LBB67_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
@@ -25580,15 +25577,14 @@ define inreg <4 x double> @bitcast_v16bf16_to_v4f64_scalar(<16 x bfloat> inreg %
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX9-NEXT: v_mov_b32_e32 v8, 0xffff
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v8, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s22, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v7, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v7, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -25600,14 +25596,14 @@ define inreg <4 x double> @bitcast_v16bf16_to_v4f64_scalar(<16 x bfloat> inreg %
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v8, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s21, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v6, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v6, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -25619,14 +25615,14 @@ define inreg <4 x double> @bitcast_v16bf16_to_v4f64_scalar(<16 x bfloat> inreg %
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v8, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s20, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v5, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -25638,14 +25634,14 @@ define inreg <4 x double> @bitcast_v16bf16_to_v4f64_scalar(<16 x bfloat> inreg %
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v8, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s19, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v4, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v4, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -25657,70 +25653,70 @@ define inreg <4 x double> @bitcast_v16bf16_to_v4f64_scalar(<16 x bfloat> inreg %
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
-; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX9-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v9, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v8, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v8, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s18, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v3, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v3, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
-; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX9-NEXT: v_or_b32_e32 v8, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: s_lshl_b32 s4, s18, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v9, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v8, vcc
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
-; GFX9-NEXT: v_bfe_u32 v9, v2, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v9, v9, v2
-; GFX9-NEXT: v_add_u32_e32 v9, 0x7fff, v9
-; GFX9-NEXT: v_or_b32_e32 v10, 0x400000, v2
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v9, v10, vcc
+; GFX9-NEXT: v_bfe_u32 v8, v2, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v8, v8, v2
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v8, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
+; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v8, v9, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s17, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v2, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
-; GFX9-NEXT: v_bfe_u32 v9, v1, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v9, v9, v1
-; GFX9-NEXT: v_add_u32_e32 v9, 0x7fff, v9
-; GFX9-NEXT: v_or_b32_e32 v10, 0x400000, v1
+; GFX9-NEXT: v_bfe_u32 v8, v1, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v8, v8, v1
+; GFX9-NEXT: v_add_u32_e32 v8, 0x7fff, v8
+; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: s_lshl_b32 s4, s17, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v9, v10, vcc
-; GFX9-NEXT: v_add_f32_e32 v9, s4, v0
-; GFX9-NEXT: v_bfe_u32 v10, v9, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v10, v10, v9
-; GFX9-NEXT: v_add_u32_e32 v10, 0x7fff, v10
-; GFX9-NEXT: v_or_b32_e32 v11, 0x400000, v9
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v9, v9
-; GFX9-NEXT: v_cndmask_b32_e32 v9, v10, v11, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v8, v9, vcc
+; GFX9-NEXT: v_add_f32_e32 v8, s4, v0
+; GFX9-NEXT: v_bfe_u32 v9, v8, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v9, v9, v8
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v9, v8, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_u32_e32 v9, 0x7fff, v9
+; GFX9-NEXT: v_or_b32_e32 v10, 0x400000, v8
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v8, v8
+; GFX9-NEXT: v_cndmask_b32_e32 v8, v9, v10, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s16, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v9
-; GFX9-NEXT: v_add_f32_e32 v9, s4, v0
-; GFX9-NEXT: v_bfe_u32 v10, v9, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v10, v10, v9
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f32_e32 v8, s4, v0
+; GFX9-NEXT: v_bfe_u32 v9, v8, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v9, v9, v8
; GFX9-NEXT: s_lshl_b32 s4, s16, 16
-; GFX9-NEXT: v_add_u32_e32 v10, 0x7fff, v10
-; GFX9-NEXT: v_or_b32_e32 v11, 0x400000, v9
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v9, v9
+; GFX9-NEXT: v_add_u32_e32 v9, 0x7fff, v9
+; GFX9-NEXT: v_or_b32_e32 v10, 0x400000, v8
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v8, v8
; GFX9-NEXT: v_add_f32_e32 v0, s4, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v9, v10, v11, vcc
-; GFX9-NEXT: v_bfe_u32 v10, v0, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v10, v10, v0
-; GFX9-NEXT: v_add_u32_e32 v10, 0x7fff, v10
-; GFX9-NEXT: v_or_b32_e32 v11, 0x400000, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v8, v9, v10, vcc
+; GFX9-NEXT: v_bfe_u32 v9, v0, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v9, v9, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; GFX9-NEXT: v_add_u32_e32 v9, 0x7fff, v9
+; GFX9-NEXT: v_or_b32_e32 v10, 0x400000, v0
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v10, v11, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v9, 16, v9
-; GFX9-NEXT: v_and_b32_sdwa v0, v8, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v0, v9, 16, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v9, v10, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX9-NEXT: v_or_b32_sdwa v0, v8, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_setpc_b64 s[30:31]
; GFX9-NEXT: .LBB83_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
@@ -34908,59 +34904,59 @@ define inreg <16 x i16> @bitcast_v32i8_to_v16i16_scalar(<32 x i8> inreg %a, i32
; GFX9-NEXT: v_readfirstlane_b32 s8, v0
; GFX9-NEXT: s_cbranch_scc0 .LBB99_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v4, 0xc0c0004
-; GFX9-NEXT: v_perm_b32 v0, s16, v0, v4
; GFX9-NEXT: v_mov_b32_e32 v1, s19
+; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_perm_b32 v1, s18, v1, v4
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX9-NEXT: v_mov_b32_e32 v1, s21
-; GFX9-NEXT: v_perm_b32 v1, s20, v1, v4
+; GFX9-NEXT: v_perm_b32 v0, s16, v0, v4
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_mov_b32_e32 v2, s23
-; GFX9-NEXT: v_perm_b32 v2, s22, v2, v4
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, s25
-; GFX9-NEXT: v_perm_b32 v2, s24, v2, v4
-; GFX9-NEXT: v_mov_b32_e32 v3, s27
-; GFX9-NEXT: v_perm_b32 v3, s26, v3, v4
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_lshl_or_b32 v2, v3, 16, v2
-; GFX9-NEXT: v_mov_b32_e32 v3, s29
-; GFX9-NEXT: v_perm_b32 v3, s28, v3, v4
-; GFX9-NEXT: v_mov_b32_e32 v5, s6
; GFX9-NEXT: s_and_b32 s4, s14, 0xff
; GFX9-NEXT: s_lshl_b32 s5, s12, 8
-; GFX9-NEXT: v_perm_b32 v5, s8, v5, v4
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v1, s21
+; GFX9-NEXT: v_perm_b32 v2, s22, v2, v4
; GFX9-NEXT: s_or_b32 s4, s4, s5
; GFX9-NEXT: s_and_b32 s5, s9, 0xff
; GFX9-NEXT: s_lshl_b32 s56, s7, 8
-; GFX9-NEXT: v_lshl_or_b32 v3, v5, 16, v3
+; GFX9-NEXT: v_perm_b32 v1, s20, v1, v4
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_mov_b32_e32 v3, s27
; GFX9-NEXT: s_or_b32 s5, s5, s56
-; GFX9-NEXT: v_mov_b32_e32 v5, s40
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v2, s25
+; GFX9-NEXT: v_perm_b32 v3, s26, v3, v4
; GFX9-NEXT: s_pack_ll_b32_b16 s4, s4, s5
-; GFX9-NEXT: v_perm_b32 v5, s42, v5, v4
; GFX9-NEXT: s_and_b32 s5, s11, 0xff
; GFX9-NEXT: s_lshl_b32 s56, s10, 8
+; GFX9-NEXT: v_perm_b32 v2, s24, v2, v4
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_mov_b32_e32 v5, s6
; GFX9-NEXT: s_or_b32 s5, s5, s56
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_lshl_or_b32 v5, s5, 16, v5
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v3, s29
+; GFX9-NEXT: v_perm_b32 v5, s8, v5, v4
+; GFX9-NEXT: v_lshlrev_b32_e64 v6, 16, s5
; GFX9-NEXT: s_and_b32 s5, s45, 0xff
; GFX9-NEXT: s_lshl_b32 s56, s44, 8
+; GFX9-NEXT: v_perm_b32 v3, s28, v3, v4
+; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX9-NEXT: s_or_b32 s5, s5, s56
; GFX9-NEXT: s_and_b32 s56, s15, 0xff
; GFX9-NEXT: s_lshl_b32 s57, s13, 8
+; GFX9-NEXT: v_or_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v5, s40
; GFX9-NEXT: s_or_b32 s56, s56, s57
-; GFX9-NEXT: v_mov_b32_e32 v6, s46
+; GFX9-NEXT: v_perm_b32 v5, s42, v5, v4
; GFX9-NEXT: s_pack_ll_b32_b16 s5, s5, s56
-; GFX9-NEXT: v_perm_b32 v4, s47, v6, v4
; GFX9-NEXT: s_and_b32 s56, s43, 0xff
; GFX9-NEXT: s_lshl_b32 s57, s41, 8
+; GFX9-NEXT: v_or_b32_sdwa v5, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v6, s46
; GFX9-NEXT: s_or_b32 s56, s56, s57
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_lshl_or_b32 v7, s56, 16, v4
+; GFX9-NEXT: v_perm_b32 v4, s47, v6, v4
+; GFX9-NEXT: v_lshlrev_b32_e64 v6, 16, s56
+; GFX9-NEXT: v_or_b32_sdwa v7, v6, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v4, s4
; GFX9-NEXT: v_mov_b32_e32 v6, s5
; GFX9-NEXT: s_mov_b64 s[4:5], 0
@@ -35008,53 +35004,46 @@ define inreg <16 x i16> @bitcast_v32i8_to_v16i16_scalar(<32 x i8> inreg %a, i32
; GFX9-NEXT: v_add_u32_e32 v1, 0x300, v0
; GFX9-NEXT: s_add_i32 s22, s22, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s23
+; GFX9-NEXT: s_movk_i32 s4, 0x300
; GFX9-NEXT: v_perm_b32 v0, s22, v0, v2
-; GFX9-NEXT: v_add_u32_e32 v12, 0x300, v0
+; GFX9-NEXT: v_add_u32_sdwa v12, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s16, s16, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s17
-; GFX9-NEXT: v_perm_b32 v0, s16, v0, v2
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: v_mov_b32_e32 v13, s19
-; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
+; GFX9-NEXT: v_perm_b32 v0, s16, v0, v2
; GFX9-NEXT: v_perm_b32 v13, s18, v13, v2
-; GFX9-NEXT: v_add_u32_e32 v13, 0x300, v13
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v0, v13, 16, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: s_add_i32 s24, s24, 3
-; GFX9-NEXT: v_mov_b32_e32 v13, s25
-; GFX9-NEXT: v_add_u32_e32 v16, 0x300, v5
-; GFX9-NEXT: v_add_u32_e32 v5, 0x300, v7
+; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
+; GFX9-NEXT: v_add_u32_sdwa v13, v13, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s8, s8, 3
-; GFX9-NEXT: v_lshl_or_b32 v1, v12, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v0, v13, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v12, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v12, s6
-; GFX9-NEXT: v_perm_b32 v13, s24, v13, v2
+; GFX9-NEXT: s_add_i32 s24, s24, 3
+; GFX9-NEXT: v_mov_b32_e32 v13, s25
; GFX9-NEXT: s_add_i32 s26, s26, 3
; GFX9-NEXT: v_mov_b32_e32 v14, s27
-; GFX9-NEXT: v_add_u32_e32 v7, 0x300, v8
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
; GFX9-NEXT: v_perm_b32 v12, s8, v12, v2
+; GFX9-NEXT: v_perm_b32 v13, s24, v13, v2
; GFX9-NEXT: v_perm_b32 v2, s26, v14, v2
; GFX9-NEXT: v_add_u32_e32 v14, 0x300, v3
-; GFX9-NEXT: v_add_u32_e32 v15, 0x300, v4
-; GFX9-NEXT: v_add_u32_e32 v6, 0x300, v6
+; GFX9-NEXT: v_add_u32_sdwa v15, v4, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_add_u32_e32 v16, 0x300, v5
+; GFX9-NEXT: v_add_u32_sdwa v6, v6, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_add_u32_e32 v5, 0x300, v7
+; GFX9-NEXT: v_add_u32_sdwa v7, v8, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_add_u32_e32 v4, 0x300, v9
-; GFX9-NEXT: v_add_u32_e32 v8, 0x300, v10
+; GFX9-NEXT: v_add_u32_sdwa v8, v10, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_add_u32_e32 v3, 0x300, v11
+; GFX9-NEXT: v_add_u32_sdwa v9, v12, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_add_u32_e32 v10, 0x300, v13
-; GFX9-NEXT: v_lshl_or_b32 v5, v7, 16, v5
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v16
-; GFX9-NEXT: v_add_u32_e32 v9, 0x300, v12
-; GFX9-NEXT: v_add_u32_e32 v2, 0x300, v2
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_lshl_or_b32 v6, v6, 16, v7
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v14
-; GFX9-NEXT: v_lshl_or_b32 v2, v2, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v3, v9, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v8, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v7, v15, 16, v7
+; GFX9-NEXT: v_add_u32_sdwa v2, v2, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v2, v2, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v9, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v4, v8, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v7, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v6, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v15, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: .LBB99_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -37169,107 +37158,106 @@ define inreg <16 x half> @bitcast_v16bf16_to_v16f16_scalar(<16 x bfloat> inreg %
; GFX9-NEXT: v_cndmask_b32_e32 v9, v6, v7, vcc
; GFX9-NEXT: v_bfe_u32 v6, v5, 16, 1
; GFX9-NEXT: v_add_u32_e32 v6, v6, v5
-; GFX9-NEXT: s_lshl_b32 s4, s19, 16
-; GFX9-NEXT: v_add_u32_e32 v6, 0x7fff, v6
-; GFX9-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX9-NEXT: v_add_f32_e32 v5, s4, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v10, v6, v7, vcc
-; GFX9-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v6, v6, v5
-; GFX9-NEXT: s_and_b32 s5, s22, 0xffff0000
; GFX9-NEXT: v_add_u32_e32 v6, 0x7fff, v6
; GFX9-NEXT: v_or_b32_e32 v7, 0x400000, v5
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX9-NEXT: v_add_f32_e32 v5, s5, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v11, v6, v7, vcc
-; GFX9-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v6, v6, v5
-; GFX9-NEXT: v_add_u32_e32 v6, 0x7fff, v6
-; GFX9-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX9-NEXT: s_lshl_b32 s5, s22, 16
+; GFX9-NEXT: s_lshl_b32 s4, s19, 16
; GFX9-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc
+; GFX9-NEXT: v_add_f32_e32 v6, s4, v1
+; GFX9-NEXT: v_bfe_u32 v7, v6, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v7, v7, v6
+; GFX9-NEXT: s_and_b32 s4, s20, 0xffff0000
+; GFX9-NEXT: v_add_u32_e32 v7, 0x7fff, v7
+; GFX9-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX9-NEXT: v_add_f32_e32 v6, s4, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v10, v7, v10, vcc
+; GFX9-NEXT: v_bfe_u32 v7, v6, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v7, v7, v6
+; GFX9-NEXT: s_and_b32 s5, s22, 0xffff0000
+; GFX9-NEXT: v_add_u32_e32 v7, 0x7fff, v7
+; GFX9-NEXT: v_or_b32_e32 v11, 0x400000, v6
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
; GFX9-NEXT: v_add_f32_e32 v6, s5, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v11, v7, v11, vcc
; GFX9-NEXT: v_bfe_u32 v7, v6, 16, 1
; GFX9-NEXT: v_add_u32_e32 v7, v7, v6
; GFX9-NEXT: v_add_u32_e32 v7, 0x7fff, v7
; GFX9-NEXT: v_or_b32_e32 v12, 0x400000, v6
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
-; GFX9-NEXT: s_and_b32 s5, s23, 0xffff0000
+; GFX9-NEXT: s_lshl_b32 s5, s22, 16
; GFX9-NEXT: v_cndmask_b32_e32 v6, v7, v12, vcc
; GFX9-NEXT: v_add_f32_e32 v7, s5, v1
; GFX9-NEXT: v_bfe_u32 v12, v7, 16, 1
; GFX9-NEXT: v_add_u32_e32 v12, v12, v7
+; GFX9-NEXT: s_and_b32 s5, s23, 0xffff0000
; GFX9-NEXT: v_add_u32_e32 v12, 0x7fff, v12
; GFX9-NEXT: v_or_b32_e32 v13, 0x400000, v7
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
+; GFX9-NEXT: v_add_f32_e32 v7, s5, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v12, v12, v13, vcc
+; GFX9-NEXT: v_bfe_u32 v13, v7, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v13, v13, v7
+; GFX9-NEXT: v_add_u32_e32 v13, 0x7fff, v13
+; GFX9-NEXT: v_or_b32_e32 v14, 0x400000, v7
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX9-NEXT: s_lshl_b32 s5, s23, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v7, v12, v13, vcc
-; GFX9-NEXT: v_add_f32_e32 v12, s5, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v7, v13, v14, vcc
+; GFX9-NEXT: v_add_f32_e32 v13, s5, v1
+; GFX9-NEXT: v_bfe_u32 v14, v13, 16, 1
+; GFX9-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX9-NEXT: v_add_u32_e32 v14, v14, v13
+; GFX9-NEXT: s_lshl_b32 s4, s20, 16
+; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v7
+; GFX9-NEXT: v_add_u32_e32 v14, 0x7fff, v14
+; GFX9-NEXT: v_or_b32_e32 v15, 0x400000, v13
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v13, v13
+; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX9-NEXT: v_cndmask_b32_e32 v13, v14, v15, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX9-NEXT: v_or_b32_sdwa v6, v6, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f32_e32 v12, s4, v1
+; GFX9-NEXT: v_or_b32_sdwa v7, v7, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_bfe_u32 v13, v12, 16, 1
; GFX9-NEXT: v_add_u32_e32 v13, v13, v12
; GFX9-NEXT: v_add_u32_e32 v13, 0x7fff, v13
; GFX9-NEXT: v_or_b32_e32 v14, 0x400000, v12
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v12, v12
-; GFX9-NEXT: v_cndmask_b32_e32 v12, v13, v14, vcc
-; GFX9-NEXT: v_mov_b32_e32 v13, 0xffff
; GFX9-NEXT: s_and_b32 s4, s21, 0xffff0000
-; GFX9-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX9-NEXT: v_and_b32_sdwa v6, v13, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX9-NEXT: v_and_b32_sdwa v12, v13, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v6, v5, 16, v6
-; GFX9-NEXT: v_add_f32_e32 v5, s4, v1
-; GFX9-NEXT: v_lshl_or_b32 v7, v7, 16, v12
-; GFX9-NEXT: v_bfe_u32 v12, v5, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v12, v12, v5
-; GFX9-NEXT: v_add_u32_e32 v12, 0x7fff, v12
-; GFX9-NEXT: v_or_b32_e32 v14, 0x400000, v5
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX9-NEXT: v_cndmask_b32_e32 v12, v13, v14, vcc
+; GFX9-NEXT: v_add_f32_e32 v13, s4, v1
+; GFX9-NEXT: v_bfe_u32 v14, v13, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v14, v14, v13
; GFX9-NEXT: s_lshl_b32 s4, s21, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v5, v12, v14, vcc
-; GFX9-NEXT: v_add_f32_e32 v12, s4, v1
-; GFX9-NEXT: v_bfe_u32 v14, v12, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v14, v14, v12
-; GFX9-NEXT: v_add_u32_e32 v14, 0x7fff, v14
-; GFX9-NEXT: v_or_b32_e32 v15, 0x400000, v12
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v12, v12
-; GFX9-NEXT: v_cndmask_b32_e32 v12, v14, v15, vcc
-; GFX9-NEXT: s_and_b32 s4, s20, 0xffff0000
-; GFX9-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX9-NEXT: v_and_b32_sdwa v12, v13, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v5, v5, 16, v12
-; GFX9-NEXT: v_add_f32_e32 v12, s4, v1
-; GFX9-NEXT: v_bfe_u32 v14, v12, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v14, v14, v12
-; GFX9-NEXT: s_lshl_b32 s4, s20, 16
; GFX9-NEXT: v_add_u32_e32 v14, 0x7fff, v14
-; GFX9-NEXT: v_or_b32_e32 v15, 0x400000, v12
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v12, v12
+; GFX9-NEXT: v_or_b32_e32 v15, 0x400000, v13
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v13, v13
; GFX9-NEXT: v_add_f32_e32 v1, s4, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v12, v14, v15, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v13, v14, v15, vcc
; GFX9-NEXT: v_bfe_u32 v14, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v14, v14, v1
; GFX9-NEXT: v_add_u32_e32 v14, 0x7fff, v14
; GFX9-NEXT: v_or_b32_e32 v15, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v3
-; GFX9-NEXT: v_lshrrev_b32_e32 v3, 16, v10
-; GFX9-NEXT: v_cndmask_b32_e32 v10, v14, v15, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v16, 16, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v14, v15, vcc
+; GFX9-NEXT: v_lshrrev_b32_e32 v15, 16, v3
+; GFX9-NEXT: v_lshrrev_b32_e32 v3, 16, v5
+; GFX9-NEXT: v_lshrrev_b32_e32 v5, 16, v13
+; GFX9-NEXT: v_lshrrev_b32_e32 v14, 16, v2
; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v4
-; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v12
-; GFX9-NEXT: v_and_b32_sdwa v10, v13, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v4, v4, 16, v10
-; GFX9-NEXT: v_and_b32_sdwa v10, v13, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_and_b32_sdwa v9, v13, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_and_b32_sdwa v8, v13, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_and_b32_sdwa v0, v13, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v3, v3, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v2, v2, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v0, v16, 16, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v11
+; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX9-NEXT: v_or_b32_sdwa v5, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX9-NEXT: v_or_b32_sdwa v4, v1, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v3, v1, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v1, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v15
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v14
+; GFX9-NEXT: v_or_b32_sdwa v0, v8, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_setpc_b64 s[30:31]
; GFX9-NEXT: .LBB103_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
@@ -40481,59 +40469,59 @@ define inreg <16 x half> @bitcast_v32i8_to_v16f16_scalar(<32 x i8> inreg %a, i32
; GFX9-NEXT: v_readfirstlane_b32 s8, v0
; GFX9-NEXT: s_cbranch_scc0 .LBB107_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v4, 0xc0c0004
-; GFX9-NEXT: v_perm_b32 v0, s16, v0, v4
; GFX9-NEXT: v_mov_b32_e32 v1, s19
+; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_perm_b32 v1, s18, v1, v4
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX9-NEXT: v_mov_b32_e32 v1, s21
-; GFX9-NEXT: v_perm_b32 v1, s20, v1, v4
+; GFX9-NEXT: v_perm_b32 v0, s16, v0, v4
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_mov_b32_e32 v2, s23
-; GFX9-NEXT: v_perm_b32 v2, s22, v2, v4
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, s25
-; GFX9-NEXT: v_perm_b32 v2, s24, v2, v4
-; GFX9-NEXT: v_mov_b32_e32 v3, s27
-; GFX9-NEXT: v_perm_b32 v3, s26, v3, v4
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_lshl_or_b32 v2, v3, 16, v2
-; GFX9-NEXT: v_mov_b32_e32 v3, s29
-; GFX9-NEXT: v_perm_b32 v3, s28, v3, v4
-; GFX9-NEXT: v_mov_b32_e32 v5, s6
; GFX9-NEXT: s_and_b32 s4, s14, 0xff
; GFX9-NEXT: s_lshl_b32 s5, s12, 8
-; GFX9-NEXT: v_perm_b32 v5, s8, v5, v4
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v1, s21
+; GFX9-NEXT: v_perm_b32 v2, s22, v2, v4
; GFX9-NEXT: s_or_b32 s4, s4, s5
; GFX9-NEXT: s_and_b32 s5, s9, 0xff
; GFX9-NEXT: s_lshl_b32 s56, s7, 8
-; GFX9-NEXT: v_lshl_or_b32 v3, v5, 16, v3
+; GFX9-NEXT: v_perm_b32 v1, s20, v1, v4
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_mov_b32_e32 v3, s27
; GFX9-NEXT: s_or_b32 s5, s5, s56
-; GFX9-NEXT: v_mov_b32_e32 v5, s40
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v2, s25
+; GFX9-NEXT: v_perm_b32 v3, s26, v3, v4
; GFX9-NEXT: s_pack_ll_b32_b16 s4, s4, s5
-; GFX9-NEXT: v_perm_b32 v5, s42, v5, v4
; GFX9-NEXT: s_and_b32 s5, s11, 0xff
; GFX9-NEXT: s_lshl_b32 s56, s10, 8
+; GFX9-NEXT: v_perm_b32 v2, s24, v2, v4
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_mov_b32_e32 v5, s6
; GFX9-NEXT: s_or_b32 s5, s5, s56
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_lshl_or_b32 v5, s5, 16, v5
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v3, s29
+; GFX9-NEXT: v_perm_b32 v5, s8, v5, v4
+; GFX9-NEXT: v_lshlrev_b32_e64 v6, 16, s5
; GFX9-NEXT: s_and_b32 s5, s45, 0xff
; GFX9-NEXT: s_lshl_b32 s56, s44, 8
+; GFX9-NEXT: v_perm_b32 v3, s28, v3, v4
+; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX9-NEXT: s_or_b32 s5, s5, s56
; GFX9-NEXT: s_and_b32 s56, s15, 0xff
; GFX9-NEXT: s_lshl_b32 s57, s13, 8
+; GFX9-NEXT: v_or_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v5, s40
; GFX9-NEXT: s_or_b32 s56, s56, s57
-; GFX9-NEXT: v_mov_b32_e32 v6, s46
+; GFX9-NEXT: v_perm_b32 v5, s42, v5, v4
; GFX9-NEXT: s_pack_ll_b32_b16 s5, s5, s56
-; GFX9-NEXT: v_perm_b32 v4, s47, v6, v4
; GFX9-NEXT: s_and_b32 s56, s43, 0xff
; GFX9-NEXT: s_lshl_b32 s57, s41, 8
+; GFX9-NEXT: v_or_b32_sdwa v5, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v6, s46
; GFX9-NEXT: s_or_b32 s56, s56, s57
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_lshl_or_b32 v7, s56, 16, v4
+; GFX9-NEXT: v_perm_b32 v4, s47, v6, v4
+; GFX9-NEXT: v_lshlrev_b32_e64 v6, 16, s56
+; GFX9-NEXT: v_or_b32_sdwa v7, v6, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v4, s4
; GFX9-NEXT: v_mov_b32_e32 v6, s5
; GFX9-NEXT: s_mov_b64 s[4:5], 0
@@ -40581,53 +40569,46 @@ define inreg <16 x half> @bitcast_v32i8_to_v16f16_scalar(<32 x i8> inreg %a, i32
; GFX9-NEXT: v_add_u32_e32 v1, 0x300, v0
; GFX9-NEXT: s_add_i32 s22, s22, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s23
+; GFX9-NEXT: s_movk_i32 s4, 0x300
; GFX9-NEXT: v_perm_b32 v0, s22, v0, v2
-; GFX9-NEXT: v_add_u32_e32 v12, 0x300, v0
+; GFX9-NEXT: v_add_u32_sdwa v12, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s16, s16, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s17
-; GFX9-NEXT: v_perm_b32 v0, s16, v0, v2
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: v_mov_b32_e32 v13, s19
-; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
+; GFX9-NEXT: v_perm_b32 v0, s16, v0, v2
; GFX9-NEXT: v_perm_b32 v13, s18, v13, v2
-; GFX9-NEXT: v_add_u32_e32 v13, 0x300, v13
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v0, v13, 16, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: s_add_i32 s24, s24, 3
-; GFX9-NEXT: v_mov_b32_e32 v13, s25
-; GFX9-NEXT: v_add_u32_e32 v16, 0x300, v5
-; GFX9-NEXT: v_add_u32_e32 v5, 0x300, v7
+; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
+; GFX9-NEXT: v_add_u32_sdwa v13, v13, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s8, s8, 3
-; GFX9-NEXT: v_lshl_or_b32 v1, v12, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v0, v13, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v12, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v12, s6
-; GFX9-NEXT: v_perm_b32 v13, s24, v13, v2
+; GFX9-NEXT: s_add_i32 s24, s24, 3
+; GFX9-NEXT: v_mov_b32_e32 v13, s25
; GFX9-NEXT: s_add_i32 s26, s26, 3
; GFX9-NEXT: v_mov_b32_e32 v14, s27
-; GFX9-NEXT: v_add_u32_e32 v7, 0x300, v8
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
; GFX9-NEXT: v_perm_b32 v12, s8, v12, v2
+; GFX9-NEXT: v_perm_b32 v13, s24, v13, v2
; GFX9-NEXT: v_perm_b32 v2, s26, v14, v2
; GFX9-NEXT: v_add_u32_e32 v14, 0x300, v3
-; GFX9-NEXT: v_add_u32_e32 v15, 0x300, v4
-; GFX9-NEXT: v_add_u32_e32 v6, 0x300, v6
+; GFX9-NEXT: v_add_u32_sdwa v15, v4, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_add_u32_e32 v16, 0x300, v5
+; GFX9-NEXT: v_add_u32_sdwa v6, v6, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_add_u32_e32 v5, 0x300, v7
+; GFX9-NEXT: v_add_u32_sdwa v7, v8, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_add_u32_e32 v4, 0x300, v9
-; GFX9-NEXT: v_add_u32_e32 v8, 0x300, v10
+; GFX9-NEXT: v_add_u32_sdwa v8, v10, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_add_u32_e32 v3, 0x300, v11
+; GFX9-NEXT: v_add_u32_sdwa v9, v12, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_add_u32_e32 v10, 0x300, v13
-; GFX9-NEXT: v_lshl_or_b32 v5, v7, 16, v5
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v16
-; GFX9-NEXT: v_add_u32_e32 v9, 0x300, v12
-; GFX9-NEXT: v_add_u32_e32 v2, 0x300, v2
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_lshl_or_b32 v6, v6, 16, v7
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v14
-; GFX9-NEXT: v_lshl_or_b32 v2, v2, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v3, v9, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v8, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v7, v15, 16, v7
+; GFX9-NEXT: v_add_u32_sdwa v2, v2, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v2, v2, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v9, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v4, v8, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v7, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v6, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v15, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: .LBB107_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -42692,9 +42673,9 @@ define inreg <32 x i8> @bitcast_v16bf16_to_v32i8_scalar(<16 x bfloat> inreg %a,
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v35, 16, v0
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v6
; GFX9-NEXT: s_and_b32 s4, s16, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v4, v6, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v4, v0, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v0, s4, v1
; GFX9-NEXT: v_bfe_u32 v2, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v0
@@ -42729,9 +42710,9 @@ define inreg <32 x i8> @bitcast_v16bf16_to_v32i8_scalar(<16 x bfloat> inreg %a,
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
; GFX9-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v34, 16, v5
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v14
; GFX9-NEXT: s_and_b32 s4, s18, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v12, v14, 16, v5
+; GFX9-NEXT: v_or_b32_sdwa v12, v5, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v5, s4, v1
; GFX9-NEXT: v_bfe_u32 v7, v5, 16, 1
; GFX9-NEXT: v_add_u32_e32 v7, v7, v5
@@ -42749,9 +42730,9 @@ define inreg <32 x i8> @bitcast_v16bf16_to_v32i8_scalar(<16 x bfloat> inreg %a,
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
; GFX9-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v5
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v10
; GFX9-NEXT: s_and_b32 s4, s21, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v11, v10, 16, v5
+; GFX9-NEXT: v_or_b32_sdwa v11, v5, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v5, s4, v1
; GFX9-NEXT: v_bfe_u32 v7, v5, 16, 1
; GFX9-NEXT: v_add_u32_e32 v7, v7, v5
@@ -42769,9 +42750,9 @@ define inreg <32 x i8> @bitcast_v16bf16_to_v32i8_scalar(<16 x bfloat> inreg %a,
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
; GFX9-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v33, 16, v5
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v22
; GFX9-NEXT: s_and_b32 s4, s20, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v20, v22, 16, v5
+; GFX9-NEXT: v_or_b32_sdwa v20, v5, v33 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v5, s4, v1
; GFX9-NEXT: v_bfe_u32 v7, v5, 16, 1
; GFX9-NEXT: v_add_u32_e32 v7, v7, v5
@@ -42789,9 +42770,9 @@ define inreg <32 x i8> @bitcast_v16bf16_to_v32i8_scalar(<16 x bfloat> inreg %a,
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
; GFX9-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v16, 16, v5
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v16
+; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v18
; GFX9-NEXT: s_and_b32 s4, s23, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v19, v18, 16, v5
+; GFX9-NEXT: v_or_b32_sdwa v19, v5, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v5, s4, v1
; GFX9-NEXT: v_bfe_u32 v7, v5, 16, 1
; GFX9-NEXT: v_add_u32_e32 v7, v7, v5
@@ -42823,17 +42804,17 @@ define inreg <32 x i8> @bitcast_v16bf16_to_v32i8_scalar(<16 x bfloat> inreg %a,
; GFX9-NEXT: v_or_b32_e32 v15, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: v_cndmask_b32_e32 v1, v13, v15, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v32, 16, v7
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_lshrrev_b32_e32 v30, 16, v5
-; GFX9-NEXT: v_lshrrev_b32_e32 v24, 16, v1
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v32
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v32, 16, v7
; GFX9-NEXT: v_lshrrev_b32_e32 v26, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v28, v30, 16, v1
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v24
-; GFX9-NEXT: v_lshl_or_b32 v3, v2, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v27, v26, 16, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v24, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v30
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v28, v1, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v26
+; GFX9-NEXT: v_or_b32_sdwa v3, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v27, v1, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_lshrrev_b32_e32 v31, 24, v28
; GFX9-NEXT: v_lshrrev_b32_e32 v29, 8, v28
; GFX9-NEXT: v_lshrrev_b32_e32 v25, 8, v27
@@ -44915,59 +44896,59 @@ define inreg <16 x bfloat> @bitcast_v32i8_to_v16bf16_scalar(<32 x i8> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s8, v0
; GFX9-NEXT: s_cbranch_scc0 .LBB111_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v4, 0xc0c0004
-; GFX9-NEXT: v_perm_b32 v0, s16, v0, v4
; GFX9-NEXT: v_mov_b32_e32 v1, s19
+; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_perm_b32 v1, s18, v1, v4
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX9-NEXT: v_mov_b32_e32 v1, s21
-; GFX9-NEXT: v_perm_b32 v1, s20, v1, v4
+; GFX9-NEXT: v_perm_b32 v0, s16, v0, v4
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_mov_b32_e32 v2, s23
-; GFX9-NEXT: v_perm_b32 v2, s22, v2, v4
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, s25
-; GFX9-NEXT: v_perm_b32 v2, s24, v2, v4
-; GFX9-NEXT: v_mov_b32_e32 v3, s27
-; GFX9-NEXT: v_perm_b32 v3, s26, v3, v4
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_lshl_or_b32 v2, v3, 16, v2
-; GFX9-NEXT: v_mov_b32_e32 v3, s29
-; GFX9-NEXT: v_perm_b32 v3, s28, v3, v4
-; GFX9-NEXT: v_mov_b32_e32 v5, s6
; GFX9-NEXT: s_and_b32 s4, s14, 0xff
; GFX9-NEXT: s_lshl_b32 s5, s12, 8
-; GFX9-NEXT: v_perm_b32 v5, s8, v5, v4
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v1, s21
+; GFX9-NEXT: v_perm_b32 v2, s22, v2, v4
; GFX9-NEXT: s_or_b32 s4, s4, s5
; GFX9-NEXT: s_and_b32 s5, s9, 0xff
; GFX9-NEXT: s_lshl_b32 s56, s7, 8
-; GFX9-NEXT: v_lshl_or_b32 v3, v5, 16, v3
+; GFX9-NEXT: v_perm_b32 v1, s20, v1, v4
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_mov_b32_e32 v3, s27
; GFX9-NEXT: s_or_b32 s5, s5, s56
-; GFX9-NEXT: v_mov_b32_e32 v5, s40
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v2, s25
+; GFX9-NEXT: v_perm_b32 v3, s26, v3, v4
; GFX9-NEXT: s_pack_ll_b32_b16 s4, s4, s5
-; GFX9-NEXT: v_perm_b32 v5, s42, v5, v4
; GFX9-NEXT: s_and_b32 s5, s11, 0xff
; GFX9-NEXT: s_lshl_b32 s56, s10, 8
+; GFX9-NEXT: v_perm_b32 v2, s24, v2, v4
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_mov_b32_e32 v5, s6
; GFX9-NEXT: s_or_b32 s5, s5, s56
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_lshl_or_b32 v5, s5, 16, v5
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v3, s29
+; GFX9-NEXT: v_perm_b32 v5, s8, v5, v4
+; GFX9-NEXT: v_lshlrev_b32_e64 v6, 16, s5
; GFX9-NEXT: s_and_b32 s5, s45, 0xff
; GFX9-NEXT: s_lshl_b32 s56, s44, 8
+; GFX9-NEXT: v_perm_b32 v3, s28, v3, v4
+; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX9-NEXT: s_or_b32 s5, s5, s56
; GFX9-NEXT: s_and_b32 s56, s15, 0xff
; GFX9-NEXT: s_lshl_b32 s57, s13, 8
+; GFX9-NEXT: v_or_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v5, s40
; GFX9-NEXT: s_or_b32 s56, s56, s57
-; GFX9-NEXT: v_mov_b32_e32 v6, s46
+; GFX9-NEXT: v_perm_b32 v5, s42, v5, v4
; GFX9-NEXT: s_pack_ll_b32_b16 s5, s5, s56
-; GFX9-NEXT: v_perm_b32 v4, s47, v6, v4
; GFX9-NEXT: s_and_b32 s56, s43, 0xff
; GFX9-NEXT: s_lshl_b32 s57, s41, 8
+; GFX9-NEXT: v_or_b32_sdwa v5, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v6, s46
; GFX9-NEXT: s_or_b32 s56, s56, s57
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_lshl_or_b32 v7, s56, 16, v4
+; GFX9-NEXT: v_perm_b32 v4, s47, v6, v4
+; GFX9-NEXT: v_lshlrev_b32_e64 v6, 16, s56
+; GFX9-NEXT: v_or_b32_sdwa v7, v6, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v4, s4
; GFX9-NEXT: v_mov_b32_e32 v6, s5
; GFX9-NEXT: s_mov_b64 s[4:5], 0
@@ -45015,53 +44996,46 @@ define inreg <16 x bfloat> @bitcast_v32i8_to_v16bf16_scalar(<32 x i8> inreg %a,
; GFX9-NEXT: v_add_u32_e32 v1, 0x300, v0
; GFX9-NEXT: s_add_i32 s22, s22, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s23
+; GFX9-NEXT: s_movk_i32 s4, 0x300
; GFX9-NEXT: v_perm_b32 v0, s22, v0, v2
-; GFX9-NEXT: v_add_u32_e32 v12, 0x300, v0
+; GFX9-NEXT: v_add_u32_sdwa v12, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s16, s16, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s17
-; GFX9-NEXT: v_perm_b32 v0, s16, v0, v2
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: v_mov_b32_e32 v13, s19
-; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
+; GFX9-NEXT: v_perm_b32 v0, s16, v0, v2
; GFX9-NEXT: v_perm_b32 v13, s18, v13, v2
-; GFX9-NEXT: v_add_u32_e32 v13, 0x300, v13
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v0, v13, 16, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: s_add_i32 s24, s24, 3
-; GFX9-NEXT: v_mov_b32_e32 v13, s25
-; GFX9-NEXT: v_add_u32_e32 v16, 0x300, v5
-; GFX9-NEXT: v_add_u32_e32 v5, 0x300, v7
+; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
+; GFX9-NEXT: v_add_u32_sdwa v13, v13, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s8, s8, 3
-; GFX9-NEXT: v_lshl_or_b32 v1, v12, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v0, v13, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v12, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v12, s6
-; GFX9-NEXT: v_perm_b32 v13, s24, v13, v2
+; GFX9-NEXT: s_add_i32 s24, s24, 3
+; GFX9-NEXT: v_mov_b32_e32 v13, s25
; GFX9-NEXT: s_add_i32 s26, s26, 3
; GFX9-NEXT: v_mov_b32_e32 v14, s27
-; GFX9-NEXT: v_add_u32_e32 v7, 0x300, v8
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
; GFX9-NEXT: v_perm_b32 v12, s8, v12, v2
+; GFX9-NEXT: v_perm_b32 v13, s24, v13, v2
; GFX9-NEXT: v_perm_b32 v2, s26, v14, v2
; GFX9-NEXT: v_add_u32_e32 v14, 0x300, v3
-; GFX9-NEXT: v_add_u32_e32 v15, 0x300, v4
-; GFX9-NEXT: v_add_u32_e32 v6, 0x300, v6
+; GFX9-NEXT: v_add_u32_sdwa v15, v4, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_add_u32_e32 v16, 0x300, v5
+; GFX9-NEXT: v_add_u32_sdwa v6, v6, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_add_u32_e32 v5, 0x300, v7
+; GFX9-NEXT: v_add_u32_sdwa v7, v8, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_add_u32_e32 v4, 0x300, v9
-; GFX9-NEXT: v_add_u32_e32 v8, 0x300, v10
+; GFX9-NEXT: v_add_u32_sdwa v8, v10, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_add_u32_e32 v3, 0x300, v11
+; GFX9-NEXT: v_add_u32_sdwa v9, v12, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_add_u32_e32 v10, 0x300, v13
-; GFX9-NEXT: v_lshl_or_b32 v5, v7, 16, v5
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v16
-; GFX9-NEXT: v_add_u32_e32 v9, 0x300, v12
-; GFX9-NEXT: v_add_u32_e32 v2, 0x300, v2
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_lshl_or_b32 v6, v6, 16, v7
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v14
-; GFX9-NEXT: v_lshl_or_b32 v2, v2, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v3, v9, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v8, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v7, v15, 16, v7
+; GFX9-NEXT: v_add_u32_sdwa v2, v2, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v2, v2, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v9, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v4, v8, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v7, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v6, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v15, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: .LBB111_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.320bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.320bit.ll
index ba58cfc5ea143..c0fcebcf00814 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.320bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.320bit.ll
@@ -19490,69 +19490,69 @@ define inreg <20 x i16> @bitcast_v40i8_to_v20i16_scalar(<40 x i8> inreg %a, i32
; GFX9-NEXT: v_readfirstlane_b32 s42, v0
; GFX9-NEXT: s_cbranch_scc0 .LBB51_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v8, 0xc0c0004
-; GFX9-NEXT: v_perm_b32 v0, s16, v0, v8
; GFX9-NEXT: v_mov_b32_e32 v1, s19
+; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_perm_b32 v1, s18, v1, v8
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX9-NEXT: v_mov_b32_e32 v1, s21
-; GFX9-NEXT: v_perm_b32 v1, s20, v1, v8
+; GFX9-NEXT: v_perm_b32 v0, s16, v0, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_mov_b32_e32 v2, s23
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v1, s21
; GFX9-NEXT: v_perm_b32 v2, s22, v2, v8
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, s25
-; GFX9-NEXT: v_perm_b32 v2, s24, v2, v8
+; GFX9-NEXT: v_perm_b32 v1, s20, v1, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX9-NEXT: v_mov_b32_e32 v3, s27
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v2, s25
; GFX9-NEXT: v_perm_b32 v3, s26, v3, v8
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_lshl_or_b32 v2, v3, 16, v2
-; GFX9-NEXT: v_mov_b32_e32 v3, s29
-; GFX9-NEXT: v_perm_b32 v3, s28, v3, v8
+; GFX9-NEXT: v_perm_b32 v2, s24, v2, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_mov_b32_e32 v4, s41
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v3, s29
; GFX9-NEXT: v_perm_b32 v4, s42, v4, v8
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_lshl_or_b32 v3, v4, 16, v3
-; GFX9-NEXT: v_mov_b32_e32 v4, s40
-; GFX9-NEXT: v_perm_b32 v4, s43, v4, v8
+; GFX9-NEXT: v_perm_b32 v3, s28, v3, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX9-NEXT: v_mov_b32_e32 v5, s14
+; GFX9-NEXT: v_or_b32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v4, s40
; GFX9-NEXT: v_perm_b32 v5, s15, v5, v8
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_lshl_or_b32 v4, v5, 16, v4
-; GFX9-NEXT: v_mov_b32_e32 v5, s13
-; GFX9-NEXT: v_perm_b32 v5, s44, v5, v8
+; GFX9-NEXT: v_perm_b32 v4, s43, v4, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX9-NEXT: v_mov_b32_e32 v6, s11
+; GFX9-NEXT: v_or_b32_sdwa v4, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v5, s13
; GFX9-NEXT: v_perm_b32 v6, s12, v6, v8
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_lshl_or_b32 v5, v6, 16, v5
-; GFX9-NEXT: v_mov_b32_e32 v6, s9
-; GFX9-NEXT: v_perm_b32 v6, s10, v6, v8
+; GFX9-NEXT: v_perm_b32 v5, s44, v5, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v6
; GFX9-NEXT: v_mov_b32_e32 v7, s6
+; GFX9-NEXT: v_or_b32_sdwa v5, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v6, s9
; GFX9-NEXT: v_perm_b32 v7, s8, v7, v8
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_lshl_or_b32 v6, v7, 16, v6
-; GFX9-NEXT: v_mov_b32_e32 v7, s56
-; GFX9-NEXT: v_perm_b32 v7, s58, v7, v8
-; GFX9-NEXT: v_mov_b32_e32 v9, s7
; GFX9-NEXT: s_and_b32 s4, s61, 0xff
; GFX9-NEXT: s_lshl_b32 s5, s60, 8
-; GFX9-NEXT: v_perm_b32 v9, s45, v9, v8
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX9-NEXT: v_perm_b32 v6, s10, v6, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX9-NEXT: v_mov_b32_e32 v9, s7
; GFX9-NEXT: s_or_b32 s4, s4, s5
; GFX9-NEXT: s_and_b32 s5, s47, 0xff
; GFX9-NEXT: s_lshl_b32 s72, s46, 8
-; GFX9-NEXT: v_lshl_or_b32 v7, v9, 16, v7
+; GFX9-NEXT: v_or_b32_sdwa v6, v7, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v7, s56
+; GFX9-NEXT: v_perm_b32 v9, s45, v9, v8
; GFX9-NEXT: s_or_b32 s5, s5, s72
-; GFX9-NEXT: v_mov_b32_e32 v9, s62
+; GFX9-NEXT: v_perm_b32 v7, s58, v7, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v9
; GFX9-NEXT: s_pack_ll_b32_b16 s4, s4, s5
-; GFX9-NEXT: v_perm_b32 v8, s63, v9, v8
; GFX9-NEXT: s_and_b32 s5, s59, 0xff
; GFX9-NEXT: s_lshl_b32 s72, s57, 8
+; GFX9-NEXT: v_or_b32_sdwa v7, v9, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v9, s62
; GFX9-NEXT: s_or_b32 s5, s5, s72
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, s5, 16, v8
+; GFX9-NEXT: v_perm_b32 v8, s63, v9, v8
+; GFX9-NEXT: v_lshlrev_b32_e64 v9, 16, s5
+; GFX9-NEXT: v_or_b32_sdwa v9, v9, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v8, s4
; GFX9-NEXT: s_mov_b64 s[4:5], 0
; GFX9-NEXT: s_branch .LBB51_3
@@ -19587,85 +19587,76 @@ define inreg <20 x i16> @bitcast_v40i8_to_v20i16_scalar(<40 x i8> inreg %a, i32
; GFX9-NEXT: v_add_u32_e32 v1, 0x300, v0
; GFX9-NEXT: s_add_i32 s22, s22, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s23
+; GFX9-NEXT: s_movk_i32 s4, 0x300
; GFX9-NEXT: v_perm_b32 v0, s22, v0, v6
-; GFX9-NEXT: v_add_u32_e32 v2, 0x300, v0
+; GFX9-NEXT: v_add_u32_sdwa v2, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s16, s16, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s17
-; GFX9-NEXT: v_perm_b32 v0, s16, v0, v6
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: v_mov_b32_e32 v3, s19
-; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
+; GFX9-NEXT: v_perm_b32 v0, s16, v0, v6
; GFX9-NEXT: v_perm_b32 v3, s18, v3, v6
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
+; GFX9-NEXT: v_add_u32_sdwa v3, v3, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s24, s24, 3
-; GFX9-NEXT: v_add_u32_e32 v3, 0x300, v3
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v2, s25
-; GFX9-NEXT: v_lshl_or_b32 v0, v3, 16, v0
-; GFX9-NEXT: v_perm_b32 v2, s24, v2, v6
; GFX9-NEXT: s_add_i32 s26, s26, 3
; GFX9-NEXT: v_mov_b32_e32 v3, s27
-; GFX9-NEXT: v_add_u32_e32 v2, 0x300, v2
+; GFX9-NEXT: v_perm_b32 v2, s24, v2, v6
; GFX9-NEXT: v_perm_b32 v3, s26, v3, v6
-; GFX9-NEXT: v_add_u32_e32 v3, 0x300, v3
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX9-NEXT: v_add_u32_e32 v2, 0x300, v2
+; GFX9-NEXT: v_add_u32_sdwa v3, v3, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s28, s28, 3
-; GFX9-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v3, s29
-; GFX9-NEXT: v_perm_b32 v3, s28, v3, v6
; GFX9-NEXT: s_add_i32 s42, s42, 3
; GFX9-NEXT: v_mov_b32_e32 v4, s41
-; GFX9-NEXT: v_add_u32_e32 v3, 0x300, v3
+; GFX9-NEXT: v_perm_b32 v3, s28, v3, v6
; GFX9-NEXT: v_perm_b32 v4, s42, v4, v6
-; GFX9-NEXT: v_add_u32_e32 v4, 0x300, v4
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX9-NEXT: v_add_u32_e32 v3, 0x300, v3
+; GFX9-NEXT: v_add_u32_sdwa v4, v4, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s43, s43, 3
-; GFX9-NEXT: v_lshl_or_b32 v3, v4, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v4, s40
-; GFX9-NEXT: v_perm_b32 v4, s43, v4, v6
; GFX9-NEXT: s_add_i32 s15, s15, 3
; GFX9-NEXT: v_mov_b32_e32 v5, s14
-; GFX9-NEXT: v_add_u32_e32 v4, 0x300, v4
+; GFX9-NEXT: v_perm_b32 v4, s43, v4, v6
; GFX9-NEXT: v_perm_b32 v5, s15, v5, v6
-; GFX9-NEXT: v_add_u32_e32 v5, 0x300, v5
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX9-NEXT: v_add_u32_e32 v4, 0x300, v4
+; GFX9-NEXT: v_add_u32_sdwa v5, v5, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s44, s44, 3
-; GFX9-NEXT: v_lshl_or_b32 v4, v5, 16, v4
+; GFX9-NEXT: v_or_b32_sdwa v4, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v5, s13
-; GFX9-NEXT: v_perm_b32 v5, s44, v5, v6
; GFX9-NEXT: s_add_i32 s12, s12, 3
; GFX9-NEXT: v_mov_b32_e32 v12, s11
-; GFX9-NEXT: v_add_u32_e32 v5, 0x300, v5
+; GFX9-NEXT: v_perm_b32 v5, s44, v5, v6
; GFX9-NEXT: v_perm_b32 v12, s12, v12, v6
-; GFX9-NEXT: v_add_u32_e32 v12, 0x300, v12
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: s_add_i32 s10, s10, 3
-; GFX9-NEXT: v_mov_b32_e32 v13, s9
+; GFX9-NEXT: v_add_u32_e32 v5, 0x300, v5
+; GFX9-NEXT: v_add_u32_sdwa v12, v12, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s45, s45, 3
-; GFX9-NEXT: v_lshl_or_b32 v5, v12, 16, v5
+; GFX9-NEXT: v_or_b32_sdwa v5, v12, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v12, s7
-; GFX9-NEXT: v_perm_b32 v13, s10, v13, v6
+; GFX9-NEXT: s_add_i32 s10, s10, 3
+; GFX9-NEXT: v_mov_b32_e32 v13, s9
; GFX9-NEXT: s_add_i32 s8, s8, 3
; GFX9-NEXT: v_mov_b32_e32 v14, s6
-; GFX9-NEXT: v_add_u32_e32 v15, 0x300, v8
-; GFX9-NEXT: v_add_u32_e32 v8, 0x300, v9
; GFX9-NEXT: v_perm_b32 v12, s45, v12, v6
+; GFX9-NEXT: v_perm_b32 v13, s10, v13, v6
; GFX9-NEXT: v_perm_b32 v6, s8, v14, v6
; GFX9-NEXT: v_add_u32_e32 v14, 0x300, v7
-; GFX9-NEXT: v_add_u32_e32 v9, 0x300, v10
+; GFX9-NEXT: v_add_u32_sdwa v15, v8, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_add_u32_e32 v8, 0x300, v9
+; GFX9-NEXT: v_add_u32_sdwa v9, v10, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_add_u32_e32 v7, 0x300, v11
+; GFX9-NEXT: v_add_u32_sdwa v10, v12, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_add_u32_e32 v11, 0x300, v13
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_add_u32_e32 v10, 0x300, v12
-; GFX9-NEXT: v_add_u32_e32 v6, 0x300, v6
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v9, 16, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v14
-; GFX9-NEXT: v_lshl_or_b32 v6, v6, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v7, v10, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v9, v15, 16, v9
+; GFX9-NEXT: v_add_u32_sdwa v6, v6, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v6, v6, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v10, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v9, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v15, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: .LBB51_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -25950,69 +25941,69 @@ define inreg <20 x half> @bitcast_v40i8_to_v20f16_scalar(<40 x i8> inreg %a, i32
; GFX9-NEXT: v_readfirstlane_b32 s42, v0
; GFX9-NEXT: s_cbranch_scc0 .LBB63_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v8, 0xc0c0004
-; GFX9-NEXT: v_perm_b32 v0, s16, v0, v8
; GFX9-NEXT: v_mov_b32_e32 v1, s19
+; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_perm_b32 v1, s18, v1, v8
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX9-NEXT: v_mov_b32_e32 v1, s21
-; GFX9-NEXT: v_perm_b32 v1, s20, v1, v8
+; GFX9-NEXT: v_perm_b32 v0, s16, v0, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_mov_b32_e32 v2, s23
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v1, s21
; GFX9-NEXT: v_perm_b32 v2, s22, v2, v8
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, s25
-; GFX9-NEXT: v_perm_b32 v2, s24, v2, v8
+; GFX9-NEXT: v_perm_b32 v1, s20, v1, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX9-NEXT: v_mov_b32_e32 v3, s27
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v2, s25
; GFX9-NEXT: v_perm_b32 v3, s26, v3, v8
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_lshl_or_b32 v2, v3, 16, v2
-; GFX9-NEXT: v_mov_b32_e32 v3, s29
-; GFX9-NEXT: v_perm_b32 v3, s28, v3, v8
+; GFX9-NEXT: v_perm_b32 v2, s24, v2, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_mov_b32_e32 v4, s41
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v3, s29
; GFX9-NEXT: v_perm_b32 v4, s42, v4, v8
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_lshl_or_b32 v3, v4, 16, v3
-; GFX9-NEXT: v_mov_b32_e32 v4, s40
-; GFX9-NEXT: v_perm_b32 v4, s43, v4, v8
+; GFX9-NEXT: v_perm_b32 v3, s28, v3, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX9-NEXT: v_mov_b32_e32 v5, s14
+; GFX9-NEXT: v_or_b32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v4, s40
; GFX9-NEXT: v_perm_b32 v5, s15, v5, v8
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_lshl_or_b32 v4, v5, 16, v4
-; GFX9-NEXT: v_mov_b32_e32 v5, s13
-; GFX9-NEXT: v_perm_b32 v5, s44, v5, v8
+; GFX9-NEXT: v_perm_b32 v4, s43, v4, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX9-NEXT: v_mov_b32_e32 v6, s11
+; GFX9-NEXT: v_or_b32_sdwa v4, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v5, s13
; GFX9-NEXT: v_perm_b32 v6, s12, v6, v8
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_lshl_or_b32 v5, v6, 16, v5
-; GFX9-NEXT: v_mov_b32_e32 v6, s9
-; GFX9-NEXT: v_perm_b32 v6, s10, v6, v8
+; GFX9-NEXT: v_perm_b32 v5, s44, v5, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v6
; GFX9-NEXT: v_mov_b32_e32 v7, s6
+; GFX9-NEXT: v_or_b32_sdwa v5, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v6, s9
; GFX9-NEXT: v_perm_b32 v7, s8, v7, v8
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_lshl_or_b32 v6, v7, 16, v6
-; GFX9-NEXT: v_mov_b32_e32 v7, s56
-; GFX9-NEXT: v_perm_b32 v7, s58, v7, v8
-; GFX9-NEXT: v_mov_b32_e32 v9, s7
; GFX9-NEXT: s_and_b32 s4, s61, 0xff
; GFX9-NEXT: s_lshl_b32 s5, s60, 8
-; GFX9-NEXT: v_perm_b32 v9, s45, v9, v8
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX9-NEXT: v_perm_b32 v6, s10, v6, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX9-NEXT: v_mov_b32_e32 v9, s7
; GFX9-NEXT: s_or_b32 s4, s4, s5
; GFX9-NEXT: s_and_b32 s5, s47, 0xff
; GFX9-NEXT: s_lshl_b32 s72, s46, 8
-; GFX9-NEXT: v_lshl_or_b32 v7, v9, 16, v7
+; GFX9-NEXT: v_or_b32_sdwa v6, v7, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v7, s56
+; GFX9-NEXT: v_perm_b32 v9, s45, v9, v8
; GFX9-NEXT: s_or_b32 s5, s5, s72
-; GFX9-NEXT: v_mov_b32_e32 v9, s62
+; GFX9-NEXT: v_perm_b32 v7, s58, v7, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v9
; GFX9-NEXT: s_pack_ll_b32_b16 s4, s4, s5
-; GFX9-NEXT: v_perm_b32 v8, s63, v9, v8
; GFX9-NEXT: s_and_b32 s5, s59, 0xff
; GFX9-NEXT: s_lshl_b32 s72, s57, 8
+; GFX9-NEXT: v_or_b32_sdwa v7, v9, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v9, s62
; GFX9-NEXT: s_or_b32 s5, s5, s72
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, s5, 16, v8
+; GFX9-NEXT: v_perm_b32 v8, s63, v9, v8
+; GFX9-NEXT: v_lshlrev_b32_e64 v9, 16, s5
+; GFX9-NEXT: v_or_b32_sdwa v9, v9, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v8, s4
; GFX9-NEXT: s_mov_b64 s[4:5], 0
; GFX9-NEXT: s_branch .LBB63_3
@@ -26047,85 +26038,76 @@ define inreg <20 x half> @bitcast_v40i8_to_v20f16_scalar(<40 x i8> inreg %a, i32
; GFX9-NEXT: v_add_u32_e32 v1, 0x300, v0
; GFX9-NEXT: s_add_i32 s22, s22, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s23
+; GFX9-NEXT: s_movk_i32 s4, 0x300
; GFX9-NEXT: v_perm_b32 v0, s22, v0, v6
-; GFX9-NEXT: v_add_u32_e32 v2, 0x300, v0
+; GFX9-NEXT: v_add_u32_sdwa v2, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s16, s16, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s17
-; GFX9-NEXT: v_perm_b32 v0, s16, v0, v6
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: v_mov_b32_e32 v3, s19
-; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
+; GFX9-NEXT: v_perm_b32 v0, s16, v0, v6
; GFX9-NEXT: v_perm_b32 v3, s18, v3, v6
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
+; GFX9-NEXT: v_add_u32_sdwa v3, v3, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s24, s24, 3
-; GFX9-NEXT: v_add_u32_e32 v3, 0x300, v3
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v2, s25
-; GFX9-NEXT: v_lshl_or_b32 v0, v3, 16, v0
-; GFX9-NEXT: v_perm_b32 v2, s24, v2, v6
; GFX9-NEXT: s_add_i32 s26, s26, 3
; GFX9-NEXT: v_mov_b32_e32 v3, s27
-; GFX9-NEXT: v_add_u32_e32 v2, 0x300, v2
+; GFX9-NEXT: v_perm_b32 v2, s24, v2, v6
; GFX9-NEXT: v_perm_b32 v3, s26, v3, v6
-; GFX9-NEXT: v_add_u32_e32 v3, 0x300, v3
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX9-NEXT: v_add_u32_e32 v2, 0x300, v2
+; GFX9-NEXT: v_add_u32_sdwa v3, v3, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s28, s28, 3
-; GFX9-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v3, s29
-; GFX9-NEXT: v_perm_b32 v3, s28, v3, v6
; GFX9-NEXT: s_add_i32 s42, s42, 3
; GFX9-NEXT: v_mov_b32_e32 v4, s41
-; GFX9-NEXT: v_add_u32_e32 v3, 0x300, v3
+; GFX9-NEXT: v_perm_b32 v3, s28, v3, v6
; GFX9-NEXT: v_perm_b32 v4, s42, v4, v6
-; GFX9-NEXT: v_add_u32_e32 v4, 0x300, v4
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX9-NEXT: v_add_u32_e32 v3, 0x300, v3
+; GFX9-NEXT: v_add_u32_sdwa v4, v4, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s43, s43, 3
-; GFX9-NEXT: v_lshl_or_b32 v3, v4, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v4, s40
-; GFX9-NEXT: v_perm_b32 v4, s43, v4, v6
; GFX9-NEXT: s_add_i32 s15, s15, 3
; GFX9-NEXT: v_mov_b32_e32 v5, s14
-; GFX9-NEXT: v_add_u32_e32 v4, 0x300, v4
+; GFX9-NEXT: v_perm_b32 v4, s43, v4, v6
; GFX9-NEXT: v_perm_b32 v5, s15, v5, v6
-; GFX9-NEXT: v_add_u32_e32 v5, 0x300, v5
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX9-NEXT: v_add_u32_e32 v4, 0x300, v4
+; GFX9-NEXT: v_add_u32_sdwa v5, v5, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s44, s44, 3
-; GFX9-NEXT: v_lshl_or_b32 v4, v5, 16, v4
+; GFX9-NEXT: v_or_b32_sdwa v4, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v5, s13
-; GFX9-NEXT: v_perm_b32 v5, s44, v5, v6
; GFX9-NEXT: s_add_i32 s12, s12, 3
; GFX9-NEXT: v_mov_b32_e32 v12, s11
-; GFX9-NEXT: v_add_u32_e32 v5, 0x300, v5
+; GFX9-NEXT: v_perm_b32 v5, s44, v5, v6
; GFX9-NEXT: v_perm_b32 v12, s12, v12, v6
-; GFX9-NEXT: v_add_u32_e32 v12, 0x300, v12
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: s_add_i32 s10, s10, 3
-; GFX9-NEXT: v_mov_b32_e32 v13, s9
+; GFX9-NEXT: v_add_u32_e32 v5, 0x300, v5
+; GFX9-NEXT: v_add_u32_sdwa v12, v12, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s45, s45, 3
-; GFX9-NEXT: v_lshl_or_b32 v5, v12, 16, v5
+; GFX9-NEXT: v_or_b32_sdwa v5, v12, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v12, s7
-; GFX9-NEXT: v_perm_b32 v13, s10, v13, v6
+; GFX9-NEXT: s_add_i32 s10, s10, 3
+; GFX9-NEXT: v_mov_b32_e32 v13, s9
; GFX9-NEXT: s_add_i32 s8, s8, 3
; GFX9-NEXT: v_mov_b32_e32 v14, s6
-; GFX9-NEXT: v_add_u32_e32 v15, 0x300, v8
-; GFX9-NEXT: v_add_u32_e32 v8, 0x300, v9
; GFX9-NEXT: v_perm_b32 v12, s45, v12, v6
+; GFX9-NEXT: v_perm_b32 v13, s10, v13, v6
; GFX9-NEXT: v_perm_b32 v6, s8, v14, v6
; GFX9-NEXT: v_add_u32_e32 v14, 0x300, v7
-; GFX9-NEXT: v_add_u32_e32 v9, 0x300, v10
+; GFX9-NEXT: v_add_u32_sdwa v15, v8, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_add_u32_e32 v8, 0x300, v9
+; GFX9-NEXT: v_add_u32_sdwa v9, v10, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_add_u32_e32 v7, 0x300, v11
+; GFX9-NEXT: v_add_u32_sdwa v10, v12, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_add_u32_e32 v11, 0x300, v13
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_add_u32_e32 v10, 0x300, v12
-; GFX9-NEXT: v_add_u32_e32 v6, 0x300, v6
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v9, 16, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v14
-; GFX9-NEXT: v_lshl_or_b32 v6, v6, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v7, v10, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v9, v15, 16, v9
+; GFX9-NEXT: v_add_u32_sdwa v6, v6, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v6, v6, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v10, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v9, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v15, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: .LBB63_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.32bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.32bit.ll
index 50d16e783332a..38a3d781676ef 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.32bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.32bit.ll
@@ -1617,14 +1617,13 @@ define inreg i32 @bitcast_v2bf16_to_i32_scalar(<2 x bfloat> inreg %a, i32 inreg
; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_bfe_u32 v2, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v0
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
-; GFX9-NEXT: v_mov_b32_e32 v2, 0xffff
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_setpc_b64 s[30:31]
; GFX9-NEXT: .LBB15_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
@@ -4085,14 +4084,13 @@ define inreg float @bitcast_v2bf16_to_f32_scalar(<2 x bfloat> inreg %a, i32 inre
; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_bfe_u32 v2, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v0
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
-; GFX9-NEXT: v_mov_b32_e32 v2, 0xffff
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_setpc_b64 s[30:31]
; GFX9-NEXT: .LBB35_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
@@ -7946,14 +7944,13 @@ define inreg <2 x half> @bitcast_v2bf16_to_v2f16_scalar(<2 x bfloat> inreg %a, i
; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_bfe_u32 v2, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v0
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
-; GFX9-NEXT: v_mov_b32_e32 v2, 0xffff
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_setpc_b64 s[30:31]
; GFX9-NEXT: .LBB63_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
@@ -9508,14 +9505,13 @@ define inreg <1 x i32> @bitcast_v2bf16_to_v1i32_scalar(<2 x bfloat> inreg %a, i3
; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_bfe_u32 v2, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v0
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
-; GFX9-NEXT: v_mov_b32_e32 v2, 0xffff
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_setpc_b64 s[30:31]
; GFX9-NEXT: .LBB73_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
@@ -10203,8 +10199,8 @@ define inreg <4 x i8> @bitcast_v2bf16_to_v4i8_scalar(<2 x bfloat> inreg %a, i32
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_lshrrev_b32_e32 v3, 24, v1
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.48bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.48bit.ll
index f9c3283bffce6..ffff9da12fe9f 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.48bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.48bit.ll
@@ -375,17 +375,15 @@ define inreg <3 x half> @bitcast_v3bf16_to_v3f16_scalar(<3 x bfloat> inreg %a, i
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_bfe_u32 v3, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v0
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc
-; GFX9-NEXT: v_mov_b32_e32 v3, 0xffff
-; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX9-NEXT: v_and_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: s_movk_i32 s4, 0x7fc0
-; GFX9-NEXT: v_and_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v0, v2, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, s4, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_mov_b32_e32 v2, 0x7fc00000
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_setpc_b64 s[30:31]
; GFX9-NEXT: .LBB1_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
@@ -1143,10 +1141,8 @@ define inreg <3 x i16> @bitcast_v3bf16_to_v3i16_scalar(<3 x bfloat> inreg %a, i3
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_mov_b32_e32 v3, 0xffff0000
; GFX9-NEXT: v_and_or_b32 v0, v2, v3, v0
-; GFX9-NEXT: v_mov_b32_e32 v2, 0xffff
-; GFX9-NEXT: s_movk_i32 s4, 0x7fc0
-; GFX9-NEXT: v_and_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v1, s4, 16, v1
+; GFX9-NEXT: v_mov_b32_e32 v2, 0x7fc00000
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_setpc_b64 s[30:31]
; GFX9-NEXT: .LBB5_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll
index 78585c9150ddb..7a26f3497f492 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll
@@ -7664,10 +7664,10 @@ define inreg <16 x i32> @bitcast_v32bf16_to_v16i32_scalar(<32 x bfloat> inreg %a
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
-; GFX9-NEXT: buffer_store_dword v20, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v19, off, s[0:3], s32 ; 4-byte Folded Spill
; GFX9-NEXT: s_mov_b64 exec, s[4:5]
-; GFX9-NEXT: v_writelane_b32 v20, s30, 0
-; GFX9-NEXT: v_writelane_b32 v20, s31, 1
+; GFX9-NEXT: v_writelane_b32 v19, s30, 0
+; GFX9-NEXT: v_writelane_b32 v19, s31, 1
; GFX9-NEXT: v_readfirstlane_b32 s4, v2
; GFX9-NEXT: v_readfirstlane_b32 s31, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
@@ -7711,20 +7711,19 @@ define inreg <16 x i32> @bitcast_v32bf16_to_v16i32_scalar(<32 x bfloat> inreg %a
; GFX9-NEXT: s_lshl_b32 s4, s30, 16
; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
; GFX9-NEXT: v_add_f32_e32 v4, s4, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v5, v5, v4
-; GFX9-NEXT: v_mov_b32_e32 v16, 0xffff
; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v4
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc
-; GFX9-NEXT: v_lshl_or_b32 v15, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v15, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v3
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s29, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v14, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v14, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -7739,11 +7738,11 @@ define inreg <16 x i32> @bitcast_v32bf16_to_v16i32_scalar(<32 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s28, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v13, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v13, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -7758,11 +7757,11 @@ define inreg <16 x i32> @bitcast_v32bf16_to_v16i32_scalar(<32 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s27, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v12, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v12, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -7777,11 +7776,11 @@ define inreg <16 x i32> @bitcast_v32bf16_to_v16i32_scalar(<32 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s26, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v11, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v11, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -7796,11 +7795,11 @@ define inreg <16 x i32> @bitcast_v32bf16_to_v16i32_scalar(<32 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s25, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v10, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v10, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -7815,11 +7814,11 @@ define inreg <16 x i32> @bitcast_v32bf16_to_v16i32_scalar(<32 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s24, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v9, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v9, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -7834,11 +7833,11 @@ define inreg <16 x i32> @bitcast_v32bf16_to_v16i32_scalar(<32 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s23, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v8, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v8, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -7853,11 +7852,11 @@ define inreg <16 x i32> @bitcast_v32bf16_to_v16i32_scalar(<32 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s22, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v7, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v7, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -7872,11 +7871,11 @@ define inreg <16 x i32> @bitcast_v32bf16_to_v16i32_scalar(<32 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s21, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v6, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v6, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -7891,11 +7890,11 @@ define inreg <16 x i32> @bitcast_v32bf16_to_v16i32_scalar(<32 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s20, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v5, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -7910,11 +7909,11 @@ define inreg <16 x i32> @bitcast_v32bf16_to_v16i32_scalar(<32 x bfloat> inreg %a
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s19, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v4, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v4, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -7927,69 +7926,69 @@ define inreg <16 x i32> @bitcast_v32bf16_to_v16i32_scalar(<32 x bfloat> inreg %a
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
-; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v2
+; GFX9-NEXT: v_or_b32_e32 v16, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v17, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v16, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s18, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v3, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v3, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
-; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v1
+; GFX9-NEXT: v_or_b32_e32 v16, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: s_lshl_b32 s4, s18, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v17, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v16, vcc
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
-; GFX9-NEXT: v_bfe_u32 v17, v2, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v17, v17, v2
-; GFX9-NEXT: v_add_u32_e32 v17, 0x7fff, v17
-; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v2
+; GFX9-NEXT: v_bfe_u32 v16, v2, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v16, v16, v2
+; GFX9-NEXT: v_add_u32_e32 v16, 0x7fff, v16
+; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v17, v18, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v16, v17, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s17, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v2, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
-; GFX9-NEXT: v_bfe_u32 v17, v1, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v17, v17, v1
-; GFX9-NEXT: v_add_u32_e32 v17, 0x7fff, v17
-; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v1
+; GFX9-NEXT: v_bfe_u32 v16, v1, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v16, v16, v1
+; GFX9-NEXT: v_add_u32_e32 v16, 0x7fff, v16
+; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: s_lshl_b32 s4, s17, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v17, v18, vcc
-; GFX9-NEXT: v_add_f32_e32 v17, s4, v0
-; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
-; GFX9-NEXT: v_add_u32_e32 v18, 0x7fff, v18
-; GFX9-NEXT: v_or_b32_e32 v19, 0x400000, v17
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
-; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v19, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v16, v17, vcc
+; GFX9-NEXT: v_add_f32_e32 v16, s4, v0
+; GFX9-NEXT: v_bfe_u32 v17, v16, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v17, v17, v16
+; GFX9-NEXT: v_add_u32_e32 v17, 0x7fff, v17
+; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v16
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v16, v16
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v17, v16, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v16, v17, v18, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s16, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v17
-; GFX9-NEXT: v_add_f32_e32 v17, s4, v0
-; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f32_e32 v16, s4, v0
+; GFX9-NEXT: v_bfe_u32 v17, v16, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v17, v17, v16
; GFX9-NEXT: s_lshl_b32 s4, s16, 16
-; GFX9-NEXT: v_add_u32_e32 v18, 0x7fff, v18
-; GFX9-NEXT: v_or_b32_e32 v19, 0x400000, v17
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
+; GFX9-NEXT: v_add_u32_e32 v17, 0x7fff, v17
+; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v16
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v16, v16
; GFX9-NEXT: v_add_f32_e32 v0, s4, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v19, vcc
-; GFX9-NEXT: v_bfe_u32 v18, v0, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v18, v18, v0
-; GFX9-NEXT: v_add_u32_e32 v18, 0x7fff, v18
-; GFX9-NEXT: v_or_b32_e32 v19, 0x400000, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v16, v17, v18, vcc
+; GFX9-NEXT: v_bfe_u32 v17, v0, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v17, v17, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v16, 16, v16
+; GFX9-NEXT: v_add_u32_e32 v17, 0x7fff, v17
+; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v0
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v17
-; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v19, vcc
-; GFX9-NEXT: v_and_b32_sdwa v16, v16, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v16
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v17, v18, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v16
+; GFX9-NEXT: v_or_b32_sdwa v0, v16, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_branch .LBB23_6
; GFX9-NEXT: .LBB23_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
@@ -8009,10 +8008,10 @@ define inreg <16 x i32> @bitcast_v32bf16_to_v16i32_scalar(<32 x bfloat> inreg %a
; GFX9-NEXT: v_mov_b32_e32 v14, s30
; GFX9-NEXT: v_mov_b32_e32 v15, s31
; GFX9-NEXT: .LBB23_6: ; %end
-; GFX9-NEXT: v_readlane_b32 s30, v20, 0
-; GFX9-NEXT: v_readlane_b32 s31, v20, 1
+; GFX9-NEXT: v_readlane_b32 s30, v19, 0
+; GFX9-NEXT: v_readlane_b32 s31, v19, 1
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
-; GFX9-NEXT: buffer_load_dword v20, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v19, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-NEXT: s_mov_b64 exec, s[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -22637,10 +22636,10 @@ define inreg <16 x float> @bitcast_v32bf16_to_v16f32_scalar(<32 x bfloat> inreg
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
-; GFX9-NEXT: buffer_store_dword v20, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v19, off, s[0:3], s32 ; 4-byte Folded Spill
; GFX9-NEXT: s_mov_b64 exec, s[4:5]
-; GFX9-NEXT: v_writelane_b32 v20, s30, 0
-; GFX9-NEXT: v_writelane_b32 v20, s31, 1
+; GFX9-NEXT: v_writelane_b32 v19, s30, 0
+; GFX9-NEXT: v_writelane_b32 v19, s31, 1
; GFX9-NEXT: v_readfirstlane_b32 s4, v2
; GFX9-NEXT: v_readfirstlane_b32 s31, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
@@ -22684,20 +22683,19 @@ define inreg <16 x float> @bitcast_v32bf16_to_v16f32_scalar(<32 x bfloat> inreg
; GFX9-NEXT: s_lshl_b32 s4, s30, 16
; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
; GFX9-NEXT: v_add_f32_e32 v4, s4, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v5, v5, v4
-; GFX9-NEXT: v_mov_b32_e32 v16, 0xffff
; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v4
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc
-; GFX9-NEXT: v_lshl_or_b32 v15, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v15, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v3
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s29, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v14, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v14, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -22712,11 +22710,11 @@ define inreg <16 x float> @bitcast_v32bf16_to_v16f32_scalar(<32 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s28, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v13, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v13, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -22731,11 +22729,11 @@ define inreg <16 x float> @bitcast_v32bf16_to_v16f32_scalar(<32 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s27, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v12, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v12, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -22750,11 +22748,11 @@ define inreg <16 x float> @bitcast_v32bf16_to_v16f32_scalar(<32 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s26, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v11, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v11, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -22769,11 +22767,11 @@ define inreg <16 x float> @bitcast_v32bf16_to_v16f32_scalar(<32 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s25, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v10, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v10, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -22788,11 +22786,11 @@ define inreg <16 x float> @bitcast_v32bf16_to_v16f32_scalar(<32 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s24, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v9, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v9, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -22807,11 +22805,11 @@ define inreg <16 x float> @bitcast_v32bf16_to_v16f32_scalar(<32 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s23, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v8, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v8, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -22826,11 +22824,11 @@ define inreg <16 x float> @bitcast_v32bf16_to_v16f32_scalar(<32 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s22, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v7, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v7, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -22845,11 +22843,11 @@ define inreg <16 x float> @bitcast_v32bf16_to_v16f32_scalar(<32 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s21, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v6, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v6, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -22864,11 +22862,11 @@ define inreg <16 x float> @bitcast_v32bf16_to_v16f32_scalar(<32 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s20, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v5, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -22883,11 +22881,11 @@ define inreg <16 x float> @bitcast_v32bf16_to_v16f32_scalar(<32 x bfloat> inreg
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s19, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v4, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v4, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -22900,69 +22898,69 @@ define inreg <16 x float> @bitcast_v32bf16_to_v16f32_scalar(<32 x bfloat> inreg
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
-; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v2
+; GFX9-NEXT: v_or_b32_e32 v16, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v17, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v16, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s18, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v3, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v3, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
-; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v1
+; GFX9-NEXT: v_or_b32_e32 v16, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: s_lshl_b32 s4, s18, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v17, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v16, vcc
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
-; GFX9-NEXT: v_bfe_u32 v17, v2, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v17, v17, v2
-; GFX9-NEXT: v_add_u32_e32 v17, 0x7fff, v17
-; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v2
+; GFX9-NEXT: v_bfe_u32 v16, v2, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v16, v16, v2
+; GFX9-NEXT: v_add_u32_e32 v16, 0x7fff, v16
+; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v17, v18, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v16, v17, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s17, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v2, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
-; GFX9-NEXT: v_bfe_u32 v17, v1, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v17, v17, v1
-; GFX9-NEXT: v_add_u32_e32 v17, 0x7fff, v17
-; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v1
+; GFX9-NEXT: v_bfe_u32 v16, v1, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v16, v16, v1
+; GFX9-NEXT: v_add_u32_e32 v16, 0x7fff, v16
+; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: s_lshl_b32 s4, s17, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v17, v18, vcc
-; GFX9-NEXT: v_add_f32_e32 v17, s4, v0
-; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
-; GFX9-NEXT: v_add_u32_e32 v18, 0x7fff, v18
-; GFX9-NEXT: v_or_b32_e32 v19, 0x400000, v17
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
-; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v19, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v16, v17, vcc
+; GFX9-NEXT: v_add_f32_e32 v16, s4, v0
+; GFX9-NEXT: v_bfe_u32 v17, v16, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v17, v17, v16
+; GFX9-NEXT: v_add_u32_e32 v17, 0x7fff, v17
+; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v16
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v16, v16
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v17, v16, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v16, v17, v18, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s16, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v17
-; GFX9-NEXT: v_add_f32_e32 v17, s4, v0
-; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f32_e32 v16, s4, v0
+; GFX9-NEXT: v_bfe_u32 v17, v16, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v17, v17, v16
; GFX9-NEXT: s_lshl_b32 s4, s16, 16
-; GFX9-NEXT: v_add_u32_e32 v18, 0x7fff, v18
-; GFX9-NEXT: v_or_b32_e32 v19, 0x400000, v17
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
+; GFX9-NEXT: v_add_u32_e32 v17, 0x7fff, v17
+; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v16
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v16, v16
; GFX9-NEXT: v_add_f32_e32 v0, s4, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v19, vcc
-; GFX9-NEXT: v_bfe_u32 v18, v0, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v18, v18, v0
-; GFX9-NEXT: v_add_u32_e32 v18, 0x7fff, v18
-; GFX9-NEXT: v_or_b32_e32 v19, 0x400000, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v16, v17, v18, vcc
+; GFX9-NEXT: v_bfe_u32 v17, v0, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v17, v17, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v16, 16, v16
+; GFX9-NEXT: v_add_u32_e32 v17, 0x7fff, v17
+; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v0
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v17
-; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v19, vcc
-; GFX9-NEXT: v_and_b32_sdwa v16, v16, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v16
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v17, v18, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v16
+; GFX9-NEXT: v_or_b32_sdwa v0, v16, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_branch .LBB47_6
; GFX9-NEXT: .LBB47_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
@@ -22982,10 +22980,10 @@ define inreg <16 x float> @bitcast_v32bf16_to_v16f32_scalar(<32 x bfloat> inreg
; GFX9-NEXT: v_mov_b32_e32 v14, s30
; GFX9-NEXT: v_mov_b32_e32 v15, s31
; GFX9-NEXT: .LBB47_6: ; %end
-; GFX9-NEXT: v_readlane_b32 s30, v20, 0
-; GFX9-NEXT: v_readlane_b32 s31, v20, 1
+; GFX9-NEXT: v_readlane_b32 s30, v19, 0
+; GFX9-NEXT: v_readlane_b32 s31, v19, 1
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
-; GFX9-NEXT: buffer_load_dword v20, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v19, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-NEXT: s_mov_b64 exec, s[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -36924,10 +36922,10 @@ define inreg <8 x i64> @bitcast_v32bf16_to_v8i64_scalar(<32 x bfloat> inreg %a,
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
-; GFX9-NEXT: buffer_store_dword v20, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v19, off, s[0:3], s32 ; 4-byte Folded Spill
; GFX9-NEXT: s_mov_b64 exec, s[4:5]
-; GFX9-NEXT: v_writelane_b32 v20, s30, 0
-; GFX9-NEXT: v_writelane_b32 v20, s31, 1
+; GFX9-NEXT: v_writelane_b32 v19, s30, 0
+; GFX9-NEXT: v_writelane_b32 v19, s31, 1
; GFX9-NEXT: v_readfirstlane_b32 s4, v2
; GFX9-NEXT: v_readfirstlane_b32 s31, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
@@ -36971,20 +36969,19 @@ define inreg <8 x i64> @bitcast_v32bf16_to_v8i64_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: s_lshl_b32 s4, s30, 16
; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
; GFX9-NEXT: v_add_f32_e32 v4, s4, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v5, v5, v4
-; GFX9-NEXT: v_mov_b32_e32 v16, 0xffff
; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v4
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc
-; GFX9-NEXT: v_lshl_or_b32 v15, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v15, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v3
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s29, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v14, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v14, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -36999,11 +36996,11 @@ define inreg <8 x i64> @bitcast_v32bf16_to_v8i64_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s28, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v13, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v13, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -37018,11 +37015,11 @@ define inreg <8 x i64> @bitcast_v32bf16_to_v8i64_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s27, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v12, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v12, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -37037,11 +37034,11 @@ define inreg <8 x i64> @bitcast_v32bf16_to_v8i64_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s26, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v11, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v11, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -37056,11 +37053,11 @@ define inreg <8 x i64> @bitcast_v32bf16_to_v8i64_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s25, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v10, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v10, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -37075,11 +37072,11 @@ define inreg <8 x i64> @bitcast_v32bf16_to_v8i64_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s24, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v9, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v9, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -37094,11 +37091,11 @@ define inreg <8 x i64> @bitcast_v32bf16_to_v8i64_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s23, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v8, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v8, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -37113,11 +37110,11 @@ define inreg <8 x i64> @bitcast_v32bf16_to_v8i64_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s22, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v7, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v7, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -37132,11 +37129,11 @@ define inreg <8 x i64> @bitcast_v32bf16_to_v8i64_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s21, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v6, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v6, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -37151,11 +37148,11 @@ define inreg <8 x i64> @bitcast_v32bf16_to_v8i64_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s20, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v5, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -37170,11 +37167,11 @@ define inreg <8 x i64> @bitcast_v32bf16_to_v8i64_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s19, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v4, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v4, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -37187,69 +37184,69 @@ define inreg <8 x i64> @bitcast_v32bf16_to_v8i64_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
-; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v2
+; GFX9-NEXT: v_or_b32_e32 v16, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v17, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v16, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s18, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v3, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v3, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
-; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v1
+; GFX9-NEXT: v_or_b32_e32 v16, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: s_lshl_b32 s4, s18, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v17, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v16, vcc
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
-; GFX9-NEXT: v_bfe_u32 v17, v2, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v17, v17, v2
-; GFX9-NEXT: v_add_u32_e32 v17, 0x7fff, v17
-; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v2
+; GFX9-NEXT: v_bfe_u32 v16, v2, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v16, v16, v2
+; GFX9-NEXT: v_add_u32_e32 v16, 0x7fff, v16
+; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v17, v18, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v16, v17, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s17, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v2, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
-; GFX9-NEXT: v_bfe_u32 v17, v1, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v17, v17, v1
-; GFX9-NEXT: v_add_u32_e32 v17, 0x7fff, v17
-; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v1
+; GFX9-NEXT: v_bfe_u32 v16, v1, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v16, v16, v1
+; GFX9-NEXT: v_add_u32_e32 v16, 0x7fff, v16
+; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: s_lshl_b32 s4, s17, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v17, v18, vcc
-; GFX9-NEXT: v_add_f32_e32 v17, s4, v0
-; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
-; GFX9-NEXT: v_add_u32_e32 v18, 0x7fff, v18
-; GFX9-NEXT: v_or_b32_e32 v19, 0x400000, v17
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
-; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v19, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v16, v17, vcc
+; GFX9-NEXT: v_add_f32_e32 v16, s4, v0
+; GFX9-NEXT: v_bfe_u32 v17, v16, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v17, v17, v16
+; GFX9-NEXT: v_add_u32_e32 v17, 0x7fff, v17
+; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v16
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v16, v16
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v17, v16, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v16, v17, v18, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s16, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v17
-; GFX9-NEXT: v_add_f32_e32 v17, s4, v0
-; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f32_e32 v16, s4, v0
+; GFX9-NEXT: v_bfe_u32 v17, v16, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v17, v17, v16
; GFX9-NEXT: s_lshl_b32 s4, s16, 16
-; GFX9-NEXT: v_add_u32_e32 v18, 0x7fff, v18
-; GFX9-NEXT: v_or_b32_e32 v19, 0x400000, v17
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
+; GFX9-NEXT: v_add_u32_e32 v17, 0x7fff, v17
+; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v16
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v16, v16
; GFX9-NEXT: v_add_f32_e32 v0, s4, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v19, vcc
-; GFX9-NEXT: v_bfe_u32 v18, v0, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v18, v18, v0
-; GFX9-NEXT: v_add_u32_e32 v18, 0x7fff, v18
-; GFX9-NEXT: v_or_b32_e32 v19, 0x400000, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v16, v17, v18, vcc
+; GFX9-NEXT: v_bfe_u32 v17, v0, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v17, v17, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v16, 16, v16
+; GFX9-NEXT: v_add_u32_e32 v17, 0x7fff, v17
+; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v0
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v17
-; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v19, vcc
-; GFX9-NEXT: v_and_b32_sdwa v16, v16, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v16
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v17, v18, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v16
+; GFX9-NEXT: v_or_b32_sdwa v0, v16, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_branch .LBB67_6
; GFX9-NEXT: .LBB67_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
@@ -37269,10 +37266,10 @@ define inreg <8 x i64> @bitcast_v32bf16_to_v8i64_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v14, s30
; GFX9-NEXT: v_mov_b32_e32 v15, s31
; GFX9-NEXT: .LBB67_6: ; %end
-; GFX9-NEXT: v_readlane_b32 s30, v20, 0
-; GFX9-NEXT: v_readlane_b32 s31, v20, 1
+; GFX9-NEXT: v_readlane_b32 s30, v19, 0
+; GFX9-NEXT: v_readlane_b32 s31, v19, 1
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
-; GFX9-NEXT: buffer_load_dword v20, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v19, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-NEXT: s_mov_b64 exec, s[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -50251,10 +50248,10 @@ define inreg <8 x double> @bitcast_v32bf16_to_v8f64_scalar(<32 x bfloat> inreg %
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
-; GFX9-NEXT: buffer_store_dword v20, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v19, off, s[0:3], s32 ; 4-byte Folded Spill
; GFX9-NEXT: s_mov_b64 exec, s[4:5]
-; GFX9-NEXT: v_writelane_b32 v20, s30, 0
-; GFX9-NEXT: v_writelane_b32 v20, s31, 1
+; GFX9-NEXT: v_writelane_b32 v19, s30, 0
+; GFX9-NEXT: v_writelane_b32 v19, s31, 1
; GFX9-NEXT: v_readfirstlane_b32 s4, v2
; GFX9-NEXT: v_readfirstlane_b32 s31, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
@@ -50298,20 +50295,19 @@ define inreg <8 x double> @bitcast_v32bf16_to_v8f64_scalar(<32 x bfloat> inreg %
; GFX9-NEXT: s_lshl_b32 s4, s30, 16
; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
; GFX9-NEXT: v_add_f32_e32 v4, s4, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v5, v5, v4
-; GFX9-NEXT: v_mov_b32_e32 v16, 0xffff
; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v4
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc
-; GFX9-NEXT: v_lshl_or_b32 v15, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v15, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v3
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s29, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v14, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v14, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -50326,11 +50322,11 @@ define inreg <8 x double> @bitcast_v32bf16_to_v8f64_scalar(<32 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s28, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v13, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v13, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -50345,11 +50341,11 @@ define inreg <8 x double> @bitcast_v32bf16_to_v8f64_scalar(<32 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s27, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v12, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v12, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -50364,11 +50360,11 @@ define inreg <8 x double> @bitcast_v32bf16_to_v8f64_scalar(<32 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s26, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v11, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v11, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -50383,11 +50379,11 @@ define inreg <8 x double> @bitcast_v32bf16_to_v8f64_scalar(<32 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s25, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v10, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v10, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -50402,11 +50398,11 @@ define inreg <8 x double> @bitcast_v32bf16_to_v8f64_scalar(<32 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s24, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v9, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v9, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -50421,11 +50417,11 @@ define inreg <8 x double> @bitcast_v32bf16_to_v8f64_scalar(<32 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s23, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v8, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v8, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -50440,11 +50436,11 @@ define inreg <8 x double> @bitcast_v32bf16_to_v8f64_scalar(<32 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s22, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v7, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v7, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -50459,11 +50455,11 @@ define inreg <8 x double> @bitcast_v32bf16_to_v8f64_scalar(<32 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s21, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v6, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v6, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -50478,11 +50474,11 @@ define inreg <8 x double> @bitcast_v32bf16_to_v8f64_scalar(<32 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s20, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v5, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -50497,11 +50493,11 @@ define inreg <8 x double> @bitcast_v32bf16_to_v8f64_scalar(<32 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s19, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v4, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v4, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
@@ -50514,69 +50510,69 @@ define inreg <8 x double> @bitcast_v32bf16_to_v8f64_scalar(<32 x bfloat> inreg %
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
-; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v2
+; GFX9-NEXT: v_or_b32_e32 v16, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v17, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v16, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s18, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v3, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v3, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
-; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v1
+; GFX9-NEXT: v_or_b32_e32 v16, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: s_lshl_b32 s4, s18, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v17, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v16, vcc
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
-; GFX9-NEXT: v_bfe_u32 v17, v2, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v17, v17, v2
-; GFX9-NEXT: v_add_u32_e32 v17, 0x7fff, v17
-; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v2
+; GFX9-NEXT: v_bfe_u32 v16, v2, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v16, v16, v2
+; GFX9-NEXT: v_add_u32_e32 v16, 0x7fff, v16
+; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v17, v18, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v16, v17, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s17, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v2, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
-; GFX9-NEXT: v_bfe_u32 v17, v1, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v17, v17, v1
-; GFX9-NEXT: v_add_u32_e32 v17, 0x7fff, v17
-; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v1
+; GFX9-NEXT: v_bfe_u32 v16, v1, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v16, v16, v1
+; GFX9-NEXT: v_add_u32_e32 v16, 0x7fff, v16
+; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: s_lshl_b32 s4, s17, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v17, v18, vcc
-; GFX9-NEXT: v_add_f32_e32 v17, s4, v0
-; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
-; GFX9-NEXT: v_add_u32_e32 v18, 0x7fff, v18
-; GFX9-NEXT: v_or_b32_e32 v19, 0x400000, v17
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
-; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v19, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v16, v17, vcc
+; GFX9-NEXT: v_add_f32_e32 v16, s4, v0
+; GFX9-NEXT: v_bfe_u32 v17, v16, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v17, v17, v16
+; GFX9-NEXT: v_add_u32_e32 v17, 0x7fff, v17
+; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v16
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v16, v16
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v17, v16, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_cndmask_b32_e32 v16, v17, v18, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_and_b32 s4, s16, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v17
-; GFX9-NEXT: v_add_f32_e32 v17, s4, v0
-; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f32_e32 v16, s4, v0
+; GFX9-NEXT: v_bfe_u32 v17, v16, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v17, v17, v16
; GFX9-NEXT: s_lshl_b32 s4, s16, 16
-; GFX9-NEXT: v_add_u32_e32 v18, 0x7fff, v18
-; GFX9-NEXT: v_or_b32_e32 v19, 0x400000, v17
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
+; GFX9-NEXT: v_add_u32_e32 v17, 0x7fff, v17
+; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v16
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v16, v16
; GFX9-NEXT: v_add_f32_e32 v0, s4, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v19, vcc
-; GFX9-NEXT: v_bfe_u32 v18, v0, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v18, v18, v0
-; GFX9-NEXT: v_add_u32_e32 v18, 0x7fff, v18
-; GFX9-NEXT: v_or_b32_e32 v19, 0x400000, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v16, v17, v18, vcc
+; GFX9-NEXT: v_bfe_u32 v17, v0, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v17, v17, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v16, 16, v16
+; GFX9-NEXT: v_add_u32_e32 v17, 0x7fff, v17
+; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v0
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v17
-; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v19, vcc
-; GFX9-NEXT: v_and_b32_sdwa v16, v16, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v16
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v17, v18, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v16
+; GFX9-NEXT: v_or_b32_sdwa v0, v16, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_branch .LBB83_6
; GFX9-NEXT: .LBB83_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
@@ -50596,10 +50592,10 @@ define inreg <8 x double> @bitcast_v32bf16_to_v8f64_scalar(<32 x bfloat> inreg %
; GFX9-NEXT: v_mov_b32_e32 v14, s30
; GFX9-NEXT: v_mov_b32_e32 v15, s31
; GFX9-NEXT: .LBB83_6: ; %end
-; GFX9-NEXT: v_readlane_b32 s30, v20, 0
-; GFX9-NEXT: v_readlane_b32 s31, v20, 1
+; GFX9-NEXT: v_readlane_b32 s30, v19, 0
+; GFX9-NEXT: v_readlane_b32 s31, v19, 1
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
-; GFX9-NEXT: buffer_load_dword v20, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v19, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-NEXT: s_mov_b64 exec, s[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -71581,11 +71577,11 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
; GFX9-NEXT: v_readfirstlane_b32 s56, v29
; GFX9-NEXT: v_readfirstlane_b32 s57, v28
; GFX9-NEXT: v_readfirstlane_b32 s61, v27
-; GFX9-NEXT: v_readfirstlane_b32 s63, v26
+; GFX9-NEXT: v_readfirstlane_b32 s62, v26
; GFX9-NEXT: v_readfirstlane_b32 s7, v25
-; GFX9-NEXT: v_readfirstlane_b32 s47, v24
+; GFX9-NEXT: v_readfirstlane_b32 s46, v24
; GFX9-NEXT: v_readfirstlane_b32 s58, v23
-; GFX9-NEXT: v_readfirstlane_b32 s60, v22
+; GFX9-NEXT: v_readfirstlane_b32 s59, v22
; GFX9-NEXT: v_readfirstlane_b32 s6, v21
; GFX9-NEXT: v_readfirstlane_b32 s8, v20
; GFX9-NEXT: v_readfirstlane_b32 s9, v19
@@ -71599,12 +71595,12 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
; GFX9-NEXT: buffer_load_ushort v18, off, s[0:3], s32
; GFX9-NEXT: buffer_load_ushort v24, off, s[0:3], s32 offset:72
; GFX9-NEXT: buffer_load_ushort v27, off, s[0:3], s32 offset:68
-; GFX9-NEXT: buffer_load_ushort v32, off, s[0:3], s32 offset:64
+; GFX9-NEXT: buffer_load_ushort v33, off, s[0:3], s32 offset:64
; GFX9-NEXT: buffer_load_ushort v34, off, s[0:3], s32 offset:60
; GFX9-NEXT: buffer_load_ushort v22, off, s[0:3], s32 offset:56
; GFX9-NEXT: buffer_load_ushort v25, off, s[0:3], s32 offset:52
; GFX9-NEXT: buffer_load_ushort v30, off, s[0:3], s32 offset:48
-; GFX9-NEXT: buffer_load_ushort v33, off, s[0:3], s32 offset:44
+; GFX9-NEXT: buffer_load_ushort v32, off, s[0:3], s32 offset:44
; GFX9-NEXT: buffer_load_ushort v20, off, s[0:3], s32 offset:40
; GFX9-NEXT: buffer_load_ushort v23, off, s[0:3], s32 offset:36
; GFX9-NEXT: buffer_load_ushort v28, off, s[0:3], s32 offset:32
@@ -71624,103 +71620,95 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
; GFX9-NEXT: v_readfirstlane_b32 s75, v6
; GFX9-NEXT: v_readfirstlane_b32 s44, v5
; GFX9-NEXT: v_readfirstlane_b32 s45, v4
-; GFX9-NEXT: v_readfirstlane_b32 s46, v3
+; GFX9-NEXT: v_readfirstlane_b32 s47, v3
; GFX9-NEXT: v_readfirstlane_b32 s74, v2
-; GFX9-NEXT: v_readfirstlane_b32 s59, v1
-; GFX9-NEXT: v_readfirstlane_b32 s62, v0
+; GFX9-NEXT: v_readfirstlane_b32 s60, v1
+; GFX9-NEXT: v_readfirstlane_b32 s63, v0
; GFX9-NEXT: s_waitcnt vmcnt(19)
; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v12
; GFX9-NEXT: s_and_b64 s[4:5], vcc, exec
; GFX9-NEXT: s_cbranch_scc0 .LBB99_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v8, 0xc0c0004
-; GFX9-NEXT: v_mov_b32_e32 v2, s21
-; GFX9-NEXT: v_perm_b32 v0, s16, v0, v8
; GFX9-NEXT: v_mov_b32_e32 v1, s19
-; GFX9-NEXT: v_perm_b32 v2, s20, v2, v8
; GFX9-NEXT: v_mov_b32_e32 v3, s23
+; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_perm_b32 v1, s18, v1, v8
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: v_mov_b32_e32 v2, s21
; GFX9-NEXT: v_perm_b32 v3, s22, v3, v8
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v3, 16, v2
-; GFX9-NEXT: v_mov_b32_e32 v2, s25
-; GFX9-NEXT: v_perm_b32 v2, s24, v2, v8
+; GFX9-NEXT: v_perm_b32 v0, s16, v0, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_perm_b32 v2, s20, v2, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v3, s27
+; GFX9-NEXT: v_mov_b32_e32 v2, s25
; GFX9-NEXT: v_perm_b32 v3, s26, v3, v8
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX9-NEXT: v_perm_b32 v2, s24, v2, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_mov_b32_e32 v4, s60
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v3, s29
+; GFX9-NEXT: v_perm_b32 v4, s63, v4, v8
; GFX9-NEXT: v_perm_b32 v3, s28, v3, v8
-; GFX9-NEXT: v_mov_b32_e32 v4, s59
-; GFX9-NEXT: v_perm_b32 v4, s62, v4, v8
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_lshl_or_b32 v3, v4, 16, v3
-; GFX9-NEXT: v_mov_b32_e32 v4, s46
-; GFX9-NEXT: v_perm_b32 v4, s74, v4, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX9-NEXT: v_mov_b32_e32 v5, s44
+; GFX9-NEXT: v_or_b32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v4, s47
; GFX9-NEXT: v_perm_b32 v5, s45, v5, v8
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_lshl_or_b32 v4, v5, 16, v4
-; GFX9-NEXT: v_mov_b32_e32 v5, s43
-; GFX9-NEXT: v_perm_b32 v5, s75, v5, v8
+; GFX9-NEXT: v_perm_b32 v4, s74, v4, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX9-NEXT: v_mov_b32_e32 v6, s41
+; GFX9-NEXT: v_or_b32_sdwa v4, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v5, s43
; GFX9-NEXT: v_perm_b32 v6, s42, v6, v8
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_lshl_or_b32 v5, v6, 16, v5
-; GFX9-NEXT: v_mov_b32_e32 v6, s40
-; GFX9-NEXT: v_perm_b32 v6, s76, v6, v8
+; GFX9-NEXT: v_perm_b32 v5, s75, v5, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v6
; GFX9-NEXT: v_mov_b32_e32 v7, s14
+; GFX9-NEXT: v_or_b32_sdwa v5, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v6, s40
; GFX9-NEXT: v_perm_b32 v7, s15, v7, v8
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_lshl_or_b32 v6, v7, 16, v6
-; GFX9-NEXT: v_mov_b32_e32 v7, s13
-; GFX9-NEXT: v_perm_b32 v7, s73, v7, v8
+; GFX9-NEXT: v_perm_b32 v6, s76, v6, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v7
; GFX9-NEXT: v_mov_b32_e32 v9, s11
+; GFX9-NEXT: v_or_b32_sdwa v6, v7, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v7, s13
; GFX9-NEXT: v_perm_b32 v9, s12, v9, v8
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_lshl_or_b32 v7, v9, 16, v7
+; GFX9-NEXT: v_perm_b32 v7, s73, v7, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX9-NEXT: s_and_b32 s4, s46, 0xff
+; GFX9-NEXT: s_lshl_b32 s5, s7, 8
+; GFX9-NEXT: v_or_b32_sdwa v7, v9, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v9, s58
+; GFX9-NEXT: s_or_b32 s4, s4, s5
+; GFX9-NEXT: v_perm_b32 v9, s59, v9, v8
+; GFX9-NEXT: v_lshlrev_b32_e64 v10, 16, s4
+; GFX9-NEXT: v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(18)
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v16
+; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v16
; GFX9-NEXT: s_waitcnt vmcnt(16)
-; GFX9-NEXT: v_perm_b32 v10, s72, v18, v8
-; GFX9-NEXT: v_or_b32_sdwa v9, v17, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v9, 16, v10
+; GFX9-NEXT: v_perm_b32 v8, s72, v18, v8
+; GFX9-NEXT: v_or_b32_sdwa v10, v17, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v11, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v26
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_or_b32_sdwa v9, v29, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v26
; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v19
-; GFX9-NEXT: v_or_b32_sdwa v10, v21, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_or_b32_sdwa v8, v29, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v10, v21, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v12, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_mov_b32 s4, 0xc0c0004
-; GFX9-NEXT: v_lshl_or_b32 v12, v10, 16, v9
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v20
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v20
+; GFX9-NEXT: v_or_b32_sdwa v8, v23, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_perm_b32 v10, v31, v28, s4
-; GFX9-NEXT: v_or_b32_sdwa v9, v23, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v13, v9, 16, v10
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v30
-; GFX9-NEXT: v_or_b32_sdwa v9, v33, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v13, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v30
; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v22
-; GFX9-NEXT: v_or_b32_sdwa v10, v25, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_lshl_or_b32 v14, v10, 16, v9
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v24
-; GFX9-NEXT: v_perm_b32 v10, v34, v32, s4
-; GFX9-NEXT: v_or_b32_sdwa v9, v27, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v15, v9, 16, v10
-; GFX9-NEXT: v_mov_b32_e32 v9, s58
-; GFX9-NEXT: v_perm_b32 v8, s60, v9, v8
-; GFX9-NEXT: s_and_b32 s4, s47, 0xff
-; GFX9-NEXT: s_lshl_b32 s5, s7, 8
-; GFX9-NEXT: s_or_b32 s4, s4, s5
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, s4, 16, v8
+; GFX9-NEXT: v_or_b32_sdwa v8, v32, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v10, v25, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v14, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_perm_b32 v10, v34, v33, s4
; GFX9-NEXT: s_and_b32 s4, s10, 0xff
; GFX9-NEXT: s_lshl_b32 s5, s9, 8
; GFX9-NEXT: s_or_b32 s4, s4, s5
@@ -71728,13 +71716,16 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
; GFX9-NEXT: s_lshl_b32 s77, s6, 8
; GFX9-NEXT: s_or_b32 s5, s5, s77
; GFX9-NEXT: s_pack_ll_b32_b16 s4, s4, s5
-; GFX9-NEXT: s_and_b32 s5, s63, 0xff
+; GFX9-NEXT: s_and_b32 s5, s62, 0xff
; GFX9-NEXT: s_lshl_b32 s77, s61, 8
; GFX9-NEXT: s_or_b32 s5, s5, s77
; GFX9-NEXT: s_and_b32 s77, s57, 0xff
; GFX9-NEXT: s_lshl_b32 s78, s56, 8
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v24
; GFX9-NEXT: s_or_b32 s77, s77, s78
+; GFX9-NEXT: v_or_b32_sdwa v8, v27, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: s_pack_ll_b32_b16 s5, s5, s77
+; GFX9-NEXT: v_or_b32_sdwa v15, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v8, s4
; GFX9-NEXT: v_mov_b32_e32 v10, s5
; GFX9-NEXT: s_mov_b64 s[4:5], 0
@@ -71751,11 +71742,11 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
; GFX9-NEXT: s_waitcnt vmcnt(12)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v34
; GFX9-NEXT: s_mov_b32 s4, 0xc0c0004
-; GFX9-NEXT: v_perm_b32 v8, v0, v32, s4
+; GFX9-NEXT: v_perm_b32 v8, v0, v33, s4
; GFX9-NEXT: v_add_u32_e32 v0, 3, v27
; GFX9-NEXT: v_perm_b32 v9, v0, v24, s4
; GFX9-NEXT: s_waitcnt vmcnt(8)
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v33
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v32
; GFX9-NEXT: v_perm_b32 v10, v0, v30, s4
; GFX9-NEXT: v_add_u32_e32 v0, 3, v25
; GFX9-NEXT: v_perm_b32 v11, v0, v22, s4
@@ -71772,138 +71763,123 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
; GFX9-NEXT: v_add_u32_e32 v0, 3, v17
; GFX9-NEXT: v_mov_b32_e32 v19, 0xc0c0004
; GFX9-NEXT: v_perm_b32 v16, v0, v16, s4
-; GFX9-NEXT: s_add_i32 s63, s63, 3
+; GFX9-NEXT: s_add_i32 s62, s62, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s61
-; GFX9-NEXT: v_perm_b32 v17, s63, v0, v19
+; GFX9-NEXT: v_perm_b32 v17, s62, v0, v19
; GFX9-NEXT: s_add_i32 s57, s57, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s56
; GFX9-NEXT: v_perm_b32 v20, s57, v0, v19
-; GFX9-NEXT: s_add_i32 s60, s60, 3
+; GFX9-NEXT: s_add_i32 s59, s59, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s58
-; GFX9-NEXT: v_perm_b32 v21, s60, v0, v19
+; GFX9-NEXT: v_perm_b32 v21, s59, v0, v19
; GFX9-NEXT: s_add_i32 s20, s20, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s21
; GFX9-NEXT: v_perm_b32 v0, s20, v0, v19
; GFX9-NEXT: v_add_u32_e32 v1, 0x300, v0
; GFX9-NEXT: s_add_i32 s22, s22, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s23
+; GFX9-NEXT: s_movk_i32 s5, 0x300
; GFX9-NEXT: v_perm_b32 v0, s22, v0, v19
-; GFX9-NEXT: v_add_u32_e32 v2, 0x300, v0
+; GFX9-NEXT: v_add_u32_sdwa v2, v0, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s16, s16, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s17
-; GFX9-NEXT: v_perm_b32 v0, s16, v0, v19
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: v_mov_b32_e32 v3, s19
-; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
+; GFX9-NEXT: v_perm_b32 v0, s16, v0, v19
; GFX9-NEXT: v_perm_b32 v3, s18, v3, v19
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
+; GFX9-NEXT: v_add_u32_sdwa v3, v3, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s24, s24, 3
-; GFX9-NEXT: v_add_u32_e32 v3, 0x300, v3
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v2, s25
-; GFX9-NEXT: v_lshl_or_b32 v0, v3, 16, v0
-; GFX9-NEXT: v_perm_b32 v2, s24, v2, v19
; GFX9-NEXT: s_add_i32 s26, s26, 3
; GFX9-NEXT: v_mov_b32_e32 v3, s27
-; GFX9-NEXT: v_add_u32_e32 v2, 0x300, v2
+; GFX9-NEXT: v_perm_b32 v2, s24, v2, v19
; GFX9-NEXT: v_perm_b32 v3, s26, v3, v19
-; GFX9-NEXT: v_add_u32_e32 v3, 0x300, v3
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX9-NEXT: v_add_u32_e32 v2, 0x300, v2
+; GFX9-NEXT: v_add_u32_sdwa v3, v3, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s28, s28, 3
-; GFX9-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v3, s29
+; GFX9-NEXT: s_add_i32 s63, s63, 3
+; GFX9-NEXT: v_mov_b32_e32 v4, s60
; GFX9-NEXT: v_perm_b32 v3, s28, v3, v19
-; GFX9-NEXT: s_add_i32 s62, s62, 3
-; GFX9-NEXT: v_mov_b32_e32 v4, s59
+; GFX9-NEXT: v_perm_b32 v4, s63, v4, v19
; GFX9-NEXT: v_add_u32_e32 v3, 0x300, v3
-; GFX9-NEXT: v_perm_b32 v4, s62, v4, v19
-; GFX9-NEXT: v_add_u32_e32 v4, 0x300, v4
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX9-NEXT: v_add_u32_sdwa v4, v4, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s74, s74, 3
-; GFX9-NEXT: v_lshl_or_b32 v3, v4, 16, v3
-; GFX9-NEXT: v_mov_b32_e32 v4, s46
-; GFX9-NEXT: v_perm_b32 v4, s74, v4, v19
+; GFX9-NEXT: v_or_b32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v4, s47
; GFX9-NEXT: s_add_i32 s45, s45, 3
; GFX9-NEXT: v_mov_b32_e32 v5, s44
-; GFX9-NEXT: v_add_u32_e32 v4, 0x300, v4
+; GFX9-NEXT: v_perm_b32 v4, s74, v4, v19
; GFX9-NEXT: v_perm_b32 v5, s45, v5, v19
-; GFX9-NEXT: v_add_u32_e32 v5, 0x300, v5
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX9-NEXT: v_add_u32_e32 v4, 0x300, v4
+; GFX9-NEXT: v_add_u32_sdwa v5, v5, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s75, s75, 3
-; GFX9-NEXT: v_lshl_or_b32 v4, v5, 16, v4
+; GFX9-NEXT: v_or_b32_sdwa v4, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v5, s43
-; GFX9-NEXT: v_perm_b32 v5, s75, v5, v19
; GFX9-NEXT: s_add_i32 s42, s42, 3
; GFX9-NEXT: v_mov_b32_e32 v6, s41
-; GFX9-NEXT: v_add_u32_e32 v5, 0x300, v5
+; GFX9-NEXT: v_perm_b32 v5, s75, v5, v19
; GFX9-NEXT: v_perm_b32 v6, s42, v6, v19
-; GFX9-NEXT: v_add_u32_e32 v6, 0x300, v6
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX9-NEXT: v_add_u32_e32 v5, 0x300, v5
+; GFX9-NEXT: v_add_u32_sdwa v6, v6, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s76, s76, 3
-; GFX9-NEXT: v_lshl_or_b32 v5, v6, 16, v5
+; GFX9-NEXT: v_or_b32_sdwa v5, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v6, s40
-; GFX9-NEXT: v_perm_b32 v6, s76, v6, v19
; GFX9-NEXT: s_add_i32 s15, s15, 3
; GFX9-NEXT: v_mov_b32_e32 v7, s14
-; GFX9-NEXT: v_add_u32_e32 v6, 0x300, v6
+; GFX9-NEXT: v_perm_b32 v6, s76, v6, v19
; GFX9-NEXT: v_perm_b32 v7, s15, v7, v19
-; GFX9-NEXT: v_add_u32_e32 v7, 0x300, v7
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; GFX9-NEXT: v_add_u32_e32 v6, 0x300, v6
+; GFX9-NEXT: v_add_u32_sdwa v7, v7, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s73, s73, 3
-; GFX9-NEXT: v_lshl_or_b32 v6, v7, 16, v6
+; GFX9-NEXT: v_or_b32_sdwa v6, v7, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v7, s13
-; GFX9-NEXT: v_perm_b32 v7, s73, v7, v19
; GFX9-NEXT: s_add_i32 s12, s12, 3
; GFX9-NEXT: v_mov_b32_e32 v22, s11
-; GFX9-NEXT: s_add_i32 s72, s72, 3
-; GFX9-NEXT: v_add_u32_e32 v7, 0x300, v7
+; GFX9-NEXT: v_perm_b32 v7, s73, v7, v19
; GFX9-NEXT: v_perm_b32 v22, s12, v22, v19
-; GFX9-NEXT: v_perm_b32 v18, s72, v18, v19
-; GFX9-NEXT: v_add_u32_e32 v22, 0x300, v22
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX9-NEXT: v_add_u32_e32 v7, 0x300, v7
+; GFX9-NEXT: v_add_u32_sdwa v22, v22, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: s_add_i32 s72, s72, 3
+; GFX9-NEXT: s_add_i32 s46, s46, 3
+; GFX9-NEXT: v_or_b32_sdwa v7, v22, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v22, s7
; GFX9-NEXT: s_add_i32 s10, s10, 3
; GFX9-NEXT: v_mov_b32_e32 v23, s9
-; GFX9-NEXT: v_add_u32_e32 v28, 0x300, v12
-; GFX9-NEXT: v_add_u32_e32 v12, 0x300, v14
-; GFX9-NEXT: s_add_i32 s47, s47, 3
-; GFX9-NEXT: v_lshl_or_b32 v7, v22, 16, v7
-; GFX9-NEXT: v_mov_b32_e32 v22, s7
-; GFX9-NEXT: v_perm_b32 v23, s10, v23, v19
; GFX9-NEXT: s_add_i32 s8, s8, 3
; GFX9-NEXT: v_mov_b32_e32 v24, s6
-; GFX9-NEXT: v_add_u32_e32 v27, 0x300, v11
-; GFX9-NEXT: v_add_u32_e32 v14, 0x300, v15
-; GFX9-NEXT: v_add_u32_e32 v11, 0x300, v18
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_perm_b32 v22, s47, v22, v19
+; GFX9-NEXT: v_perm_b32 v18, s72, v18, v19
+; GFX9-NEXT: v_perm_b32 v22, s46, v22, v19
+; GFX9-NEXT: v_perm_b32 v23, s10, v23, v19
; GFX9-NEXT: v_perm_b32 v19, s8, v24, v19
; GFX9-NEXT: v_add_u32_e32 v24, 0x300, v8
-; GFX9-NEXT: v_add_u32_e32 v25, 0x300, v9
+; GFX9-NEXT: v_add_u32_sdwa v25, v9, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_add_u32_e32 v26, 0x300, v10
-; GFX9-NEXT: v_add_u32_e32 v13, 0x300, v13
-; GFX9-NEXT: v_add_u32_e32 v15, 0x300, v16
+; GFX9-NEXT: v_add_u32_sdwa v27, v11, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_add_u32_e32 v28, 0x300, v12
+; GFX9-NEXT: v_add_u32_sdwa v13, v13, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_add_u32_e32 v12, 0x300, v14
+; GFX9-NEXT: v_add_u32_sdwa v14, v15, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_add_u32_e32 v11, 0x300, v18
+; GFX9-NEXT: v_add_u32_sdwa v15, v16, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_add_u32_e32 v10, 0x300, v17
+; GFX9-NEXT: v_add_u32_sdwa v16, v20, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_add_u32_e32 v9, 0x300, v21
+; GFX9-NEXT: v_add_u32_sdwa v17, v22, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_add_u32_e32 v8, 0x300, v23
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v14, 16, v12
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v28
-; GFX9-NEXT: v_add_u32_e32 v16, 0x300, v20
-; GFX9-NEXT: v_add_u32_e32 v17, 0x300, v22
-; GFX9-NEXT: v_add_u32_e32 v18, 0x300, v19
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v15, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v13, v13, 16, v14
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v26
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v24
-; GFX9-NEXT: v_lshl_or_b32 v8, v18, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v17, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v16, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v14, v27, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v25, 16, v15
+; GFX9-NEXT: v_add_u32_sdwa v18, v19, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v8, v18, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v17, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v16, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v15, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v12, v14, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v13, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v14, v27, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v25, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: .LBB99_5: ; %end
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -76046,15 +76022,15 @@ define inreg <32 x half> @bitcast_v32bf16_to_v32f16_scalar(<32 x bfloat> inreg %
; GFX9-NEXT: s_cmp_lg_u32 s4, 1
; GFX9-NEXT: s_cbranch_scc1 .LBB103_5
; GFX9-NEXT: ; %bb.4: ; %cmp.true
+; GFX9-NEXT: s_and_b32 s4, s16, 0xffff0000
; GFX9-NEXT: v_mov_b32_e32 v0, 0x40c00000
-; GFX9-NEXT: s_and_b32 s5, s30, 0xffff0000
-; GFX9-NEXT: v_add_f32_e32 v1, s5, v0
+; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: s_lshl_b32 s5, s30, 16
+; GFX9-NEXT: s_and_b32 s5, s30, 0xffff0000
; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_add_f32_e32 v2, s5, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
@@ -76062,7 +76038,7 @@ define inreg <32 x half> @bitcast_v32bf16_to_v32f16_scalar(<32 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: s_and_b32 s5, s31, 0xffff0000
+; GFX9-NEXT: s_lshl_b32 s5, s30, 16
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_add_f32_e32 v3, s5, v0
; GFX9-NEXT: v_bfe_u32 v4, v3, 16, 1
@@ -76070,274 +76046,270 @@ define inreg <32 x half> @bitcast_v32bf16_to_v32f16_scalar(<32 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v3
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX9-NEXT: s_lshl_b32 s5, s31, 16
+; GFX9-NEXT: s_and_b32 s5, s31, 0xffff0000
; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
; GFX9-NEXT: v_add_f32_e32 v4, s5, v0
; GFX9-NEXT: v_bfe_u32 v5, v4, 16, 1
-; GFX9-NEXT: v_mov_b32_e32 v16, 0xffff
-; GFX9-NEXT: s_and_b32 s4, s29, 0xffff0000
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v5, v5, v4
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v4
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX9-NEXT: v_lshl_or_b32 v14, v1, 16, v2
-; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
+; GFX9-NEXT: s_lshl_b32 s5, s31, 16
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX9-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc
-; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
-; GFX9-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX9-NEXT: v_and_b32_sdwa v4, v16, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
-; GFX9-NEXT: v_lshl_or_b32 v15, v3, 16, v4
-; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
-; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: s_lshl_b32 s4, s29, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_add_f32_e32 v5, s5, v0
+; GFX9-NEXT: s_and_b32 s4, s29, 0xffff0000
+; GFX9-NEXT: v_bfe_u32 v6, v5, 16, 1
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_add_u32_e32 v6, v6, v5
+; GFX9-NEXT: v_or_b32_sdwa v14, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX9-NEXT: v_add_u32_e32 v6, 0x7fff, v6
+; GFX9-NEXT: v_or_b32_e32 v7, 0x400000, v5
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
+; GFX9-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_or_b32_sdwa v15, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT: s_lshl_b32 s4, s29, 16
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_add_f32_e32 v3, s4, v0
+; GFX9-NEXT: v_bfe_u32 v4, v3, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v4, v4, v3
+; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
; GFX9-NEXT: s_and_b32 s4, s28, 0xffff0000
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v13, v1, 16, v2
-; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
-; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
-; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
-; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: s_lshl_b32 s4, s28, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v13, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT: s_lshl_b32 s4, s28, 16
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_add_f32_e32 v3, s4, v0
+; GFX9-NEXT: v_bfe_u32 v4, v3, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v4, v4, v3
+; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
; GFX9-NEXT: s_and_b32 s4, s27, 0xffff0000
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v12, v1, 16, v2
-; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
-; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
-; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
-; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: s_lshl_b32 s4, s27, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v12, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT: s_lshl_b32 s4, s27, 16
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_add_f32_e32 v3, s4, v0
+; GFX9-NEXT: v_bfe_u32 v4, v3, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v4, v4, v3
+; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
; GFX9-NEXT: s_and_b32 s4, s26, 0xffff0000
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v11, v1, 16, v2
-; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
-; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
-; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
-; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: s_lshl_b32 s4, s26, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v11, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT: s_lshl_b32 s4, s26, 16
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_add_f32_e32 v3, s4, v0
+; GFX9-NEXT: v_bfe_u32 v4, v3, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v4, v4, v3
+; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
; GFX9-NEXT: s_and_b32 s4, s25, 0xffff0000
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v10, v1, 16, v2
-; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
-; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
-; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
-; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: s_lshl_b32 s4, s25, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v10, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT: s_lshl_b32 s4, s25, 16
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_add_f32_e32 v3, s4, v0
+; GFX9-NEXT: v_bfe_u32 v4, v3, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v4, v4, v3
+; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
; GFX9-NEXT: s_and_b32 s4, s24, 0xffff0000
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v9, v1, 16, v2
-; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
-; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
-; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
-; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: s_lshl_b32 s4, s24, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v9, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT: s_lshl_b32 s4, s24, 16
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_add_f32_e32 v3, s4, v0
+; GFX9-NEXT: v_bfe_u32 v4, v3, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v4, v4, v3
+; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
; GFX9-NEXT: s_and_b32 s4, s23, 0xffff0000
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v8, v1, 16, v2
-; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
-; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
-; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
-; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: s_lshl_b32 s4, s23, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v8, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT: s_lshl_b32 s4, s23, 16
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_add_f32_e32 v3, s4, v0
+; GFX9-NEXT: v_bfe_u32 v4, v3, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v4, v4, v3
+; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
; GFX9-NEXT: s_and_b32 s4, s22, 0xffff0000
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v7, v1, 16, v2
-; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
-; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
-; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
-; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: s_lshl_b32 s4, s22, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v7, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT: s_lshl_b32 s4, s22, 16
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_add_f32_e32 v3, s4, v0
+; GFX9-NEXT: v_bfe_u32 v4, v3, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v4, v4, v3
+; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
; GFX9-NEXT: s_and_b32 s4, s21, 0xffff0000
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v6, v1, 16, v2
-; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
-; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
-; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
-; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: s_lshl_b32 s4, s21, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v6, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT: s_lshl_b32 s4, s21, 16
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_add_f32_e32 v3, s4, v0
+; GFX9-NEXT: v_bfe_u32 v4, v3, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v4, v4, v3
+; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
; GFX9-NEXT: s_and_b32 s4, s20, 0xffff0000
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v5, v1, 16, v2
-; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
-; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
-; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
-; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: s_lshl_b32 s4, s20, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v5, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT: s_lshl_b32 s4, s20, 16
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_add_f32_e32 v3, s4, v0
+; GFX9-NEXT: v_bfe_u32 v4, v3, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v4, v4, v3
+; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
+; GFX9-NEXT: v_or_b32_e32 v16, 0x400000, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v16, vcc
; GFX9-NEXT: s_and_b32 s4, s19, 0xffff0000
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v4, v1, 16, v2
-; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
-; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
-; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
-; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: s_lshl_b32 s4, s19, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v4, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
-; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v2
+; GFX9-NEXT: v_or_b32_e32 v16, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v17, vcc
+; GFX9-NEXT: s_lshl_b32 s4, s19, 16
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v16, vcc
+; GFX9-NEXT: v_add_f32_e32 v3, s4, v0
+; GFX9-NEXT: v_bfe_u32 v16, v3, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v16, v16, v3
+; GFX9-NEXT: v_add_u32_e32 v16, 0x7fff, v16
+; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v16, v17, vcc
; GFX9-NEXT: s_and_b32 s4, s18, 0xffff0000
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v3, v1, 16, v2
-; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
-; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
-; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
-; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v1
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: s_lshl_b32 s4, s18, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v17, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v3, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
-; GFX9-NEXT: v_bfe_u32 v17, v2, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v17, v17, v2
-; GFX9-NEXT: v_add_u32_e32 v17, 0x7fff, v17
-; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v2
+; GFX9-NEXT: v_bfe_u32 v16, v2, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v16, v16, v2
+; GFX9-NEXT: v_add_u32_e32 v16, 0x7fff, v16
+; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v17, v18, vcc
-; GFX9-NEXT: s_and_b32 s4, s17, 0xffff0000
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v2, v1, 16, v2
-; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
-; GFX9-NEXT: v_bfe_u32 v17, v1, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v17, v17, v1
+; GFX9-NEXT: s_lshl_b32 s4, s18, 16
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v16, v17, vcc
+; GFX9-NEXT: v_add_f32_e32 v16, s4, v0
+; GFX9-NEXT: v_bfe_u32 v17, v16, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v17, v17, v16
; GFX9-NEXT: v_add_u32_e32 v17, 0x7fff, v17
-; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v1
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: s_lshl_b32 s4, s17, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v17, v18, vcc
-; GFX9-NEXT: v_add_f32_e32 v17, s4, v0
-; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
-; GFX9-NEXT: v_add_u32_e32 v18, 0x7fff, v18
-; GFX9-NEXT: v_or_b32_e32 v19, 0x400000, v17
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
-; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v19, vcc
-; GFX9-NEXT: s_and_b32 s4, s16, 0xffff0000
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v17, v16, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v17
+; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v16
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v16, v16
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v16, v17, v18, vcc
+; GFX9-NEXT: s_lshl_b32 s4, s16, 16
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v2, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f32_e32 v16, s4, v0
+; GFX9-NEXT: v_bfe_u32 v17, v16, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v17, v17, v16
+; GFX9-NEXT: v_add_u32_e32 v17, 0x7fff, v17
+; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v16
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v16, v16
+; GFX9-NEXT: s_and_b32 s4, s17, 0xffff0000
+; GFX9-NEXT: v_cndmask_b32_e32 v16, v17, v18, vcc
; GFX9-NEXT: v_add_f32_e32 v17, s4, v0
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add_u32_e32 v18, v18, v17
-; GFX9-NEXT: s_lshl_b32 s4, s16, 16
+; GFX9-NEXT: s_lshl_b32 s4, s17, 16
; GFX9-NEXT: v_add_u32_e32 v18, 0x7fff, v18
; GFX9-NEXT: v_or_b32_e32 v19, 0x400000, v17
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
@@ -76348,10 +76320,13 @@ define inreg <32 x half> @bitcast_v32bf16_to_v32f16_scalar(<32 x bfloat> inreg %
; GFX9-NEXT: v_add_u32_e32 v18, 0x7fff, v18
; GFX9-NEXT: v_or_b32_e32 v19, 0x400000, v0
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v17
-; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v19, vcc
-; GFX9-NEXT: v_and_b32_sdwa v16, v16, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v16
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v18, v19, vcc
+; GFX9-NEXT: v_lshrrev_b32_e32 v18, 16, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v17
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_branch .LBB103_6
; GFX9-NEXT: .LBB103_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
@@ -84391,11 +84366,11 @@ define inreg <32 x half> @bitcast_v64i8_to_v32f16_scalar(<64 x i8> inreg %a, i32
; GFX9-NEXT: v_readfirstlane_b32 s56, v29
; GFX9-NEXT: v_readfirstlane_b32 s57, v28
; GFX9-NEXT: v_readfirstlane_b32 s61, v27
-; GFX9-NEXT: v_readfirstlane_b32 s63, v26
+; GFX9-NEXT: v_readfirstlane_b32 s62, v26
; GFX9-NEXT: v_readfirstlane_b32 s7, v25
-; GFX9-NEXT: v_readfirstlane_b32 s47, v24
+; GFX9-NEXT: v_readfirstlane_b32 s46, v24
; GFX9-NEXT: v_readfirstlane_b32 s58, v23
-; GFX9-NEXT: v_readfirstlane_b32 s60, v22
+; GFX9-NEXT: v_readfirstlane_b32 s59, v22
; GFX9-NEXT: v_readfirstlane_b32 s6, v21
; GFX9-NEXT: v_readfirstlane_b32 s8, v20
; GFX9-NEXT: v_readfirstlane_b32 s9, v19
@@ -84409,12 +84384,12 @@ define inreg <32 x half> @bitcast_v64i8_to_v32f16_scalar(<64 x i8> inreg %a, i32
; GFX9-NEXT: buffer_load_ushort v18, off, s[0:3], s32
; GFX9-NEXT: buffer_load_ushort v24, off, s[0:3], s32 offset:72
; GFX9-NEXT: buffer_load_ushort v27, off, s[0:3], s32 offset:68
-; GFX9-NEXT: buffer_load_ushort v32, off, s[0:3], s32 offset:64
+; GFX9-NEXT: buffer_load_ushort v33, off, s[0:3], s32 offset:64
; GFX9-NEXT: buffer_load_ushort v34, off, s[0:3], s32 offset:60
; GFX9-NEXT: buffer_load_ushort v22, off, s[0:3], s32 offset:56
; GFX9-NEXT: buffer_load_ushort v25, off, s[0:3], s32 offset:52
; GFX9-NEXT: buffer_load_ushort v30, off, s[0:3], s32 offset:48
-; GFX9-NEXT: buffer_load_ushort v33, off, s[0:3], s32 offset:44
+; GFX9-NEXT: buffer_load_ushort v32, off, s[0:3], s32 offset:44
; GFX9-NEXT: buffer_load_ushort v20, off, s[0:3], s32 offset:40
; GFX9-NEXT: buffer_load_ushort v23, off, s[0:3], s32 offset:36
; GFX9-NEXT: buffer_load_ushort v28, off, s[0:3], s32 offset:32
@@ -84434,103 +84409,95 @@ define inreg <32 x half> @bitcast_v64i8_to_v32f16_scalar(<64 x i8> inreg %a, i32
; GFX9-NEXT: v_readfirstlane_b32 s75, v6
; GFX9-NEXT: v_readfirstlane_b32 s44, v5
; GFX9-NEXT: v_readfirstlane_b32 s45, v4
-; GFX9-NEXT: v_readfirstlane_b32 s46, v3
+; GFX9-NEXT: v_readfirstlane_b32 s47, v3
; GFX9-NEXT: v_readfirstlane_b32 s74, v2
-; GFX9-NEXT: v_readfirstlane_b32 s59, v1
-; GFX9-NEXT: v_readfirstlane_b32 s62, v0
+; GFX9-NEXT: v_readfirstlane_b32 s60, v1
+; GFX9-NEXT: v_readfirstlane_b32 s63, v0
; GFX9-NEXT: s_waitcnt vmcnt(19)
; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v12
; GFX9-NEXT: s_and_b64 s[4:5], vcc, exec
; GFX9-NEXT: s_cbranch_scc0 .LBB107_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v8, 0xc0c0004
-; GFX9-NEXT: v_mov_b32_e32 v2, s21
-; GFX9-NEXT: v_perm_b32 v0, s16, v0, v8
; GFX9-NEXT: v_mov_b32_e32 v1, s19
-; GFX9-NEXT: v_perm_b32 v2, s20, v2, v8
; GFX9-NEXT: v_mov_b32_e32 v3, s23
+; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_perm_b32 v1, s18, v1, v8
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: v_mov_b32_e32 v2, s21
; GFX9-NEXT: v_perm_b32 v3, s22, v3, v8
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v3, 16, v2
-; GFX9-NEXT: v_mov_b32_e32 v2, s25
-; GFX9-NEXT: v_perm_b32 v2, s24, v2, v8
+; GFX9-NEXT: v_perm_b32 v0, s16, v0, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_perm_b32 v2, s20, v2, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v3, s27
+; GFX9-NEXT: v_mov_b32_e32 v2, s25
; GFX9-NEXT: v_perm_b32 v3, s26, v3, v8
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX9-NEXT: v_perm_b32 v2, s24, v2, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_mov_b32_e32 v4, s60
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v3, s29
+; GFX9-NEXT: v_perm_b32 v4, s63, v4, v8
; GFX9-NEXT: v_perm_b32 v3, s28, v3, v8
-; GFX9-NEXT: v_mov_b32_e32 v4, s59
-; GFX9-NEXT: v_perm_b32 v4, s62, v4, v8
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_lshl_or_b32 v3, v4, 16, v3
-; GFX9-NEXT: v_mov_b32_e32 v4, s46
-; GFX9-NEXT: v_perm_b32 v4, s74, v4, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX9-NEXT: v_mov_b32_e32 v5, s44
+; GFX9-NEXT: v_or_b32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v4, s47
; GFX9-NEXT: v_perm_b32 v5, s45, v5, v8
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_lshl_or_b32 v4, v5, 16, v4
-; GFX9-NEXT: v_mov_b32_e32 v5, s43
-; GFX9-NEXT: v_perm_b32 v5, s75, v5, v8
+; GFX9-NEXT: v_perm_b32 v4, s74, v4, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX9-NEXT: v_mov_b32_e32 v6, s41
+; GFX9-NEXT: v_or_b32_sdwa v4, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v5, s43
; GFX9-NEXT: v_perm_b32 v6, s42, v6, v8
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_lshl_or_b32 v5, v6, 16, v5
-; GFX9-NEXT: v_mov_b32_e32 v6, s40
-; GFX9-NEXT: v_perm_b32 v6, s76, v6, v8
+; GFX9-NEXT: v_perm_b32 v5, s75, v5, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v6
; GFX9-NEXT: v_mov_b32_e32 v7, s14
+; GFX9-NEXT: v_or_b32_sdwa v5, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v6, s40
; GFX9-NEXT: v_perm_b32 v7, s15, v7, v8
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_lshl_or_b32 v6, v7, 16, v6
-; GFX9-NEXT: v_mov_b32_e32 v7, s13
-; GFX9-NEXT: v_perm_b32 v7, s73, v7, v8
+; GFX9-NEXT: v_perm_b32 v6, s76, v6, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v7
; GFX9-NEXT: v_mov_b32_e32 v9, s11
+; GFX9-NEXT: v_or_b32_sdwa v6, v7, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v7, s13
; GFX9-NEXT: v_perm_b32 v9, s12, v9, v8
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_lshl_or_b32 v7, v9, 16, v7
+; GFX9-NEXT: v_perm_b32 v7, s73, v7, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX9-NEXT: s_and_b32 s4, s46, 0xff
+; GFX9-NEXT: s_lshl_b32 s5, s7, 8
+; GFX9-NEXT: v_or_b32_sdwa v7, v9, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v9, s58
+; GFX9-NEXT: s_or_b32 s4, s4, s5
+; GFX9-NEXT: v_perm_b32 v9, s59, v9, v8
+; GFX9-NEXT: v_lshlrev_b32_e64 v10, 16, s4
+; GFX9-NEXT: v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(18)
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v16
+; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v16
; GFX9-NEXT: s_waitcnt vmcnt(16)
-; GFX9-NEXT: v_perm_b32 v10, s72, v18, v8
-; GFX9-NEXT: v_or_b32_sdwa v9, v17, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v9, 16, v10
+; GFX9-NEXT: v_perm_b32 v8, s72, v18, v8
+; GFX9-NEXT: v_or_b32_sdwa v10, v17, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v11, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v26
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_or_b32_sdwa v9, v29, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v26
; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v19
-; GFX9-NEXT: v_or_b32_sdwa v10, v21, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_or_b32_sdwa v8, v29, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v10, v21, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v12, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_mov_b32 s4, 0xc0c0004
-; GFX9-NEXT: v_lshl_or_b32 v12, v10, 16, v9
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v20
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v20
+; GFX9-NEXT: v_or_b32_sdwa v8, v23, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_perm_b32 v10, v31, v28, s4
-; GFX9-NEXT: v_or_b32_sdwa v9, v23, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v13, v9, 16, v10
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v30
-; GFX9-NEXT: v_or_b32_sdwa v9, v33, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v13, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v30
; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v22
-; GFX9-NEXT: v_or_b32_sdwa v10, v25, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_lshl_or_b32 v14, v10, 16, v9
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v24
-; GFX9-NEXT: v_perm_b32 v10, v34, v32, s4
-; GFX9-NEXT: v_or_b32_sdwa v9, v27, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v15, v9, 16, v10
-; GFX9-NEXT: v_mov_b32_e32 v9, s58
-; GFX9-NEXT: v_perm_b32 v8, s60, v9, v8
-; GFX9-NEXT: s_and_b32 s4, s47, 0xff
-; GFX9-NEXT: s_lshl_b32 s5, s7, 8
-; GFX9-NEXT: s_or_b32 s4, s4, s5
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, s4, 16, v8
+; GFX9-NEXT: v_or_b32_sdwa v8, v32, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v10, v25, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v14, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_perm_b32 v10, v34, v33, s4
; GFX9-NEXT: s_and_b32 s4, s10, 0xff
; GFX9-NEXT: s_lshl_b32 s5, s9, 8
; GFX9-NEXT: s_or_b32 s4, s4, s5
@@ -84538,13 +84505,16 @@ define inreg <32 x half> @bitcast_v64i8_to_v32f16_scalar(<64 x i8> inreg %a, i32
; GFX9-NEXT: s_lshl_b32 s77, s6, 8
; GFX9-NEXT: s_or_b32 s5, s5, s77
; GFX9-NEXT: s_pack_ll_b32_b16 s4, s4, s5
-; GFX9-NEXT: s_and_b32 s5, s63, 0xff
+; GFX9-NEXT: s_and_b32 s5, s62, 0xff
; GFX9-NEXT: s_lshl_b32 s77, s61, 8
; GFX9-NEXT: s_or_b32 s5, s5, s77
; GFX9-NEXT: s_and_b32 s77, s57, 0xff
; GFX9-NEXT: s_lshl_b32 s78, s56, 8
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v24
; GFX9-NEXT: s_or_b32 s77, s77, s78
+; GFX9-NEXT: v_or_b32_sdwa v8, v27, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: s_pack_ll_b32_b16 s5, s5, s77
+; GFX9-NEXT: v_or_b32_sdwa v15, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v8, s4
; GFX9-NEXT: v_mov_b32_e32 v10, s5
; GFX9-NEXT: s_mov_b64 s[4:5], 0
@@ -84561,11 +84531,11 @@ define inreg <32 x half> @bitcast_v64i8_to_v32f16_scalar(<64 x i8> inreg %a, i32
; GFX9-NEXT: s_waitcnt vmcnt(12)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v34
; GFX9-NEXT: s_mov_b32 s4, 0xc0c0004
-; GFX9-NEXT: v_perm_b32 v8, v0, v32, s4
+; GFX9-NEXT: v_perm_b32 v8, v0, v33, s4
; GFX9-NEXT: v_add_u32_e32 v0, 3, v27
; GFX9-NEXT: v_perm_b32 v9, v0, v24, s4
; GFX9-NEXT: s_waitcnt vmcnt(8)
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v33
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v32
; GFX9-NEXT: v_perm_b32 v10, v0, v30, s4
; GFX9-NEXT: v_add_u32_e32 v0, 3, v25
; GFX9-NEXT: v_perm_b32 v11, v0, v22, s4
@@ -84582,138 +84552,123 @@ define inreg <32 x half> @bitcast_v64i8_to_v32f16_scalar(<64 x i8> inreg %a, i32
; GFX9-NEXT: v_add_u32_e32 v0, 3, v17
; GFX9-NEXT: v_mov_b32_e32 v19, 0xc0c0004
; GFX9-NEXT: v_perm_b32 v16, v0, v16, s4
-; GFX9-NEXT: s_add_i32 s63, s63, 3
+; GFX9-NEXT: s_add_i32 s62, s62, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s61
-; GFX9-NEXT: v_perm_b32 v17, s63, v0, v19
+; GFX9-NEXT: v_perm_b32 v17, s62, v0, v19
; GFX9-NEXT: s_add_i32 s57, s57, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s56
; GFX9-NEXT: v_perm_b32 v20, s57, v0, v19
-; GFX9-NEXT: s_add_i32 s60, s60, 3
+; GFX9-NEXT: s_add_i32 s59, s59, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s58
-; GFX9-NEXT: v_perm_b32 v21, s60, v0, v19
+; GFX9-NEXT: v_perm_b32 v21, s59, v0, v19
; GFX9-NEXT: s_add_i32 s20, s20, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s21
; GFX9-NEXT: v_perm_b32 v0, s20, v0, v19
; GFX9-NEXT: v_add_u32_e32 v1, 0x300, v0
; GFX9-NEXT: s_add_i32 s22, s22, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s23
+; GFX9-NEXT: s_movk_i32 s5, 0x300
; GFX9-NEXT: v_perm_b32 v0, s22, v0, v19
-; GFX9-NEXT: v_add_u32_e32 v2, 0x300, v0
+; GFX9-NEXT: v_add_u32_sdwa v2, v0, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s16, s16, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s17
-; GFX9-NEXT: v_perm_b32 v0, s16, v0, v19
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: v_mov_b32_e32 v3, s19
-; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
+; GFX9-NEXT: v_perm_b32 v0, s16, v0, v19
; GFX9-NEXT: v_perm_b32 v3, s18, v3, v19
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
+; GFX9-NEXT: v_add_u32_sdwa v3, v3, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s24, s24, 3
-; GFX9-NEXT: v_add_u32_e32 v3, 0x300, v3
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v2, s25
-; GFX9-NEXT: v_lshl_or_b32 v0, v3, 16, v0
-; GFX9-NEXT: v_perm_b32 v2, s24, v2, v19
; GFX9-NEXT: s_add_i32 s26, s26, 3
; GFX9-NEXT: v_mov_b32_e32 v3, s27
-; GFX9-NEXT: v_add_u32_e32 v2, 0x300, v2
+; GFX9-NEXT: v_perm_b32 v2, s24, v2, v19
; GFX9-NEXT: v_perm_b32 v3, s26, v3, v19
-; GFX9-NEXT: v_add_u32_e32 v3, 0x300, v3
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX9-NEXT: v_add_u32_e32 v2, 0x300, v2
+; GFX9-NEXT: v_add_u32_sdwa v3, v3, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s28, s28, 3
-; GFX9-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v3, s29
+; GFX9-NEXT: s_add_i32 s63, s63, 3
+; GFX9-NEXT: v_mov_b32_e32 v4, s60
; GFX9-NEXT: v_perm_b32 v3, s28, v3, v19
-; GFX9-NEXT: s_add_i32 s62, s62, 3
-; GFX9-NEXT: v_mov_b32_e32 v4, s59
+; GFX9-NEXT: v_perm_b32 v4, s63, v4, v19
; GFX9-NEXT: v_add_u32_e32 v3, 0x300, v3
-; GFX9-NEXT: v_perm_b32 v4, s62, v4, v19
-; GFX9-NEXT: v_add_u32_e32 v4, 0x300, v4
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX9-NEXT: v_add_u32_sdwa v4, v4, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s74, s74, 3
-; GFX9-NEXT: v_lshl_or_b32 v3, v4, 16, v3
-; GFX9-NEXT: v_mov_b32_e32 v4, s46
-; GFX9-NEXT: v_perm_b32 v4, s74, v4, v19
+; GFX9-NEXT: v_or_b32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v4, s47
; GFX9-NEXT: s_add_i32 s45, s45, 3
; GFX9-NEXT: v_mov_b32_e32 v5, s44
-; GFX9-NEXT: v_add_u32_e32 v4, 0x300, v4
+; GFX9-NEXT: v_perm_b32 v4, s74, v4, v19
; GFX9-NEXT: v_perm_b32 v5, s45, v5, v19
-; GFX9-NEXT: v_add_u32_e32 v5, 0x300, v5
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX9-NEXT: v_add_u32_e32 v4, 0x300, v4
+; GFX9-NEXT: v_add_u32_sdwa v5, v5, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s75, s75, 3
-; GFX9-NEXT: v_lshl_or_b32 v4, v5, 16, v4
+; GFX9-NEXT: v_or_b32_sdwa v4, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v5, s43
-; GFX9-NEXT: v_perm_b32 v5, s75, v5, v19
; GFX9-NEXT: s_add_i32 s42, s42, 3
; GFX9-NEXT: v_mov_b32_e32 v6, s41
-; GFX9-NEXT: v_add_u32_e32 v5, 0x300, v5
+; GFX9-NEXT: v_perm_b32 v5, s75, v5, v19
; GFX9-NEXT: v_perm_b32 v6, s42, v6, v19
-; GFX9-NEXT: v_add_u32_e32 v6, 0x300, v6
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX9-NEXT: v_add_u32_e32 v5, 0x300, v5
+; GFX9-NEXT: v_add_u32_sdwa v6, v6, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s76, s76, 3
-; GFX9-NEXT: v_lshl_or_b32 v5, v6, 16, v5
+; GFX9-NEXT: v_or_b32_sdwa v5, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v6, s40
-; GFX9-NEXT: v_perm_b32 v6, s76, v6, v19
; GFX9-NEXT: s_add_i32 s15, s15, 3
; GFX9-NEXT: v_mov_b32_e32 v7, s14
-; GFX9-NEXT: v_add_u32_e32 v6, 0x300, v6
+; GFX9-NEXT: v_perm_b32 v6, s76, v6, v19
; GFX9-NEXT: v_perm_b32 v7, s15, v7, v19
-; GFX9-NEXT: v_add_u32_e32 v7, 0x300, v7
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; GFX9-NEXT: v_add_u32_e32 v6, 0x300, v6
+; GFX9-NEXT: v_add_u32_sdwa v7, v7, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s73, s73, 3
-; GFX9-NEXT: v_lshl_or_b32 v6, v7, 16, v6
+; GFX9-NEXT: v_or_b32_sdwa v6, v7, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v7, s13
-; GFX9-NEXT: v_perm_b32 v7, s73, v7, v19
; GFX9-NEXT: s_add_i32 s12, s12, 3
; GFX9-NEXT: v_mov_b32_e32 v22, s11
-; GFX9-NEXT: s_add_i32 s72, s72, 3
-; GFX9-NEXT: v_add_u32_e32 v7, 0x300, v7
+; GFX9-NEXT: v_perm_b32 v7, s73, v7, v19
; GFX9-NEXT: v_perm_b32 v22, s12, v22, v19
-; GFX9-NEXT: v_perm_b32 v18, s72, v18, v19
-; GFX9-NEXT: v_add_u32_e32 v22, 0x300, v22
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX9-NEXT: v_add_u32_e32 v7, 0x300, v7
+; GFX9-NEXT: v_add_u32_sdwa v22, v22, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: s_add_i32 s72, s72, 3
+; GFX9-NEXT: s_add_i32 s46, s46, 3
+; GFX9-NEXT: v_or_b32_sdwa v7, v22, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v22, s7
; GFX9-NEXT: s_add_i32 s10, s10, 3
; GFX9-NEXT: v_mov_b32_e32 v23, s9
-; GFX9-NEXT: v_add_u32_e32 v28, 0x300, v12
-; GFX9-NEXT: v_add_u32_e32 v12, 0x300, v14
-; GFX9-NEXT: s_add_i32 s47, s47, 3
-; GFX9-NEXT: v_lshl_or_b32 v7, v22, 16, v7
-; GFX9-NEXT: v_mov_b32_e32 v22, s7
-; GFX9-NEXT: v_perm_b32 v23, s10, v23, v19
; GFX9-NEXT: s_add_i32 s8, s8, 3
; GFX9-NEXT: v_mov_b32_e32 v24, s6
-; GFX9-NEXT: v_add_u32_e32 v27, 0x300, v11
-; GFX9-NEXT: v_add_u32_e32 v14, 0x300, v15
-; GFX9-NEXT: v_add_u32_e32 v11, 0x300, v18
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_perm_b32 v22, s47, v22, v19
+; GFX9-NEXT: v_perm_b32 v18, s72, v18, v19
+; GFX9-NEXT: v_perm_b32 v22, s46, v22, v19
+; GFX9-NEXT: v_perm_b32 v23, s10, v23, v19
; GFX9-NEXT: v_perm_b32 v19, s8, v24, v19
; GFX9-NEXT: v_add_u32_e32 v24, 0x300, v8
-; GFX9-NEXT: v_add_u32_e32 v25, 0x300, v9
+; GFX9-NEXT: v_add_u32_sdwa v25, v9, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_add_u32_e32 v26, 0x300, v10
-; GFX9-NEXT: v_add_u32_e32 v13, 0x300, v13
-; GFX9-NEXT: v_add_u32_e32 v15, 0x300, v16
+; GFX9-NEXT: v_add_u32_sdwa v27, v11, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_add_u32_e32 v28, 0x300, v12
+; GFX9-NEXT: v_add_u32_sdwa v13, v13, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_add_u32_e32 v12, 0x300, v14
+; GFX9-NEXT: v_add_u32_sdwa v14, v15, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_add_u32_e32 v11, 0x300, v18
+; GFX9-NEXT: v_add_u32_sdwa v15, v16, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_add_u32_e32 v10, 0x300, v17
+; GFX9-NEXT: v_add_u32_sdwa v16, v20, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_add_u32_e32 v9, 0x300, v21
+; GFX9-NEXT: v_add_u32_sdwa v17, v22, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_add_u32_e32 v8, 0x300, v23
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v14, 16, v12
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v28
-; GFX9-NEXT: v_add_u32_e32 v16, 0x300, v20
-; GFX9-NEXT: v_add_u32_e32 v17, 0x300, v22
-; GFX9-NEXT: v_add_u32_e32 v18, 0x300, v19
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v15, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v13, v13, 16, v14
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v26
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v24
-; GFX9-NEXT: v_lshl_or_b32 v8, v18, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v17, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v16, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v14, v27, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v25, 16, v15
+; GFX9-NEXT: v_add_u32_sdwa v18, v19, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v8, v18, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v17, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v16, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v15, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v12, v14, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v13, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v14, v27, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v25, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: .LBB107_5: ; %end
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -89792,12 +89747,12 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX9-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v5
; GFX9-NEXT: s_and_b32 s6, s16, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v4, v5, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v4, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v2, s6, v1
+; GFX9-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
; GFX9-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
@@ -89815,9 +89770,9 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v5, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v30, 16, v2
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v30
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v31
; GFX9-NEXT: s_and_b32 s6, s19, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v3, v31, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v3, v2, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v2, s6, v1
; GFX9-NEXT: v_bfe_u32 v5, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v5, v5, v2
@@ -89835,9 +89790,9 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v32, 16, v2
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v29
; GFX9-NEXT: s_and_b32 s6, s18, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v11, v29, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v11, v2, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v2, s6, v1
; GFX9-NEXT: v_bfe_u32 v5, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v5, v5, v2
@@ -89855,9 +89810,9 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v35, 16, v2
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v37
; GFX9-NEXT: s_and_b32 s6, s21, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v10, v37, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v10, v2, v35 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v2, s6, v1
; GFX9-NEXT: v_bfe_u32 v5, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v5, v5, v2
@@ -89875,9 +89830,9 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v28, 16, v2
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v28
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v27
; GFX9-NEXT: s_and_b32 s6, s20, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v15, v27, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v15, v2, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v2, s6, v1
; GFX9-NEXT: v_bfe_u32 v5, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v5, v5, v2
@@ -89895,9 +89850,9 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v50, 16, v2
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v50
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v36
; GFX9-NEXT: s_and_b32 s6, s23, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v14, v36, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v14, v2, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v2, s6, v1
; GFX9-NEXT: v_bfe_u32 v5, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v5, v5, v2
@@ -89915,9 +89870,9 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v34, 16, v2
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v33
; GFX9-NEXT: s_and_b32 s6, s22, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v20, v33, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v20, v2, v34 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v2, s6, v1
; GFX9-NEXT: v_bfe_u32 v5, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v5, v5, v2
@@ -89935,9 +89890,9 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v48, 16, v2
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v48
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v38
; GFX9-NEXT: s_and_b32 s6, s25, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v19, v38, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v19, v2, v48 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v2, s6, v1
; GFX9-NEXT: v_bfe_u32 v5, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v5, v5, v2
@@ -89955,9 +89910,9 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v49, 16, v2
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v49
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v39
; GFX9-NEXT: s_and_b32 s6, s24, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v22, v39, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v22, v2, v49 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v2, s6, v1
; GFX9-NEXT: v_bfe_u32 v5, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v5, v5, v2
@@ -89975,9 +89930,9 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v53, 16, v2
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v53
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v51
; GFX9-NEXT: s_and_b32 s6, s27, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v21, v51, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v21, v2, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v2, s6, v1
; GFX9-NEXT: v_bfe_u32 v5, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v5, v5, v2
@@ -89995,9 +89950,9 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v54, 16, v2
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v54
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v52
; GFX9-NEXT: s_and_b32 s6, s26, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v24, v52, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v24, v2, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v2, s6, v1
; GFX9-NEXT: v_bfe_u32 v5, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v5, v5, v2
@@ -90015,9 +89970,9 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v41, 16, v2
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v41
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v55
; GFX9-NEXT: s_and_b32 s6, s29, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v23, v55, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v23, v2, v41 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v2, s6, v1
; GFX9-NEXT: v_bfe_u32 v5, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v5, v5, v2
@@ -90035,9 +89990,9 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v42, 16, v2
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v42
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v40
; GFX9-NEXT: s_and_b32 s6, s28, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v26, v40, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v26, v2, v42 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v2, s6, v1
; GFX9-NEXT: v_bfe_u32 v5, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v5, v5, v2
@@ -90055,9 +90010,9 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v45, 16, v2
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v45
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v43
; GFX9-NEXT: s_and_b32 s6, s5, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v25, v43, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v25, v2, v45 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v2, s6, v1
; GFX9-NEXT: v_bfe_u32 v5, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v5, v5, v2
@@ -90075,9 +90030,9 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v46, 16, v2
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v46
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v44
; GFX9-NEXT: s_and_b32 s5, s4, 0xffff0000
-; GFX9-NEXT: v_lshl_or_b32 v58, v44, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v58, v2, v46 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v2, s5, v1
; GFX9-NEXT: v_bfe_u32 v5, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v5, v5, v2
@@ -90094,11 +90049,11 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v5, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v56, 16, v1
; GFX9-NEXT: v_lshrrev_b64 v[5:6], 24, v[25:26]
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v56
+; GFX9-NEXT: v_lshrrev_b32_e32 v56, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v47
; GFX9-NEXT: v_lshrrev_b64 v[6:7], 24, v[23:24]
-; GFX9-NEXT: v_lshl_or_b32 v57, v47, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v57, v1, v56 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_lshrrev_b64 v[7:8], 24, v[21:22]
; GFX9-NEXT: v_lshrrev_b64 v[16:17], 24, v[10:11]
; GFX9-NEXT: v_lshrrev_b64 v[1:2], 24, v[57:58]
@@ -95102,11 +95057,11 @@ define inreg <32 x bfloat> @bitcast_v64i8_to_v32bf16_scalar(<64 x i8> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s56, v29
; GFX9-NEXT: v_readfirstlane_b32 s57, v28
; GFX9-NEXT: v_readfirstlane_b32 s61, v27
-; GFX9-NEXT: v_readfirstlane_b32 s63, v26
+; GFX9-NEXT: v_readfirstlane_b32 s62, v26
; GFX9-NEXT: v_readfirstlane_b32 s7, v25
-; GFX9-NEXT: v_readfirstlane_b32 s47, v24
+; GFX9-NEXT: v_readfirstlane_b32 s46, v24
; GFX9-NEXT: v_readfirstlane_b32 s58, v23
-; GFX9-NEXT: v_readfirstlane_b32 s60, v22
+; GFX9-NEXT: v_readfirstlane_b32 s59, v22
; GFX9-NEXT: v_readfirstlane_b32 s6, v21
; GFX9-NEXT: v_readfirstlane_b32 s8, v20
; GFX9-NEXT: v_readfirstlane_b32 s9, v19
@@ -95120,12 +95075,12 @@ define inreg <32 x bfloat> @bitcast_v64i8_to_v32bf16_scalar(<64 x i8> inreg %a,
; GFX9-NEXT: buffer_load_ushort v18, off, s[0:3], s32
; GFX9-NEXT: buffer_load_ushort v24, off, s[0:3], s32 offset:72
; GFX9-NEXT: buffer_load_ushort v27, off, s[0:3], s32 offset:68
-; GFX9-NEXT: buffer_load_ushort v32, off, s[0:3], s32 offset:64
+; GFX9-NEXT: buffer_load_ushort v33, off, s[0:3], s32 offset:64
; GFX9-NEXT: buffer_load_ushort v34, off, s[0:3], s32 offset:60
; GFX9-NEXT: buffer_load_ushort v22, off, s[0:3], s32 offset:56
; GFX9-NEXT: buffer_load_ushort v25, off, s[0:3], s32 offset:52
; GFX9-NEXT: buffer_load_ushort v30, off, s[0:3], s32 offset:48
-; GFX9-NEXT: buffer_load_ushort v33, off, s[0:3], s32 offset:44
+; GFX9-NEXT: buffer_load_ushort v32, off, s[0:3], s32 offset:44
; GFX9-NEXT: buffer_load_ushort v20, off, s[0:3], s32 offset:40
; GFX9-NEXT: buffer_load_ushort v23, off, s[0:3], s32 offset:36
; GFX9-NEXT: buffer_load_ushort v28, off, s[0:3], s32 offset:32
@@ -95145,103 +95100,95 @@ define inreg <32 x bfloat> @bitcast_v64i8_to_v32bf16_scalar(<64 x i8> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s75, v6
; GFX9-NEXT: v_readfirstlane_b32 s44, v5
; GFX9-NEXT: v_readfirstlane_b32 s45, v4
-; GFX9-NEXT: v_readfirstlane_b32 s46, v3
+; GFX9-NEXT: v_readfirstlane_b32 s47, v3
; GFX9-NEXT: v_readfirstlane_b32 s74, v2
-; GFX9-NEXT: v_readfirstlane_b32 s59, v1
-; GFX9-NEXT: v_readfirstlane_b32 s62, v0
+; GFX9-NEXT: v_readfirstlane_b32 s60, v1
+; GFX9-NEXT: v_readfirstlane_b32 s63, v0
; GFX9-NEXT: s_waitcnt vmcnt(19)
; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v12
; GFX9-NEXT: s_and_b64 s[4:5], vcc, exec
; GFX9-NEXT: s_cbranch_scc0 .LBB111_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v8, 0xc0c0004
-; GFX9-NEXT: v_mov_b32_e32 v2, s21
-; GFX9-NEXT: v_perm_b32 v0, s16, v0, v8
; GFX9-NEXT: v_mov_b32_e32 v1, s19
-; GFX9-NEXT: v_perm_b32 v2, s20, v2, v8
; GFX9-NEXT: v_mov_b32_e32 v3, s23
+; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_perm_b32 v1, s18, v1, v8
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: v_mov_b32_e32 v2, s21
; GFX9-NEXT: v_perm_b32 v3, s22, v3, v8
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v3, 16, v2
-; GFX9-NEXT: v_mov_b32_e32 v2, s25
-; GFX9-NEXT: v_perm_b32 v2, s24, v2, v8
+; GFX9-NEXT: v_perm_b32 v0, s16, v0, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_perm_b32 v2, s20, v2, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v3, s27
+; GFX9-NEXT: v_mov_b32_e32 v2, s25
; GFX9-NEXT: v_perm_b32 v3, s26, v3, v8
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX9-NEXT: v_perm_b32 v2, s24, v2, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_mov_b32_e32 v4, s60
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v3, s29
+; GFX9-NEXT: v_perm_b32 v4, s63, v4, v8
; GFX9-NEXT: v_perm_b32 v3, s28, v3, v8
-; GFX9-NEXT: v_mov_b32_e32 v4, s59
-; GFX9-NEXT: v_perm_b32 v4, s62, v4, v8
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_lshl_or_b32 v3, v4, 16, v3
-; GFX9-NEXT: v_mov_b32_e32 v4, s46
-; GFX9-NEXT: v_perm_b32 v4, s74, v4, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX9-NEXT: v_mov_b32_e32 v5, s44
+; GFX9-NEXT: v_or_b32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v4, s47
; GFX9-NEXT: v_perm_b32 v5, s45, v5, v8
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_lshl_or_b32 v4, v5, 16, v4
-; GFX9-NEXT: v_mov_b32_e32 v5, s43
-; GFX9-NEXT: v_perm_b32 v5, s75, v5, v8
+; GFX9-NEXT: v_perm_b32 v4, s74, v4, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX9-NEXT: v_mov_b32_e32 v6, s41
+; GFX9-NEXT: v_or_b32_sdwa v4, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v5, s43
; GFX9-NEXT: v_perm_b32 v6, s42, v6, v8
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_lshl_or_b32 v5, v6, 16, v5
-; GFX9-NEXT: v_mov_b32_e32 v6, s40
-; GFX9-NEXT: v_perm_b32 v6, s76, v6, v8
+; GFX9-NEXT: v_perm_b32 v5, s75, v5, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v6
; GFX9-NEXT: v_mov_b32_e32 v7, s14
+; GFX9-NEXT: v_or_b32_sdwa v5, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v6, s40
; GFX9-NEXT: v_perm_b32 v7, s15, v7, v8
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_lshl_or_b32 v6, v7, 16, v6
-; GFX9-NEXT: v_mov_b32_e32 v7, s13
-; GFX9-NEXT: v_perm_b32 v7, s73, v7, v8
+; GFX9-NEXT: v_perm_b32 v6, s76, v6, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v7
; GFX9-NEXT: v_mov_b32_e32 v9, s11
+; GFX9-NEXT: v_or_b32_sdwa v6, v7, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v7, s13
; GFX9-NEXT: v_perm_b32 v9, s12, v9, v8
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_lshl_or_b32 v7, v9, 16, v7
+; GFX9-NEXT: v_perm_b32 v7, s73, v7, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX9-NEXT: s_and_b32 s4, s46, 0xff
+; GFX9-NEXT: s_lshl_b32 s5, s7, 8
+; GFX9-NEXT: v_or_b32_sdwa v7, v9, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v9, s58
+; GFX9-NEXT: s_or_b32 s4, s4, s5
+; GFX9-NEXT: v_perm_b32 v9, s59, v9, v8
+; GFX9-NEXT: v_lshlrev_b32_e64 v10, 16, s4
+; GFX9-NEXT: v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(18)
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v16
+; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v16
; GFX9-NEXT: s_waitcnt vmcnt(16)
-; GFX9-NEXT: v_perm_b32 v10, s72, v18, v8
-; GFX9-NEXT: v_or_b32_sdwa v9, v17, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v9, 16, v10
+; GFX9-NEXT: v_perm_b32 v8, s72, v18, v8
+; GFX9-NEXT: v_or_b32_sdwa v10, v17, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v11, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v26
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_or_b32_sdwa v9, v29, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v26
; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v19
-; GFX9-NEXT: v_or_b32_sdwa v10, v21, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_or_b32_sdwa v8, v29, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v10, v21, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v12, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_mov_b32 s4, 0xc0c0004
-; GFX9-NEXT: v_lshl_or_b32 v12, v10, 16, v9
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v20
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v20
+; GFX9-NEXT: v_or_b32_sdwa v8, v23, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_perm_b32 v10, v31, v28, s4
-; GFX9-NEXT: v_or_b32_sdwa v9, v23, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v13, v9, 16, v10
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v30
-; GFX9-NEXT: v_or_b32_sdwa v9, v33, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v13, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v30
; GFX9-NEXT: v_lshlrev_b32_e32 v10, 8, v22
-; GFX9-NEXT: v_or_b32_sdwa v10, v25, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_lshl_or_b32 v14, v10, 16, v9
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 8, v24
-; GFX9-NEXT: v_perm_b32 v10, v34, v32, s4
-; GFX9-NEXT: v_or_b32_sdwa v9, v27, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v15, v9, 16, v10
-; GFX9-NEXT: v_mov_b32_e32 v9, s58
-; GFX9-NEXT: v_perm_b32 v8, s60, v9, v8
-; GFX9-NEXT: s_and_b32 s4, s47, 0xff
-; GFX9-NEXT: s_lshl_b32 s5, s7, 8
-; GFX9-NEXT: s_or_b32 s4, s4, s5
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, s4, 16, v8
+; GFX9-NEXT: v_or_b32_sdwa v8, v32, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v10, v25, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v14, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_perm_b32 v10, v34, v33, s4
; GFX9-NEXT: s_and_b32 s4, s10, 0xff
; GFX9-NEXT: s_lshl_b32 s5, s9, 8
; GFX9-NEXT: s_or_b32 s4, s4, s5
@@ -95249,13 +95196,16 @@ define inreg <32 x bfloat> @bitcast_v64i8_to_v32bf16_scalar(<64 x i8> inreg %a,
; GFX9-NEXT: s_lshl_b32 s77, s6, 8
; GFX9-NEXT: s_or_b32 s5, s5, s77
; GFX9-NEXT: s_pack_ll_b32_b16 s4, s4, s5
-; GFX9-NEXT: s_and_b32 s5, s63, 0xff
+; GFX9-NEXT: s_and_b32 s5, s62, 0xff
; GFX9-NEXT: s_lshl_b32 s77, s61, 8
; GFX9-NEXT: s_or_b32 s5, s5, s77
; GFX9-NEXT: s_and_b32 s77, s57, 0xff
; GFX9-NEXT: s_lshl_b32 s78, s56, 8
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 8, v24
; GFX9-NEXT: s_or_b32 s77, s77, s78
+; GFX9-NEXT: v_or_b32_sdwa v8, v27, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: s_pack_ll_b32_b16 s5, s5, s77
+; GFX9-NEXT: v_or_b32_sdwa v15, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v8, s4
; GFX9-NEXT: v_mov_b32_e32 v10, s5
; GFX9-NEXT: s_mov_b64 s[4:5], 0
@@ -95272,11 +95222,11 @@ define inreg <32 x bfloat> @bitcast_v64i8_to_v32bf16_scalar(<64 x i8> inreg %a,
; GFX9-NEXT: s_waitcnt vmcnt(12)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v34
; GFX9-NEXT: s_mov_b32 s4, 0xc0c0004
-; GFX9-NEXT: v_perm_b32 v8, v0, v32, s4
+; GFX9-NEXT: v_perm_b32 v8, v0, v33, s4
; GFX9-NEXT: v_add_u32_e32 v0, 3, v27
; GFX9-NEXT: v_perm_b32 v9, v0, v24, s4
; GFX9-NEXT: s_waitcnt vmcnt(8)
-; GFX9-NEXT: v_add_u32_e32 v0, 3, v33
+; GFX9-NEXT: v_add_u32_e32 v0, 3, v32
; GFX9-NEXT: v_perm_b32 v10, v0, v30, s4
; GFX9-NEXT: v_add_u32_e32 v0, 3, v25
; GFX9-NEXT: v_perm_b32 v11, v0, v22, s4
@@ -95293,138 +95243,123 @@ define inreg <32 x bfloat> @bitcast_v64i8_to_v32bf16_scalar(<64 x i8> inreg %a,
; GFX9-NEXT: v_add_u32_e32 v0, 3, v17
; GFX9-NEXT: v_mov_b32_e32 v19, 0xc0c0004
; GFX9-NEXT: v_perm_b32 v16, v0, v16, s4
-; GFX9-NEXT: s_add_i32 s63, s63, 3
+; GFX9-NEXT: s_add_i32 s62, s62, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s61
-; GFX9-NEXT: v_perm_b32 v17, s63, v0, v19
+; GFX9-NEXT: v_perm_b32 v17, s62, v0, v19
; GFX9-NEXT: s_add_i32 s57, s57, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s56
; GFX9-NEXT: v_perm_b32 v20, s57, v0, v19
-; GFX9-NEXT: s_add_i32 s60, s60, 3
+; GFX9-NEXT: s_add_i32 s59, s59, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s58
-; GFX9-NEXT: v_perm_b32 v21, s60, v0, v19
+; GFX9-NEXT: v_perm_b32 v21, s59, v0, v19
; GFX9-NEXT: s_add_i32 s20, s20, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s21
; GFX9-NEXT: v_perm_b32 v0, s20, v0, v19
; GFX9-NEXT: v_add_u32_e32 v1, 0x300, v0
; GFX9-NEXT: s_add_i32 s22, s22, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s23
+; GFX9-NEXT: s_movk_i32 s5, 0x300
; GFX9-NEXT: v_perm_b32 v0, s22, v0, v19
-; GFX9-NEXT: v_add_u32_e32 v2, 0x300, v0
+; GFX9-NEXT: v_add_u32_sdwa v2, v0, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s16, s16, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s17
-; GFX9-NEXT: v_perm_b32 v0, s16, v0, v19
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: v_mov_b32_e32 v3, s19
-; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
+; GFX9-NEXT: v_perm_b32 v0, s16, v0, v19
; GFX9-NEXT: v_perm_b32 v3, s18, v3, v19
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
+; GFX9-NEXT: v_add_u32_sdwa v3, v3, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s24, s24, 3
-; GFX9-NEXT: v_add_u32_e32 v3, 0x300, v3
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v2, s25
-; GFX9-NEXT: v_lshl_or_b32 v0, v3, 16, v0
-; GFX9-NEXT: v_perm_b32 v2, s24, v2, v19
; GFX9-NEXT: s_add_i32 s26, s26, 3
; GFX9-NEXT: v_mov_b32_e32 v3, s27
-; GFX9-NEXT: v_add_u32_e32 v2, 0x300, v2
+; GFX9-NEXT: v_perm_b32 v2, s24, v2, v19
; GFX9-NEXT: v_perm_b32 v3, s26, v3, v19
-; GFX9-NEXT: v_add_u32_e32 v3, 0x300, v3
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX9-NEXT: v_add_u32_e32 v2, 0x300, v2
+; GFX9-NEXT: v_add_u32_sdwa v3, v3, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s28, s28, 3
-; GFX9-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v3, s29
+; GFX9-NEXT: s_add_i32 s63, s63, 3
+; GFX9-NEXT: v_mov_b32_e32 v4, s60
; GFX9-NEXT: v_perm_b32 v3, s28, v3, v19
-; GFX9-NEXT: s_add_i32 s62, s62, 3
-; GFX9-NEXT: v_mov_b32_e32 v4, s59
+; GFX9-NEXT: v_perm_b32 v4, s63, v4, v19
; GFX9-NEXT: v_add_u32_e32 v3, 0x300, v3
-; GFX9-NEXT: v_perm_b32 v4, s62, v4, v19
-; GFX9-NEXT: v_add_u32_e32 v4, 0x300, v4
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX9-NEXT: v_add_u32_sdwa v4, v4, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s74, s74, 3
-; GFX9-NEXT: v_lshl_or_b32 v3, v4, 16, v3
-; GFX9-NEXT: v_mov_b32_e32 v4, s46
-; GFX9-NEXT: v_perm_b32 v4, s74, v4, v19
+; GFX9-NEXT: v_or_b32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v4, s47
; GFX9-NEXT: s_add_i32 s45, s45, 3
; GFX9-NEXT: v_mov_b32_e32 v5, s44
-; GFX9-NEXT: v_add_u32_e32 v4, 0x300, v4
+; GFX9-NEXT: v_perm_b32 v4, s74, v4, v19
; GFX9-NEXT: v_perm_b32 v5, s45, v5, v19
-; GFX9-NEXT: v_add_u32_e32 v5, 0x300, v5
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX9-NEXT: v_add_u32_e32 v4, 0x300, v4
+; GFX9-NEXT: v_add_u32_sdwa v5, v5, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s75, s75, 3
-; GFX9-NEXT: v_lshl_or_b32 v4, v5, 16, v4
+; GFX9-NEXT: v_or_b32_sdwa v4, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v5, s43
-; GFX9-NEXT: v_perm_b32 v5, s75, v5, v19
; GFX9-NEXT: s_add_i32 s42, s42, 3
; GFX9-NEXT: v_mov_b32_e32 v6, s41
-; GFX9-NEXT: v_add_u32_e32 v5, 0x300, v5
+; GFX9-NEXT: v_perm_b32 v5, s75, v5, v19
; GFX9-NEXT: v_perm_b32 v6, s42, v6, v19
-; GFX9-NEXT: v_add_u32_e32 v6, 0x300, v6
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX9-NEXT: v_add_u32_e32 v5, 0x300, v5
+; GFX9-NEXT: v_add_u32_sdwa v6, v6, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s76, s76, 3
-; GFX9-NEXT: v_lshl_or_b32 v5, v6, 16, v5
+; GFX9-NEXT: v_or_b32_sdwa v5, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v6, s40
-; GFX9-NEXT: v_perm_b32 v6, s76, v6, v19
; GFX9-NEXT: s_add_i32 s15, s15, 3
; GFX9-NEXT: v_mov_b32_e32 v7, s14
-; GFX9-NEXT: v_add_u32_e32 v6, 0x300, v6
+; GFX9-NEXT: v_perm_b32 v6, s76, v6, v19
; GFX9-NEXT: v_perm_b32 v7, s15, v7, v19
-; GFX9-NEXT: v_add_u32_e32 v7, 0x300, v7
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; GFX9-NEXT: v_add_u32_e32 v6, 0x300, v6
+; GFX9-NEXT: v_add_u32_sdwa v7, v7, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_add_i32 s73, s73, 3
-; GFX9-NEXT: v_lshl_or_b32 v6, v7, 16, v6
+; GFX9-NEXT: v_or_b32_sdwa v6, v7, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v7, s13
-; GFX9-NEXT: v_perm_b32 v7, s73, v7, v19
; GFX9-NEXT: s_add_i32 s12, s12, 3
; GFX9-NEXT: v_mov_b32_e32 v22, s11
-; GFX9-NEXT: s_add_i32 s72, s72, 3
-; GFX9-NEXT: v_add_u32_e32 v7, 0x300, v7
+; GFX9-NEXT: v_perm_b32 v7, s73, v7, v19
; GFX9-NEXT: v_perm_b32 v22, s12, v22, v19
-; GFX9-NEXT: v_perm_b32 v18, s72, v18, v19
-; GFX9-NEXT: v_add_u32_e32 v22, 0x300, v22
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX9-NEXT: v_add_u32_e32 v7, 0x300, v7
+; GFX9-NEXT: v_add_u32_sdwa v22, v22, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: s_add_i32 s72, s72, 3
+; GFX9-NEXT: s_add_i32 s46, s46, 3
+; GFX9-NEXT: v_or_b32_sdwa v7, v22, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_mov_b32_e32 v22, s7
; GFX9-NEXT: s_add_i32 s10, s10, 3
; GFX9-NEXT: v_mov_b32_e32 v23, s9
-; GFX9-NEXT: v_add_u32_e32 v28, 0x300, v12
-; GFX9-NEXT: v_add_u32_e32 v12, 0x300, v14
-; GFX9-NEXT: s_add_i32 s47, s47, 3
-; GFX9-NEXT: v_lshl_or_b32 v7, v22, 16, v7
-; GFX9-NEXT: v_mov_b32_e32 v22, s7
-; GFX9-NEXT: v_perm_b32 v23, s10, v23, v19
; GFX9-NEXT: s_add_i32 s8, s8, 3
; GFX9-NEXT: v_mov_b32_e32 v24, s6
-; GFX9-NEXT: v_add_u32_e32 v27, 0x300, v11
-; GFX9-NEXT: v_add_u32_e32 v14, 0x300, v15
-; GFX9-NEXT: v_add_u32_e32 v11, 0x300, v18
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_perm_b32 v22, s47, v22, v19
+; GFX9-NEXT: v_perm_b32 v18, s72, v18, v19
+; GFX9-NEXT: v_perm_b32 v22, s46, v22, v19
+; GFX9-NEXT: v_perm_b32 v23, s10, v23, v19
; GFX9-NEXT: v_perm_b32 v19, s8, v24, v19
; GFX9-NEXT: v_add_u32_e32 v24, 0x300, v8
-; GFX9-NEXT: v_add_u32_e32 v25, 0x300, v9
+; GFX9-NEXT: v_add_u32_sdwa v25, v9, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_add_u32_e32 v26, 0x300, v10
-; GFX9-NEXT: v_add_u32_e32 v13, 0x300, v13
-; GFX9-NEXT: v_add_u32_e32 v15, 0x300, v16
+; GFX9-NEXT: v_add_u32_sdwa v27, v11, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_add_u32_e32 v28, 0x300, v12
+; GFX9-NEXT: v_add_u32_sdwa v13, v13, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_add_u32_e32 v12, 0x300, v14
+; GFX9-NEXT: v_add_u32_sdwa v14, v15, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_add_u32_e32 v11, 0x300, v18
+; GFX9-NEXT: v_add_u32_sdwa v15, v16, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_add_u32_e32 v10, 0x300, v17
+; GFX9-NEXT: v_add_u32_sdwa v16, v20, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_add_u32_e32 v9, 0x300, v21
+; GFX9-NEXT: v_add_u32_sdwa v17, v22, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_add_u32_e32 v8, 0x300, v23
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v14, 16, v12
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v28
-; GFX9-NEXT: v_add_u32_e32 v16, 0x300, v20
-; GFX9-NEXT: v_add_u32_e32 v17, 0x300, v22
-; GFX9-NEXT: v_add_u32_e32 v18, 0x300, v19
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v15, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v13, v13, 16, v14
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v26
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v24
-; GFX9-NEXT: v_lshl_or_b32 v8, v18, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v17, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v16, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v14, v27, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v25, 16, v15
+; GFX9-NEXT: v_add_u32_sdwa v18, v19, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v8, v18, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v17, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v16, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v15, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v12, v14, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v13, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v14, v27, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v25, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: .LBB111_5: ; %end
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.576bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.576bit.ll
index 98632843b3abd..9b079dccf1023 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.576bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.576bit.ll
@@ -11585,42 +11585,42 @@ define inreg <36 x i16> @bitcast_v18f32_to_v36i16_scalar(<18 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v19, s11
; GFX9-NEXT: v_mov_b32_e32 v18, s10
; GFX9-NEXT: .LBB29_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_lshl_or_b32 v0, v35, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v34, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v33, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v32, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v31, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v30, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v29, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v28, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v27, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v26, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v25, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v24, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v23, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v22, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v21, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v20, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v19, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v18, 16, v17
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; GFX9-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; GFX9-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; GFX9-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX9-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; GFX9-NEXT: v_lshlrev_b32_e32 v20, 16, v20
+; GFX9-NEXT: v_lshlrev_b32_e32 v19, 16, v19
+; GFX9-NEXT: v_lshlrev_b32_e32 v18, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v0, v35, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v34, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v33, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v32, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v4, v31, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v30, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v29, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v28, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v27, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v26, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v25, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v24, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v12, v23, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v22, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v14, v21, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v20, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v16, v19, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v18, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: bitcast_v18f32_to_v36i16_scalar:
@@ -14553,42 +14553,42 @@ define inreg <36 x half> @bitcast_v18f32_to_v36f16_scalar(<18 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v19, s11
; GFX9-NEXT: v_mov_b32_e32 v18, s10
; GFX9-NEXT: .LBB33_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_lshl_or_b32 v0, v35, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v34, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v33, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v32, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v31, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v30, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v29, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v28, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v27, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v26, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v25, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v24, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v23, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v22, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v21, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v20, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v19, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v18, 16, v17
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; GFX9-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; GFX9-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; GFX9-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX9-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; GFX9-NEXT: v_lshlrev_b32_e32 v20, 16, v20
+; GFX9-NEXT: v_lshlrev_b32_e32 v19, 16, v19
+; GFX9-NEXT: v_lshlrev_b32_e32 v18, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v0, v35, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v34, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v33, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v32, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v4, v31, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v30, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v29, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v28, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v27, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v26, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v25, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v24, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v12, v23, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v22, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v14, v21, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v20, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v16, v19, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v18, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: bitcast_v18f32_to_v36f16_scalar:
@@ -24236,42 +24236,42 @@ define inreg <36 x i16> @bitcast_v9f64_to_v36i16_scalar(<9 x double> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v22, s11
; GFX9-NEXT: v_mov_b32_e32 v20, s10
; GFX9-NEXT: .LBB49_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_lshl_or_b32 v0, v33, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v35, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v31, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v34, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v29, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v32, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v28, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v30, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v27, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v26, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v23, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v25, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v21, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v24, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v19, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v22, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v18, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v20, 16, v17
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; GFX9-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; GFX9-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; GFX9-NEXT: v_lshlrev_b32_e32 v19, 16, v19
+; GFX9-NEXT: v_lshlrev_b32_e32 v18, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v0, v33, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v35
+; GFX9-NEXT: v_or_b32_sdwa v2, v31, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v34
+; GFX9-NEXT: v_or_b32_sdwa v4, v29, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v32
+; GFX9-NEXT: v_or_b32_sdwa v6, v28, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v30
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX9-NEXT: v_or_b32_sdwa v10, v23, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v23, 16, v25
+; GFX9-NEXT: v_or_b32_sdwa v12, v21, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v21, 16, v24
+; GFX9-NEXT: v_or_b32_sdwa v14, v19, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v19, 16, v22
+; GFX9-NEXT: v_or_b32_sdwa v16, v18, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v18, 16, v20
+; GFX9-NEXT: v_or_b32_sdwa v1, v33, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v31, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v29, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v28, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v27, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v26, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v23, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v21, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v19, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v18, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: bitcast_v9f64_to_v36i16_scalar:
@@ -27132,42 +27132,42 @@ define inreg <36 x half> @bitcast_v9f64_to_v36f16_scalar(<9 x double> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v22, s11
; GFX9-NEXT: v_mov_b32_e32 v20, s10
; GFX9-NEXT: .LBB53_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_lshl_or_b32 v0, v33, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v35, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v31, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v34, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v29, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v32, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v28, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v30, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v27, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v26, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v23, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v25, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v21, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v24, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v19, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v22, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v18, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v20, 16, v17
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; GFX9-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; GFX9-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; GFX9-NEXT: v_lshlrev_b32_e32 v19, 16, v19
+; GFX9-NEXT: v_lshlrev_b32_e32 v18, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v0, v33, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v35
+; GFX9-NEXT: v_or_b32_sdwa v2, v31, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v34
+; GFX9-NEXT: v_or_b32_sdwa v4, v29, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v32
+; GFX9-NEXT: v_or_b32_sdwa v6, v28, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v30
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX9-NEXT: v_or_b32_sdwa v10, v23, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v23, 16, v25
+; GFX9-NEXT: v_or_b32_sdwa v12, v21, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v21, 16, v24
+; GFX9-NEXT: v_or_b32_sdwa v14, v19, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v19, 16, v22
+; GFX9-NEXT: v_or_b32_sdwa v16, v18, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v18, 16, v20
+; GFX9-NEXT: v_or_b32_sdwa v1, v33, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v31, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v29, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v28, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v27, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v26, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v23, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v21, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v19, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v18, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: bitcast_v9f64_to_v36f16_scalar:
@@ -30604,42 +30604,42 @@ define inreg <36 x half> @bitcast_v36i16_to_v36f16_scalar(<36 x i16> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v34, s7
; GFX9-NEXT: v_mov_b32_e32 v35, s6
; GFX9-NEXT: .LBB57_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_lshl_or_b32 v0, v35, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v34, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v33, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v32, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v31, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v30, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v29, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v28, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v27, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v26, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v25, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v24, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v23, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v22, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v21, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v20, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v19, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v18, 16, v17
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; GFX9-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; GFX9-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; GFX9-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX9-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; GFX9-NEXT: v_lshlrev_b32_e32 v20, 16, v20
+; GFX9-NEXT: v_lshlrev_b32_e32 v19, 16, v19
+; GFX9-NEXT: v_lshlrev_b32_e32 v18, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v0, v35, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v34, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v33, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v32, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v4, v31, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v30, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v29, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v28, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v27, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v26, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v25, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v24, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v12, v23, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v22, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v14, v21, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v20, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v16, v19, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v18, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: bitcast_v36i16_to_v36f16_scalar:
@@ -32200,42 +32200,42 @@ define inreg <36 x i16> @bitcast_v36f16_to_v36i16_scalar(<36 x half> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v34, s7
; GFX9-NEXT: v_mov_b32_e32 v35, s6
; GFX9-NEXT: .LBB59_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_lshl_or_b32 v0, v35, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v34, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v33, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v32, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v31, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v30, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v29, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v28, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v27, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v26, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v25, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v24, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v23, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v22, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v21, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v20, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v19, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v18, 16, v17
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; GFX9-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; GFX9-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; GFX9-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX9-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; GFX9-NEXT: v_lshlrev_b32_e32 v20, 16, v20
+; GFX9-NEXT: v_lshlrev_b32_e32 v19, 16, v19
+; GFX9-NEXT: v_lshlrev_b32_e32 v18, 16, v18
+; GFX9-NEXT: v_or_b32_sdwa v0, v35, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v34, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v33, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v32, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v4, v31, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v30, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v29, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v28, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v27, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v26, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v25, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v24, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v12, v23, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v22, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v14, v21, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v20, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v16, v19, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v18, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: bitcast_v36f16_to_v36i16_scalar:
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.640bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.640bit.ll
index 04dfa46f6dba9..78b54755887fc 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.640bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.640bit.ll
@@ -12642,46 +12642,46 @@ define inreg <40 x i16> @bitcast_v20f32_to_v40i16_scalar(<20 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v21, s13
; GFX9-NEXT: v_mov_b32_e32 v20, s12
; GFX9-NEXT: .LBB29_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_lshl_or_b32 v0, v39, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v38, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v37, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v36, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v35, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v34, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v33, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v32, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v31, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v30, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v29, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v28, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v27, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v26, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v25, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v24, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v23, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v22, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v21, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v20, 16, v19
+; GFX9-NEXT: v_lshlrev_b32_e32 v39, 16, v39
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; GFX9-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; GFX9-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; GFX9-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; GFX9-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX9-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; GFX9-NEXT: v_lshlrev_b32_e32 v20, 16, v20
+; GFX9-NEXT: v_or_b32_sdwa v0, v39, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v38, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v37, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v36, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v4, v35, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v34, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v33, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v32, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v31, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v30, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v29, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v28, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v12, v27, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v26, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v14, v25, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v24, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v16, v23, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v22, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v18, v21, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v20, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: bitcast_v20f32_to_v40i16_scalar:
@@ -15933,46 +15933,46 @@ define inreg <40 x half> @bitcast_v20f32_to_v40f16_scalar(<20 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v21, s13
; GFX9-NEXT: v_mov_b32_e32 v20, s12
; GFX9-NEXT: .LBB33_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_lshl_or_b32 v0, v39, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v38, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v37, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v36, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v35, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v34, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v33, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v32, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v31, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v30, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v29, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v28, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v27, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v26, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v25, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v24, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v23, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v22, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v21, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v20, 16, v19
+; GFX9-NEXT: v_lshlrev_b32_e32 v39, 16, v39
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; GFX9-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; GFX9-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; GFX9-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; GFX9-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX9-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; GFX9-NEXT: v_lshlrev_b32_e32 v20, 16, v20
+; GFX9-NEXT: v_or_b32_sdwa v0, v39, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v38, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v37, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v36, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v4, v35, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v34, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v33, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v32, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v31, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v30, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v29, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v28, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v12, v27, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v26, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v14, v25, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v24, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v16, v23, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v22, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v18, v21, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v20, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: bitcast_v20f32_to_v40f16_scalar:
@@ -26622,46 +26622,46 @@ define inreg <40 x i16> @bitcast_v10f64_to_v40i16_scalar(<10 x double> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v24, s13
; GFX9-NEXT: v_mov_b32_e32 v22, s12
; GFX9-NEXT: .LBB49_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_lshl_or_b32 v0, v37, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v39, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v35, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v38, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v33, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v36, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v32, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v34, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v31, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v30, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v27, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v29, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v25, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v28, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v23, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v26, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v21, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v24, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v20, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v22, 16, v19
+; GFX9-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; GFX9-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; GFX9-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; GFX9-NEXT: v_lshlrev_b32_e32 v20, 16, v20
+; GFX9-NEXT: v_or_b32_sdwa v0, v37, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v37, 16, v39
+; GFX9-NEXT: v_or_b32_sdwa v2, v35, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v38
+; GFX9-NEXT: v_or_b32_sdwa v4, v33, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v36
+; GFX9-NEXT: v_or_b32_sdwa v6, v32, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; GFX9-NEXT: v_or_b32_sdwa v10, v27, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v29
+; GFX9-NEXT: v_or_b32_sdwa v12, v25, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v28
+; GFX9-NEXT: v_or_b32_sdwa v14, v23, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v23, 16, v26
+; GFX9-NEXT: v_or_b32_sdwa v16, v21, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v21, 16, v24
+; GFX9-NEXT: v_or_b32_sdwa v18, v20, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v20, 16, v22
+; GFX9-NEXT: v_or_b32_sdwa v1, v37, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v35, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v33, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v32, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v31, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v30, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v27, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v25, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v23, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v21, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v20, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: bitcast_v10f64_to_v40i16_scalar:
@@ -29833,46 +29833,46 @@ define inreg <40 x half> @bitcast_v10f64_to_v40f16_scalar(<10 x double> inreg %a
; GFX9-NEXT: v_mov_b32_e32 v24, s13
; GFX9-NEXT: v_mov_b32_e32 v22, s12
; GFX9-NEXT: .LBB53_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_lshl_or_b32 v0, v37, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v39, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v35, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v38, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v33, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v36, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v32, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v34, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v31, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v30, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v27, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v29, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v25, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v28, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v23, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v26, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v21, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v24, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v20, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v22, 16, v19
+; GFX9-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; GFX9-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; GFX9-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; GFX9-NEXT: v_lshlrev_b32_e32 v20, 16, v20
+; GFX9-NEXT: v_or_b32_sdwa v0, v37, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v37, 16, v39
+; GFX9-NEXT: v_or_b32_sdwa v2, v35, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v38
+; GFX9-NEXT: v_or_b32_sdwa v4, v33, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v36
+; GFX9-NEXT: v_or_b32_sdwa v6, v32, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; GFX9-NEXT: v_or_b32_sdwa v10, v27, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v29
+; GFX9-NEXT: v_or_b32_sdwa v12, v25, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v28
+; GFX9-NEXT: v_or_b32_sdwa v14, v23, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v23, 16, v26
+; GFX9-NEXT: v_or_b32_sdwa v16, v21, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v21, 16, v24
+; GFX9-NEXT: v_or_b32_sdwa v18, v20, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v20, 16, v22
+; GFX9-NEXT: v_or_b32_sdwa v1, v37, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v35, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v33, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v32, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v31, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v30, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v27, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v25, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v23, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v21, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v20, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: bitcast_v10f64_to_v40f16_scalar:
@@ -33733,46 +33733,46 @@ define inreg <40 x half> @bitcast_v40i16_to_v40f16_scalar(<40 x i16> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v38, s7
; GFX9-NEXT: v_mov_b32_e32 v39, s6
; GFX9-NEXT: .LBB57_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_lshl_or_b32 v0, v39, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v38, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v37, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v36, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v35, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v34, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v33, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v32, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v31, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v30, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v29, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v28, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v27, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v26, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v25, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v24, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v23, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v22, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v21, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v20, 16, v19
+; GFX9-NEXT: v_lshlrev_b32_e32 v39, 16, v39
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; GFX9-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; GFX9-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; GFX9-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; GFX9-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX9-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; GFX9-NEXT: v_lshlrev_b32_e32 v20, 16, v20
+; GFX9-NEXT: v_or_b32_sdwa v0, v39, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v38, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v37, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v36, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v4, v35, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v34, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v33, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v32, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v31, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v30, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v29, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v28, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v12, v27, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v26, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v14, v25, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v24, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v16, v23, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v22, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v18, v21, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v20, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: bitcast_v40i16_to_v40f16_scalar:
@@ -35501,46 +35501,46 @@ define inreg <40 x i16> @bitcast_v40f16_to_v40i16_scalar(<40 x half> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v38, s7
; GFX9-NEXT: v_mov_b32_e32 v39, s6
; GFX9-NEXT: .LBB59_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_lshl_or_b32 v0, v39, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v38, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v37, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v36, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v35, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v34, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v33, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v32, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v31, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v30, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v29, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v28, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v27, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v26, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v25, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v24, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v23, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v22, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v21, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v20, 16, v19
+; GFX9-NEXT: v_lshlrev_b32_e32 v39, 16, v39
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; GFX9-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; GFX9-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; GFX9-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; GFX9-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX9-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; GFX9-NEXT: v_lshlrev_b32_e32 v20, 16, v20
+; GFX9-NEXT: v_or_b32_sdwa v0, v39, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v38, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v37, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v36, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v4, v35, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v34, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v33, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v32, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v31, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v30, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v29, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v28, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v12, v27, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v26, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v14, v25, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v24, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v16, v23, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v22, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v18, v21, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v20, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: bitcast_v40f16_to_v40i16_scalar:
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll
index 2c4eb8b3f66d2..275118e33eeab 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll
@@ -2642,33 +2642,32 @@ define inreg i64 @bitcast_v4bf16_to_i64_scalar(<4 x bfloat> inreg %a, i32 inreg
; GFX9-NEXT: v_add_f32_e32 v2, s4, v1
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX9-NEXT: v_mov_b32_e32 v3, 0xffff
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s17
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v2, s4, v1
-; GFX9-NEXT: v_bfe_u32 v4, v2, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v4, v4, v2
+; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
; GFX9-NEXT: s_lshl_b32 s4, s17, 16
-; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
-; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v2
+; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_add_f32_e32 v1, s4, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc
-; GFX9-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v4, v4, v1
-; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
-; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v5, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v3, v3, v1
; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX9-NEXT: v_and_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v1
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_setpc_b64 s[30:31]
; GFX9-NEXT: .LBB23_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
@@ -6072,33 +6071,32 @@ define inreg double @bitcast_v4bf16_to_f64_scalar(<4 x bfloat> inreg %a, i32 inr
; GFX9-NEXT: v_add_f32_e32 v2, s4, v1
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX9-NEXT: v_mov_b32_e32 v3, 0xffff
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s17
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v2, s4, v1
-; GFX9-NEXT: v_bfe_u32 v4, v2, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v4, v4, v2
+; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
; GFX9-NEXT: s_lshl_b32 s4, s17, 16
-; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
-; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v2
+; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_add_f32_e32 v1, s4, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc
-; GFX9-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v4, v4, v1
-; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
-; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v5, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v3, v3, v1
; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX9-NEXT: v_and_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v1
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_setpc_b64 s[30:31]
; GFX9-NEXT: .LBB47_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
@@ -9154,33 +9152,32 @@ define inreg <2 x i32> @bitcast_v4bf16_to_v2i32_scalar(<4 x bfloat> inreg %a, i3
; GFX9-NEXT: v_add_f32_e32 v2, s4, v1
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX9-NEXT: v_mov_b32_e32 v3, 0xffff
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s17
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v2, s4, v1
-; GFX9-NEXT: v_bfe_u32 v4, v2, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v4, v4, v2
+; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
; GFX9-NEXT: s_lshl_b32 s4, s17, 16
-; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
-; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v2
+; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_add_f32_e32 v1, s4, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc
-; GFX9-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v4, v4, v1
-; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
-; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v5, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v3, v3, v1
; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX9-NEXT: v_and_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v1
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_setpc_b64 s[30:31]
; GFX9-NEXT: .LBB67_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
@@ -11879,33 +11876,32 @@ define inreg <2 x float> @bitcast_v4bf16_to_v2f32_scalar(<4 x bfloat> inreg %a,
; GFX9-NEXT: v_add_f32_e32 v2, s4, v1
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX9-NEXT: v_mov_b32_e32 v3, 0xffff
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s17
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v2, s4, v1
-; GFX9-NEXT: v_bfe_u32 v4, v2, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v4, v4, v2
+; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
; GFX9-NEXT: s_lshl_b32 s4, s17, 16
-; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
-; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v2
+; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_add_f32_e32 v1, s4, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc
-; GFX9-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v4, v4, v1
-; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
-; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v5, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v3, v3, v1
; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX9-NEXT: v_and_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v1
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_setpc_b64 s[30:31]
; GFX9-NEXT: .LBB83_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
@@ -16189,17 +16185,16 @@ define inreg <4 x half> @bitcast_v4bf16_to_v4f16_scalar(<4 x bfloat> inreg %a, i
; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
; GFX9-NEXT: v_bfe_u32 v4, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v4, v4, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v0
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc
-; GFX9-NEXT: v_mov_b32_e32 v4, 0xffff
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX9-NEXT: v_and_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_and_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v0, v3, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_setpc_b64 s[30:31]
; GFX9-NEXT: .LBB103_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
@@ -18021,9 +18016,9 @@ define inreg <8 x i8> @bitcast_v4bf16_to_v8i8_scalar(<4 x bfloat> inreg %a, i32
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s17
-; GFX9-NEXT: v_lshl_or_b32 v9, v2, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v9, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v3, s4, v1
; GFX9-NEXT: v_bfe_u32 v4, v3, 16, 1
; GFX9-NEXT: v_add_u32_e32 v4, v4, v3
@@ -18041,8 +18036,8 @@ define inreg <8 x i8> @bitcast_v4bf16_to_v8i8_scalar(<4 x bfloat> inreg %a, i32
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v1
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v8
-; GFX9-NEXT: v_lshl_or_b32 v10, v6, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v6
+; GFX9-NEXT: v_or_b32_sdwa v10, v1, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_lshrrev_b64 v[3:4], 24, v[9:10]
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 8, v9
; GFX9-NEXT: v_lshrrev_b32_e32 v7, 24, v10
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.704bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.704bit.ll
index ccbfc67dd61b7..2a5574397fad8 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.704bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.704bit.ll
@@ -13618,50 +13618,50 @@ define inreg <44 x i16> @bitcast_v22f32_to_v44i16_scalar(<22 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v23, s15
; GFX9-NEXT: v_mov_b32_e32 v22, s14
; GFX9-NEXT: .LBB29_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_and_b32_e32 v21, 0xffff, v21
-; GFX9-NEXT: v_lshl_or_b32 v0, v51, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v50, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v49, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v48, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v39, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v38, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v37, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v36, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v35, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v34, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v33, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v32, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v31, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v30, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v29, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v28, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v27, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v26, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v25, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v24, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v23, 16, v20
-; GFX9-NEXT: v_lshl_or_b32 v21, v22, 16, v21
+; GFX9-NEXT: v_lshlrev_b32_e32 v51, 16, v51
+; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v50
+; GFX9-NEXT: v_lshlrev_b32_e32 v49, 16, v49
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; GFX9-NEXT: v_lshlrev_b32_e32 v39, 16, v39
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; GFX9-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; GFX9-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; GFX9-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; GFX9-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX9-NEXT: v_or_b32_sdwa v0, v51, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v50, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v49, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v48, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v4, v39, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v38, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v37, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v36, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v35, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v34, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v33, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v32, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v12, v31, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v30, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v14, v29, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v28, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v16, v27, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v26, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v18, v25, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v24, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v20, v23, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v21, v22, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: bitcast_v22f32_to_v44i16_scalar:
@@ -17229,50 +17229,50 @@ define inreg <44 x half> @bitcast_v22f32_to_v44f16_scalar(<22 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v23, s15
; GFX9-NEXT: v_mov_b32_e32 v22, s14
; GFX9-NEXT: .LBB33_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_and_b32_e32 v21, 0xffff, v21
-; GFX9-NEXT: v_lshl_or_b32 v0, v51, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v50, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v49, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v48, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v39, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v38, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v37, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v36, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v35, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v34, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v33, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v32, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v31, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v30, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v29, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v28, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v27, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v26, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v25, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v24, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v23, 16, v20
-; GFX9-NEXT: v_lshl_or_b32 v21, v22, 16, v21
+; GFX9-NEXT: v_lshlrev_b32_e32 v51, 16, v51
+; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v50
+; GFX9-NEXT: v_lshlrev_b32_e32 v49, 16, v49
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; GFX9-NEXT: v_lshlrev_b32_e32 v39, 16, v39
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; GFX9-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; GFX9-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; GFX9-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; GFX9-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX9-NEXT: v_or_b32_sdwa v0, v51, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v50, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v49, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v48, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v4, v39, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v38, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v37, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v36, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v35, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v34, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v33, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v32, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v12, v31, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v30, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v14, v29, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v28, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v16, v27, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v26, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v18, v25, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v24, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v20, v23, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v21, v22, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: bitcast_v22f32_to_v44f16_scalar:
@@ -28910,50 +28910,50 @@ define inreg <44 x i16> @bitcast_v11f64_to_v44i16_scalar(<11 x double> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v26, s15
; GFX9-NEXT: v_mov_b32_e32 v24, s14
; GFX9-NEXT: .LBB49_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_and_b32_e32 v21, 0xffff, v21
-; GFX9-NEXT: v_lshl_or_b32 v0, v49, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v51, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v39, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v50, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v37, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v48, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v35, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v38, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v33, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v36, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v31, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v34, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v29, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v32, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v27, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v30, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v25, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v28, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v23, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v26, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v22, 16, v20
-; GFX9-NEXT: v_lshl_or_b32 v21, v24, 16, v21
+; GFX9-NEXT: v_lshlrev_b32_e32 v49, 16, v49
+; GFX9-NEXT: v_lshlrev_b32_e32 v39, 16, v39
+; GFX9-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; GFX9-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; GFX9-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX9-NEXT: v_or_b32_sdwa v0, v49, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v49, 16, v51
+; GFX9-NEXT: v_or_b32_sdwa v2, v39, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v39, 16, v50
+; GFX9-NEXT: v_or_b32_sdwa v4, v37, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v37, 16, v48
+; GFX9-NEXT: v_or_b32_sdwa v6, v35, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v38
+; GFX9-NEXT: v_or_b32_sdwa v8, v33, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v36
+; GFX9-NEXT: v_or_b32_sdwa v10, v31, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v34
+; GFX9-NEXT: v_or_b32_sdwa v12, v29, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v32
+; GFX9-NEXT: v_or_b32_sdwa v14, v27, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v30
+; GFX9-NEXT: v_or_b32_sdwa v16, v25, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v28
+; GFX9-NEXT: v_or_b32_sdwa v18, v23, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v23, 16, v26
+; GFX9-NEXT: v_or_b32_sdwa v20, v22, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v22, 16, v24
+; GFX9-NEXT: v_or_b32_sdwa v1, v49, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v39, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v37, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v35, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v33, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v31, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v29, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v27, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v25, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v23, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v21, v22, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: bitcast_v11f64_to_v44i16_scalar:
@@ -32433,50 +32433,50 @@ define inreg <44 x half> @bitcast_v11f64_to_v44f16_scalar(<11 x double> inreg %a
; GFX9-NEXT: v_mov_b32_e32 v26, s15
; GFX9-NEXT: v_mov_b32_e32 v24, s14
; GFX9-NEXT: .LBB53_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_and_b32_e32 v21, 0xffff, v21
-; GFX9-NEXT: v_lshl_or_b32 v0, v49, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v51, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v39, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v50, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v37, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v48, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v35, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v38, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v33, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v36, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v31, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v34, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v29, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v32, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v27, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v30, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v25, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v28, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v23, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v26, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v22, 16, v20
-; GFX9-NEXT: v_lshl_or_b32 v21, v24, 16, v21
+; GFX9-NEXT: v_lshlrev_b32_e32 v49, 16, v49
+; GFX9-NEXT: v_lshlrev_b32_e32 v39, 16, v39
+; GFX9-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; GFX9-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; GFX9-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX9-NEXT: v_or_b32_sdwa v0, v49, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v49, 16, v51
+; GFX9-NEXT: v_or_b32_sdwa v2, v39, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v39, 16, v50
+; GFX9-NEXT: v_or_b32_sdwa v4, v37, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v37, 16, v48
+; GFX9-NEXT: v_or_b32_sdwa v6, v35, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v38
+; GFX9-NEXT: v_or_b32_sdwa v8, v33, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v36
+; GFX9-NEXT: v_or_b32_sdwa v10, v31, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v34
+; GFX9-NEXT: v_or_b32_sdwa v12, v29, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v32
+; GFX9-NEXT: v_or_b32_sdwa v14, v27, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v30
+; GFX9-NEXT: v_or_b32_sdwa v16, v25, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v28
+; GFX9-NEXT: v_or_b32_sdwa v18, v23, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v23, 16, v26
+; GFX9-NEXT: v_or_b32_sdwa v20, v22, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v22, 16, v24
+; GFX9-NEXT: v_or_b32_sdwa v1, v49, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v39, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v37, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v35, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v33, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v31, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v29, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v27, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v25, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v23, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v21, v22, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: bitcast_v11f64_to_v44f16_scalar:
@@ -36764,50 +36764,50 @@ define inreg <44 x half> @bitcast_v44i16_to_v44f16_scalar(<44 x i16> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v50, s7
; GFX9-NEXT: v_mov_b32_e32 v51, s6
; GFX9-NEXT: .LBB57_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_and_b32_e32 v21, 0xffff, v21
-; GFX9-NEXT: v_lshl_or_b32 v0, v51, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v50, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v49, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v48, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v39, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v38, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v37, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v36, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v35, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v34, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v33, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v32, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v31, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v30, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v29, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v28, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v27, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v26, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v25, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v24, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v23, 16, v20
-; GFX9-NEXT: v_lshl_or_b32 v21, v22, 16, v21
+; GFX9-NEXT: v_lshlrev_b32_e32 v51, 16, v51
+; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v50
+; GFX9-NEXT: v_lshlrev_b32_e32 v49, 16, v49
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; GFX9-NEXT: v_lshlrev_b32_e32 v39, 16, v39
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; GFX9-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; GFX9-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; GFX9-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; GFX9-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX9-NEXT: v_or_b32_sdwa v0, v51, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v50, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v49, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v48, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v4, v39, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v38, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v37, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v36, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v35, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v34, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v33, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v32, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v12, v31, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v30, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v14, v29, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v28, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v16, v27, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v26, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v18, v25, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v24, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v20, v23, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v21, v22, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: bitcast_v44i16_to_v44f16_scalar:
@@ -38708,50 +38708,50 @@ define inreg <44 x i16> @bitcast_v44f16_to_v44i16_scalar(<44 x half> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v50, s7
; GFX9-NEXT: v_mov_b32_e32 v51, s6
; GFX9-NEXT: .LBB59_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_and_b32_e32 v21, 0xffff, v21
-; GFX9-NEXT: v_lshl_or_b32 v0, v51, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v50, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v49, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v48, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v39, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v38, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v37, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v36, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v35, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v34, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v33, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v32, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v31, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v30, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v29, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v28, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v27, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v26, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v25, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v24, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v23, 16, v20
-; GFX9-NEXT: v_lshl_or_b32 v21, v22, 16, v21
+; GFX9-NEXT: v_lshlrev_b32_e32 v51, 16, v51
+; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v50
+; GFX9-NEXT: v_lshlrev_b32_e32 v49, 16, v49
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; GFX9-NEXT: v_lshlrev_b32_e32 v39, 16, v39
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; GFX9-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; GFX9-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; GFX9-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; GFX9-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX9-NEXT: v_or_b32_sdwa v0, v51, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v50, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v49, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v48, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v4, v39, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v38, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v37, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v36, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v35, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v34, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v33, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v32, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v12, v31, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v30, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v14, v29, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v28, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v16, v27, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v26, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v18, v25, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v24, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v20, v23, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v21, v22, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: bitcast_v44f16_to_v44i16_scalar:
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.768bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.768bit.ll
index 1183a0d8569fc..d7b8411c8f146 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.768bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.768bit.ll
@@ -14633,54 +14633,54 @@ define inreg <48 x i16> @bitcast_v24f32_to_v48i16_scalar(<24 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v25, s41
; GFX9-NEXT: v_mov_b32_e32 v24, s40
; GFX9-NEXT: .LBB29_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_and_b32_e32 v21, 0xffff, v21
-; GFX9-NEXT: v_and_b32_e32 v22, 0xffff, v22
-; GFX9-NEXT: v_and_b32_e32 v23, 0xffff, v23
-; GFX9-NEXT: v_lshl_or_b32 v0, v55, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v54, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v53, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v52, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v51, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v50, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v49, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v48, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v39, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v38, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v37, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v36, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v35, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v34, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v33, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v32, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v31, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v30, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v29, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v28, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v27, 16, v20
-; GFX9-NEXT: v_lshl_or_b32 v21, v26, 16, v21
-; GFX9-NEXT: v_lshl_or_b32 v22, v25, 16, v22
-; GFX9-NEXT: v_lshl_or_b32 v23, v24, 16, v23
+; GFX9-NEXT: v_lshlrev_b32_e32 v55, 16, v55
+; GFX9-NEXT: v_lshlrev_b32_e32 v54, 16, v54
+; GFX9-NEXT: v_lshlrev_b32_e32 v53, 16, v53
+; GFX9-NEXT: v_lshlrev_b32_e32 v52, 16, v52
+; GFX9-NEXT: v_lshlrev_b32_e32 v51, 16, v51
+; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v50
+; GFX9-NEXT: v_lshlrev_b32_e32 v49, 16, v49
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; GFX9-NEXT: v_lshlrev_b32_e32 v39, 16, v39
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; GFX9-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; GFX9-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; GFX9-NEXT: v_or_b32_sdwa v0, v55, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v54, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v53, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v52, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v4, v51, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v50, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v49, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v48, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v39, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v38, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v37, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v36, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v12, v35, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v34, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v14, v33, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v32, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v16, v31, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v30, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v18, v29, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v28, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v20, v27, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v21, v26, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v22, v25, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v23, v24, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: bitcast_v24f32_to_v48i16_scalar:
@@ -18579,54 +18579,54 @@ define inreg <48 x half> @bitcast_v24f32_to_v48f16_scalar(<24 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v25, s41
; GFX9-NEXT: v_mov_b32_e32 v24, s40
; GFX9-NEXT: .LBB33_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_and_b32_e32 v21, 0xffff, v21
-; GFX9-NEXT: v_and_b32_e32 v22, 0xffff, v22
-; GFX9-NEXT: v_and_b32_e32 v23, 0xffff, v23
-; GFX9-NEXT: v_lshl_or_b32 v0, v55, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v54, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v53, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v52, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v51, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v50, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v49, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v48, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v39, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v38, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v37, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v36, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v35, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v34, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v33, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v32, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v31, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v30, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v29, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v28, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v27, 16, v20
-; GFX9-NEXT: v_lshl_or_b32 v21, v26, 16, v21
-; GFX9-NEXT: v_lshl_or_b32 v22, v25, 16, v22
-; GFX9-NEXT: v_lshl_or_b32 v23, v24, 16, v23
+; GFX9-NEXT: v_lshlrev_b32_e32 v55, 16, v55
+; GFX9-NEXT: v_lshlrev_b32_e32 v54, 16, v54
+; GFX9-NEXT: v_lshlrev_b32_e32 v53, 16, v53
+; GFX9-NEXT: v_lshlrev_b32_e32 v52, 16, v52
+; GFX9-NEXT: v_lshlrev_b32_e32 v51, 16, v51
+; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v50
+; GFX9-NEXT: v_lshlrev_b32_e32 v49, 16, v49
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; GFX9-NEXT: v_lshlrev_b32_e32 v39, 16, v39
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; GFX9-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; GFX9-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; GFX9-NEXT: v_or_b32_sdwa v0, v55, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v54, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v53, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v52, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v4, v51, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v50, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v49, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v48, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v39, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v38, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v37, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v36, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v12, v35, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v34, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v14, v33, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v32, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v16, v31, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v30, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v18, v29, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v28, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v20, v27, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v21, v26, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v22, v25, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v23, v24, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: bitcast_v24f32_to_v48f16_scalar:
@@ -31290,54 +31290,54 @@ define inreg <48 x i16> @bitcast_v12f64_to_v48i16_scalar(<12 x double> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v28, s41
; GFX9-NEXT: v_mov_b32_e32 v26, s40
; GFX9-NEXT: .LBB49_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_and_b32_e32 v21, 0xffff, v21
-; GFX9-NEXT: v_and_b32_e32 v22, 0xffff, v22
-; GFX9-NEXT: v_and_b32_e32 v23, 0xffff, v23
-; GFX9-NEXT: v_lshl_or_b32 v0, v53, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v55, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v51, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v54, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v49, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v52, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v48, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v50, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v38, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v39, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v35, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v37, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v33, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v36, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v31, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v34, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v29, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v32, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v27, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v30, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v25, 16, v20
-; GFX9-NEXT: v_lshl_or_b32 v21, v28, 16, v21
-; GFX9-NEXT: v_lshl_or_b32 v22, v24, 16, v22
-; GFX9-NEXT: v_lshl_or_b32 v23, v26, 16, v23
+; GFX9-NEXT: v_lshlrev_b32_e32 v53, 16, v53
+; GFX9-NEXT: v_lshlrev_b32_e32 v51, 16, v51
+; GFX9-NEXT: v_lshlrev_b32_e32 v49, 16, v49
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; GFX9-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; GFX9-NEXT: v_or_b32_sdwa v0, v53, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v53, 16, v55
+; GFX9-NEXT: v_or_b32_sdwa v2, v51, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v51, 16, v54
+; GFX9-NEXT: v_or_b32_sdwa v4, v49, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v49, 16, v52
+; GFX9-NEXT: v_or_b32_sdwa v6, v48, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v50
+; GFX9-NEXT: v_or_b32_sdwa v8, v38, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v39
+; GFX9-NEXT: v_or_b32_sdwa v10, v35, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v37
+; GFX9-NEXT: v_or_b32_sdwa v12, v33, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v36
+; GFX9-NEXT: v_or_b32_sdwa v14, v31, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v34
+; GFX9-NEXT: v_or_b32_sdwa v16, v29, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v32
+; GFX9-NEXT: v_or_b32_sdwa v18, v27, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v30
+; GFX9-NEXT: v_or_b32_sdwa v20, v25, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v28
+; GFX9-NEXT: v_or_b32_sdwa v22, v24, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v24, 16, v26
+; GFX9-NEXT: v_or_b32_sdwa v1, v53, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v51, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v49, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v48, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v38, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v35, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v33, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v31, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v29, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v27, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v21, v25, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v23, v24, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: bitcast_v12f64_to_v48i16_scalar:
@@ -35140,54 +35140,54 @@ define inreg <48 x half> @bitcast_v12f64_to_v48f16_scalar(<12 x double> inreg %a
; GFX9-NEXT: v_mov_b32_e32 v28, s41
; GFX9-NEXT: v_mov_b32_e32 v26, s40
; GFX9-NEXT: .LBB53_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_and_b32_e32 v21, 0xffff, v21
-; GFX9-NEXT: v_and_b32_e32 v22, 0xffff, v22
-; GFX9-NEXT: v_and_b32_e32 v23, 0xffff, v23
-; GFX9-NEXT: v_lshl_or_b32 v0, v53, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v55, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v51, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v54, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v49, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v52, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v48, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v50, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v38, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v39, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v35, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v37, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v33, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v36, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v31, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v34, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v29, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v32, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v27, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v30, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v25, 16, v20
-; GFX9-NEXT: v_lshl_or_b32 v21, v28, 16, v21
-; GFX9-NEXT: v_lshl_or_b32 v22, v24, 16, v22
-; GFX9-NEXT: v_lshl_or_b32 v23, v26, 16, v23
+; GFX9-NEXT: v_lshlrev_b32_e32 v53, 16, v53
+; GFX9-NEXT: v_lshlrev_b32_e32 v51, 16, v51
+; GFX9-NEXT: v_lshlrev_b32_e32 v49, 16, v49
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; GFX9-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; GFX9-NEXT: v_or_b32_sdwa v0, v53, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v53, 16, v55
+; GFX9-NEXT: v_or_b32_sdwa v2, v51, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v51, 16, v54
+; GFX9-NEXT: v_or_b32_sdwa v4, v49, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v49, 16, v52
+; GFX9-NEXT: v_or_b32_sdwa v6, v48, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v50
+; GFX9-NEXT: v_or_b32_sdwa v8, v38, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v39
+; GFX9-NEXT: v_or_b32_sdwa v10, v35, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v37
+; GFX9-NEXT: v_or_b32_sdwa v12, v33, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v36
+; GFX9-NEXT: v_or_b32_sdwa v14, v31, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v34
+; GFX9-NEXT: v_or_b32_sdwa v16, v29, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v32
+; GFX9-NEXT: v_or_b32_sdwa v18, v27, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v30
+; GFX9-NEXT: v_or_b32_sdwa v20, v25, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v28
+; GFX9-NEXT: v_or_b32_sdwa v22, v24, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v24, 16, v26
+; GFX9-NEXT: v_or_b32_sdwa v1, v53, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v51, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v49, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v48, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v38, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v35, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v33, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v31, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v29, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v27, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v21, v25, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v23, v24, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: bitcast_v12f64_to_v48f16_scalar:
@@ -39894,54 +39894,54 @@ define inreg <48 x half> @bitcast_v48i16_to_v48f16_scalar(<48 x i16> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v54, s7
; GFX9-NEXT: v_mov_b32_e32 v55, s6
; GFX9-NEXT: .LBB57_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_and_b32_e32 v21, 0xffff, v21
-; GFX9-NEXT: v_and_b32_e32 v22, 0xffff, v22
-; GFX9-NEXT: v_and_b32_e32 v23, 0xffff, v23
-; GFX9-NEXT: v_lshl_or_b32 v0, v55, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v54, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v53, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v52, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v51, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v50, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v49, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v48, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v39, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v38, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v37, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v36, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v35, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v34, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v33, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v32, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v31, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v30, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v29, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v28, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v27, 16, v20
-; GFX9-NEXT: v_lshl_or_b32 v21, v26, 16, v21
-; GFX9-NEXT: v_lshl_or_b32 v22, v25, 16, v22
-; GFX9-NEXT: v_lshl_or_b32 v23, v24, 16, v23
+; GFX9-NEXT: v_lshlrev_b32_e32 v55, 16, v55
+; GFX9-NEXT: v_lshlrev_b32_e32 v54, 16, v54
+; GFX9-NEXT: v_lshlrev_b32_e32 v53, 16, v53
+; GFX9-NEXT: v_lshlrev_b32_e32 v52, 16, v52
+; GFX9-NEXT: v_lshlrev_b32_e32 v51, 16, v51
+; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v50
+; GFX9-NEXT: v_lshlrev_b32_e32 v49, 16, v49
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; GFX9-NEXT: v_lshlrev_b32_e32 v39, 16, v39
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; GFX9-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; GFX9-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; GFX9-NEXT: v_or_b32_sdwa v0, v55, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v54, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v53, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v52, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v4, v51, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v50, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v49, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v48, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v39, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v38, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v37, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v36, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v12, v35, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v34, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v14, v33, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v32, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v16, v31, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v30, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v18, v29, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v28, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v20, v27, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v21, v26, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v22, v25, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v23, v24, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: bitcast_v48i16_to_v48f16_scalar:
@@ -42017,54 +42017,54 @@ define inreg <48 x i16> @bitcast_v48f16_to_v48i16_scalar(<48 x half> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v54, s7
; GFX9-NEXT: v_mov_b32_e32 v55, s6
; GFX9-NEXT: .LBB59_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_and_b32_e32 v21, 0xffff, v21
-; GFX9-NEXT: v_and_b32_e32 v22, 0xffff, v22
-; GFX9-NEXT: v_and_b32_e32 v23, 0xffff, v23
-; GFX9-NEXT: v_lshl_or_b32 v0, v55, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v54, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v53, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v52, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v51, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v50, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v49, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v48, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v39, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v38, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v37, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v36, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v35, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v34, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v33, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v32, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v31, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v30, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v29, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v28, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v27, 16, v20
-; GFX9-NEXT: v_lshl_or_b32 v21, v26, 16, v21
-; GFX9-NEXT: v_lshl_or_b32 v22, v25, 16, v22
-; GFX9-NEXT: v_lshl_or_b32 v23, v24, 16, v23
+; GFX9-NEXT: v_lshlrev_b32_e32 v55, 16, v55
+; GFX9-NEXT: v_lshlrev_b32_e32 v54, 16, v54
+; GFX9-NEXT: v_lshlrev_b32_e32 v53, 16, v53
+; GFX9-NEXT: v_lshlrev_b32_e32 v52, 16, v52
+; GFX9-NEXT: v_lshlrev_b32_e32 v51, 16, v51
+; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v50
+; GFX9-NEXT: v_lshlrev_b32_e32 v49, 16, v49
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; GFX9-NEXT: v_lshlrev_b32_e32 v39, 16, v39
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; GFX9-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; GFX9-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; GFX9-NEXT: v_or_b32_sdwa v0, v55, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v54, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v53, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v52, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v4, v51, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v50, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v49, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v48, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v39, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v38, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v37, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v36, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v12, v35, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v34, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v14, v33, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v32, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v16, v31, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v30, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v18, v29, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v28, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v20, v27, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v21, v26, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v22, v25, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v23, v24, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: bitcast_v48f16_to_v48i16_scalar:
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.832bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.832bit.ll
index 3803d21d588d8..d070d09aa3dc3 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.832bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.832bit.ll
@@ -15783,62 +15783,62 @@ define inreg <52 x i16> @bitcast_v26f32_to_v52i16_scalar(<26 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v27, s43
; GFX9-NEXT: v_mov_b32_e32 v26, s42
; GFX9-NEXT: .LBB29_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_lshl_or_b32 v0, v43, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v42, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v41, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v40, 16, v3
+; GFX9-NEXT: v_lshlrev_b32_e32 v43, 16, v43
+; GFX9-NEXT: v_lshlrev_b32_e32 v42, 16, v42
+; GFX9-NEXT: v_lshlrev_b32_e32 v41, 16, v41
+; GFX9-NEXT: v_lshlrev_b32_e32 v40, 16, v40
+; GFX9-NEXT: v_or_b32_sdwa v0, v43, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v42, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v41, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v40, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: buffer_load_dword v43, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_and_b32_e32 v21, 0xffff, v21
-; GFX9-NEXT: v_and_b32_e32 v22, 0xffff, v22
-; GFX9-NEXT: v_and_b32_e32 v23, 0xffff, v23
-; GFX9-NEXT: v_and_b32_e32 v24, 0xffff, v24
-; GFX9-NEXT: v_and_b32_e32 v25, 0xffff, v25
-; GFX9-NEXT: v_lshl_or_b32 v4, v55, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v54, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v53, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v52, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v51, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v50, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v49, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v48, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v39, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v38, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v37, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v36, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v35, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v34, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v33, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v32, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v31, 16, v20
-; GFX9-NEXT: v_lshl_or_b32 v21, v30, 16, v21
-; GFX9-NEXT: v_lshl_or_b32 v22, v29, 16, v22
-; GFX9-NEXT: v_lshl_or_b32 v23, v28, 16, v23
-; GFX9-NEXT: v_lshl_or_b32 v24, v27, 16, v24
-; GFX9-NEXT: v_lshl_or_b32 v25, v26, 16, v25
+; GFX9-NEXT: v_lshlrev_b32_e32 v55, 16, v55
+; GFX9-NEXT: v_lshlrev_b32_e32 v54, 16, v54
+; GFX9-NEXT: v_lshlrev_b32_e32 v53, 16, v53
+; GFX9-NEXT: v_lshlrev_b32_e32 v52, 16, v52
+; GFX9-NEXT: v_lshlrev_b32_e32 v51, 16, v51
+; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v50
+; GFX9-NEXT: v_lshlrev_b32_e32 v49, 16, v49
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; GFX9-NEXT: v_lshlrev_b32_e32 v39, 16, v39
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; GFX9-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX9-NEXT: v_or_b32_sdwa v4, v55, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v54, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v53, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v52, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v51, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v50, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v49, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v48, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v12, v39, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v38, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v14, v37, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v36, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v16, v35, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v34, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v18, v33, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v32, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v20, v31, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v21, v30, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v22, v29, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v23, v28, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v24, v27, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v25, v26, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -20135,62 +20135,62 @@ define inreg <52 x half> @bitcast_v26f32_to_v52f16_scalar(<26 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v27, s43
; GFX9-NEXT: v_mov_b32_e32 v26, s42
; GFX9-NEXT: .LBB33_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_lshl_or_b32 v0, v43, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v42, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v41, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v40, 16, v3
+; GFX9-NEXT: v_lshlrev_b32_e32 v43, 16, v43
+; GFX9-NEXT: v_lshlrev_b32_e32 v42, 16, v42
+; GFX9-NEXT: v_lshlrev_b32_e32 v41, 16, v41
+; GFX9-NEXT: v_lshlrev_b32_e32 v40, 16, v40
+; GFX9-NEXT: v_or_b32_sdwa v0, v43, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v42, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v41, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v40, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: buffer_load_dword v43, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_and_b32_e32 v21, 0xffff, v21
-; GFX9-NEXT: v_and_b32_e32 v22, 0xffff, v22
-; GFX9-NEXT: v_and_b32_e32 v23, 0xffff, v23
-; GFX9-NEXT: v_and_b32_e32 v24, 0xffff, v24
-; GFX9-NEXT: v_and_b32_e32 v25, 0xffff, v25
-; GFX9-NEXT: v_lshl_or_b32 v4, v55, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v54, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v53, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v52, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v51, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v50, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v49, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v48, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v39, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v38, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v37, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v36, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v35, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v34, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v33, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v32, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v31, 16, v20
-; GFX9-NEXT: v_lshl_or_b32 v21, v30, 16, v21
-; GFX9-NEXT: v_lshl_or_b32 v22, v29, 16, v22
-; GFX9-NEXT: v_lshl_or_b32 v23, v28, 16, v23
-; GFX9-NEXT: v_lshl_or_b32 v24, v27, 16, v24
-; GFX9-NEXT: v_lshl_or_b32 v25, v26, 16, v25
+; GFX9-NEXT: v_lshlrev_b32_e32 v55, 16, v55
+; GFX9-NEXT: v_lshlrev_b32_e32 v54, 16, v54
+; GFX9-NEXT: v_lshlrev_b32_e32 v53, 16, v53
+; GFX9-NEXT: v_lshlrev_b32_e32 v52, 16, v52
+; GFX9-NEXT: v_lshlrev_b32_e32 v51, 16, v51
+; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v50
+; GFX9-NEXT: v_lshlrev_b32_e32 v49, 16, v49
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; GFX9-NEXT: v_lshlrev_b32_e32 v39, 16, v39
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; GFX9-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX9-NEXT: v_or_b32_sdwa v4, v55, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v54, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v53, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v52, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v51, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v50, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v49, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v48, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v12, v39, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v38, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v14, v37, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v36, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v16, v35, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v34, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v18, v33, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v32, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v20, v31, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v21, v30, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v22, v29, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v23, v28, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v24, v27, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v25, v26, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -34022,62 +34022,62 @@ define inreg <52 x i16> @bitcast_v13f64_to_v52i16_scalar(<13 x double> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v31, s43
; GFX9-NEXT: v_mov_b32_e32 v28, s42
; GFX9-NEXT: .LBB49_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_lshl_or_b32 v0, v40, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v43, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v3, v42, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v5, v41, 16, v5
+; GFX9-NEXT: v_lshlrev_b32_e32 v40, 16, v40
+; GFX9-NEXT: v_or_b32_sdwa v0, v40, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v40, 16, v43
+; GFX9-NEXT: v_lshlrev_b32_e32 v54, 16, v54
+; GFX9-NEXT: v_lshlrev_b32_e32 v52, 16, v52
+; GFX9-NEXT: v_or_b32_sdwa v1, v40, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v54, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v54, 16, v42
+; GFX9-NEXT: v_or_b32_sdwa v4, v52, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v52, 16, v41
; GFX9-NEXT: buffer_load_dword v43, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_and_b32_e32 v21, 0xffff, v21
-; GFX9-NEXT: v_and_b32_e32 v22, 0xffff, v22
-; GFX9-NEXT: v_and_b32_e32 v23, 0xffff, v23
-; GFX9-NEXT: v_and_b32_e32 v24, 0xffff, v24
-; GFX9-NEXT: v_and_b32_e32 v25, 0xffff, v25
-; GFX9-NEXT: v_lshl_or_b32 v2, v54, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v4, v52, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v6, v50, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v55, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v48, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v53, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v38, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v51, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v36, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v49, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v34, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v39, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v32, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v37, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v30, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v35, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v29, 16, v20
-; GFX9-NEXT: v_lshl_or_b32 v21, v33, 16, v21
-; GFX9-NEXT: v_lshl_or_b32 v22, v27, 16, v22
-; GFX9-NEXT: v_lshl_or_b32 v23, v31, 16, v23
-; GFX9-NEXT: v_lshl_or_b32 v24, v26, 16, v24
-; GFX9-NEXT: v_lshl_or_b32 v25, v28, 16, v25
+; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v50
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX9-NEXT: v_or_b32_sdwa v6, v50, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v55
+; GFX9-NEXT: v_or_b32_sdwa v8, v48, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v53
+; GFX9-NEXT: v_or_b32_sdwa v10, v38, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v51
+; GFX9-NEXT: v_or_b32_sdwa v12, v36, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v49
+; GFX9-NEXT: v_or_b32_sdwa v14, v34, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v39
+; GFX9-NEXT: v_or_b32_sdwa v16, v32, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v37
+; GFX9-NEXT: v_or_b32_sdwa v18, v30, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v35
+; GFX9-NEXT: v_or_b32_sdwa v20, v29, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v33
+; GFX9-NEXT: v_or_b32_sdwa v22, v27, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v31
+; GFX9-NEXT: v_or_b32_sdwa v24, v26, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v26, 16, v28
+; GFX9-NEXT: v_or_b32_sdwa v3, v54, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v52, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v50, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v48, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v38, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v36, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v34, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v32, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v30, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v21, v29, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v23, v27, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v25, v26, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -38274,62 +38274,62 @@ define inreg <52 x half> @bitcast_v13f64_to_v52f16_scalar(<13 x double> inreg %a
; GFX9-NEXT: v_mov_b32_e32 v31, s43
; GFX9-NEXT: v_mov_b32_e32 v28, s42
; GFX9-NEXT: .LBB53_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_lshl_or_b32 v0, v40, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v43, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v3, v42, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v5, v41, 16, v5
+; GFX9-NEXT: v_lshlrev_b32_e32 v40, 16, v40
+; GFX9-NEXT: v_or_b32_sdwa v0, v40, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v40, 16, v43
+; GFX9-NEXT: v_lshlrev_b32_e32 v54, 16, v54
+; GFX9-NEXT: v_lshlrev_b32_e32 v52, 16, v52
+; GFX9-NEXT: v_or_b32_sdwa v1, v40, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v54, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v54, 16, v42
+; GFX9-NEXT: v_or_b32_sdwa v4, v52, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v52, 16, v41
; GFX9-NEXT: buffer_load_dword v43, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_and_b32_e32 v21, 0xffff, v21
-; GFX9-NEXT: v_and_b32_e32 v22, 0xffff, v22
-; GFX9-NEXT: v_and_b32_e32 v23, 0xffff, v23
-; GFX9-NEXT: v_and_b32_e32 v24, 0xffff, v24
-; GFX9-NEXT: v_and_b32_e32 v25, 0xffff, v25
-; GFX9-NEXT: v_lshl_or_b32 v2, v54, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v4, v52, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v6, v50, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v55, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v48, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v53, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v38, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v51, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v36, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v49, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v34, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v39, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v32, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v37, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v30, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v35, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v29, 16, v20
-; GFX9-NEXT: v_lshl_or_b32 v21, v33, 16, v21
-; GFX9-NEXT: v_lshl_or_b32 v22, v27, 16, v22
-; GFX9-NEXT: v_lshl_or_b32 v23, v31, 16, v23
-; GFX9-NEXT: v_lshl_or_b32 v24, v26, 16, v24
-; GFX9-NEXT: v_lshl_or_b32 v25, v28, 16, v25
+; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v50
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX9-NEXT: v_or_b32_sdwa v6, v50, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v55
+; GFX9-NEXT: v_or_b32_sdwa v8, v48, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v53
+; GFX9-NEXT: v_or_b32_sdwa v10, v38, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v51
+; GFX9-NEXT: v_or_b32_sdwa v12, v36, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v49
+; GFX9-NEXT: v_or_b32_sdwa v14, v34, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v39
+; GFX9-NEXT: v_or_b32_sdwa v16, v32, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v37
+; GFX9-NEXT: v_or_b32_sdwa v18, v30, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v35
+; GFX9-NEXT: v_or_b32_sdwa v20, v29, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v33
+; GFX9-NEXT: v_or_b32_sdwa v22, v27, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v31
+; GFX9-NEXT: v_or_b32_sdwa v24, v26, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v26, 16, v28
+; GFX9-NEXT: v_or_b32_sdwa v3, v54, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v52, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v50, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v48, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v38, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v36, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v34, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v32, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v30, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v21, v29, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v23, v27, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v25, v26, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -43486,62 +43486,62 @@ define inreg <52 x half> @bitcast_v52i16_to_v52f16_scalar(<52 x i16> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v42, s7
; GFX9-NEXT: v_mov_b32_e32 v43, s6
; GFX9-NEXT: .LBB57_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_lshl_or_b32 v0, v43, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v42, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v41, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v40, 16, v3
+; GFX9-NEXT: v_lshlrev_b32_e32 v43, 16, v43
+; GFX9-NEXT: v_lshlrev_b32_e32 v42, 16, v42
+; GFX9-NEXT: v_lshlrev_b32_e32 v41, 16, v41
+; GFX9-NEXT: v_lshlrev_b32_e32 v40, 16, v40
+; GFX9-NEXT: v_or_b32_sdwa v0, v43, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v42, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v41, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v40, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: buffer_load_dword v43, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_and_b32_e32 v21, 0xffff, v21
-; GFX9-NEXT: v_and_b32_e32 v22, 0xffff, v22
-; GFX9-NEXT: v_and_b32_e32 v23, 0xffff, v23
-; GFX9-NEXT: v_and_b32_e32 v24, 0xffff, v24
-; GFX9-NEXT: v_and_b32_e32 v25, 0xffff, v25
-; GFX9-NEXT: v_lshl_or_b32 v4, v55, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v54, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v53, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v52, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v51, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v50, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v49, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v48, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v39, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v38, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v37, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v36, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v35, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v34, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v33, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v32, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v31, 16, v20
-; GFX9-NEXT: v_lshl_or_b32 v21, v30, 16, v21
-; GFX9-NEXT: v_lshl_or_b32 v22, v29, 16, v22
-; GFX9-NEXT: v_lshl_or_b32 v23, v28, 16, v23
-; GFX9-NEXT: v_lshl_or_b32 v24, v27, 16, v24
-; GFX9-NEXT: v_lshl_or_b32 v25, v26, 16, v25
+; GFX9-NEXT: v_lshlrev_b32_e32 v55, 16, v55
+; GFX9-NEXT: v_lshlrev_b32_e32 v54, 16, v54
+; GFX9-NEXT: v_lshlrev_b32_e32 v53, 16, v53
+; GFX9-NEXT: v_lshlrev_b32_e32 v52, 16, v52
+; GFX9-NEXT: v_lshlrev_b32_e32 v51, 16, v51
+; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v50
+; GFX9-NEXT: v_lshlrev_b32_e32 v49, 16, v49
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; GFX9-NEXT: v_lshlrev_b32_e32 v39, 16, v39
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; GFX9-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX9-NEXT: v_or_b32_sdwa v4, v55, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v54, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v53, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v52, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v51, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v50, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v49, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v48, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v12, v39, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v38, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v14, v37, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v36, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v16, v35, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v34, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v18, v33, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v32, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v20, v31, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v21, v30, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v22, v29, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v23, v28, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v24, v27, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v25, v26, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -45825,62 +45825,62 @@ define inreg <52 x i16> @bitcast_v52f16_to_v52i16_scalar(<52 x half> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v42, s7
; GFX9-NEXT: v_mov_b32_e32 v43, s6
; GFX9-NEXT: .LBB59_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_lshl_or_b32 v0, v43, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v42, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v41, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v40, 16, v3
+; GFX9-NEXT: v_lshlrev_b32_e32 v43, 16, v43
+; GFX9-NEXT: v_lshlrev_b32_e32 v42, 16, v42
+; GFX9-NEXT: v_lshlrev_b32_e32 v41, 16, v41
+; GFX9-NEXT: v_lshlrev_b32_e32 v40, 16, v40
+; GFX9-NEXT: v_or_b32_sdwa v0, v43, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v42, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v41, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v40, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: buffer_load_dword v43, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_and_b32_e32 v21, 0xffff, v21
-; GFX9-NEXT: v_and_b32_e32 v22, 0xffff, v22
-; GFX9-NEXT: v_and_b32_e32 v23, 0xffff, v23
-; GFX9-NEXT: v_and_b32_e32 v24, 0xffff, v24
-; GFX9-NEXT: v_and_b32_e32 v25, 0xffff, v25
-; GFX9-NEXT: v_lshl_or_b32 v4, v55, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v54, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v53, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v52, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v51, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v50, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v49, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v48, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v39, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v38, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v37, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v36, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v35, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v34, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v33, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v32, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v31, 16, v20
-; GFX9-NEXT: v_lshl_or_b32 v21, v30, 16, v21
-; GFX9-NEXT: v_lshl_or_b32 v22, v29, 16, v22
-; GFX9-NEXT: v_lshl_or_b32 v23, v28, 16, v23
-; GFX9-NEXT: v_lshl_or_b32 v24, v27, 16, v24
-; GFX9-NEXT: v_lshl_or_b32 v25, v26, 16, v25
+; GFX9-NEXT: v_lshlrev_b32_e32 v55, 16, v55
+; GFX9-NEXT: v_lshlrev_b32_e32 v54, 16, v54
+; GFX9-NEXT: v_lshlrev_b32_e32 v53, 16, v53
+; GFX9-NEXT: v_lshlrev_b32_e32 v52, 16, v52
+; GFX9-NEXT: v_lshlrev_b32_e32 v51, 16, v51
+; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v50
+; GFX9-NEXT: v_lshlrev_b32_e32 v49, 16, v49
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; GFX9-NEXT: v_lshlrev_b32_e32 v39, 16, v39
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; GFX9-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX9-NEXT: v_or_b32_sdwa v4, v55, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v54, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v53, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v52, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v51, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v50, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v49, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v48, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v12, v39, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v38, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v14, v37, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v36, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v16, v35, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v34, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v18, v33, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v32, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v20, v31, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v21, v30, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v22, v29, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v23, v28, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v24, v27, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v25, v26, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.896bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.896bit.ll
index 7d7da2df094d7..8319358c1e356 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.896bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.896bit.ll
@@ -17030,22 +17030,22 @@ define inreg <56 x i16> @bitcast_v28f32_to_v56i16_scalar(<28 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v29, s45
; GFX9-NEXT: v_mov_b32_e32 v28, s44
; GFX9-NEXT: .LBB29_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_lshl_or_b32 v0, v47, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v46, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v45, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v44, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v43, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v42, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v41, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v40, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v47, 16, v47
+; GFX9-NEXT: v_lshlrev_b32_e32 v46, 16, v46
+; GFX9-NEXT: v_lshlrev_b32_e32 v45, 16, v45
+; GFX9-NEXT: v_lshlrev_b32_e32 v44, 16, v44
+; GFX9-NEXT: v_lshlrev_b32_e32 v43, 16, v43
+; GFX9-NEXT: v_lshlrev_b32_e32 v42, 16, v42
+; GFX9-NEXT: v_lshlrev_b32_e32 v41, 16, v41
+; GFX9-NEXT: v_lshlrev_b32_e32 v40, 16, v40
+; GFX9-NEXT: v_or_b32_sdwa v0, v47, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v46, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v45, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v44, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v4, v43, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v42, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v41, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v40, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: buffer_load_dword v47, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
@@ -17054,46 +17054,46 @@ define inreg <56 x i16> @bitcast_v28f32_to_v56i16_scalar(<28 x float> inreg %a,
; GFX9-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_and_b32_e32 v21, 0xffff, v21
-; GFX9-NEXT: v_and_b32_e32 v22, 0xffff, v22
-; GFX9-NEXT: v_and_b32_e32 v23, 0xffff, v23
-; GFX9-NEXT: v_and_b32_e32 v24, 0xffff, v24
-; GFX9-NEXT: v_and_b32_e32 v25, 0xffff, v25
-; GFX9-NEXT: v_and_b32_e32 v26, 0xffff, v26
-; GFX9-NEXT: v_and_b32_e32 v27, 0xffff, v27
-; GFX9-NEXT: v_lshl_or_b32 v8, v55, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v54, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v53, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v52, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v51, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v50, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v49, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v48, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v39, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v38, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v37, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v36, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v35, 16, v20
-; GFX9-NEXT: v_lshl_or_b32 v21, v34, 16, v21
-; GFX9-NEXT: v_lshl_or_b32 v22, v33, 16, v22
-; GFX9-NEXT: v_lshl_or_b32 v23, v32, 16, v23
-; GFX9-NEXT: v_lshl_or_b32 v24, v31, 16, v24
-; GFX9-NEXT: v_lshl_or_b32 v25, v30, 16, v25
-; GFX9-NEXT: v_lshl_or_b32 v26, v29, 16, v26
-; GFX9-NEXT: v_lshl_or_b32 v27, v28, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v55, 16, v55
+; GFX9-NEXT: v_lshlrev_b32_e32 v54, 16, v54
+; GFX9-NEXT: v_lshlrev_b32_e32 v53, 16, v53
+; GFX9-NEXT: v_lshlrev_b32_e32 v52, 16, v52
+; GFX9-NEXT: v_lshlrev_b32_e32 v51, 16, v51
+; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v50
+; GFX9-NEXT: v_lshlrev_b32_e32 v49, 16, v49
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; GFX9-NEXT: v_lshlrev_b32_e32 v39, 16, v39
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; GFX9-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; GFX9-NEXT: v_or_b32_sdwa v8, v55, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v54, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v53, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v52, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v12, v51, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v50, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v14, v49, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v48, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v16, v39, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v38, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v18, v37, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v36, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v20, v35, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v21, v34, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v22, v33, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v23, v32, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v24, v31, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v25, v30, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v26, v29, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v27, v28, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -21856,22 +21856,22 @@ define inreg <56 x half> @bitcast_v28f32_to_v56f16_scalar(<28 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v29, s45
; GFX9-NEXT: v_mov_b32_e32 v28, s44
; GFX9-NEXT: .LBB33_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_lshl_or_b32 v0, v47, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v46, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v45, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v44, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v43, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v42, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v41, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v40, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v47, 16, v47
+; GFX9-NEXT: v_lshlrev_b32_e32 v46, 16, v46
+; GFX9-NEXT: v_lshlrev_b32_e32 v45, 16, v45
+; GFX9-NEXT: v_lshlrev_b32_e32 v44, 16, v44
+; GFX9-NEXT: v_lshlrev_b32_e32 v43, 16, v43
+; GFX9-NEXT: v_lshlrev_b32_e32 v42, 16, v42
+; GFX9-NEXT: v_lshlrev_b32_e32 v41, 16, v41
+; GFX9-NEXT: v_lshlrev_b32_e32 v40, 16, v40
+; GFX9-NEXT: v_or_b32_sdwa v0, v47, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v46, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v45, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v44, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v4, v43, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v42, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v41, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v40, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: buffer_load_dword v47, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
@@ -21880,46 +21880,46 @@ define inreg <56 x half> @bitcast_v28f32_to_v56f16_scalar(<28 x float> inreg %a,
; GFX9-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_and_b32_e32 v21, 0xffff, v21
-; GFX9-NEXT: v_and_b32_e32 v22, 0xffff, v22
-; GFX9-NEXT: v_and_b32_e32 v23, 0xffff, v23
-; GFX9-NEXT: v_and_b32_e32 v24, 0xffff, v24
-; GFX9-NEXT: v_and_b32_e32 v25, 0xffff, v25
-; GFX9-NEXT: v_and_b32_e32 v26, 0xffff, v26
-; GFX9-NEXT: v_and_b32_e32 v27, 0xffff, v27
-; GFX9-NEXT: v_lshl_or_b32 v8, v55, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v54, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v53, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v52, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v51, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v50, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v49, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v48, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v39, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v38, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v37, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v36, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v35, 16, v20
-; GFX9-NEXT: v_lshl_or_b32 v21, v34, 16, v21
-; GFX9-NEXT: v_lshl_or_b32 v22, v33, 16, v22
-; GFX9-NEXT: v_lshl_or_b32 v23, v32, 16, v23
-; GFX9-NEXT: v_lshl_or_b32 v24, v31, 16, v24
-; GFX9-NEXT: v_lshl_or_b32 v25, v30, 16, v25
-; GFX9-NEXT: v_lshl_or_b32 v26, v29, 16, v26
-; GFX9-NEXT: v_lshl_or_b32 v27, v28, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v55, 16, v55
+; GFX9-NEXT: v_lshlrev_b32_e32 v54, 16, v54
+; GFX9-NEXT: v_lshlrev_b32_e32 v53, 16, v53
+; GFX9-NEXT: v_lshlrev_b32_e32 v52, 16, v52
+; GFX9-NEXT: v_lshlrev_b32_e32 v51, 16, v51
+; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v50
+; GFX9-NEXT: v_lshlrev_b32_e32 v49, 16, v49
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; GFX9-NEXT: v_lshlrev_b32_e32 v39, 16, v39
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; GFX9-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; GFX9-NEXT: v_or_b32_sdwa v8, v55, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v54, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v53, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v52, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v12, v51, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v50, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v14, v49, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v48, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v16, v39, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v38, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v18, v37, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v36, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v20, v35, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v21, v34, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v22, v33, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v23, v32, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v24, v31, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v25, v30, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v26, v29, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v27, v28, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -37030,22 +37030,24 @@ define inreg <56 x i16> @bitcast_v14f64_to_v56i16_scalar(<14 x double> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v33, s45
; GFX9-NEXT: v_mov_b32_e32 v31, s44
; GFX9-NEXT: .LBB49_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_lshl_or_b32 v0, v44, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v47, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v42, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v46, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v40, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v45, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v7, v43, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v9, v41, 16, v9
+; GFX9-NEXT: v_lshlrev_b32_e32 v44, 16, v44
+; GFX9-NEXT: v_lshlrev_b32_e32 v42, 16, v42
+; GFX9-NEXT: v_lshlrev_b32_e32 v40, 16, v40
+; GFX9-NEXT: v_or_b32_sdwa v0, v44, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v44, 16, v47
+; GFX9-NEXT: v_or_b32_sdwa v2, v42, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v42, 16, v46
+; GFX9-NEXT: v_or_b32_sdwa v4, v40, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v40, 16, v45
+; GFX9-NEXT: v_lshlrev_b32_e32 v54, 16, v54
+; GFX9-NEXT: v_lshlrev_b32_e32 v52, 16, v52
+; GFX9-NEXT: v_or_b32_sdwa v1, v44, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v42, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v40, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v54, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v54, 16, v43
+; GFX9-NEXT: v_or_b32_sdwa v8, v52, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v52, 16, v41
; GFX9-NEXT: buffer_load_dword v47, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
@@ -37054,46 +37056,44 @@ define inreg <56 x i16> @bitcast_v14f64_to_v56i16_scalar(<14 x double> inreg %a,
; GFX9-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_and_b32_e32 v21, 0xffff, v21
-; GFX9-NEXT: v_and_b32_e32 v22, 0xffff, v22
-; GFX9-NEXT: v_and_b32_e32 v23, 0xffff, v23
-; GFX9-NEXT: v_and_b32_e32 v24, 0xffff, v24
-; GFX9-NEXT: v_and_b32_e32 v25, 0xffff, v25
-; GFX9-NEXT: v_and_b32_e32 v26, 0xffff, v26
-; GFX9-NEXT: v_and_b32_e32 v27, 0xffff, v27
-; GFX9-NEXT: v_lshl_or_b32 v6, v54, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v8, v52, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v10, v50, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v55, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v48, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v53, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v38, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v51, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v36, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v49, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v34, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v39, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v32, 16, v20
-; GFX9-NEXT: v_lshl_or_b32 v21, v37, 16, v21
-; GFX9-NEXT: v_lshl_or_b32 v22, v30, 16, v22
-; GFX9-NEXT: v_lshl_or_b32 v23, v35, 16, v23
-; GFX9-NEXT: v_lshl_or_b32 v24, v29, 16, v24
-; GFX9-NEXT: v_lshl_or_b32 v25, v33, 16, v25
-; GFX9-NEXT: v_lshl_or_b32 v26, v28, 16, v26
-; GFX9-NEXT: v_lshl_or_b32 v27, v31, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v50
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; GFX9-NEXT: v_or_b32_sdwa v10, v50, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v55
+; GFX9-NEXT: v_or_b32_sdwa v12, v48, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v53
+; GFX9-NEXT: v_or_b32_sdwa v14, v38, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v51
+; GFX9-NEXT: v_or_b32_sdwa v16, v36, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v49
+; GFX9-NEXT: v_or_b32_sdwa v18, v34, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v39
+; GFX9-NEXT: v_or_b32_sdwa v20, v32, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v37
+; GFX9-NEXT: v_or_b32_sdwa v22, v30, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v35
+; GFX9-NEXT: v_or_b32_sdwa v24, v29, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v33
+; GFX9-NEXT: v_or_b32_sdwa v26, v28, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v31
+; GFX9-NEXT: v_or_b32_sdwa v7, v54, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v52, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v50, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v48, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v38, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v36, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v34, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v21, v32, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v23, v30, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v25, v29, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v27, v28, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -41751,22 +41751,24 @@ define inreg <56 x half> @bitcast_v14f64_to_v56f16_scalar(<14 x double> inreg %a
; GFX9-NEXT: v_mov_b32_e32 v33, s45
; GFX9-NEXT: v_mov_b32_e32 v31, s44
; GFX9-NEXT: .LBB53_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_lshl_or_b32 v0, v44, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v47, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v42, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v46, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v40, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v45, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v7, v43, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v9, v41, 16, v9
+; GFX9-NEXT: v_lshlrev_b32_e32 v44, 16, v44
+; GFX9-NEXT: v_lshlrev_b32_e32 v42, 16, v42
+; GFX9-NEXT: v_lshlrev_b32_e32 v40, 16, v40
+; GFX9-NEXT: v_or_b32_sdwa v0, v44, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v44, 16, v47
+; GFX9-NEXT: v_or_b32_sdwa v2, v42, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v42, 16, v46
+; GFX9-NEXT: v_or_b32_sdwa v4, v40, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v40, 16, v45
+; GFX9-NEXT: v_lshlrev_b32_e32 v54, 16, v54
+; GFX9-NEXT: v_lshlrev_b32_e32 v52, 16, v52
+; GFX9-NEXT: v_or_b32_sdwa v1, v44, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v42, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v40, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v54, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v54, 16, v43
+; GFX9-NEXT: v_or_b32_sdwa v8, v52, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v52, 16, v41
; GFX9-NEXT: buffer_load_dword v47, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
@@ -41775,46 +41777,44 @@ define inreg <56 x half> @bitcast_v14f64_to_v56f16_scalar(<14 x double> inreg %a
; GFX9-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_and_b32_e32 v21, 0xffff, v21
-; GFX9-NEXT: v_and_b32_e32 v22, 0xffff, v22
-; GFX9-NEXT: v_and_b32_e32 v23, 0xffff, v23
-; GFX9-NEXT: v_and_b32_e32 v24, 0xffff, v24
-; GFX9-NEXT: v_and_b32_e32 v25, 0xffff, v25
-; GFX9-NEXT: v_and_b32_e32 v26, 0xffff, v26
-; GFX9-NEXT: v_and_b32_e32 v27, 0xffff, v27
-; GFX9-NEXT: v_lshl_or_b32 v6, v54, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v8, v52, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v10, v50, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v55, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v48, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v53, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v38, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v51, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v36, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v49, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v34, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v39, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v32, 16, v20
-; GFX9-NEXT: v_lshl_or_b32 v21, v37, 16, v21
-; GFX9-NEXT: v_lshl_or_b32 v22, v30, 16, v22
-; GFX9-NEXT: v_lshl_or_b32 v23, v35, 16, v23
-; GFX9-NEXT: v_lshl_or_b32 v24, v29, 16, v24
-; GFX9-NEXT: v_lshl_or_b32 v25, v33, 16, v25
-; GFX9-NEXT: v_lshl_or_b32 v26, v28, 16, v26
-; GFX9-NEXT: v_lshl_or_b32 v27, v31, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v50
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; GFX9-NEXT: v_or_b32_sdwa v10, v50, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v55
+; GFX9-NEXT: v_or_b32_sdwa v12, v48, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v53
+; GFX9-NEXT: v_or_b32_sdwa v14, v38, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v51
+; GFX9-NEXT: v_or_b32_sdwa v16, v36, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v49
+; GFX9-NEXT: v_or_b32_sdwa v18, v34, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v39
+; GFX9-NEXT: v_or_b32_sdwa v20, v32, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v37
+; GFX9-NEXT: v_or_b32_sdwa v22, v30, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v35
+; GFX9-NEXT: v_or_b32_sdwa v24, v29, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v33
+; GFX9-NEXT: v_or_b32_sdwa v26, v28, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v31
+; GFX9-NEXT: v_or_b32_sdwa v7, v54, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v52, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v50, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v48, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v38, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v36, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v34, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v21, v32, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v23, v30, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v25, v29, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v27, v28, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -47446,22 +47446,22 @@ define inreg <56 x half> @bitcast_v56i16_to_v56f16_scalar(<56 x i16> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v46, s7
; GFX9-NEXT: v_mov_b32_e32 v47, s6
; GFX9-NEXT: .LBB57_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_lshl_or_b32 v0, v47, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v46, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v45, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v44, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v43, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v42, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v41, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v40, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v47, 16, v47
+; GFX9-NEXT: v_lshlrev_b32_e32 v46, 16, v46
+; GFX9-NEXT: v_lshlrev_b32_e32 v45, 16, v45
+; GFX9-NEXT: v_lshlrev_b32_e32 v44, 16, v44
+; GFX9-NEXT: v_lshlrev_b32_e32 v43, 16, v43
+; GFX9-NEXT: v_lshlrev_b32_e32 v42, 16, v42
+; GFX9-NEXT: v_lshlrev_b32_e32 v41, 16, v41
+; GFX9-NEXT: v_lshlrev_b32_e32 v40, 16, v40
+; GFX9-NEXT: v_or_b32_sdwa v0, v47, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v46, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v45, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v44, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v4, v43, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v42, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v41, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v40, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: buffer_load_dword v47, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
@@ -47470,46 +47470,46 @@ define inreg <56 x half> @bitcast_v56i16_to_v56f16_scalar(<56 x i16> inreg %a, i
; GFX9-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_and_b32_e32 v21, 0xffff, v21
-; GFX9-NEXT: v_and_b32_e32 v22, 0xffff, v22
-; GFX9-NEXT: v_and_b32_e32 v23, 0xffff, v23
-; GFX9-NEXT: v_and_b32_e32 v24, 0xffff, v24
-; GFX9-NEXT: v_and_b32_e32 v25, 0xffff, v25
-; GFX9-NEXT: v_and_b32_e32 v26, 0xffff, v26
-; GFX9-NEXT: v_and_b32_e32 v27, 0xffff, v27
-; GFX9-NEXT: v_lshl_or_b32 v8, v55, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v54, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v53, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v52, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v51, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v50, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v49, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v48, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v39, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v38, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v37, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v36, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v35, 16, v20
-; GFX9-NEXT: v_lshl_or_b32 v21, v34, 16, v21
-; GFX9-NEXT: v_lshl_or_b32 v22, v33, 16, v22
-; GFX9-NEXT: v_lshl_or_b32 v23, v32, 16, v23
-; GFX9-NEXT: v_lshl_or_b32 v24, v31, 16, v24
-; GFX9-NEXT: v_lshl_or_b32 v25, v30, 16, v25
-; GFX9-NEXT: v_lshl_or_b32 v26, v29, 16, v26
-; GFX9-NEXT: v_lshl_or_b32 v27, v28, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v55, 16, v55
+; GFX9-NEXT: v_lshlrev_b32_e32 v54, 16, v54
+; GFX9-NEXT: v_lshlrev_b32_e32 v53, 16, v53
+; GFX9-NEXT: v_lshlrev_b32_e32 v52, 16, v52
+; GFX9-NEXT: v_lshlrev_b32_e32 v51, 16, v51
+; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v50
+; GFX9-NEXT: v_lshlrev_b32_e32 v49, 16, v49
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; GFX9-NEXT: v_lshlrev_b32_e32 v39, 16, v39
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; GFX9-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; GFX9-NEXT: v_or_b32_sdwa v8, v55, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v54, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v53, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v52, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v12, v51, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v50, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v14, v49, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v48, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v16, v39, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v38, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v18, v37, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v36, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v20, v35, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v21, v34, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v22, v33, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v23, v32, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v24, v31, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v25, v30, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v26, v29, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v27, v28, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -50002,22 +50002,22 @@ define inreg <56 x i16> @bitcast_v56f16_to_v56i16_scalar(<56 x half> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v46, s7
; GFX9-NEXT: v_mov_b32_e32 v47, s6
; GFX9-NEXT: .LBB59_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_lshl_or_b32 v0, v47, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v46, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v45, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v44, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v43, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v42, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v41, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v40, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v47, 16, v47
+; GFX9-NEXT: v_lshlrev_b32_e32 v46, 16, v46
+; GFX9-NEXT: v_lshlrev_b32_e32 v45, 16, v45
+; GFX9-NEXT: v_lshlrev_b32_e32 v44, 16, v44
+; GFX9-NEXT: v_lshlrev_b32_e32 v43, 16, v43
+; GFX9-NEXT: v_lshlrev_b32_e32 v42, 16, v42
+; GFX9-NEXT: v_lshlrev_b32_e32 v41, 16, v41
+; GFX9-NEXT: v_lshlrev_b32_e32 v40, 16, v40
+; GFX9-NEXT: v_or_b32_sdwa v0, v47, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v46, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v45, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v44, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v4, v43, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v42, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v41, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v40, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: buffer_load_dword v47, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
@@ -50026,46 +50026,46 @@ define inreg <56 x i16> @bitcast_v56f16_to_v56i16_scalar(<56 x half> inreg %a, i
; GFX9-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_and_b32_e32 v21, 0xffff, v21
-; GFX9-NEXT: v_and_b32_e32 v22, 0xffff, v22
-; GFX9-NEXT: v_and_b32_e32 v23, 0xffff, v23
-; GFX9-NEXT: v_and_b32_e32 v24, 0xffff, v24
-; GFX9-NEXT: v_and_b32_e32 v25, 0xffff, v25
-; GFX9-NEXT: v_and_b32_e32 v26, 0xffff, v26
-; GFX9-NEXT: v_and_b32_e32 v27, 0xffff, v27
-; GFX9-NEXT: v_lshl_or_b32 v8, v55, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v54, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v53, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v52, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v12, v51, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v50, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v49, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v48, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v39, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v38, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v37, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v36, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v35, 16, v20
-; GFX9-NEXT: v_lshl_or_b32 v21, v34, 16, v21
-; GFX9-NEXT: v_lshl_or_b32 v22, v33, 16, v22
-; GFX9-NEXT: v_lshl_or_b32 v23, v32, 16, v23
-; GFX9-NEXT: v_lshl_or_b32 v24, v31, 16, v24
-; GFX9-NEXT: v_lshl_or_b32 v25, v30, 16, v25
-; GFX9-NEXT: v_lshl_or_b32 v26, v29, 16, v26
-; GFX9-NEXT: v_lshl_or_b32 v27, v28, 16, v27
+; GFX9-NEXT: v_lshlrev_b32_e32 v55, 16, v55
+; GFX9-NEXT: v_lshlrev_b32_e32 v54, 16, v54
+; GFX9-NEXT: v_lshlrev_b32_e32 v53, 16, v53
+; GFX9-NEXT: v_lshlrev_b32_e32 v52, 16, v52
+; GFX9-NEXT: v_lshlrev_b32_e32 v51, 16, v51
+; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v50
+; GFX9-NEXT: v_lshlrev_b32_e32 v49, 16, v49
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; GFX9-NEXT: v_lshlrev_b32_e32 v39, 16, v39
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; GFX9-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; GFX9-NEXT: v_or_b32_sdwa v8, v55, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v54, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v53, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v52, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v12, v51, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v50, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v14, v49, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v48, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v16, v39, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v38, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v18, v37, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v36, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v20, v35, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v21, v34, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v22, v33, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v23, v32, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v24, v31, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v25, v30, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v26, v29, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v27, v28, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.960bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.960bit.ll
index d3bb79fe0084e..deced731f537b 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.960bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.960bit.ll
@@ -18268,30 +18268,30 @@ define inreg <60 x i16> @bitcast_v30f32_to_v60i16_scalar(<30 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v31, s47
; GFX9-NEXT: v_mov_b32_e32 v30, s46
; GFX9-NEXT: .LBB29_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_lshl_or_b32 v0, v59, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v58, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v57, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v56, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v47, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v46, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v45, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v44, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v43, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v42, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v41, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v40, 16, v11
+; GFX9-NEXT: v_lshlrev_b32_e32 v59, 16, v59
+; GFX9-NEXT: v_lshlrev_b32_e32 v58, 16, v58
+; GFX9-NEXT: v_lshlrev_b32_e32 v57, 16, v57
+; GFX9-NEXT: v_lshlrev_b32_e32 v56, 16, v56
+; GFX9-NEXT: v_lshlrev_b32_e32 v47, 16, v47
+; GFX9-NEXT: v_lshlrev_b32_e32 v46, 16, v46
+; GFX9-NEXT: v_lshlrev_b32_e32 v45, 16, v45
+; GFX9-NEXT: v_lshlrev_b32_e32 v44, 16, v44
+; GFX9-NEXT: v_lshlrev_b32_e32 v43, 16, v43
+; GFX9-NEXT: v_lshlrev_b32_e32 v42, 16, v42
+; GFX9-NEXT: v_lshlrev_b32_e32 v41, 16, v41
+; GFX9-NEXT: v_lshlrev_b32_e32 v40, 16, v40
+; GFX9-NEXT: v_or_b32_sdwa v0, v59, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v58, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v57, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v56, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v4, v47, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v46, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v45, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v44, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v43, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v42, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v41, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v40, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: buffer_load_dword v59, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
@@ -18304,43 +18304,43 @@ define inreg <60 x i16> @bitcast_v30f32_to_v60i16_scalar(<30 x float> inreg %a,
; GFX9-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_and_b32_e32 v21, 0xffff, v21
-; GFX9-NEXT: v_and_b32_e32 v22, 0xffff, v22
-; GFX9-NEXT: v_and_b32_e32 v23, 0xffff, v23
-; GFX9-NEXT: v_and_b32_e32 v24, 0xffff, v24
-; GFX9-NEXT: v_and_b32_e32 v25, 0xffff, v25
-; GFX9-NEXT: v_and_b32_e32 v26, 0xffff, v26
-; GFX9-NEXT: v_and_b32_e32 v27, 0xffff, v27
-; GFX9-NEXT: v_and_b32_e32 v28, 0xffff, v28
-; GFX9-NEXT: v_and_b32_e32 v29, 0xffff, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v55, 16, v55
+; GFX9-NEXT: v_lshlrev_b32_e32 v54, 16, v54
+; GFX9-NEXT: v_lshlrev_b32_e32 v53, 16, v53
+; GFX9-NEXT: v_lshlrev_b32_e32 v52, 16, v52
+; GFX9-NEXT: v_lshlrev_b32_e32 v51, 16, v51
+; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v50
+; GFX9-NEXT: v_lshlrev_b32_e32 v49, 16, v49
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; GFX9-NEXT: v_lshlrev_b32_e32 v39, 16, v39
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; GFX9-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
; GFX9-NEXT: v_readlane_b32 s30, v60, 0
-; GFX9-NEXT: v_lshl_or_b32 v12, v55, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v54, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v53, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v52, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v51, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v50, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v49, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v48, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v39, 16, v20
-; GFX9-NEXT: v_lshl_or_b32 v21, v38, 16, v21
-; GFX9-NEXT: v_lshl_or_b32 v22, v37, 16, v22
-; GFX9-NEXT: v_lshl_or_b32 v23, v36, 16, v23
-; GFX9-NEXT: v_lshl_or_b32 v24, v35, 16, v24
-; GFX9-NEXT: v_lshl_or_b32 v25, v34, 16, v25
-; GFX9-NEXT: v_lshl_or_b32 v26, v33, 16, v26
-; GFX9-NEXT: v_lshl_or_b32 v27, v32, 16, v27
-; GFX9-NEXT: v_lshl_or_b32 v28, v31, 16, v28
-; GFX9-NEXT: v_lshl_or_b32 v29, v30, 16, v29
+; GFX9-NEXT: v_or_b32_sdwa v12, v55, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v54, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v14, v53, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v52, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v16, v51, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v50, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v18, v49, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v48, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v20, v39, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v21, v38, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v22, v37, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v23, v36, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v24, v35, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v25, v34, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v26, v33, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v27, v32, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v28, v31, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v29, v30, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_readlane_b32 s31, v60, 1
; GFX9-NEXT: s_or_saveexec_b64 s[4:5], -1
; GFX9-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
@@ -23502,30 +23502,30 @@ define inreg <60 x half> @bitcast_v30f32_to_v60f16_scalar(<30 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v31, s47
; GFX9-NEXT: v_mov_b32_e32 v30, s46
; GFX9-NEXT: .LBB33_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_lshl_or_b32 v0, v59, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v58, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v57, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v56, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v47, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v46, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v45, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v44, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v43, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v42, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v41, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v40, 16, v11
+; GFX9-NEXT: v_lshlrev_b32_e32 v59, 16, v59
+; GFX9-NEXT: v_lshlrev_b32_e32 v58, 16, v58
+; GFX9-NEXT: v_lshlrev_b32_e32 v57, 16, v57
+; GFX9-NEXT: v_lshlrev_b32_e32 v56, 16, v56
+; GFX9-NEXT: v_lshlrev_b32_e32 v47, 16, v47
+; GFX9-NEXT: v_lshlrev_b32_e32 v46, 16, v46
+; GFX9-NEXT: v_lshlrev_b32_e32 v45, 16, v45
+; GFX9-NEXT: v_lshlrev_b32_e32 v44, 16, v44
+; GFX9-NEXT: v_lshlrev_b32_e32 v43, 16, v43
+; GFX9-NEXT: v_lshlrev_b32_e32 v42, 16, v42
+; GFX9-NEXT: v_lshlrev_b32_e32 v41, 16, v41
+; GFX9-NEXT: v_lshlrev_b32_e32 v40, 16, v40
+; GFX9-NEXT: v_or_b32_sdwa v0, v59, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v58, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v57, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v56, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v4, v47, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v46, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v45, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v44, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v43, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v42, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v41, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v40, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: buffer_load_dword v59, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
@@ -23538,43 +23538,43 @@ define inreg <60 x half> @bitcast_v30f32_to_v60f16_scalar(<30 x float> inreg %a,
; GFX9-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_and_b32_e32 v21, 0xffff, v21
-; GFX9-NEXT: v_and_b32_e32 v22, 0xffff, v22
-; GFX9-NEXT: v_and_b32_e32 v23, 0xffff, v23
-; GFX9-NEXT: v_and_b32_e32 v24, 0xffff, v24
-; GFX9-NEXT: v_and_b32_e32 v25, 0xffff, v25
-; GFX9-NEXT: v_and_b32_e32 v26, 0xffff, v26
-; GFX9-NEXT: v_and_b32_e32 v27, 0xffff, v27
-; GFX9-NEXT: v_and_b32_e32 v28, 0xffff, v28
-; GFX9-NEXT: v_and_b32_e32 v29, 0xffff, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v55, 16, v55
+; GFX9-NEXT: v_lshlrev_b32_e32 v54, 16, v54
+; GFX9-NEXT: v_lshlrev_b32_e32 v53, 16, v53
+; GFX9-NEXT: v_lshlrev_b32_e32 v52, 16, v52
+; GFX9-NEXT: v_lshlrev_b32_e32 v51, 16, v51
+; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v50
+; GFX9-NEXT: v_lshlrev_b32_e32 v49, 16, v49
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; GFX9-NEXT: v_lshlrev_b32_e32 v39, 16, v39
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; GFX9-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
; GFX9-NEXT: v_readlane_b32 s30, v60, 0
-; GFX9-NEXT: v_lshl_or_b32 v12, v55, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v54, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v53, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v52, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v51, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v50, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v49, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v48, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v39, 16, v20
-; GFX9-NEXT: v_lshl_or_b32 v21, v38, 16, v21
-; GFX9-NEXT: v_lshl_or_b32 v22, v37, 16, v22
-; GFX9-NEXT: v_lshl_or_b32 v23, v36, 16, v23
-; GFX9-NEXT: v_lshl_or_b32 v24, v35, 16, v24
-; GFX9-NEXT: v_lshl_or_b32 v25, v34, 16, v25
-; GFX9-NEXT: v_lshl_or_b32 v26, v33, 16, v26
-; GFX9-NEXT: v_lshl_or_b32 v27, v32, 16, v27
-; GFX9-NEXT: v_lshl_or_b32 v28, v31, 16, v28
-; GFX9-NEXT: v_lshl_or_b32 v29, v30, 16, v29
+; GFX9-NEXT: v_or_b32_sdwa v12, v55, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v54, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v14, v53, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v52, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v16, v51, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v50, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v18, v49, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v48, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v20, v39, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v21, v38, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v22, v37, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v23, v36, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v24, v35, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v25, v34, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v26, v33, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v27, v32, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v28, v31, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v29, v30, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_readlane_b32 s31, v60, 1
; GFX9-NEXT: s_or_saveexec_b64 s[4:5], -1
; GFX9-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
@@ -39872,30 +39872,32 @@ define inreg <60 x i16> @bitcast_v15f64_to_v60i16_scalar(<15 x double> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v35, s47
; GFX9-NEXT: v_mov_b32_e32 v33, s46
; GFX9-NEXT: .LBB49_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_lshl_or_b32 v0, v56, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v59, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v46, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v58, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v44, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v57, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v42, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v47, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v40, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v45, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v11, v43, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v13, v41, 16, v13
+; GFX9-NEXT: v_lshlrev_b32_e32 v56, 16, v56
+; GFX9-NEXT: v_lshlrev_b32_e32 v46, 16, v46
+; GFX9-NEXT: v_lshlrev_b32_e32 v44, 16, v44
+; GFX9-NEXT: v_lshlrev_b32_e32 v42, 16, v42
+; GFX9-NEXT: v_lshlrev_b32_e32 v40, 16, v40
+; GFX9-NEXT: v_or_b32_sdwa v0, v56, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v56, 16, v59
+; GFX9-NEXT: v_or_b32_sdwa v2, v46, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v46, 16, v58
+; GFX9-NEXT: v_or_b32_sdwa v4, v44, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v44, 16, v57
+; GFX9-NEXT: v_or_b32_sdwa v6, v42, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v42, 16, v47
+; GFX9-NEXT: v_or_b32_sdwa v8, v40, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v40, 16, v45
+; GFX9-NEXT: v_lshlrev_b32_e32 v54, 16, v54
+; GFX9-NEXT: v_lshlrev_b32_e32 v52, 16, v52
+; GFX9-NEXT: v_or_b32_sdwa v1, v56, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v46, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v44, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v42, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v40, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v54, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v54, 16, v43
+; GFX9-NEXT: v_or_b32_sdwa v12, v52, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v52, 16, v41
; GFX9-NEXT: buffer_load_dword v59, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
@@ -39908,43 +39910,41 @@ define inreg <60 x i16> @bitcast_v15f64_to_v60i16_scalar(<15 x double> inreg %a,
; GFX9-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_and_b32_e32 v21, 0xffff, v21
-; GFX9-NEXT: v_and_b32_e32 v22, 0xffff, v22
-; GFX9-NEXT: v_and_b32_e32 v23, 0xffff, v23
-; GFX9-NEXT: v_and_b32_e32 v24, 0xffff, v24
-; GFX9-NEXT: v_and_b32_e32 v25, 0xffff, v25
-; GFX9-NEXT: v_and_b32_e32 v26, 0xffff, v26
-; GFX9-NEXT: v_and_b32_e32 v27, 0xffff, v27
-; GFX9-NEXT: v_and_b32_e32 v28, 0xffff, v28
-; GFX9-NEXT: v_and_b32_e32 v29, 0xffff, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v50
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; GFX9-NEXT: v_or_b32_sdwa v14, v50, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v55
+; GFX9-NEXT: v_or_b32_sdwa v16, v48, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v53
+; GFX9-NEXT: v_or_b32_sdwa v18, v38, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v51
+; GFX9-NEXT: v_or_b32_sdwa v20, v36, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v49
+; GFX9-NEXT: v_or_b32_sdwa v22, v34, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v39
+; GFX9-NEXT: v_or_b32_sdwa v24, v32, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v37
+; GFX9-NEXT: v_or_b32_sdwa v26, v31, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v35
+; GFX9-NEXT: v_or_b32_sdwa v28, v30, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v33
; GFX9-NEXT: v_readlane_b32 s30, v60, 0
-; GFX9-NEXT: v_lshl_or_b32 v10, v54, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v12, v52, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v14, v50, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v55, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v48, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v53, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v38, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v51, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v36, 16, v20
-; GFX9-NEXT: v_lshl_or_b32 v21, v49, 16, v21
-; GFX9-NEXT: v_lshl_or_b32 v22, v34, 16, v22
-; GFX9-NEXT: v_lshl_or_b32 v23, v39, 16, v23
-; GFX9-NEXT: v_lshl_or_b32 v24, v32, 16, v24
-; GFX9-NEXT: v_lshl_or_b32 v25, v37, 16, v25
-; GFX9-NEXT: v_lshl_or_b32 v26, v31, 16, v26
-; GFX9-NEXT: v_lshl_or_b32 v27, v35, 16, v27
-; GFX9-NEXT: v_lshl_or_b32 v28, v30, 16, v28
-; GFX9-NEXT: v_lshl_or_b32 v29, v33, 16, v29
+; GFX9-NEXT: v_or_b32_sdwa v11, v54, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v52, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v50, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v48, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v38, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v21, v36, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v23, v34, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v25, v32, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v27, v31, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v29, v30, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_readlane_b32 s31, v60, 1
; GFX9-NEXT: s_or_saveexec_b64 s[4:5], -1
; GFX9-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
@@ -44993,30 +44993,32 @@ define inreg <60 x half> @bitcast_v15f64_to_v60f16_scalar(<15 x double> inreg %a
; GFX9-NEXT: v_mov_b32_e32 v35, s47
; GFX9-NEXT: v_mov_b32_e32 v33, s46
; GFX9-NEXT: .LBB53_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_lshl_or_b32 v0, v56, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v59, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v46, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v58, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v44, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v57, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v42, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v47, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v40, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v45, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v11, v43, 16, v11
-; GFX9-NEXT: v_lshl_or_b32 v13, v41, 16, v13
+; GFX9-NEXT: v_lshlrev_b32_e32 v56, 16, v56
+; GFX9-NEXT: v_lshlrev_b32_e32 v46, 16, v46
+; GFX9-NEXT: v_lshlrev_b32_e32 v44, 16, v44
+; GFX9-NEXT: v_lshlrev_b32_e32 v42, 16, v42
+; GFX9-NEXT: v_lshlrev_b32_e32 v40, 16, v40
+; GFX9-NEXT: v_or_b32_sdwa v0, v56, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v56, 16, v59
+; GFX9-NEXT: v_or_b32_sdwa v2, v46, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v46, 16, v58
+; GFX9-NEXT: v_or_b32_sdwa v4, v44, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v44, 16, v57
+; GFX9-NEXT: v_or_b32_sdwa v6, v42, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v42, 16, v47
+; GFX9-NEXT: v_or_b32_sdwa v8, v40, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v40, 16, v45
+; GFX9-NEXT: v_lshlrev_b32_e32 v54, 16, v54
+; GFX9-NEXT: v_lshlrev_b32_e32 v52, 16, v52
+; GFX9-NEXT: v_or_b32_sdwa v1, v56, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v46, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v44, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v42, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v40, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v54, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v54, 16, v43
+; GFX9-NEXT: v_or_b32_sdwa v12, v52, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v52, 16, v41
; GFX9-NEXT: buffer_load_dword v59, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
@@ -45029,43 +45031,41 @@ define inreg <60 x half> @bitcast_v15f64_to_v60f16_scalar(<15 x double> inreg %a
; GFX9-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_and_b32_e32 v21, 0xffff, v21
-; GFX9-NEXT: v_and_b32_e32 v22, 0xffff, v22
-; GFX9-NEXT: v_and_b32_e32 v23, 0xffff, v23
-; GFX9-NEXT: v_and_b32_e32 v24, 0xffff, v24
-; GFX9-NEXT: v_and_b32_e32 v25, 0xffff, v25
-; GFX9-NEXT: v_and_b32_e32 v26, 0xffff, v26
-; GFX9-NEXT: v_and_b32_e32 v27, 0xffff, v27
-; GFX9-NEXT: v_and_b32_e32 v28, 0xffff, v28
-; GFX9-NEXT: v_and_b32_e32 v29, 0xffff, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v50
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; GFX9-NEXT: v_or_b32_sdwa v14, v50, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v55
+; GFX9-NEXT: v_or_b32_sdwa v16, v48, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v53
+; GFX9-NEXT: v_or_b32_sdwa v18, v38, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v51
+; GFX9-NEXT: v_or_b32_sdwa v20, v36, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v49
+; GFX9-NEXT: v_or_b32_sdwa v22, v34, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v39
+; GFX9-NEXT: v_or_b32_sdwa v24, v32, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v37
+; GFX9-NEXT: v_or_b32_sdwa v26, v31, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v35
+; GFX9-NEXT: v_or_b32_sdwa v28, v30, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v33
; GFX9-NEXT: v_readlane_b32 s30, v60, 0
-; GFX9-NEXT: v_lshl_or_b32 v10, v54, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v12, v52, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v14, v50, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v55, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v48, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v53, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v38, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v51, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v36, 16, v20
-; GFX9-NEXT: v_lshl_or_b32 v21, v49, 16, v21
-; GFX9-NEXT: v_lshl_or_b32 v22, v34, 16, v22
-; GFX9-NEXT: v_lshl_or_b32 v23, v39, 16, v23
-; GFX9-NEXT: v_lshl_or_b32 v24, v32, 16, v24
-; GFX9-NEXT: v_lshl_or_b32 v25, v37, 16, v25
-; GFX9-NEXT: v_lshl_or_b32 v26, v31, 16, v26
-; GFX9-NEXT: v_lshl_or_b32 v27, v35, 16, v27
-; GFX9-NEXT: v_lshl_or_b32 v28, v30, 16, v28
-; GFX9-NEXT: v_lshl_or_b32 v29, v33, 16, v29
+; GFX9-NEXT: v_or_b32_sdwa v11, v54, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v52, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v50, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v48, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v38, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v21, v36, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v23, v34, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v25, v32, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v27, v31, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v29, v30, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_readlane_b32 s31, v60, 1
; GFX9-NEXT: s_or_saveexec_b64 s[4:5], -1
; GFX9-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
@@ -51157,30 +51157,30 @@ define inreg <60 x half> @bitcast_v60i16_to_v60f16_scalar(<60 x i16> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v58, s7
; GFX9-NEXT: v_mov_b32_e32 v59, s6
; GFX9-NEXT: .LBB57_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_lshl_or_b32 v0, v59, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v58, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v57, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v56, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v47, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v46, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v45, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v44, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v43, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v42, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v41, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v40, 16, v11
+; GFX9-NEXT: v_lshlrev_b32_e32 v59, 16, v59
+; GFX9-NEXT: v_lshlrev_b32_e32 v58, 16, v58
+; GFX9-NEXT: v_lshlrev_b32_e32 v57, 16, v57
+; GFX9-NEXT: v_lshlrev_b32_e32 v56, 16, v56
+; GFX9-NEXT: v_lshlrev_b32_e32 v47, 16, v47
+; GFX9-NEXT: v_lshlrev_b32_e32 v46, 16, v46
+; GFX9-NEXT: v_lshlrev_b32_e32 v45, 16, v45
+; GFX9-NEXT: v_lshlrev_b32_e32 v44, 16, v44
+; GFX9-NEXT: v_lshlrev_b32_e32 v43, 16, v43
+; GFX9-NEXT: v_lshlrev_b32_e32 v42, 16, v42
+; GFX9-NEXT: v_lshlrev_b32_e32 v41, 16, v41
+; GFX9-NEXT: v_lshlrev_b32_e32 v40, 16, v40
+; GFX9-NEXT: v_or_b32_sdwa v0, v59, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v58, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v57, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v56, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v4, v47, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v46, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v45, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v44, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v43, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v42, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v41, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v40, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: buffer_load_dword v59, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
@@ -51193,43 +51193,43 @@ define inreg <60 x half> @bitcast_v60i16_to_v60f16_scalar(<60 x i16> inreg %a, i
; GFX9-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_and_b32_e32 v21, 0xffff, v21
-; GFX9-NEXT: v_and_b32_e32 v22, 0xffff, v22
-; GFX9-NEXT: v_and_b32_e32 v23, 0xffff, v23
-; GFX9-NEXT: v_and_b32_e32 v24, 0xffff, v24
-; GFX9-NEXT: v_and_b32_e32 v25, 0xffff, v25
-; GFX9-NEXT: v_and_b32_e32 v26, 0xffff, v26
-; GFX9-NEXT: v_and_b32_e32 v27, 0xffff, v27
-; GFX9-NEXT: v_and_b32_e32 v28, 0xffff, v28
-; GFX9-NEXT: v_and_b32_e32 v29, 0xffff, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v55, 16, v55
+; GFX9-NEXT: v_lshlrev_b32_e32 v54, 16, v54
+; GFX9-NEXT: v_lshlrev_b32_e32 v53, 16, v53
+; GFX9-NEXT: v_lshlrev_b32_e32 v52, 16, v52
+; GFX9-NEXT: v_lshlrev_b32_e32 v51, 16, v51
+; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v50
+; GFX9-NEXT: v_lshlrev_b32_e32 v49, 16, v49
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; GFX9-NEXT: v_lshlrev_b32_e32 v39, 16, v39
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; GFX9-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
; GFX9-NEXT: v_readlane_b32 s30, v60, 0
-; GFX9-NEXT: v_lshl_or_b32 v12, v55, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v54, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v53, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v52, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v51, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v50, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v49, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v48, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v39, 16, v20
-; GFX9-NEXT: v_lshl_or_b32 v21, v38, 16, v21
-; GFX9-NEXT: v_lshl_or_b32 v22, v37, 16, v22
-; GFX9-NEXT: v_lshl_or_b32 v23, v36, 16, v23
-; GFX9-NEXT: v_lshl_or_b32 v24, v35, 16, v24
-; GFX9-NEXT: v_lshl_or_b32 v25, v34, 16, v25
-; GFX9-NEXT: v_lshl_or_b32 v26, v33, 16, v26
-; GFX9-NEXT: v_lshl_or_b32 v27, v32, 16, v27
-; GFX9-NEXT: v_lshl_or_b32 v28, v31, 16, v28
-; GFX9-NEXT: v_lshl_or_b32 v29, v30, 16, v29
+; GFX9-NEXT: v_or_b32_sdwa v12, v55, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v54, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v14, v53, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v52, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v16, v51, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v50, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v18, v49, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v48, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v20, v39, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v21, v38, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v22, v37, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v23, v36, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v24, v35, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v25, v34, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v26, v33, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v27, v32, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v28, v31, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v29, v30, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_readlane_b32 s31, v60, 1
; GFX9-NEXT: s_or_saveexec_b64 s[4:5], -1
; GFX9-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
@@ -53961,30 +53961,30 @@ define inreg <60 x i16> @bitcast_v60f16_to_v60i16_scalar(<60 x half> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v58, s7
; GFX9-NEXT: v_mov_b32_e32 v59, s6
; GFX9-NEXT: .LBB59_6: ; %end
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX9-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX9-NEXT: v_lshl_or_b32 v0, v59, 16, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v58, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v57, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v3, v56, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v47, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v46, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v6, v45, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, v44, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v8, v43, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, v42, 16, v9
-; GFX9-NEXT: v_lshl_or_b32 v10, v41, 16, v10
-; GFX9-NEXT: v_lshl_or_b32 v11, v40, 16, v11
+; GFX9-NEXT: v_lshlrev_b32_e32 v59, 16, v59
+; GFX9-NEXT: v_lshlrev_b32_e32 v58, 16, v58
+; GFX9-NEXT: v_lshlrev_b32_e32 v57, 16, v57
+; GFX9-NEXT: v_lshlrev_b32_e32 v56, 16, v56
+; GFX9-NEXT: v_lshlrev_b32_e32 v47, 16, v47
+; GFX9-NEXT: v_lshlrev_b32_e32 v46, 16, v46
+; GFX9-NEXT: v_lshlrev_b32_e32 v45, 16, v45
+; GFX9-NEXT: v_lshlrev_b32_e32 v44, 16, v44
+; GFX9-NEXT: v_lshlrev_b32_e32 v43, 16, v43
+; GFX9-NEXT: v_lshlrev_b32_e32 v42, 16, v42
+; GFX9-NEXT: v_lshlrev_b32_e32 v41, 16, v41
+; GFX9-NEXT: v_lshlrev_b32_e32 v40, 16, v40
+; GFX9-NEXT: v_or_b32_sdwa v0, v59, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v58, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v57, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v56, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v4, v47, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v5, v46, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v6, v45, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v7, v44, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v8, v43, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v9, v42, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v10, v41, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v11, v40, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: buffer_load_dword v59, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
@@ -53997,43 +53997,43 @@ define inreg <60 x i16> @bitcast_v60f16_to_v60i16_scalar(<60 x half> inreg %a, i
; GFX9-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
-; GFX9-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX9-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX9-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX9-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; GFX9-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NEXT: v_and_b32_e32 v21, 0xffff, v21
-; GFX9-NEXT: v_and_b32_e32 v22, 0xffff, v22
-; GFX9-NEXT: v_and_b32_e32 v23, 0xffff, v23
-; GFX9-NEXT: v_and_b32_e32 v24, 0xffff, v24
-; GFX9-NEXT: v_and_b32_e32 v25, 0xffff, v25
-; GFX9-NEXT: v_and_b32_e32 v26, 0xffff, v26
-; GFX9-NEXT: v_and_b32_e32 v27, 0xffff, v27
-; GFX9-NEXT: v_and_b32_e32 v28, 0xffff, v28
-; GFX9-NEXT: v_and_b32_e32 v29, 0xffff, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v55, 16, v55
+; GFX9-NEXT: v_lshlrev_b32_e32 v54, 16, v54
+; GFX9-NEXT: v_lshlrev_b32_e32 v53, 16, v53
+; GFX9-NEXT: v_lshlrev_b32_e32 v52, 16, v52
+; GFX9-NEXT: v_lshlrev_b32_e32 v51, 16, v51
+; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v50
+; GFX9-NEXT: v_lshlrev_b32_e32 v49, 16, v49
+; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; GFX9-NEXT: v_lshlrev_b32_e32 v39, 16, v39
+; GFX9-NEXT: v_lshlrev_b32_e32 v38, 16, v38
+; GFX9-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; GFX9-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; GFX9-NEXT: v_lshlrev_b32_e32 v35, 16, v35
+; GFX9-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v30
; GFX9-NEXT: v_readlane_b32 s30, v60, 0
-; GFX9-NEXT: v_lshl_or_b32 v12, v55, 16, v12
-; GFX9-NEXT: v_lshl_or_b32 v13, v54, 16, v13
-; GFX9-NEXT: v_lshl_or_b32 v14, v53, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v15, v52, 16, v15
-; GFX9-NEXT: v_lshl_or_b32 v16, v51, 16, v16
-; GFX9-NEXT: v_lshl_or_b32 v17, v50, 16, v17
-; GFX9-NEXT: v_lshl_or_b32 v18, v49, 16, v18
-; GFX9-NEXT: v_lshl_or_b32 v19, v48, 16, v19
-; GFX9-NEXT: v_lshl_or_b32 v20, v39, 16, v20
-; GFX9-NEXT: v_lshl_or_b32 v21, v38, 16, v21
-; GFX9-NEXT: v_lshl_or_b32 v22, v37, 16, v22
-; GFX9-NEXT: v_lshl_or_b32 v23, v36, 16, v23
-; GFX9-NEXT: v_lshl_or_b32 v24, v35, 16, v24
-; GFX9-NEXT: v_lshl_or_b32 v25, v34, 16, v25
-; GFX9-NEXT: v_lshl_or_b32 v26, v33, 16, v26
-; GFX9-NEXT: v_lshl_or_b32 v27, v32, 16, v27
-; GFX9-NEXT: v_lshl_or_b32 v28, v31, 16, v28
-; GFX9-NEXT: v_lshl_or_b32 v29, v30, 16, v29
+; GFX9-NEXT: v_or_b32_sdwa v12, v55, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v13, v54, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v14, v53, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v15, v52, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v16, v51, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v17, v50, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v18, v49, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v19, v48, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v20, v39, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v21, v38, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v22, v37, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v23, v36, v23 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v24, v35, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v25, v34, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v26, v33, v26 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v27, v32, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v28, v31, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v29, v30, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_readlane_b32 s31, v60, 1
; GFX9-NEXT: s_or_saveexec_b64 s[4:5], -1
; GFX9-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll
index ece52bde4d948..005d3672f784b 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll
@@ -2427,52 +2427,51 @@ define inreg <3 x i32> @bitcast_v6bf16_to_v3i32_scalar(<6 x bfloat> inreg %a, i3
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX9-NEXT: v_mov_b32_e32 v3, 0xffff
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s17
-; GFX9-NEXT: v_lshl_or_b32 v2, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
-; GFX9-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v4, v4, v1
-; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
-; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v3, v3, v1
+; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: s_lshl_b32 s4, s17, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v5, vcc
-; GFX9-NEXT: v_add_f32_e32 v4, s4, v0
-; GFX9-NEXT: v_bfe_u32 v5, v4, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v5, v5, v4
-; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
-; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v4
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
+; GFX9-NEXT: v_add_f32_e32 v3, s4, v0
+; GFX9-NEXT: v_bfe_u32 v4, v3, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v4, v4, v3
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v4, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s16
-; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v4
-; GFX9-NEXT: v_add_f32_e32 v4, s4, v0
-; GFX9-NEXT: v_bfe_u32 v5, v4, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v5, v5, v4
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f32_e32 v3, s4, v0
+; GFX9-NEXT: v_bfe_u32 v4, v3, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v4, v4, v3
; GFX9-NEXT: s_lshl_b32 s4, s16, 16
-; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
-; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v4
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
+; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
; GFX9-NEXT: v_add_f32_e32 v0, s4, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc
-; GFX9-NEXT: v_bfe_u32 v5, v0, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v5, v5, v0
-; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
-; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
+; GFX9-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v4, v4, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v0
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v5, v6, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX9-NEXT: v_and_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v0, v4, 16, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_setpc_b64 s[30:31]
; GFX9-NEXT: .LBB11_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
@@ -5776,52 +5775,51 @@ define inreg <3 x float> @bitcast_v6bf16_to_v3f32_scalar(<6 x bfloat> inreg %a,
; GFX9-NEXT: v_add_f32_e32 v2, s4, v0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX9-NEXT: v_mov_b32_e32 v3, 0xffff
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s17
-; GFX9-NEXT: v_lshl_or_b32 v2, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
-; GFX9-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v4, v4, v1
-; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
-; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v3, v3, v1
+; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: s_lshl_b32 s4, s17, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v5, vcc
-; GFX9-NEXT: v_add_f32_e32 v4, s4, v0
-; GFX9-NEXT: v_bfe_u32 v5, v4, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v5, v5, v4
-; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
-; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v4
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
+; GFX9-NEXT: v_add_f32_e32 v3, s4, v0
+; GFX9-NEXT: v_bfe_u32 v4, v3, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v4, v4, v3
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v4, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s16
-; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v4
-; GFX9-NEXT: v_add_f32_e32 v4, s4, v0
-; GFX9-NEXT: v_bfe_u32 v5, v4, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v5, v5, v4
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_f32_e32 v3, s4, v0
+; GFX9-NEXT: v_bfe_u32 v4, v3, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v4, v4, v3
; GFX9-NEXT: s_lshl_b32 s4, s16, 16
-; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
-; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v4
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
+; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
; GFX9-NEXT: v_add_f32_e32 v0, s4, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc
-; GFX9-NEXT: v_bfe_u32 v5, v0, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v5, v5, v0
-; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
-; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
+; GFX9-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v4, v4, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v0
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v5, v6, vcc
-; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX9-NEXT: v_and_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v0, v4, 16, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_setpc_b64 s[30:31]
; GFX9-NEXT: .LBB27_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
@@ -8627,9 +8625,9 @@ define inreg <12 x i8> @bitcast_v6bf16_to_v12i8_scalar(<6 x bfloat> inreg %a, i3
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v13, 16, v0
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v13
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v6
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s16
-; GFX9-NEXT: v_lshl_or_b32 v15, v6, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v15, v0, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v0, s4, v1
; GFX9-NEXT: v_bfe_u32 v2, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v0
@@ -8647,9 +8645,9 @@ define inreg <12 x i8> @bitcast_v6bf16_to_v12i8_scalar(<6 x bfloat> inreg %a, i3
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s18
-; GFX9-NEXT: v_lshl_or_b32 v14, v2, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v14, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_add_f32_e32 v3, s4, v1
; GFX9-NEXT: v_bfe_u32 v4, v3, 16, 1
; GFX9-NEXT: v_add_u32_e32 v4, v4, v3
@@ -8667,8 +8665,8 @@ define inreg <12 x i8> @bitcast_v6bf16_to_v12i8_scalar(<6 x bfloat> inreg %a, i3
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v1
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v8
-; GFX9-NEXT: v_lshl_or_b32 v11, v10, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v10
+; GFX9-NEXT: v_or_b32_sdwa v11, v1, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_mov_b32_e32 v12, 0x7fc07fc0
; GFX9-NEXT: v_lshrrev_b64 v[3:4], 24, v[14:15]
; GFX9-NEXT: v_lshrrev_b32_e32 v9, 8, v11
@@ -12322,19 +12320,18 @@ define inreg <6 x half> @bitcast_v6bf16_to_v6f16_scalar(<6 x bfloat> inreg %a, i
; GFX9-NEXT: v_cndmask_b32_e32 v2, v6, v7, vcc
; GFX9-NEXT: v_bfe_u32 v6, v0, 16, 1
; GFX9-NEXT: v_add_u32_e32 v6, v6, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX9-NEXT: v_add_u32_e32 v6, 0x7fff, v6
; GFX9-NEXT: v_or_b32_e32 v7, 0x400000, v0
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v7, vcc
-; GFX9-NEXT: v_mov_b32_e32 v6, 0xffff
-; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX9-NEXT: v_and_b32_sdwa v0, v6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v2, v2, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v0, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v0, v6, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v0, v3, 16, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v7, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v2, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_setpc_b64 s[30:31]
; GFX9-NEXT: .LBB49_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll
index 1333df721c7f8..fb78590bd110c 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll
@@ -2474,7 +2474,6 @@ define amdgpu_kernel void @udiv_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_and_b32 s5, s2, 0xffff
; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s5
@@ -2482,41 +2481,40 @@ define amdgpu_kernel void @udiv_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s4
; GFX9-NEXT: s_lshr_b32 s2, s2, 16
; GFX9-NEXT: v_rcp_f32_e32 v0, v0
-; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s2
+; GFX9-NEXT: v_cvt_f32_u32_e32 v2, s2
; GFX9-NEXT: s_lshr_b32 s0, s0, 16
; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
; GFX9-NEXT: s_and_b32 s2, s3, 0xffff
-; GFX9-NEXT: v_cvt_f32_u32_e32 v4, s0
+; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s0
; GFX9-NEXT: v_mul_f32_e32 v0, v1, v0
; GFX9-NEXT: s_and_b32 s0, s1, 0xffff
; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s2
-; GFX9-NEXT: v_rcp_f32_e32 v3, v3
-; GFX9-NEXT: v_cvt_f32_u32_e32 v5, s0
-; GFX9-NEXT: v_add_u32_e32 v4, 1, v4
+; GFX9-NEXT: v_rcp_f32_e32 v2, v2
+; GFX9-NEXT: v_cvt_f32_u32_e32 v4, s0
+; GFX9-NEXT: v_add_u32_e32 v3, 1, v3
; GFX9-NEXT: v_rcp_f32_e32 v1, v1
; GFX9-NEXT: s_lshr_b32 s0, s3, 16
-; GFX9-NEXT: v_mul_f32_e32 v3, v4, v3
-; GFX9-NEXT: v_add_u32_e32 v4, 1, v5
-; GFX9-NEXT: v_cvt_f32_u32_e32 v5, s0
-; GFX9-NEXT: s_lshr_b32 s0, s1, 16
-; GFX9-NEXT: v_mul_f32_e32 v1, v4, v1
+; GFX9-NEXT: v_mul_f32_e32 v2, v3, v2
+; GFX9-NEXT: v_add_u32_e32 v3, 1, v4
; GFX9-NEXT: v_cvt_f32_u32_e32 v4, s0
-; GFX9-NEXT: v_rcp_f32_e32 v5, v5
+; GFX9-NEXT: s_lshr_b32 s0, s1, 16
+; GFX9-NEXT: v_mul_f32_e32 v1, v3, v1
+; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s0
+; GFX9-NEXT: v_rcp_f32_e32 v4, v4
; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: v_trunc_f32_e32 v2, v2
+; GFX9-NEXT: v_add_u32_e32 v3, 1, v3
+; GFX9-NEXT: v_mul_f32_e32 v3, v3, v4
; GFX9-NEXT: v_trunc_f32_e32 v1, v1
-; GFX9-NEXT: v_add_u32_e32 v4, 1, v4
-; GFX9-NEXT: v_mul_f32_e32 v4, v4, v5
+; GFX9-NEXT: v_trunc_f32_e32 v3, v3
; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
; GFX9-NEXT: v_cvt_u32_f32_e32 v1, v1
-; GFX9-NEXT: v_trunc_f32_e32 v4, v4
-; GFX9-NEXT: v_trunc_f32_e32 v3, v3
-; GFX9-NEXT: v_cvt_u32_f32_e32 v4, v4
-; GFX9-NEXT: v_cvt_u32_f32_e32 v3, v3
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v4, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v0, v3, 16, v0
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7]
+; GFX9-NEXT: v_cvt_u32_f32_sdwa v3, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; GFX9-NEXT: v_cvt_u32_f32_sdwa v2, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: global_store_dwordx2 v4, v[0:1], s[6:7]
; GFX9-NEXT: s_endpgm
%r = udiv <4 x i16> %x, %y
store <4 x i16> %r, ptr addrspace(1) %out
@@ -2692,25 +2690,23 @@ define amdgpu_kernel void @urem_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
; GFX9-NEXT: v_trunc_f32_e32 v0, v0
; GFX9-NEXT: v_trunc_f32_e32 v1, v1
; GFX9-NEXT: v_add_u32_e32 v4, 1, v4
-; GFX9-NEXT: v_trunc_f32_e32 v3, v3
; GFX9-NEXT: v_mul_f32_e32 v4, v4, v5
+; GFX9-NEXT: v_trunc_f32_e32 v3, v3
+; GFX9-NEXT: v_trunc_f32_e32 v4, v4
; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
; GFX9-NEXT: v_cvt_u32_f32_e32 v1, v1
; GFX9-NEXT: v_cvt_u32_f32_e32 v3, v3
-; GFX9-NEXT: v_trunc_f32_e32 v4, v4
; GFX9-NEXT: v_cvt_u32_f32_e32 v4, v4
; GFX9-NEXT: v_mul_lo_u32 v0, v0, s5
; GFX9-NEXT: v_mul_lo_u32 v1, v1, s2
; GFX9-NEXT: v_mul_lo_u32 v3, v3, s9
; GFX9-NEXT: v_mul_lo_u32 v4, v4, s3
; GFX9-NEXT: v_sub_u32_e32 v0, s4, v0
-; GFX9-NEXT: v_sub_u32_e32 v5, s0, v1
+; GFX9-NEXT: v_sub_u32_sdwa v5, s0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_sub_u32_e32 v1, s8, v3
-; GFX9-NEXT: v_sub_u32_e32 v3, s1, v4
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v3, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v0, v5, 16, v0
+; GFX9-NEXT: v_sub_u32_sdwa v3, s1, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v0, v5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7]
; GFX9-NEXT: s_endpgm
%r = urem <4 x i16> %x, %y
@@ -2840,7 +2836,6 @@ define amdgpu_kernel void @sdiv_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_sext_i32_i16 s5, s2
; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s5
@@ -2848,41 +2843,40 @@ define amdgpu_kernel void @sdiv_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s4
; GFX9-NEXT: s_ashr_i32 s2, s2, 16
; GFX9-NEXT: v_rcp_f32_e32 v0, v0
-; GFX9-NEXT: v_cvt_f32_i32_e32 v3, s2
+; GFX9-NEXT: v_cvt_f32_i32_e32 v2, s2
; GFX9-NEXT: s_ashr_i32 s0, s0, 16
; GFX9-NEXT: v_add_u32_e32 v1, 1, v1
; GFX9-NEXT: s_sext_i32_i16 s2, s3
-; GFX9-NEXT: v_cvt_f32_i32_e32 v4, s0
+; GFX9-NEXT: v_cvt_f32_i32_e32 v3, s0
; GFX9-NEXT: v_mul_f32_e32 v0, v1, v0
; GFX9-NEXT: s_sext_i32_i16 s0, s1
; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s2
-; GFX9-NEXT: v_rcp_f32_e32 v3, v3
-; GFX9-NEXT: v_cvt_f32_i32_e32 v5, s0
-; GFX9-NEXT: v_add_u32_e32 v4, 1, v4
+; GFX9-NEXT: v_rcp_f32_e32 v2, v2
+; GFX9-NEXT: v_cvt_f32_i32_e32 v4, s0
+; GFX9-NEXT: v_add_u32_e32 v3, 1, v3
; GFX9-NEXT: v_rcp_f32_e32 v1, v1
; GFX9-NEXT: s_ashr_i32 s0, s3, 16
-; GFX9-NEXT: v_mul_f32_e32 v3, v4, v3
-; GFX9-NEXT: v_add_u32_e32 v4, 1, v5
-; GFX9-NEXT: v_cvt_f32_i32_e32 v5, s0
-; GFX9-NEXT: s_ashr_i32 s0, s1, 16
-; GFX9-NEXT: v_mul_f32_e32 v1, v4, v1
+; GFX9-NEXT: v_mul_f32_e32 v2, v3, v2
+; GFX9-NEXT: v_add_u32_e32 v3, 1, v4
; GFX9-NEXT: v_cvt_f32_i32_e32 v4, s0
-; GFX9-NEXT: v_rcp_f32_e32 v5, v5
+; GFX9-NEXT: s_ashr_i32 s0, s1, 16
+; GFX9-NEXT: v_mul_f32_e32 v1, v3, v1
+; GFX9-NEXT: v_cvt_f32_i32_e32 v3, s0
+; GFX9-NEXT: v_rcp_f32_e32 v4, v4
; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: v_trunc_f32_e32 v2, v2
+; GFX9-NEXT: v_add_u32_e32 v3, 1, v3
+; GFX9-NEXT: v_mul_f32_e32 v3, v3, v4
; GFX9-NEXT: v_trunc_f32_e32 v1, v1
-; GFX9-NEXT: v_add_u32_e32 v4, 1, v4
-; GFX9-NEXT: v_mul_f32_e32 v4, v4, v5
+; GFX9-NEXT: v_trunc_f32_e32 v3, v3
; GFX9-NEXT: v_cvt_i32_f32_e32 v0, v0
; GFX9-NEXT: v_cvt_i32_f32_e32 v1, v1
-; GFX9-NEXT: v_trunc_f32_e32 v4, v4
-; GFX9-NEXT: v_trunc_f32_e32 v3, v3
-; GFX9-NEXT: v_cvt_i32_f32_e32 v4, v4
-; GFX9-NEXT: v_cvt_i32_f32_e32 v3, v3
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v4, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v0, v3, 16, v0
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7]
+; GFX9-NEXT: v_cvt_i32_f32_sdwa v3, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; GFX9-NEXT: v_cvt_i32_f32_sdwa v2, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: global_store_dwordx2 v4, v[0:1], s[6:7]
; GFX9-NEXT: s_endpgm
%r = sdiv <4 x i16> %x, %y
store <4 x i16> %r, ptr addrspace(1) %out
@@ -3062,25 +3056,23 @@ define amdgpu_kernel void @srem_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
; GFX9-NEXT: v_trunc_f32_e32 v0, v0
; GFX9-NEXT: v_trunc_f32_e32 v1, v1
; GFX9-NEXT: v_add_u32_e32 v4, 1, v4
-; GFX9-NEXT: v_trunc_f32_e32 v3, v3
; GFX9-NEXT: v_mul_f32_e32 v4, v4, v5
+; GFX9-NEXT: v_trunc_f32_e32 v3, v3
+; GFX9-NEXT: v_trunc_f32_e32 v4, v4
; GFX9-NEXT: v_cvt_i32_f32_e32 v0, v0
; GFX9-NEXT: v_cvt_i32_f32_e32 v1, v1
; GFX9-NEXT: v_cvt_i32_f32_e32 v3, v3
-; GFX9-NEXT: v_trunc_f32_e32 v4, v4
; GFX9-NEXT: v_cvt_i32_f32_e32 v4, v4
; GFX9-NEXT: v_mul_lo_u32 v0, v0, s5
; GFX9-NEXT: v_mul_lo_u32 v1, v1, s2
; GFX9-NEXT: v_mul_lo_u32 v3, v3, s9
; GFX9-NEXT: v_mul_lo_u32 v4, v4, s3
; GFX9-NEXT: v_sub_u32_e32 v0, s4, v0
-; GFX9-NEXT: v_sub_u32_e32 v5, s0, v1
+; GFX9-NEXT: v_sub_u32_sdwa v5, s0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_sub_u32_e32 v1, s8, v3
-; GFX9-NEXT: v_sub_u32_e32 v3, s1, v4
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v3, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v0, v5, 16, v0
+; GFX9-NEXT: v_sub_u32_sdwa v3, s1, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v0, v5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7]
; GFX9-NEXT: s_endpgm
%r = srem <4 x i16> %x, %y
@@ -3471,16 +3463,15 @@ define amdgpu_kernel void @udiv_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s0
; GFX9-NEXT: v_rcp_f32_e32 v1, v1
; GFX9-NEXT: v_trunc_f32_e32 v0, v0
-; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX9-NEXT: v_trunc_f32_e32 v2, v2
; GFX9-NEXT: v_add_u32_e32 v3, 1, v3
; GFX9-NEXT: v_mul_f32_e32 v1, v3, v1
-; GFX9-NEXT: v_trunc_f32_e32 v2, v2
; GFX9-NEXT: v_trunc_f32_e32 v1, v1
-; GFX9-NEXT: v_cvt_u32_f32_e32 v2, v2
+; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_u32_f32_sdwa v2, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
; GFX9-NEXT: v_cvt_u32_f32_e32 v1, v1
; GFX9-NEXT: v_mov_b32_e32 v3, 0
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v0, v2, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: global_store_short v3, v1, s[6:7] offset:4
; GFX9-NEXT: global_store_dword v3, v0, s[6:7]
; GFX9-NEXT: s_endpgm
@@ -3617,23 +3608,22 @@ define amdgpu_kernel void @urem_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s9
; GFX9-NEXT: v_rcp_f32_e32 v1, v1
; GFX9-NEXT: v_trunc_f32_e32 v0, v0
-; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX9-NEXT: v_trunc_f32_e32 v2, v2
; GFX9-NEXT: v_add_u32_e32 v3, 1, v3
; GFX9-NEXT: v_mul_f32_e32 v1, v3, v1
-; GFX9-NEXT: v_trunc_f32_e32 v2, v2
; GFX9-NEXT: v_trunc_f32_e32 v1, v1
+; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
; GFX9-NEXT: v_cvt_u32_f32_e32 v2, v2
; GFX9-NEXT: v_cvt_u32_f32_e32 v1, v1
-; GFX9-NEXT: v_mul_lo_u32 v0, v0, s7
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: v_mul_lo_u32 v0, v0, s7
; GFX9-NEXT: v_mul_lo_u32 v2, v2, s2
; GFX9-NEXT: v_mul_lo_u32 v1, v1, s3
-; GFX9-NEXT: v_sub_u32_e32 v0, s6, v0
; GFX9-NEXT: v_mov_b32_e32 v3, 0
-; GFX9-NEXT: v_sub_u32_e32 v2, s8, v2
+; GFX9-NEXT: v_sub_u32_e32 v0, s6, v0
+; GFX9-NEXT: v_sub_u32_sdwa v2, s8, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_sub_u32_e32 v1, s9, v1
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v0, v2, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_store_short v3, v1, s[0:1] offset:4
; GFX9-NEXT: global_store_dword v3, v0, s[0:1]
@@ -3760,16 +3750,15 @@ define amdgpu_kernel void @sdiv_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
; GFX9-NEXT: v_cvt_f32_i32_e32 v3, s0
; GFX9-NEXT: v_rcp_f32_e32 v1, v1
; GFX9-NEXT: v_trunc_f32_e32 v0, v0
-; GFX9-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX9-NEXT: v_trunc_f32_e32 v2, v2
; GFX9-NEXT: v_add_u32_e32 v3, 1, v3
; GFX9-NEXT: v_mul_f32_e32 v1, v3, v1
-; GFX9-NEXT: v_trunc_f32_e32 v2, v2
; GFX9-NEXT: v_trunc_f32_e32 v1, v1
-; GFX9-NEXT: v_cvt_i32_f32_e32 v2, v2
+; GFX9-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX9-NEXT: v_cvt_i32_f32_sdwa v2, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
; GFX9-NEXT: v_cvt_i32_f32_e32 v1, v1
; GFX9-NEXT: v_mov_b32_e32 v3, 0
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v0, v2, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: global_store_short v3, v1, s[6:7] offset:4
; GFX9-NEXT: global_store_dword v3, v0, s[6:7]
; GFX9-NEXT: s_endpgm
@@ -3908,23 +3897,22 @@ define amdgpu_kernel void @srem_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
; GFX9-NEXT: v_cvt_f32_i32_e32 v3, s9
; GFX9-NEXT: v_rcp_f32_e32 v1, v1
; GFX9-NEXT: v_trunc_f32_e32 v0, v0
-; GFX9-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX9-NEXT: v_trunc_f32_e32 v2, v2
; GFX9-NEXT: v_add_u32_e32 v3, 1, v3
; GFX9-NEXT: v_mul_f32_e32 v1, v3, v1
-; GFX9-NEXT: v_trunc_f32_e32 v2, v2
; GFX9-NEXT: v_trunc_f32_e32 v1, v1
+; GFX9-NEXT: v_cvt_i32_f32_e32 v0, v0
; GFX9-NEXT: v_cvt_i32_f32_e32 v2, v2
; GFX9-NEXT: v_cvt_i32_f32_e32 v1, v1
-; GFX9-NEXT: v_mul_lo_u32 v0, v0, s7
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: v_mul_lo_u32 v0, v0, s7
; GFX9-NEXT: v_mul_lo_u32 v2, v2, s2
; GFX9-NEXT: v_mul_lo_u32 v1, v1, s3
-; GFX9-NEXT: v_sub_u32_e32 v0, s6, v0
; GFX9-NEXT: v_mov_b32_e32 v3, 0
-; GFX9-NEXT: v_sub_u32_e32 v2, s8, v2
+; GFX9-NEXT: v_sub_u32_e32 v0, s6, v0
+; GFX9-NEXT: v_sub_u32_sdwa v2, s8, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_sub_u32_e32 v1, s9, v1
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v0, v2, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_store_short v3, v1, s[0:1] offset:4
; GFX9-NEXT: global_store_dword v3, v0, s[0:1]
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
index 43358550e302b..4719d6edbaaa5 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
@@ -6562,13 +6562,13 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no
; GFX10-NEXT: v_mov_b32_e32 v1, s20
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: buffer_load_ushort v3, v0, s[16:19], 0 offen offset:1024
-; GFX10-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v3, v0, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX10-NEXT: v_mov_b32_e32 v3, s20
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshl_or_b32 v0, v4, 16, v0
+; GFX10-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_mov_b32_e32 v5, v0
@@ -6600,16 +6600,16 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX908-NEXT: v_mov_b32_e32 v1, s20
; GFX908-NEXT: v_mov_b32_e32 v2, v0
; GFX908-NEXT: v_mov_b32_e32 v0, s20
-; GFX908-NEXT: v_mov_b32_e32 v1, s20
-; GFX908-NEXT: buffer_load_ushort v3, v0, s[16:19], 0 offen offset:1024
-; GFX908-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: buffer_load_ushort v4, v0, s[16:19], 0 offen offset:1024
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: s_waitcnt vmcnt(1)
-; GFX908-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshl_or_b32 v0, v4, 16, v0
+; GFX908-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX908-NEXT: v_mov_b32_e32 v3, s20
; GFX908-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -6852,13 +6852,13 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_fin
; GFX10-NEXT: v_mov_b32_e32 v2, s20
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1024
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v3
; GFX10-NEXT: v_mov_b32_e32 v3, s20
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshl_or_b32 v2, v4, 16, v1
+; GFX10-NEXT: v_or_b32_sdwa v2, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_pk_add_f16 v1, v2, v0
@@ -7761,13 +7761,13 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset(ptr addrsp
; GFX10-NEXT: v_mov_b32_e32 v1, s20
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: buffer_load_ushort v3, v0, s[16:19], 0 offen offset:1024
-; GFX10-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v3, v0, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX10-NEXT: v_mov_b32_e32 v3, s20
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshl_or_b32 v0, v4, 16, v0
+; GFX10-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_mov_b32_e32 v5, v0
@@ -7790,16 +7790,16 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset(ptr addrsp
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_mov_b32_e32 v1, s20
; GFX90A-NEXT: v_mov_b32_e32 v2, v0
; GFX90A-NEXT: v_mov_b32_e32 v0, s20
-; GFX90A-NEXT: v_mov_b32_e32 v1, s20
-; GFX90A-NEXT: buffer_load_ushort v3, v0, s[16:19], 0 offen offset:1024
-; GFX90A-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: buffer_load_ushort v4, v0, s[16:19], 0 offen offset:1024
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: s_waitcnt vmcnt(1)
-; GFX90A-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshl_or_b32 v0, v4, 16, v0
+; GFX90A-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX90A-NEXT: v_mov_b32_e32 v3, s20
; GFX90A-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -7820,16 +7820,16 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset(ptr addrsp
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX908-NEXT: v_mov_b32_e32 v1, s20
; GFX908-NEXT: v_mov_b32_e32 v2, v0
; GFX908-NEXT: v_mov_b32_e32 v0, s20
-; GFX908-NEXT: v_mov_b32_e32 v1, s20
-; GFX908-NEXT: buffer_load_ushort v3, v0, s[16:19], 0 offen offset:1024
-; GFX908-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: buffer_load_ushort v4, v0, s[16:19], 0 offen offset:1024
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: s_waitcnt vmcnt(1)
-; GFX908-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshl_or_b32 v0, v4, 16, v0
+; GFX908-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX908-NEXT: v_mov_b32_e32 v3, s20
; GFX908-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -8072,13 +8072,13 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset(ptr addrspace(
; GFX10-NEXT: v_mov_b32_e32 v2, s20
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1024
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v3
; GFX10-NEXT: v_mov_b32_e32 v3, s20
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshl_or_b32 v2, v4, 16, v1
+; GFX10-NEXT: v_or_b32_sdwa v2, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: .LBB23_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_pk_add_f16 v1, v2, v0
@@ -8101,15 +8101,15 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset(ptr addrspace(
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v1, s20
; GFX90A-NEXT: v_mov_b32_e32 v2, s20
-; GFX90A-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX90A-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: v_mov_b32_e32 v1, s20
+; GFX90A-NEXT: buffer_load_ushort v3, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: s_waitcnt vmcnt(1)
-; GFX90A-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v3
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshl_or_b32 v3, v4, 16, v1
+; GFX90A-NEXT: v_or_b32_sdwa v3, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX90A-NEXT: v_mov_b32_e32 v1, s20
; GFX90A-NEXT: .LBB23_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -8130,15 +8130,15 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset(ptr addrspace(
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v1, s20
; GFX908-NEXT: v_mov_b32_e32 v2, s20
-; GFX908-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX908-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: v_mov_b32_e32 v1, s20
+; GFX908-NEXT: buffer_load_ushort v3, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: s_waitcnt vmcnt(1)
-; GFX908-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v1, 16, v3
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshl_or_b32 v2, v4, 16, v1
+; GFX908-NEXT: v_or_b32_sdwa v2, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX908-NEXT: v_mov_b32_e32 v3, s20
; GFX908-NEXT: .LBB23_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -8375,13 +8375,13 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no
; GFX10-NEXT: v_mov_b32_e32 v1, s20
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: buffer_load_ushort v3, v0, s[16:19], 0 offen offset:1024
-; GFX10-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v3, v0, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX10-NEXT: v_mov_b32_e32 v3, s20
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshl_or_b32 v0, v4, 16, v0
+; GFX10-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_mov_b32_e32 v5, v0
@@ -8404,16 +8404,16 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no_remote_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_mov_b32_e32 v1, s20
; GFX90A-NEXT: v_mov_b32_e32 v2, v0
; GFX90A-NEXT: v_mov_b32_e32 v0, s20
-; GFX90A-NEXT: v_mov_b32_e32 v1, s20
-; GFX90A-NEXT: buffer_load_ushort v3, v0, s[16:19], 0 offen offset:1024
-; GFX90A-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: buffer_load_ushort v4, v0, s[16:19], 0 offen offset:1024
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: s_waitcnt vmcnt(1)
-; GFX90A-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshl_or_b32 v0, v4, 16, v0
+; GFX90A-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX90A-NEXT: v_mov_b32_e32 v3, s20
; GFX90A-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -8434,16 +8434,16 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no_remote_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX908-NEXT: v_mov_b32_e32 v1, s20
; GFX908-NEXT: v_mov_b32_e32 v2, v0
; GFX908-NEXT: v_mov_b32_e32 v0, s20
-; GFX908-NEXT: v_mov_b32_e32 v1, s20
-; GFX908-NEXT: buffer_load_ushort v3, v0, s[16:19], 0 offen offset:1024
-; GFX908-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: buffer_load_ushort v4, v0, s[16:19], 0 offen offset:1024
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: s_waitcnt vmcnt(1)
-; GFX908-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshl_or_b32 v0, v4, 16, v0
+; GFX908-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX908-NEXT: v_mov_b32_e32 v3, s20
; GFX908-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -8686,13 +8686,13 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_rem
; GFX10-NEXT: v_mov_b32_e32 v2, s20
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1024
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v3
; GFX10-NEXT: v_mov_b32_e32 v3, s20
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshl_or_b32 v2, v4, 16, v1
+; GFX10-NEXT: v_or_b32_sdwa v2, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_pk_add_f16 v1, v2, v0
@@ -8715,15 +8715,15 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_rem
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_remote_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v1, s20
; GFX90A-NEXT: v_mov_b32_e32 v2, s20
-; GFX90A-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX90A-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: v_mov_b32_e32 v1, s20
+; GFX90A-NEXT: buffer_load_ushort v3, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: s_waitcnt vmcnt(1)
-; GFX90A-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v3
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshl_or_b32 v3, v4, 16, v1
+; GFX90A-NEXT: v_or_b32_sdwa v3, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX90A-NEXT: v_mov_b32_e32 v1, s20
; GFX90A-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -8744,15 +8744,15 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_rem
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_remote_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v1, s20
; GFX908-NEXT: v_mov_b32_e32 v2, s20
-; GFX908-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX908-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: v_mov_b32_e32 v1, s20
+; GFX908-NEXT: buffer_load_ushort v3, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: s_waitcnt vmcnt(1)
-; GFX908-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v1, 16, v3
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshl_or_b32 v2, v4, 16, v1
+; GFX908-NEXT: v_or_b32_sdwa v2, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX908-NEXT: v_mov_b32_e32 v3, s20
; GFX908-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -8910,19 +8910,19 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v1, s16
; GFX942-NEXT: v_mov_b32_e32 v2, s16
-; GFX942-NEXT: buffer_load_ushort v4, v1, s[0:3], 0 offen offset:1024
-; GFX942-NEXT: buffer_load_ushort v5, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: v_mov_b32_e32 v1, s16
+; GFX942-NEXT: buffer_load_ushort v4, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: buffer_load_ushort v5, v1, s[0:3], 0 offen offset:1024
; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX942-NEXT: s_mov_b64 s[6:7], 0
; GFX942-NEXT: s_movk_i32 s8, 0x7fff
; GFX942-NEXT: s_mov_b32 s9, 0x7060302
; GFX942-NEXT: s_waitcnt vmcnt(1)
-; GFX942-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshl_or_b32 v0, v5, 16, v0
+; GFX942-NEXT: v_or_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX942-NEXT: v_mov_b32_e32 v4, s16
; GFX942-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -9078,14 +9078,14 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX10-NEXT: v_mov_b32_e32 v2, s20
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1024
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v3
; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshl_or_b32 v0, v4, 16, v1
+; GFX10-NEXT: v_or_b32_sdwa v0, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_mov_b32_e32 v4, s20
; GFX10-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -9123,19 +9123,19 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v1, s20
; GFX90A-NEXT: v_mov_b32_e32 v2, s20
-; GFX90A-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
-; GFX90A-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: v_mov_b32_e32 v1, s20
+; GFX90A-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: s_waitcnt vmcnt(1)
-; GFX90A-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshl_or_b32 v0, v5, 16, v0
+; GFX90A-NEXT: v_or_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX90A-NEXT: v_mov_b32_e32 v4, s20
; GFX90A-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -9170,19 +9170,19 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v1, s20
; GFX908-NEXT: v_mov_b32_e32 v2, s20
-; GFX908-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
-; GFX908-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: v_mov_b32_e32 v1, s20
+; GFX908-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX908-NEXT: s_mov_b64 s[6:7], 0
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: s_waitcnt vmcnt(1)
-; GFX908-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshl_or_b32 v0, v5, 16, v0
+; GFX908-NEXT: v_or_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX908-NEXT: v_mov_b32_e32 v4, s20
; GFX908-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -9388,19 +9388,19 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v1, s16
; GFX942-NEXT: v_mov_b32_e32 v2, s16
-; GFX942-NEXT: buffer_load_ushort v4, v1, s[0:3], 0 offen offset:1024
-; GFX942-NEXT: buffer_load_ushort v5, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: v_mov_b32_e32 v1, s16
+; GFX942-NEXT: buffer_load_ushort v4, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: buffer_load_ushort v5, v1, s[0:3], 0 offen offset:1024
; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX942-NEXT: s_mov_b64 s[6:7], 0
; GFX942-NEXT: s_movk_i32 s8, 0x7fff
; GFX942-NEXT: s_mov_b32 s9, 0x7060302
; GFX942-NEXT: s_waitcnt vmcnt(1)
-; GFX942-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshl_or_b32 v1, v5, 16, v0
+; GFX942-NEXT: v_or_b32_sdwa v1, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX942-NEXT: v_mov_b32_e32 v4, s16
; GFX942-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -9550,14 +9550,14 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX10-NEXT: v_mov_b32_e32 v2, s20
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1024
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v3
; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshl_or_b32 v1, v4, 16, v1
+; GFX10-NEXT: v_or_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_mov_b32_e32 v4, s20
; GFX10-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -9595,19 +9595,19 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v1, s20
; GFX90A-NEXT: v_mov_b32_e32 v2, s20
-; GFX90A-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
-; GFX90A-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: v_mov_b32_e32 v1, s20
+; GFX90A-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: s_waitcnt vmcnt(1)
-; GFX90A-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshl_or_b32 v1, v5, 16, v0
+; GFX90A-NEXT: v_or_b32_sdwa v1, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX90A-NEXT: v_mov_b32_e32 v4, s20
; GFX90A-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -9642,19 +9642,19 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v1, s20
; GFX908-NEXT: v_mov_b32_e32 v2, s20
-; GFX908-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
-; GFX908-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: v_mov_b32_e32 v1, s20
+; GFX908-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX908-NEXT: s_mov_b64 s[6:7], 0
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: s_waitcnt vmcnt(1)
-; GFX908-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshl_or_b32 v1, v5, 16, v0
+; GFX908-NEXT: v_or_b32_sdwa v1, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX908-NEXT: v_mov_b32_e32 v4, s20
; GFX908-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -10735,19 +10735,19 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset(ptr add
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v1, s16
; GFX942-NEXT: v_mov_b32_e32 v2, s16
-; GFX942-NEXT: buffer_load_ushort v4, v1, s[0:3], 0 offen offset:1024
-; GFX942-NEXT: buffer_load_ushort v5, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: v_mov_b32_e32 v1, s16
+; GFX942-NEXT: buffer_load_ushort v4, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: buffer_load_ushort v5, v1, s[0:3], 0 offen offset:1024
; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX942-NEXT: s_mov_b64 s[6:7], 0
; GFX942-NEXT: s_movk_i32 s8, 0x7fff
; GFX942-NEXT: s_mov_b32 s9, 0x7060302
; GFX942-NEXT: s_waitcnt vmcnt(1)
-; GFX942-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshl_or_b32 v0, v5, 16, v0
+; GFX942-NEXT: v_or_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX942-NEXT: v_mov_b32_e32 v4, s16
; GFX942-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -10903,14 +10903,14 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset(ptr add
; GFX10-NEXT: v_mov_b32_e32 v2, s20
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1024
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v3
; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshl_or_b32 v0, v4, 16, v1
+; GFX10-NEXT: v_or_b32_sdwa v0, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_mov_b32_e32 v4, s20
; GFX10-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -10948,19 +10948,19 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset(ptr add
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v1, s20
; GFX90A-NEXT: v_mov_b32_e32 v2, s20
-; GFX90A-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
-; GFX90A-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: v_mov_b32_e32 v1, s20
+; GFX90A-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: s_waitcnt vmcnt(1)
-; GFX90A-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshl_or_b32 v0, v5, 16, v0
+; GFX90A-NEXT: v_or_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX90A-NEXT: v_mov_b32_e32 v4, s20
; GFX90A-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -10995,19 +10995,19 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset(ptr add
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v1, s20
; GFX908-NEXT: v_mov_b32_e32 v2, s20
-; GFX908-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
-; GFX908-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: v_mov_b32_e32 v1, s20
+; GFX908-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX908-NEXT: s_mov_b64 s[6:7], 0
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: s_waitcnt vmcnt(1)
-; GFX908-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshl_or_b32 v0, v5, 16, v0
+; GFX908-NEXT: v_or_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX908-NEXT: v_mov_b32_e32 v4, s20
; GFX908-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -11213,19 +11213,19 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset(ptr addrspace
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v1, s16
; GFX942-NEXT: v_mov_b32_e32 v2, s16
-; GFX942-NEXT: buffer_load_ushort v4, v1, s[0:3], 0 offen offset:1024
-; GFX942-NEXT: buffer_load_ushort v5, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: v_mov_b32_e32 v1, s16
+; GFX942-NEXT: buffer_load_ushort v4, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: buffer_load_ushort v5, v1, s[0:3], 0 offen offset:1024
; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX942-NEXT: s_mov_b64 s[6:7], 0
; GFX942-NEXT: s_movk_i32 s8, 0x7fff
; GFX942-NEXT: s_mov_b32 s9, 0x7060302
; GFX942-NEXT: s_waitcnt vmcnt(1)
-; GFX942-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshl_or_b32 v1, v5, 16, v0
+; GFX942-NEXT: v_or_b32_sdwa v1, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX942-NEXT: v_mov_b32_e32 v4, s16
; GFX942-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -11375,14 +11375,14 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset(ptr addrspace
; GFX10-NEXT: v_mov_b32_e32 v2, s20
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1024
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v3
; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshl_or_b32 v1, v4, 16, v1
+; GFX10-NEXT: v_or_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_mov_b32_e32 v4, s20
; GFX10-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -11420,19 +11420,19 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset(ptr addrspace
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v1, s20
; GFX90A-NEXT: v_mov_b32_e32 v2, s20
-; GFX90A-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
-; GFX90A-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: v_mov_b32_e32 v1, s20
+; GFX90A-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: s_waitcnt vmcnt(1)
-; GFX90A-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshl_or_b32 v1, v5, 16, v0
+; GFX90A-NEXT: v_or_b32_sdwa v1, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX90A-NEXT: v_mov_b32_e32 v4, s20
; GFX90A-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -11467,19 +11467,19 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset(ptr addrspace
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v1, s20
; GFX908-NEXT: v_mov_b32_e32 v2, s20
-; GFX908-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
-; GFX908-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: v_mov_b32_e32 v1, s20
+; GFX908-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX908-NEXT: s_mov_b64 s[6:7], 0
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: s_waitcnt vmcnt(1)
-; GFX908-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshl_or_b32 v1, v5, 16, v0
+; GFX908-NEXT: v_or_b32_sdwa v1, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX908-NEXT: v_mov_b32_e32 v4, s20
; GFX908-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -11677,19 +11677,19 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_remote_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v1, s16
; GFX942-NEXT: v_mov_b32_e32 v2, s16
-; GFX942-NEXT: buffer_load_ushort v4, v1, s[0:3], 0 offen offset:1024
-; GFX942-NEXT: buffer_load_ushort v5, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: v_mov_b32_e32 v1, s16
+; GFX942-NEXT: buffer_load_ushort v4, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: buffer_load_ushort v5, v1, s[0:3], 0 offen offset:1024
; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX942-NEXT: s_mov_b64 s[6:7], 0
; GFX942-NEXT: s_movk_i32 s8, 0x7fff
; GFX942-NEXT: s_mov_b32 s9, 0x7060302
; GFX942-NEXT: s_waitcnt vmcnt(1)
-; GFX942-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshl_or_b32 v0, v5, 16, v0
+; GFX942-NEXT: v_or_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX942-NEXT: v_mov_b32_e32 v4, s16
; GFX942-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -11845,14 +11845,14 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX10-NEXT: v_mov_b32_e32 v2, s20
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1024
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v3
; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshl_or_b32 v0, v4, 16, v1
+; GFX10-NEXT: v_or_b32_sdwa v0, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_mov_b32_e32 v4, s20
; GFX10-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -11890,19 +11890,19 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_remote_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v1, s20
; GFX90A-NEXT: v_mov_b32_e32 v2, s20
-; GFX90A-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
-; GFX90A-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: v_mov_b32_e32 v1, s20
+; GFX90A-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: s_waitcnt vmcnt(1)
-; GFX90A-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshl_or_b32 v0, v5, 16, v0
+; GFX90A-NEXT: v_or_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX90A-NEXT: v_mov_b32_e32 v4, s20
; GFX90A-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -11937,19 +11937,19 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_remote_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v1, s20
; GFX908-NEXT: v_mov_b32_e32 v2, s20
-; GFX908-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
-; GFX908-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: v_mov_b32_e32 v1, s20
+; GFX908-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX908-NEXT: s_mov_b64 s[6:7], 0
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: s_waitcnt vmcnt(1)
-; GFX908-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshl_or_b32 v0, v5, 16, v0
+; GFX908-NEXT: v_or_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX908-NEXT: v_mov_b32_e32 v4, s20
; GFX908-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -12155,19 +12155,19 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_re
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_remote_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v1, s16
; GFX942-NEXT: v_mov_b32_e32 v2, s16
-; GFX942-NEXT: buffer_load_ushort v4, v1, s[0:3], 0 offen offset:1024
-; GFX942-NEXT: buffer_load_ushort v5, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: v_mov_b32_e32 v1, s16
+; GFX942-NEXT: buffer_load_ushort v4, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: buffer_load_ushort v5, v1, s[0:3], 0 offen offset:1024
; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX942-NEXT: s_mov_b64 s[6:7], 0
; GFX942-NEXT: s_movk_i32 s8, 0x7fff
; GFX942-NEXT: s_mov_b32 s9, 0x7060302
; GFX942-NEXT: s_waitcnt vmcnt(1)
-; GFX942-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshl_or_b32 v1, v5, 16, v0
+; GFX942-NEXT: v_or_b32_sdwa v1, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX942-NEXT: v_mov_b32_e32 v4, s16
; GFX942-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -12317,14 +12317,14 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_re
; GFX10-NEXT: v_mov_b32_e32 v2, s20
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1024
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v3
; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshl_or_b32 v1, v4, 16, v1
+; GFX10-NEXT: v_or_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_mov_b32_e32 v4, s20
; GFX10-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -12362,19 +12362,19 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_re
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_remote_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v1, s20
; GFX90A-NEXT: v_mov_b32_e32 v2, s20
-; GFX90A-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
-; GFX90A-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: v_mov_b32_e32 v1, s20
+; GFX90A-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: s_waitcnt vmcnt(1)
-; GFX90A-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshl_or_b32 v1, v5, 16, v0
+; GFX90A-NEXT: v_or_b32_sdwa v1, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX90A-NEXT: v_mov_b32_e32 v4, s20
; GFX90A-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -12409,19 +12409,19 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_re
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_remote_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v1, s20
; GFX908-NEXT: v_mov_b32_e32 v2, s20
-; GFX908-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
-; GFX908-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: v_mov_b32_e32 v1, s20
+; GFX908-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX908-NEXT: s_mov_b64 s[6:7], 0
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: s_waitcnt vmcnt(1)
-; GFX908-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshl_or_b32 v1, v5, 16, v0
+; GFX908-NEXT: v_or_b32_sdwa v1, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX908-NEXT: v_mov_b32_e32 v4, s20
; GFX908-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -12619,19 +12619,19 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v1, s16
; GFX942-NEXT: v_mov_b32_e32 v2, s16
-; GFX942-NEXT: buffer_load_ushort v4, v1, s[0:3], 0 offen offset:1024
-; GFX942-NEXT: buffer_load_ushort v5, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: v_mov_b32_e32 v1, s16
+; GFX942-NEXT: buffer_load_ushort v4, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: buffer_load_ushort v5, v1, s[0:3], 0 offen offset:1024
; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX942-NEXT: s_mov_b64 s[6:7], 0
; GFX942-NEXT: s_movk_i32 s8, 0x7fff
; GFX942-NEXT: s_mov_b32 s9, 0x7060302
; GFX942-NEXT: s_waitcnt vmcnt(1)
-; GFX942-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshl_or_b32 v1, v5, 16, v0
+; GFX942-NEXT: v_or_b32_sdwa v1, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX942-NEXT: v_mov_b32_e32 v4, s16
; GFX942-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -12781,14 +12781,14 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX10-NEXT: v_mov_b32_e32 v2, s20
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1024
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v3
; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshl_or_b32 v1, v4, 16, v1
+; GFX10-NEXT: v_or_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_mov_b32_e32 v4, s20
; GFX10-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -12826,19 +12826,19 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v1, s20
; GFX90A-NEXT: v_mov_b32_e32 v2, s20
-; GFX90A-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
-; GFX90A-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: v_mov_b32_e32 v1, s20
+; GFX90A-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: s_waitcnt vmcnt(1)
-; GFX90A-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshl_or_b32 v1, v5, 16, v0
+; GFX90A-NEXT: v_or_b32_sdwa v1, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX90A-NEXT: v_mov_b32_e32 v4, s20
; GFX90A-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -12873,19 +12873,19 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v1, s20
; GFX908-NEXT: v_mov_b32_e32 v2, s20
-; GFX908-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
-; GFX908-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: v_mov_b32_e32 v1, s20
+; GFX908-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX908-NEXT: s_mov_b64 s[6:7], 0
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: s_waitcnt vmcnt(1)
-; GFX908-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshl_or_b32 v1, v5, 16, v0
+; GFX908-NEXT: v_or_b32_sdwa v1, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX908-NEXT: v_mov_b32_e32 v4, s20
; GFX908-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
index a3fa25ebcc990..cecc14be3c521 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
@@ -5677,16 +5677,16 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v1, s16
; GFX942-NEXT: v_mov_b32_e32 v2, s16
-; GFX942-NEXT: buffer_load_ushort v3, v1, s[0:3], 0 offen offset:1024
-; GFX942-NEXT: buffer_load_ushort v4, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: v_mov_b32_e32 v1, s16
+; GFX942-NEXT: buffer_load_ushort v3, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: buffer_load_ushort v4, v1, s[0:3], 0 offen offset:1024
; GFX942-NEXT: v_pk_max_f16 v2, v0, v0
; GFX942-NEXT: s_mov_b64 s[4:5], 0
; GFX942-NEXT: s_waitcnt vmcnt(1)
-; GFX942-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshl_or_b32 v0, v4, 16, v0
+; GFX942-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX942-NEXT: v_mov_b32_e32 v3, s16
; GFX942-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -5786,14 +5786,14 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no
; GFX10-NEXT: v_mov_b32_e32 v2, s20
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1024
; GFX10-NEXT: v_pk_max_f16 v2, v0, v0
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v3
; GFX10-NEXT: v_mov_b32_e32 v3, s20
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshl_or_b32 v0, v4, 16, v1
+; GFX10-NEXT: v_or_b32_sdwa v0, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_mov_b32_e32 v5, v0
@@ -5817,16 +5817,16 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v1, s20
; GFX90A-NEXT: v_mov_b32_e32 v2, s20
-; GFX90A-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX90A-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: v_mov_b32_e32 v1, s20
+; GFX90A-NEXT: buffer_load_ushort v3, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
; GFX90A-NEXT: v_pk_max_f16 v2, v0, v0
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: s_waitcnt vmcnt(1)
-; GFX90A-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshl_or_b32 v0, v4, 16, v0
+; GFX90A-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX90A-NEXT: v_mov_b32_e32 v3, s20
; GFX90A-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -5848,16 +5848,16 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v1, s20
; GFX908-NEXT: v_mov_b32_e32 v2, s20
-; GFX908-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX908-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: v_mov_b32_e32 v1, s20
+; GFX908-NEXT: buffer_load_ushort v3, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
; GFX908-NEXT: v_pk_max_f16 v2, v0, v0
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: s_waitcnt vmcnt(1)
-; GFX908-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshl_or_b32 v0, v4, 16, v0
+; GFX908-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX908-NEXT: v_mov_b32_e32 v3, s20
; GFX908-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -6085,16 +6085,16 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fin
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v1, s16
; GFX942-NEXT: v_mov_b32_e32 v2, s16
-; GFX942-NEXT: buffer_load_ushort v3, v1, s[0:3], 0 offen offset:1024
-; GFX942-NEXT: buffer_load_ushort v4, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: v_mov_b32_e32 v1, s16
+; GFX942-NEXT: buffer_load_ushort v3, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: buffer_load_ushort v4, v1, s[0:3], 0 offen offset:1024
; GFX942-NEXT: v_pk_max_f16 v2, v0, v0
; GFX942-NEXT: s_mov_b64 s[4:5], 0
; GFX942-NEXT: s_waitcnt vmcnt(1)
-; GFX942-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshl_or_b32 v1, v4, 16, v0
+; GFX942-NEXT: v_or_b32_sdwa v1, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX942-NEXT: v_mov_b32_e32 v3, s16
; GFX942-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -6194,14 +6194,14 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fin
; GFX10-NEXT: v_mov_b32_e32 v2, s20
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1024
; GFX10-NEXT: v_pk_max_f16 v2, v0, v0
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v3
; GFX10-NEXT: v_mov_b32_e32 v3, s20
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshl_or_b32 v1, v4, 16, v1
+; GFX10-NEXT: v_or_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_pk_max_f16 v0, v1, v1
@@ -6225,16 +6225,16 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fin
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v1, s20
; GFX90A-NEXT: v_mov_b32_e32 v2, s20
-; GFX90A-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX90A-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: v_mov_b32_e32 v1, s20
+; GFX90A-NEXT: buffer_load_ushort v3, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
; GFX90A-NEXT: v_pk_max_f16 v2, v0, v0
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: s_waitcnt vmcnt(1)
-; GFX90A-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshl_or_b32 v1, v4, 16, v0
+; GFX90A-NEXT: v_or_b32_sdwa v1, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX90A-NEXT: v_mov_b32_e32 v3, s20
; GFX90A-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -6256,16 +6256,16 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fin
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v1, s20
; GFX908-NEXT: v_mov_b32_e32 v2, s20
-; GFX908-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX908-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: v_mov_b32_e32 v1, s20
+; GFX908-NEXT: buffer_load_ushort v3, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
; GFX908-NEXT: v_pk_max_f16 v2, v0, v0
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: s_waitcnt vmcnt(1)
-; GFX908-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshl_or_b32 v1, v4, 16, v0
+; GFX908-NEXT: v_or_b32_sdwa v1, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX908-NEXT: v_mov_b32_e32 v3, s20
; GFX908-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -7436,19 +7436,19 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v1, s16
; GFX942-NEXT: v_mov_b32_e32 v2, s16
-; GFX942-NEXT: buffer_load_ushort v4, v1, s[0:3], 0 offen offset:1024
-; GFX942-NEXT: buffer_load_ushort v5, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: v_mov_b32_e32 v1, s16
+; GFX942-NEXT: buffer_load_ushort v4, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: buffer_load_ushort v5, v1, s[0:3], 0 offen offset:1024
; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX942-NEXT: s_mov_b64 s[6:7], 0
; GFX942-NEXT: s_movk_i32 s8, 0x7fff
; GFX942-NEXT: s_mov_b32 s9, 0x7060302
; GFX942-NEXT: s_waitcnt vmcnt(1)
-; GFX942-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshl_or_b32 v0, v5, 16, v0
+; GFX942-NEXT: v_or_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX942-NEXT: v_mov_b32_e32 v4, s16
; GFX942-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -7604,14 +7604,14 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu
; GFX10-NEXT: v_mov_b32_e32 v2, s20
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1024
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v3
; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshl_or_b32 v0, v4, 16, v1
+; GFX10-NEXT: v_or_b32_sdwa v0, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_mov_b32_e32 v4, s20
; GFX10-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -7649,19 +7649,19 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v1, s20
; GFX90A-NEXT: v_mov_b32_e32 v2, s20
-; GFX90A-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
-; GFX90A-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: v_mov_b32_e32 v1, s20
+; GFX90A-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: s_waitcnt vmcnt(1)
-; GFX90A-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshl_or_b32 v0, v5, 16, v0
+; GFX90A-NEXT: v_or_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX90A-NEXT: v_mov_b32_e32 v4, s20
; GFX90A-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -7696,19 +7696,19 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v1, s20
; GFX908-NEXT: v_mov_b32_e32 v2, s20
-; GFX908-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
-; GFX908-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: v_mov_b32_e32 v1, s20
+; GFX908-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX908-NEXT: s_mov_b64 s[6:7], 0
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: s_waitcnt vmcnt(1)
-; GFX908-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshl_or_b32 v0, v5, 16, v0
+; GFX908-NEXT: v_or_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX908-NEXT: v_mov_b32_e32 v4, s20
; GFX908-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -8013,19 +8013,19 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fi
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v1, s16
; GFX942-NEXT: v_mov_b32_e32 v2, s16
-; GFX942-NEXT: buffer_load_ushort v4, v1, s[0:3], 0 offen offset:1024
-; GFX942-NEXT: buffer_load_ushort v5, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: v_mov_b32_e32 v1, s16
+; GFX942-NEXT: buffer_load_ushort v4, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: buffer_load_ushort v5, v1, s[0:3], 0 offen offset:1024
; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX942-NEXT: s_mov_b64 s[6:7], 0
; GFX942-NEXT: s_movk_i32 s8, 0x7fff
; GFX942-NEXT: s_mov_b32 s9, 0x7060302
; GFX942-NEXT: s_waitcnt vmcnt(1)
-; GFX942-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshl_or_b32 v1, v5, 16, v0
+; GFX942-NEXT: v_or_b32_sdwa v1, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX942-NEXT: v_mov_b32_e32 v4, s16
; GFX942-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -8175,14 +8175,14 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fi
; GFX10-NEXT: v_mov_b32_e32 v2, s20
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1024
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v3
; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshl_or_b32 v1, v4, 16, v1
+; GFX10-NEXT: v_or_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_mov_b32_e32 v4, s20
; GFX10-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -8220,19 +8220,19 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fi
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v1, s20
; GFX90A-NEXT: v_mov_b32_e32 v2, s20
-; GFX90A-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
-; GFX90A-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: v_mov_b32_e32 v1, s20
+; GFX90A-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: s_waitcnt vmcnt(1)
-; GFX90A-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshl_or_b32 v1, v5, 16, v0
+; GFX90A-NEXT: v_or_b32_sdwa v1, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX90A-NEXT: v_mov_b32_e32 v4, s20
; GFX90A-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -8267,19 +8267,19 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fi
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v1, s20
; GFX908-NEXT: v_mov_b32_e32 v2, s20
-; GFX908-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
-; GFX908-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: v_mov_b32_e32 v1, s20
+; GFX908-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX908-NEXT: s_mov_b64 s[6:7], 0
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: s_waitcnt vmcnt(1)
-; GFX908-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshl_or_b32 v1, v5, 16, v0
+; GFX908-NEXT: v_or_b32_sdwa v1, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX908-NEXT: v_mov_b32_e32 v4, s20
; GFX908-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
index bf50043a88098..7859ac4841a10 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
@@ -5677,16 +5677,16 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v1, s16
; GFX942-NEXT: v_mov_b32_e32 v2, s16
-; GFX942-NEXT: buffer_load_ushort v3, v1, s[0:3], 0 offen offset:1024
-; GFX942-NEXT: buffer_load_ushort v4, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: v_mov_b32_e32 v1, s16
+; GFX942-NEXT: buffer_load_ushort v3, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: buffer_load_ushort v4, v1, s[0:3], 0 offen offset:1024
; GFX942-NEXT: v_pk_max_f16 v2, v0, v0
; GFX942-NEXT: s_mov_b64 s[4:5], 0
; GFX942-NEXT: s_waitcnt vmcnt(1)
-; GFX942-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshl_or_b32 v0, v4, 16, v0
+; GFX942-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX942-NEXT: v_mov_b32_e32 v3, s16
; GFX942-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -5786,14 +5786,14 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no
; GFX10-NEXT: v_mov_b32_e32 v2, s20
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1024
; GFX10-NEXT: v_pk_max_f16 v2, v0, v0
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v3
; GFX10-NEXT: v_mov_b32_e32 v3, s20
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshl_or_b32 v0, v4, 16, v1
+; GFX10-NEXT: v_or_b32_sdwa v0, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_mov_b32_e32 v5, v0
@@ -5817,16 +5817,16 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v1, s20
; GFX90A-NEXT: v_mov_b32_e32 v2, s20
-; GFX90A-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX90A-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: v_mov_b32_e32 v1, s20
+; GFX90A-NEXT: buffer_load_ushort v3, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
; GFX90A-NEXT: v_pk_max_f16 v2, v0, v0
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: s_waitcnt vmcnt(1)
-; GFX90A-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshl_or_b32 v0, v4, 16, v0
+; GFX90A-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX90A-NEXT: v_mov_b32_e32 v3, s20
; GFX90A-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -5848,16 +5848,16 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v1, s20
; GFX908-NEXT: v_mov_b32_e32 v2, s20
-; GFX908-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX908-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: v_mov_b32_e32 v1, s20
+; GFX908-NEXT: buffer_load_ushort v3, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
; GFX908-NEXT: v_pk_max_f16 v2, v0, v0
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: s_waitcnt vmcnt(1)
-; GFX908-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshl_or_b32 v0, v4, 16, v0
+; GFX908-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX908-NEXT: v_mov_b32_e32 v3, s20
; GFX908-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -6085,16 +6085,16 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fin
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v1, s16
; GFX942-NEXT: v_mov_b32_e32 v2, s16
-; GFX942-NEXT: buffer_load_ushort v3, v1, s[0:3], 0 offen offset:1024
-; GFX942-NEXT: buffer_load_ushort v4, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: v_mov_b32_e32 v1, s16
+; GFX942-NEXT: buffer_load_ushort v3, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: buffer_load_ushort v4, v1, s[0:3], 0 offen offset:1024
; GFX942-NEXT: v_pk_max_f16 v2, v0, v0
; GFX942-NEXT: s_mov_b64 s[4:5], 0
; GFX942-NEXT: s_waitcnt vmcnt(1)
-; GFX942-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshl_or_b32 v1, v4, 16, v0
+; GFX942-NEXT: v_or_b32_sdwa v1, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX942-NEXT: v_mov_b32_e32 v3, s16
; GFX942-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -6194,14 +6194,14 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fin
; GFX10-NEXT: v_mov_b32_e32 v2, s20
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1024
; GFX10-NEXT: v_pk_max_f16 v2, v0, v0
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v3
; GFX10-NEXT: v_mov_b32_e32 v3, s20
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshl_or_b32 v1, v4, 16, v1
+; GFX10-NEXT: v_or_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_pk_max_f16 v0, v1, v1
@@ -6225,16 +6225,16 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fin
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v1, s20
; GFX90A-NEXT: v_mov_b32_e32 v2, s20
-; GFX90A-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX90A-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: v_mov_b32_e32 v1, s20
+; GFX90A-NEXT: buffer_load_ushort v3, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
; GFX90A-NEXT: v_pk_max_f16 v2, v0, v0
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: s_waitcnt vmcnt(1)
-; GFX90A-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshl_or_b32 v1, v4, 16, v0
+; GFX90A-NEXT: v_or_b32_sdwa v1, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX90A-NEXT: v_mov_b32_e32 v3, s20
; GFX90A-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -6256,16 +6256,16 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fin
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v1, s20
; GFX908-NEXT: v_mov_b32_e32 v2, s20
-; GFX908-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX908-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: v_mov_b32_e32 v1, s20
+; GFX908-NEXT: buffer_load_ushort v3, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
; GFX908-NEXT: v_pk_max_f16 v2, v0, v0
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: s_waitcnt vmcnt(1)
-; GFX908-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshl_or_b32 v1, v4, 16, v0
+; GFX908-NEXT: v_or_b32_sdwa v1, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX908-NEXT: v_mov_b32_e32 v3, s20
; GFX908-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -7436,19 +7436,19 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v1, s16
; GFX942-NEXT: v_mov_b32_e32 v2, s16
-; GFX942-NEXT: buffer_load_ushort v4, v1, s[0:3], 0 offen offset:1024
-; GFX942-NEXT: buffer_load_ushort v5, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: v_mov_b32_e32 v1, s16
+; GFX942-NEXT: buffer_load_ushort v4, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: buffer_load_ushort v5, v1, s[0:3], 0 offen offset:1024
; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX942-NEXT: s_mov_b64 s[6:7], 0
; GFX942-NEXT: s_movk_i32 s8, 0x7fff
; GFX942-NEXT: s_mov_b32 s9, 0x7060302
; GFX942-NEXT: s_waitcnt vmcnt(1)
-; GFX942-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshl_or_b32 v0, v5, 16, v0
+; GFX942-NEXT: v_or_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX942-NEXT: v_mov_b32_e32 v4, s16
; GFX942-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -7604,14 +7604,14 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu
; GFX10-NEXT: v_mov_b32_e32 v2, s20
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1024
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v3
; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshl_or_b32 v0, v4, 16, v1
+; GFX10-NEXT: v_or_b32_sdwa v0, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_mov_b32_e32 v4, s20
; GFX10-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -7649,19 +7649,19 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v1, s20
; GFX90A-NEXT: v_mov_b32_e32 v2, s20
-; GFX90A-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
-; GFX90A-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: v_mov_b32_e32 v1, s20
+; GFX90A-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: s_waitcnt vmcnt(1)
-; GFX90A-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshl_or_b32 v0, v5, 16, v0
+; GFX90A-NEXT: v_or_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX90A-NEXT: v_mov_b32_e32 v4, s20
; GFX90A-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -7696,19 +7696,19 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v1, s20
; GFX908-NEXT: v_mov_b32_e32 v2, s20
-; GFX908-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
-; GFX908-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: v_mov_b32_e32 v1, s20
+; GFX908-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX908-NEXT: s_mov_b64 s[6:7], 0
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: s_waitcnt vmcnt(1)
-; GFX908-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshl_or_b32 v0, v5, 16, v0
+; GFX908-NEXT: v_or_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX908-NEXT: v_mov_b32_e32 v4, s20
; GFX908-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -8013,19 +8013,19 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fi
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v1, s16
; GFX942-NEXT: v_mov_b32_e32 v2, s16
-; GFX942-NEXT: buffer_load_ushort v4, v1, s[0:3], 0 offen offset:1024
-; GFX942-NEXT: buffer_load_ushort v5, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: v_mov_b32_e32 v1, s16
+; GFX942-NEXT: buffer_load_ushort v4, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: buffer_load_ushort v5, v1, s[0:3], 0 offen offset:1024
; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX942-NEXT: s_mov_b64 s[6:7], 0
; GFX942-NEXT: s_movk_i32 s8, 0x7fff
; GFX942-NEXT: s_mov_b32 s9, 0x7060302
; GFX942-NEXT: s_waitcnt vmcnt(1)
-; GFX942-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshl_or_b32 v1, v5, 16, v0
+; GFX942-NEXT: v_or_b32_sdwa v1, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX942-NEXT: v_mov_b32_e32 v4, s16
; GFX942-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -8175,14 +8175,14 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fi
; GFX10-NEXT: v_mov_b32_e32 v2, s20
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
-; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1024
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v3
; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshl_or_b32 v1, v4, 16, v1
+; GFX10-NEXT: v_or_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_mov_b32_e32 v4, s20
; GFX10-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -8220,19 +8220,19 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fi
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v1, s20
; GFX90A-NEXT: v_mov_b32_e32 v2, s20
-; GFX90A-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
-; GFX90A-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: v_mov_b32_e32 v1, s20
+; GFX90A-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: s_waitcnt vmcnt(1)
-; GFX90A-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshl_or_b32 v1, v5, 16, v0
+; GFX90A-NEXT: v_or_b32_sdwa v1, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX90A-NEXT: v_mov_b32_e32 v4, s20
; GFX90A-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -8267,19 +8267,19 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fi
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v1, s20
; GFX908-NEXT: v_mov_b32_e32 v2, s20
-; GFX908-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
-; GFX908-NEXT: buffer_load_ushort v5, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: v_mov_b32_e32 v1, s20
+; GFX908-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX908-NEXT: s_mov_b64 s[6:7], 0
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: s_waitcnt vmcnt(1)
-; GFX908-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshl_or_b32 v1, v5, 16, v0
+; GFX908-NEXT: v_or_b32_sdwa v1, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX908-NEXT: v_mov_b32_e32 v4, s20
; GFX908-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-contents-legalization.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-contents-legalization.ll
index b8b323aa83851..fd469c3f069bf 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-contents-legalization.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-contents-legalization.ll
@@ -2333,11 +2333,10 @@ define void @store_v6i8(<6 x i8> %data, ptr addrspace(8) inreg %buf) {
; GISEL-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GISEL-NEXT: v_and_b32_e32 v1, 0xff, v3
; GISEL-NEXT: v_lshlrev_b16_e32 v1, 8, v1
-; GISEL-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GISEL-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GISEL-NEXT: v_and_b32_e32 v2, 0xff, v5
-; GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GISEL-NEXT: v_lshlrev_b16_e32 v2, 8, v2
-; GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GISEL-NEXT: v_or_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GISEL-NEXT: buffer_store_dword v0, off, s[16:19], 0
; GISEL-NEXT: buffer_store_short v2, off, s[16:19], 0 offset:4
@@ -3862,11 +3861,10 @@ define void @volatile_store_v6i8(<6 x i8> %data, ptr addrspace(8) inreg %buf) {
; GISEL-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GISEL-NEXT: v_and_b32_e32 v1, 0xff, v3
; GISEL-NEXT: v_lshlrev_b16_e32 v1, 8, v1
-; GISEL-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GISEL-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GISEL-NEXT: v_and_b32_e32 v2, 0xff, v5
-; GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GISEL-NEXT: v_lshlrev_b16_e32 v2, 8, v2
-; GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GISEL-NEXT: v_or_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GISEL-NEXT: buffer_store_dword v0, off, s[16:19], 0
; GISEL-NEXT: s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/build-vector-packed-partial-undef.ll b/llvm/test/CodeGen/AMDGPU/build-vector-packed-partial-undef.ll
index 841c5ad444628..f38ac4dee69c2 100644
--- a/llvm/test/CodeGen/AMDGPU/build-vector-packed-partial-undef.ll
+++ b/llvm/test/CodeGen/AMDGPU/build-vector-packed-partial-undef.ll
@@ -647,8 +647,8 @@ define void @undef_hi_v2i16(i16 %arg0) {
; GFX9-GISEL-LABEL: undef_hi_v2i16:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, s4, 16, v0
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e64 v1, 16, s4
+; GFX9-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-GISEL-NEXT: ;;#ASMSTART
; GFX9-GISEL-NEXT: ; use v0
; GFX9-GISEL-NEXT: ;;#ASMEND
@@ -714,8 +714,8 @@ define void @undef_hi_v2f16(half %arg0) {
; GFX9-GISEL-LABEL: undef_hi_v2f16:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, s4, 16, v0
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e64 v1, 16, s4
+; GFX9-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-GISEL-NEXT: ;;#ASMSTART
; GFX9-GISEL-NEXT: ; use v0
; GFX9-GISEL-NEXT: ;;#ASMEND
@@ -790,8 +790,8 @@ define void @undef_hi_op_v2f16(half %arg0) {
; GFX9-GISEL-LABEL: undef_hi_op_v2f16:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, s4, 16, v0
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e64 v1, 16, s4
+; GFX9-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-GISEL-NEXT: v_pk_add_f16 v0, v0, 1.0 op_sel_hi:[1,0]
; GFX9-GISEL-NEXT: ;;#ASMSTART
; GFX9-GISEL-NEXT: ; use v0
@@ -878,8 +878,8 @@ define void @undef_hi_op_v2i16(i16 %arg0) {
; GFX9-GISEL-LABEL: undef_hi_op_v2i16:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, s4, 16, v0
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e64 v1, 16, s4
+; GFX9-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0x630063
; GFX9-GISEL-NEXT: v_pk_add_u16 v0, v0, v1
; GFX9-GISEL-NEXT: ;;#ASMSTART
@@ -961,8 +961,8 @@ define void @undef_hi3_v4i16(i16 %arg0) {
; GFX9-GISEL-LABEL: undef_hi3_v4i16:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, s4, 16, v0
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e64 v1, 16, s4
+; GFX9-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-GISEL-NEXT: ;;#ASMSTART
; GFX9-GISEL-NEXT: ; use v[0:1]
; GFX9-GISEL-NEXT: ;;#ASMEND
@@ -1029,8 +1029,8 @@ define void @undef_hi3_v4f16(half %arg0) {
; GFX9-GISEL-LABEL: undef_hi3_v4f16:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, s4, 16, v0
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e64 v1, 16, s4
+; GFX9-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-GISEL-NEXT: ;;#ASMSTART
; GFX9-GISEL-NEXT: ; use v[0:1]
; GFX9-GISEL-NEXT: ;;#ASMEND
diff --git a/llvm/test/CodeGen/AMDGPU/constant-address-space-32bit.ll b/llvm/test/CodeGen/AMDGPU/constant-address-space-32bit.ll
index 22fbbe3fe33b8..32bfce4a2fc58 100644
--- a/llvm/test/CodeGen/AMDGPU/constant-address-space-32bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/constant-address-space-32bit.ll
@@ -2543,14 +2543,13 @@ define amdgpu_vs <2 x bfloat> @load_v2bf16(ptr addrspace(6) inreg %p0, ptr addrs
; GFX9-NEXT: v_add_f32_e32 v1, s1, v1
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
-; GFX9-NEXT: v_mov_b32_e32 v2, 0xffff
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_and_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: ; return to shader part epilog
%gep1 = getelementptr inbounds <2 x bfloat>, ptr addrspace(6) %p1, i32 2
%r0 = load <2 x bfloat>, ptr addrspace(6) %p0
diff --git a/llvm/test/CodeGen/AMDGPU/ctlz_zero_poison.ll b/llvm/test/CodeGen/AMDGPU/ctlz_zero_poison.ll
index 240cf3d042250..b9a47766cb089 100644
--- a/llvm/test/CodeGen/AMDGPU/ctlz_zero_poison.ll
+++ b/llvm/test/CodeGen/AMDGPU/ctlz_zero_poison.ll
@@ -2419,11 +2419,10 @@ define <2 x i16> @v_ctlz_zero_poison_v2i16(<2 x i16> %val) {
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v0, v0
; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v1, v1
-; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v0, v0
+; GFX9-GISEL-NEXT: v_ffbh_u32_sdwa v1, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; GFX9-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
%ctlz = call <2 x i16> @llvm.ctlz.v2i16(<2 x i16> %val, i1 true)
ret <2 x i16> %ctlz
@@ -2465,15 +2464,14 @@ define <3 x i16> @v_ctlz_zero_poison_v3i16(<3 x i16> %val) {
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v0, v0
; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v0, v0
+; GFX9-GISEL-NEXT: v_ffbh_u32_sdwa v2, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v1, v1
-; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v2, v2
-; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v2, 16, v0
-; GFX9-GISEL-NEXT: v_lshl_or_b32 v1, s4, 16, v1
+; GFX9-GISEL-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e64 v2, 16, s4
+; GFX9-GISEL-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
%ctlz = call <3 x i16> @llvm.ctlz.v3i16(<3 x i16> %val, i1 true)
ret <3 x i16> %ctlz
@@ -2524,17 +2522,15 @@ define <4 x i16> @v_ctlz_zero_poison_v4i16(<4 x i16> %val) {
; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v0, v0
-; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-GISEL-NEXT: v_ffbh_u32_sdwa v2, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v1, v1
-; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v2, v2
-; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v3, v3
-; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v2, 16, v0
-; GFX9-GISEL-NEXT: v_lshl_or_b32 v1, v3, 16, v1
+; GFX9-GISEL-NEXT: v_ffbh_u32_sdwa v3, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; GFX9-GISEL-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-GISEL-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
%ctlz = call <4 x i16> @llvm.ctlz.v4i16(<4 x i16> %val, i1 true)
ret <4 x i16> %ctlz
diff --git a/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll b/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
index 31eb0331c386c..27d66a1886d0e 100644
--- a/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
+++ b/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
@@ -1676,27 +1676,27 @@ define amdgpu_kernel void @load_v4i8_to_v4f32_2_uses(ptr addrspace(1) noalias %o
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX10-NEXT: v_mov_b32_e32 v4, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: global_load_dword v0, v0, s[0:1]
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffffff00, v0
-; GFX10-NEXT: v_add_nc_u16 v2, v0, 9
-; GFX10-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; GFX10-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffffff00, v3
-; GFX10-NEXT: v_add_nc_u16 v3, v3, 9
-; GFX10-NEXT: v_add_nc_u16 v1, 0x900, v1
-; GFX10-NEXT: v_or_b32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX10-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX10-NEXT: v_and_b32_e32 v3, 0xffffff00, v0
+; GFX10-NEXT: v_add_nc_u16 v4, v0, 9
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffffff00, v1
+; GFX10-NEXT: v_add_nc_u16 v1, v1, 9
+; GFX10-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX10-NEXT: v_or_b32_sdwa v2, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX10-NEXT: v_mov_b32_e32 v4, 0
; GFX10-NEXT: v_cvt_f32_ubyte3_e32 v3, v0
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff, v1
-; GFX10-NEXT: v_cvt_f32_ubyte1_e32 v1, v0
-; GFX10-NEXT: v_add_nc_u16 v6, 0x900, v2
+; GFX10-NEXT: v_add_nc_u16 v1, 0x900, v1
+; GFX10-NEXT: v_add_nc_u16 v5, 0x900, v2
; GFX10-NEXT: v_cvt_f32_ubyte2_e32 v2, v0
+; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v1
+; GFX10-NEXT: v_cvt_f32_ubyte1_e32 v1, v0
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
-; GFX10-NEXT: v_lshl_or_b32 v5, v6, 16, v5
+; GFX10-NEXT: v_or_b32_sdwa v5, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
; GFX10-NEXT: global_store_dword v4, v5, s[2:3]
diff --git a/llvm/test/CodeGen/AMDGPU/dagcombine-fmul-sel.ll b/llvm/test/CodeGen/AMDGPU/dagcombine-fmul-sel.ll
index 8f77f2f773ece..7dd05ff26717a 100644
--- a/llvm/test/CodeGen/AMDGPU/dagcombine-fmul-sel.ll
+++ b/llvm/test/CodeGen/AMDGPU/dagcombine-fmul-sel.ll
@@ -1828,9 +1828,8 @@ define <2 x half> @fmul_select_v2f16_test3(<2 x half> %x, <2 x i32> %bool.arg1,
; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
; GFX10-GISEL-NEXT: v_ldexp_f16_e32 v1, v0, v1
; GFX10-GISEL-NEXT: v_med3_i32 v2, 0xffff8000, v2, v3
-; GFX10-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-GISEL-NEXT: v_ldexp_f16_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-GISEL-NEXT: v_lshl_or_b32 v0, v0, 16, v1
+; GFX10-GISEL-NEXT: v_ldexp_f16_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-GISEL-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: fmul_select_v2f16_test3:
@@ -1996,9 +1995,8 @@ define <2 x half> @fmul_select_v2f16_test4(<2 x half> %x, <2 x i32> %bool.arg1,
; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v2, 0, -1, vcc_lo
; GFX10-GISEL-NEXT: v_ldexp_f16_e32 v1, v0, v1
; GFX10-GISEL-NEXT: v_med3_i32 v2, 0xffff8000, v2, v3
-; GFX10-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-GISEL-NEXT: v_ldexp_f16_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-GISEL-NEXT: v_lshl_or_b32 v0, v0, 16, v1
+; GFX10-GISEL-NEXT: v_ldexp_f16_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-GISEL-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: fmul_select_v2f16_test4:
diff --git a/llvm/test/CodeGen/AMDGPU/fmaximum.ll b/llvm/test/CodeGen/AMDGPU/fmaximum.ll
index 98e24aa3340a3..213d6ffeb9f00 100644
--- a/llvm/test/CodeGen/AMDGPU/fmaximum.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmaximum.ll
@@ -511,13 +511,12 @@ define amdgpu_ps <2 x half> @test_fmaximum_v2f16_vv(<2 x half> %a, <2 x half> %b
; GFX9-GISEL-LABEL: test_fmaximum_v2f16_vv:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: v_pk_max_f16 v2, v0, v1
+; GFX9-GISEL-NEXT: v_cmp_o_f16_sdwa vcc, v0, v1 src0_sel:WORD_1 src1_sel:WORD_1
; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, 0x7e00
; GFX9-GISEL-NEXT: v_cmp_o_f16_e64 s[0:1], v0, v1
-; GFX9-GISEL-NEXT: v_cmp_o_f16_sdwa vcc, v0, v1 src0_sel:WORD_1 src1_sel:WORD_1
; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v0, v3, v2, s[0:1]
-; GFX9-GISEL-NEXT: v_cndmask_b32_sdwa v1, v3, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX9-GISEL-NEXT: v_cndmask_b32_sdwa v1, v3, v2, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-GISEL-NEXT: ; return to shader part epilog
;
; GFX1170-LABEL: test_fmaximum_v2f16_vv:
@@ -546,9 +545,8 @@ define amdgpu_ps <2 x half> @test_fmaximum_v2f16_ss(<2 x half> inreg %a, <2 x ha
; GFX9-SDAG-NEXT: v_mov_b32_e32 v3, s1
; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
; GFX9-SDAG-NEXT: v_cmp_o_f16_e32 vcc, s0, v3
-; GFX9-SDAG-NEXT: v_cndmask_b32_sdwa v1, v2, v1, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-SDAG-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX9-SDAG-NEXT: v_cndmask_b32_sdwa v1, v2, v1, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-SDAG-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-SDAG-NEXT: ; return to shader part epilog
;
; GFX9-GISEL-LABEL: test_fmaximum_v2f16_ss:
@@ -615,10 +613,10 @@ define amdgpu_ps <3 x half> @test_fmaximum_v3f16_vv(<3 x half> %a, <3 x half> %b
; GFX9-GISEL-NEXT: v_pk_max_f16 v2, v1, v3
; GFX9-GISEL-NEXT: v_cmp_o_f16_e32 vcc, v1, v3
; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v1, v6, v2, vcc
-; GFX9-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v4
-; GFX9-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v0, 16, v2
-; GFX9-GISEL-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e64 v2, 16, s0
+; GFX9-GISEL-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-GISEL-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-GISEL-NEXT: ; return to shader part epilog
;
; GFX1170-SDAG-LABEL: test_fmaximum_v3f16_vv:
@@ -684,9 +682,8 @@ define amdgpu_ps <3 x half> @test_fmaximum_v3f16_ss(<3 x half> inreg %a, <3 x ha
; GFX9-SDAG-NEXT: v_mov_b32_e32 v4, s1
; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
; GFX9-SDAG-NEXT: v_cmp_o_f16_e32 vcc, s0, v4
-; GFX9-SDAG-NEXT: v_cndmask_b32_sdwa v2, v2, v3, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-SDAG-NEXT: v_lshl_or_b32 v0, v2, 16, v0
+; GFX9-SDAG-NEXT: v_cndmask_b32_sdwa v2, v2, v3, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-SDAG-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-SDAG-NEXT: ; return to shader part epilog
;
; GFX9-GISEL-LABEL: test_fmaximum_v3f16_ss:
@@ -791,15 +788,14 @@ define amdgpu_ps <4 x half> @test_fmaximum_v4f16(<4 x half> %a, <4 x half> %b) {
; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc
; GFX9-GISEL-NEXT: v_cmp_o_f16_sdwa vcc, v0, v2 src0_sel:WORD_1 src1_sel:WORD_1
; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v0, v6, v5, vcc
-; GFX9-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v4
-; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v0, 16, v2
; GFX9-GISEL-NEXT: v_pk_max_f16 v2, v1, v3
-; GFX9-GISEL-NEXT: v_cmp_o_f16_e64 s[0:1], v1, v3
; GFX9-GISEL-NEXT: v_cmp_o_f16_sdwa vcc, v1, v3 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-GISEL-NEXT: v_cmp_o_f16_e64 s[0:1], v1, v3
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v1, v6, v2, s[0:1]
-; GFX9-GISEL-NEXT: v_cndmask_b32_sdwa v2, v6, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-GISEL-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX9-GISEL-NEXT: v_cndmask_b32_sdwa v2, v6, v2, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-GISEL-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-GISEL-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-GISEL-NEXT: ; return to shader part epilog
;
; GFX1170-LABEL: test_fmaximum_v4f16:
@@ -841,11 +837,10 @@ define amdgpu_ps <4 x half> @test_fmaximum_v4f16_ss(<4 x half> inreg %a, <4 x ha
; GFX9-SDAG-NEXT: v_mov_b32_e32 v5, s1
; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc
; GFX9-SDAG-NEXT: v_cmp_o_f16_e32 vcc, s0, v5
-; GFX9-SDAG-NEXT: v_cndmask_b32_sdwa v2, v2, v4, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-SDAG-NEXT: v_lshl_or_b32 v0, v2, 16, v0
-; GFX9-SDAG-NEXT: v_and_b32_e32 v2, 0xffff, v3
-; GFX9-SDAG-NEXT: v_lshl_or_b32 v1, v1, 16, v2
+; GFX9-SDAG-NEXT: v_cndmask_b32_sdwa v2, v2, v4, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-SDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-SDAG-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-SDAG-NEXT: v_or_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-SDAG-NEXT: ; return to shader part epilog
;
; GFX9-GISEL-LABEL: test_fmaximum_v4f16_ss:
diff --git a/llvm/test/CodeGen/AMDGPU/fminimum.ll b/llvm/test/CodeGen/AMDGPU/fminimum.ll
index 8ba95ee3550c0..094dd76e0f945 100644
--- a/llvm/test/CodeGen/AMDGPU/fminimum.ll
+++ b/llvm/test/CodeGen/AMDGPU/fminimum.ll
@@ -511,13 +511,12 @@ define amdgpu_ps <2 x half> @test_fminimum_v2f16_vv(<2 x half> %a, <2 x half> %b
; GFX9-GISEL-LABEL: test_fminimum_v2f16_vv:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: v_pk_min_f16 v2, v0, v1
+; GFX9-GISEL-NEXT: v_cmp_o_f16_sdwa vcc, v0, v1 src0_sel:WORD_1 src1_sel:WORD_1
; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, 0x7e00
; GFX9-GISEL-NEXT: v_cmp_o_f16_e64 s[0:1], v0, v1
-; GFX9-GISEL-NEXT: v_cmp_o_f16_sdwa vcc, v0, v1 src0_sel:WORD_1 src1_sel:WORD_1
; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v0, v3, v2, s[0:1]
-; GFX9-GISEL-NEXT: v_cndmask_b32_sdwa v1, v3, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX9-GISEL-NEXT: v_cndmask_b32_sdwa v1, v3, v2, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-GISEL-NEXT: ; return to shader part epilog
;
; GFX1170-LABEL: test_fminimum_v2f16_vv:
@@ -546,9 +545,8 @@ define amdgpu_ps <2 x half> @test_fminimum_v2f16_ss(<2 x half> inreg %a, <2 x ha
; GFX9-SDAG-NEXT: v_mov_b32_e32 v3, s1
; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
; GFX9-SDAG-NEXT: v_cmp_o_f16_e32 vcc, s0, v3
-; GFX9-SDAG-NEXT: v_cndmask_b32_sdwa v1, v2, v1, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-SDAG-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX9-SDAG-NEXT: v_cndmask_b32_sdwa v1, v2, v1, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-SDAG-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-SDAG-NEXT: ; return to shader part epilog
;
; GFX9-GISEL-LABEL: test_fminimum_v2f16_ss:
@@ -615,10 +613,10 @@ define amdgpu_ps <3 x half> @test_fminimum_v3f16_vv(<3 x half> %a, <3 x half> %b
; GFX9-GISEL-NEXT: v_pk_min_f16 v2, v1, v3
; GFX9-GISEL-NEXT: v_cmp_o_f16_e32 vcc, v1, v3
; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v1, v6, v2, vcc
-; GFX9-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v4
-; GFX9-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v0, 16, v2
-; GFX9-GISEL-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e64 v2, 16, s0
+; GFX9-GISEL-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-GISEL-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-GISEL-NEXT: ; return to shader part epilog
;
; GFX1170-SDAG-LABEL: test_fminimum_v3f16_vv:
@@ -684,9 +682,8 @@ define amdgpu_ps <3 x half> @test_fminimum_v3f16_ss(<3 x half> inreg %a, <3 x ha
; GFX9-SDAG-NEXT: v_mov_b32_e32 v4, s1
; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
; GFX9-SDAG-NEXT: v_cmp_o_f16_e32 vcc, s0, v4
-; GFX9-SDAG-NEXT: v_cndmask_b32_sdwa v2, v2, v3, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-SDAG-NEXT: v_lshl_or_b32 v0, v2, 16, v0
+; GFX9-SDAG-NEXT: v_cndmask_b32_sdwa v2, v2, v3, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-SDAG-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-SDAG-NEXT: ; return to shader part epilog
;
; GFX9-GISEL-LABEL: test_fminimum_v3f16_ss:
@@ -791,15 +788,14 @@ define amdgpu_ps <4 x half> @test_fminimum_v4f16(<4 x half> %a, <4 x half> %b) {
; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc
; GFX9-GISEL-NEXT: v_cmp_o_f16_sdwa vcc, v0, v2 src0_sel:WORD_1 src1_sel:WORD_1
; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v0, v6, v5, vcc
-; GFX9-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v4
-; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v0, 16, v2
; GFX9-GISEL-NEXT: v_pk_min_f16 v2, v1, v3
-; GFX9-GISEL-NEXT: v_cmp_o_f16_e64 s[0:1], v1, v3
; GFX9-GISEL-NEXT: v_cmp_o_f16_sdwa vcc, v1, v3 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-GISEL-NEXT: v_cmp_o_f16_e64 s[0:1], v1, v3
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v1, v6, v2, s[0:1]
-; GFX9-GISEL-NEXT: v_cndmask_b32_sdwa v2, v6, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-GISEL-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX9-GISEL-NEXT: v_cndmask_b32_sdwa v2, v6, v2, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-GISEL-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-GISEL-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-GISEL-NEXT: ; return to shader part epilog
;
; GFX1170-LABEL: test_fminimum_v4f16:
@@ -841,11 +837,10 @@ define amdgpu_ps <4 x half> @test_fminimum_v4f16_ss(<4 x half> inreg %a, <4 x ha
; GFX9-SDAG-NEXT: v_mov_b32_e32 v5, s1
; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc
; GFX9-SDAG-NEXT: v_cmp_o_f16_e32 vcc, s0, v5
-; GFX9-SDAG-NEXT: v_cndmask_b32_sdwa v2, v2, v4, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-SDAG-NEXT: v_lshl_or_b32 v0, v2, 16, v0
-; GFX9-SDAG-NEXT: v_and_b32_e32 v2, 0xffff, v3
-; GFX9-SDAG-NEXT: v_lshl_or_b32 v1, v1, 16, v2
+; GFX9-SDAG-NEXT: v_cndmask_b32_sdwa v2, v2, v4, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-SDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-SDAG-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-SDAG-NEXT: v_or_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-SDAG-NEXT: ; return to shader part epilog
;
; GFX9-GISEL-LABEL: test_fminimum_v4f16_ss:
diff --git a/llvm/test/CodeGen/AMDGPU/fmul-to-ldexp.ll b/llvm/test/CodeGen/AMDGPU/fmul-to-ldexp.ll
index 24f035bd85e5c..79e76039d6777 100644
--- a/llvm/test/CodeGen/AMDGPU/fmul-to-ldexp.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmul-to-ldexp.ll
@@ -6759,9 +6759,8 @@ define <2 x half> @v_mul_v2f16_select_64_1(<2 x i32> %arg, <2 x half> %x) {
; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
; GFX10-GISEL-NEXT: v_ldexp_f16_e32 v0, v2, v0
; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 6, vcc_lo
-; GFX10-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: v_mul_v2f16_select_64_1:
@@ -6863,9 +6862,8 @@ define <2 x half> @v_mul_v2f16_select_1_64(<2 x i32> %arg, <2 x half> %x) {
; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
; GFX10-GISEL-NEXT: v_ldexp_f16_e32 v0, v2, v0
; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v1, 6, 0, vcc_lo
-; GFX10-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: v_mul_v2f16_select_1_64:
@@ -6969,9 +6967,8 @@ define <2 x half> @v_mul_v2f16_select_n1_n64(<2 x i32> %arg, <2 x half> %x) {
; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
; GFX10-GISEL-NEXT: v_ldexp_f16_e32 v0, v2, v0
; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v1, 6, 0, vcc_lo
-; GFX10-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: v_mul_v2f16_select_n1_n64:
@@ -7086,9 +7083,8 @@ define <2 x half> @v_mul_v2f16_select_128_64(<2 x i32> %arg, <2 x half> %x) {
; GFX10-GISEL-NEXT: v_add_nc_u32_e32 v1, 6, v1
; GFX10-GISEL-NEXT: v_ldexp_f16_e32 v0, v2, v0
; GFX10-GISEL-NEXT: v_med3_i32 v1, 0xffff8000, v1, v3
-; GFX10-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: v_mul_v2f16_select_128_64:
@@ -7212,9 +7208,8 @@ define <2 x half> @v_mul_v2f16_select_n128_n64(<2 x i32> %arg, <2 x half> %x) {
; GFX10-GISEL-NEXT: v_add_nc_u32_e32 v1, 6, v1
; GFX10-GISEL-NEXT: v_ldexp_f16_e32 v0, v2, v0
; GFX10-GISEL-NEXT: v_med3_i32 v1, 0xffff8000, v1, v3
-; GFX10-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: v_mul_v2f16_select_n128_n64:
@@ -7329,9 +7324,8 @@ define <2 x half> @v_mul_v2f16_select_n128_n16(<2 x i32> %arg, <2 x half> %x) {
; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
; GFX10-GISEL-NEXT: v_ldexp_f16_e32 v0, v2, v0
; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v1, 4, 7, vcc_lo
-; GFX10-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: v_mul_v2f16_select_n128_n16:
@@ -7436,9 +7430,8 @@ define <2 x half> @v_contract_mul_add_v2f16_select_64_1(<2 x i32> %arg, <2 x hal
; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
; GFX10-GISEL-NEXT: v_ldexp_f16_e32 v0, v2, v0
; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 6, vcc_lo
-; GFX10-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-GISEL-NEXT: v_pk_add_f16 v0, v0, v3
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
@@ -7545,9 +7538,8 @@ define <2 x half> @v_contract_mul_add_v2f16_select_1_64(<2 x i32> %arg, <2 x hal
; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
; GFX10-GISEL-NEXT: v_ldexp_f16_e32 v0, v2, v0
; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v1, 6, 0, vcc_lo
-; GFX10-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-GISEL-NEXT: v_pk_add_f16 v0, v0, v3
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
@@ -7656,9 +7648,8 @@ define <2 x half> @v_contract_mul_add_v2f16_select_n64_n1(<2 x i32> %arg, <2 x h
; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
; GFX10-GISEL-NEXT: v_ldexp_f16_e32 v0, v2, v0
; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 6, vcc_lo
-; GFX10-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-GISEL-NEXT: v_pk_add_f16 v0, v0, v3
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
@@ -7769,9 +7760,8 @@ define <2 x half> @v_contract_mul_add_v2f16_select_n1_n64(<2 x i32> %arg, <2 x h
; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
; GFX10-GISEL-NEXT: v_ldexp_f16_e32 v0, v2, v0
; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v1, 6, 0, vcc_lo
-; GFX10-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-GISEL-NEXT: v_pk_add_f16 v0, v0, v3
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
@@ -7891,9 +7881,8 @@ define <2 x half> @v_contract_mul_add_v2f16_select_128_64(<2 x i32> %arg, <2 x h
; GFX10-GISEL-NEXT: v_add_nc_u32_e32 v1, 6, v1
; GFX10-GISEL-NEXT: v_ldexp_f16_e32 v0, v2, v0
; GFX10-GISEL-NEXT: v_med3_i32 v1, 0xffff8000, v1, v4
-; GFX10-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-GISEL-NEXT: v_pk_add_f16 v0, v0, v3
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
@@ -8008,9 +7997,8 @@ define <2 x half> @v_contract_mul_add_v2f16_select_128_4(<2 x i32> %arg, <2 x ha
; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
; GFX10-GISEL-NEXT: v_ldexp_f16_e32 v0, v2, v0
; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v1, 2, 7, vcc_lo
-; GFX10-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-GISEL-NEXT: v_pk_add_f16 v0, v0, v3
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
@@ -8128,9 +8116,8 @@ define <2 x half> @v_contract_mul_add_v2f16_select_2_4(<2 x i32> %arg, <2 x half
; GFX10-GISEL-NEXT: v_add_nc_u32_e32 v1, 2, v1
; GFX10-GISEL-NEXT: v_ldexp_f16_e32 v0, v2, v0
; GFX10-GISEL-NEXT: v_med3_i32 v1, 0xffff8000, v1, v4
-; GFX10-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-GISEL-NEXT: v_pk_add_f16 v0, v0, v3
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
@@ -8245,9 +8232,8 @@ define <2 x half> @v_contract_mul_add_v2f16_select_4_128(<2 x i32> %arg, <2 x ha
; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
; GFX10-GISEL-NEXT: v_ldexp_f16_e32 v0, v2, v0
; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v1, 7, 2, vcc_lo
-; GFX10-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10-GISEL-NEXT: v_ldexp_f16_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-GISEL-NEXT: v_pk_add_f16 v0, v0, v3
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/fneg-fabs.bf16.ll b/llvm/test/CodeGen/AMDGPU/fneg-fabs.bf16.ll
index 59793344be87c..9b6fd0bcf703b 100644
--- a/llvm/test/CodeGen/AMDGPU/fneg-fabs.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fneg-fabs.bf16.ll
@@ -466,14 +466,13 @@ define amdgpu_kernel void @s_fneg_fabs_v2bf16_non_bc_src(ptr addrspace(1) %out,
; GFX9-NEXT: v_add_f32_e64 v2, s2, 1.0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX9-NEXT: v_mov_b32_e32 v3, 0xffff
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_or_b32_e32 v1, 0x80008000, v1
; GFX9-NEXT: global_store_dword v0, v1, s[0:1]
; GFX9-NEXT: s_endpgm
@@ -780,10 +779,9 @@ define amdgpu_kernel void @fold_user_fneg_fabs_v2bf16(ptr addrspace(1) %out, <2
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX9-NEXT: v_mov_b32_e32 v3, 0xffff
; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX9-NEXT: v_and_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: global_store_dword v0, v1, s[0:1]
; GFX9-NEXT: s_endpgm
;
@@ -1037,10 +1035,9 @@ define amdgpu_kernel void @s_fneg_multi_use_fabs_foldable_neg_v2bf16(ptr addrspa
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX9-NEXT: v_mov_b32_e32 v3, 0xffff
; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX9-NEXT: v_and_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_mov_b32_e32 v2, s5
; GFX9-NEXT: global_store_dword v0, v2, s[0:1]
; GFX9-NEXT: global_store_dword v0, v1, s[2:3]
diff --git a/llvm/test/CodeGen/AMDGPU/fneg.bf16.ll b/llvm/test/CodeGen/AMDGPU/fneg.bf16.ll
index bed203ebaf8dd..07d243c393d31 100644
--- a/llvm/test/CodeGen/AMDGPU/fneg.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fneg.bf16.ll
@@ -683,32 +683,31 @@ define amdgpu_kernel void @v_fneg_fold_v2bf16(ptr addrspace(1) %out, ptr addrspa
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: v_mov_b32_e32 v2, 0xffff
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_load_dword v1, v0, s[2:3]
; GFX9-NEXT: s_mov_b32 s2, 0x8000
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX9-NEXT: v_xor_b32_sdwa v4, v1, s2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX9-NEXT: v_xor_b32_sdwa v3, v1, s2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
; GFX9-NEXT: v_xor_b32_sdwa v1, v1, s2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: v_mul_f32_e32 v3, v4, v3
-; GFX9-NEXT: v_mul_f32_e32 v1, v1, v5
-; GFX9-NEXT: v_bfe_u32 v4, v3, 16, 1
-; GFX9-NEXT: v_bfe_u32 v6, v1, 16, 1
-; GFX9-NEXT: v_add_u32_e32 v4, v4, v3
-; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v3
-; GFX9-NEXT: v_add_u32_e32 v6, v6, v1
-; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4
-; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX9-NEXT: v_or_b32_e32 v7, 0x400000, v1
-; GFX9-NEXT: v_add_u32_e32 v6, 0x7fff, v6
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
+; GFX9-NEXT: v_mul_f32_e32 v2, v3, v2
+; GFX9-NEXT: v_mul_f32_e32 v1, v1, v4
+; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
+; GFX9-NEXT: v_bfe_u32 v5, v1, 16, 1
+; GFX9-NEXT: v_add_u32_e32 v3, v3, v2
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2
+; GFX9-NEXT: v_add_u32_e32 v5, v5, v1
+; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v1
+; GFX9-NEXT: v_add_u32_e32 v5, 0x7fff, v5
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v7, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v6, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_and_b32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: global_store_dword v0, v1, s[0:1]
; GFX9-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/fptosi-sat-vector.ll b/llvm/test/CodeGen/AMDGPU/fptosi-sat-vector.ll
index c4eabce299cee..78ee5f7356e37 100644
--- a/llvm/test/CodeGen/AMDGPU/fptosi-sat-vector.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptosi-sat-vector.ll
@@ -2104,14 +2104,14 @@ define <2 x i16> @test_s_signed_v2f64_v2i16(<2 x double> inreg %f) {
; GFX9-LABEL: test_s_signed_v2f64_v2i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_cvt_i32_f64_e32 v1, s[16:17]
; GFX9-NEXT: v_cvt_i32_f64_e32 v0, s[18:19]
+; GFX9-NEXT: v_cvt_i32_f64_e32 v1, s[16:17]
; GFX9-NEXT: s_movk_i32 s4, 0x8000
; GFX9-NEXT: v_mov_b32_e32 v2, 0x7fff
-; GFX9-NEXT: v_med3_i32 v1, v1, s4, v2
; GFX9-NEXT: v_med3_i32 v0, v0, s4, v2
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v1
+; GFX9-NEXT: v_med3_i32 v1, v1, s4, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: test_s_signed_v2f64_v2i16:
@@ -3833,15 +3833,13 @@ define <4 x i16> @test_s_signed_v4f16_v4i16(<4 x half> inreg %f) {
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_lshr_b32 s4, s17, 16
-; GFX9-NEXT: v_cvt_i16_f16_e32 v1, s4
+; GFX9-NEXT: v_cvt_i16_f16_sdwa v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
; GFX9-NEXT: s_lshr_b32 s4, s16, 16
+; GFX9-NEXT: v_cvt_i16_f16_sdwa v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
; GFX9-NEXT: v_cvt_i16_f16_e32 v2, s17
; GFX9-NEXT: v_cvt_i16_f16_e32 v3, s16
-; GFX9-NEXT: v_cvt_i16_f16_e32 v0, s4
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: test_s_signed_v4f16_v4i16:
@@ -4422,8 +4420,6 @@ define <8 x i8> @test_signed_v8f16_v8i8(<8 x half> %f) {
; GFX9-NEXT: v_med3_i16 v3, v3, s4, v7
; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
; GFX9-NEXT: v_or_b32_sdwa v8, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v8
-; GFX9-NEXT: v_lshl_or_b32 v4, v6, 16, v2
; GFX9-NEXT: v_cvt_i16_f16_e32 v2, v1
; GFX9-NEXT: v_cvt_i16_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
; GFX9-NEXT: v_med3_i16 v1, v1, s4, v7
@@ -4432,13 +4428,14 @@ define <8 x i8> @test_signed_v8f16_v8i8(<8 x half> %f) {
; GFX9-NEXT: v_or_b32_sdwa v2, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_cvt_i16_f16_e32 v1, v0
; GFX9-NEXT: v_cvt_i16_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v6
; GFX9-NEXT: v_med3_i16 v0, v0, s4, v7
+; GFX9-NEXT: v_or_b32_sdwa v4, v9, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX9-NEXT: v_med3_i16 v1, v1, s4, v7
; GFX9-NEXT: v_lshlrev_b16_e32 v7, 8, v0
; GFX9-NEXT: v_lshrrev_b32_e32 v5, 8, v4
; GFX9-NEXT: v_or_b32_sdwa v0, v1, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v6
; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v7
; GFX9-NEXT: v_lshrrev_b64 v[3:4], 24, v[3:4]
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 8, v1
@@ -6369,25 +6366,21 @@ define <8 x i16> @test_s_signed_v8f16_v8i16(<8 x half> inreg %f) {
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_lshr_b32 s4, s19, 16
-; GFX9-NEXT: v_cvt_i16_f16_e32 v3, s4
+; GFX9-NEXT: v_cvt_i16_f16_sdwa v3, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
; GFX9-NEXT: s_lshr_b32 s4, s18, 16
-; GFX9-NEXT: v_cvt_i16_f16_e32 v2, s4
+; GFX9-NEXT: v_cvt_i16_f16_sdwa v2, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
; GFX9-NEXT: s_lshr_b32 s4, s17, 16
-; GFX9-NEXT: v_cvt_i16_f16_e32 v1, s4
+; GFX9-NEXT: v_cvt_i16_f16_sdwa v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
; GFX9-NEXT: s_lshr_b32 s4, s16, 16
+; GFX9-NEXT: v_cvt_i16_f16_sdwa v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
; GFX9-NEXT: v_cvt_i16_f16_e32 v4, s19
; GFX9-NEXT: v_cvt_i16_f16_e32 v5, s18
; GFX9-NEXT: v_cvt_i16_f16_e32 v6, s17
; GFX9-NEXT: v_cvt_i16_f16_e32 v7, s16
-; GFX9-NEXT: v_cvt_i16_f16_e32 v0, s4
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v2, v2, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v3, v3, 16, v4
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: test_s_signed_v8f16_v8i16:
@@ -6888,19 +6881,19 @@ define <4 x i16> @test_s_signed_v4f32_v4i16(<4 x float> inreg %f) {
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_cvt_i32_f32_e32 v0, s19
-; GFX9-NEXT: v_cvt_i32_f32_e32 v2, s18
; GFX9-NEXT: v_cvt_i32_f32_e32 v3, s17
+; GFX9-NEXT: v_cvt_i32_f32_e32 v2, s18
; GFX9-NEXT: v_cvt_i32_f32_e32 v4, s16
; GFX9-NEXT: s_movk_i32 s4, 0x7fff
; GFX9-NEXT: v_mov_b32_e32 v1, 0xffff8000
; GFX9-NEXT: v_med3_i32 v5, v0, v1, s4
-; GFX9-NEXT: v_med3_i32 v2, v2, v1, s4
; GFX9-NEXT: v_med3_i32 v0, v3, v1, s4
+; GFX9-NEXT: v_med3_i32 v2, v2, v1, s4
; GFX9-NEXT: v_med3_i32 v1, v4, v1, s4
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v1
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v2
-; GFX9-NEXT: v_lshl_or_b32 v1, v5, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v5
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: test_s_signed_v4f32_v4i16:
diff --git a/llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll b/llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll
index 953f732c9c6b3..830e14ec13ad1 100644
--- a/llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll
@@ -1974,12 +1974,12 @@ define <2 x i16> @test_s_unsigned_v2f64_v2i16(<2 x double> inreg %f) {
; GFX9-LABEL: test_s_unsigned_v2f64_v2i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_cvt_u32_f64_e32 v1, s[16:17]
; GFX9-NEXT: v_cvt_u32_f64_e32 v0, s[18:19]
+; GFX9-NEXT: v_cvt_u32_f64_e32 v1, s[16:17]
+; GFX9-NEXT: s_mov_b32 s4, 0xffff
+; GFX9-NEXT: v_min_u32_sdwa v0, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_min_u32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_min_u32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: test_s_unsigned_v2f64_v2i16:
@@ -3537,15 +3537,13 @@ define <4 x i16> @test_s_unsigned_v4f16_v4i16(<4 x half> inreg %f) {
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_lshr_b32 s4, s17, 16
-; GFX9-NEXT: v_cvt_u16_f16_e32 v1, s4
+; GFX9-NEXT: v_cvt_u16_f16_sdwa v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
; GFX9-NEXT: s_lshr_b32 s4, s16, 16
+; GFX9-NEXT: v_cvt_u16_f16_sdwa v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
; GFX9-NEXT: v_cvt_u16_f16_e32 v2, s17
; GFX9-NEXT: v_cvt_u16_f16_e32 v3, s16
-; GFX9-NEXT: v_cvt_u16_f16_e32 v0, s4
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: test_s_unsigned_v4f16_v4i16:
@@ -4107,25 +4105,24 @@ define <8 x i8> @test_unsigned_v8f16_v8i8(<8 x half> %f) {
; GFX9-NEXT: v_min_u16_e32 v3, 0xff, v3
; GFX9-NEXT: v_min_u16_sdwa v2, v2, s4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_or_b32_e32 v8, v3, v2
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v8
-; GFX9-NEXT: v_lshl_or_b32 v4, v6, 16, v2
; GFX9-NEXT: v_cvt_u16_f16_e32 v2, v1
; GFX9-NEXT: v_cvt_u16_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
; GFX9-NEXT: v_min_u16_e32 v2, 0xff, v2
; GFX9-NEXT: v_min_u16_sdwa v1, v1, s4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v6
; GFX9-NEXT: v_or_b32_e32 v2, v2, v1
; GFX9-NEXT: v_cvt_u16_f16_e32 v1, v0
; GFX9-NEXT: v_cvt_u16_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-NEXT: v_or_b32_sdwa v4, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX9-NEXT: v_min_u16_e32 v1, 0xff, v1
-; GFX9-NEXT: v_min_u16_sdwa v7, v0, s4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_min_u16_sdwa v9, v0, s4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_lshrrev_b32_e32 v5, 8, v4
-; GFX9-NEXT: v_or_b32_e32 v0, v1, v7
-; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v7
+; GFX9-NEXT: v_or_b32_e32 v0, v1, v9
+; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v9
; GFX9-NEXT: v_lshrrev_b64 v[3:4], 24, v[3:4]
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 8, v1
-; GFX9-NEXT: v_lshrrev_b32_e32 v7, 24, v9
+; GFX9-NEXT: v_lshrrev_b32_e32 v7, 24, v7
; GFX9-NEXT: v_mov_b32_e32 v4, v8
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -5895,25 +5892,21 @@ define <8 x i16> @test_s_unsigned_v8f16_v8i16(<8 x half> inreg %f) {
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_lshr_b32 s4, s19, 16
-; GFX9-NEXT: v_cvt_u16_f16_e32 v3, s4
+; GFX9-NEXT: v_cvt_u16_f16_sdwa v3, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
; GFX9-NEXT: s_lshr_b32 s4, s18, 16
-; GFX9-NEXT: v_cvt_u16_f16_e32 v2, s4
+; GFX9-NEXT: v_cvt_u16_f16_sdwa v2, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
; GFX9-NEXT: s_lshr_b32 s4, s17, 16
-; GFX9-NEXT: v_cvt_u16_f16_e32 v1, s4
+; GFX9-NEXT: v_cvt_u16_f16_sdwa v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
; GFX9-NEXT: s_lshr_b32 s4, s16, 16
+; GFX9-NEXT: v_cvt_u16_f16_sdwa v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
; GFX9-NEXT: v_cvt_u16_f16_e32 v4, s19
; GFX9-NEXT: v_cvt_u16_f16_e32 v5, s18
; GFX9-NEXT: v_cvt_u16_f16_e32 v6, s17
; GFX9-NEXT: v_cvt_u16_f16_e32 v7, s16
-; GFX9-NEXT: v_cvt_u16_f16_e32 v0, s4
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v7
-; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v2, v2, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v3, v3, 16, v4
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: test_s_unsigned_v8f16_v8i16:
@@ -6394,14 +6387,13 @@ define <4 x i16> @test_s_unsigned_v4f32_v4i16(<4 x float> inreg %f) {
; GFX9-NEXT: v_cvt_u32_f32_e32 v1, s18
; GFX9-NEXT: v_cvt_u32_f32_e32 v2, s17
; GFX9-NEXT: v_cvt_u32_f32_e32 v3, s16
-; GFX9-NEXT: v_min_u32_e32 v4, 0xffff, v0
+; GFX9-NEXT: s_mov_b32 s4, 0xffff
+; GFX9-NEXT: v_min_u32_sdwa v4, v0, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_min_u32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_min_u32_e32 v0, 0xffff, v2
+; GFX9-NEXT: v_min_u32_sdwa v0, v2, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_min_u32_e32 v2, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v1, v4, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: test_s_unsigned_v4f32_v4i16:
diff --git a/llvm/test/CodeGen/AMDGPU/fptrunc.v2f16.no.fast.math.ll b/llvm/test/CodeGen/AMDGPU/fptrunc.v2f16.no.fast.math.ll
index 332e65874e45f..c4de6ea2e65a8 100644
--- a/llvm/test/CodeGen/AMDGPU/fptrunc.v2f16.no.fast.math.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptrunc.v2f16.no.fast.math.ll
@@ -441,7 +441,7 @@ define <2 x half> @v_test_cvt_v2f64_v2f16(<2 x double> %src) {
; GFX950-GISEL-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, v7, v2
; GFX950-GISEL-NEXT: v_lshl_or_b32 v5, v5, 9, v8
-; GFX950-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX950-GISEL-NEXT: s_nop 0
; GFX950-GISEL-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
; GFX950-GISEL-NEXT: v_or_b32_e32 v2, v10, v2
; GFX950-GISEL-NEXT: v_cmp_gt_i32_e32 vcc, 1, v1
@@ -462,7 +462,8 @@ define <2 x half> @v_test_cvt_v2f64_v2f16(<2 x double> %src) {
; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v1, v2, v5, vcc
; GFX950-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v3
; GFX950-GISEL-NEXT: v_and_or_b32 v1, v2, v4, v1
-; GFX950-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX950-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX950-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX950-GISEL-NEXT: s_setpc_b64 s[30:31]
%res = fptrunc <2 x double> %src to <2 x half>
ret <2 x half> %res
diff --git a/llvm/test/CodeGen/AMDGPU/freeze.ll b/llvm/test/CodeGen/AMDGPU/freeze.ll
index 55f78749569b4..433dc96366e62 100644
--- a/llvm/test/CodeGen/AMDGPU/freeze.ll
+++ b/llvm/test/CodeGen/AMDGPU/freeze.ll
@@ -14409,13 +14409,13 @@ define void @freeze_v3i8(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0xff
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v4, 8, v0
+; GFX9-GISEL-NEXT: v_and_b32_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX9-GISEL-NEXT: v_and_b32_e32 v4, 0xff, v4
+; GFX9-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-GISEL-NEXT: v_lshlrev_b16_e32 v4, 8, v4
-; GFX9-GISEL-NEXT: v_and_b32_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-GISEL-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX9-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-GISEL-NEXT: global_store_short v[2:3], v0, off
; GFX9-GISEL-NEXT: global_store_byte_d16_hi v[2:3], v0, off offset:2
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
@@ -14437,12 +14437,12 @@ define void @freeze_v3i8(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0xff
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10-GISEL-NEXT: v_lshrrev_b32_e32 v4, 8, v0
-; GFX10-GISEL-NEXT: v_and_b32_sdwa v4, v4, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-GISEL-NEXT: v_and_b32_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-GISEL-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10-GISEL-NEXT: v_and_b32_sdwa v5, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-GISEL-NEXT: v_and_b32_sdwa v1, v4, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v5
+; GFX10-GISEL-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX10-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-GISEL-NEXT: global_store_short v[2:3], v0, off
; GFX10-GISEL-NEXT: global_store_byte_d16_hi v[2:3], v0, off offset:2
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/frem.ll b/llvm/test/CodeGen/AMDGPU/frem.ll
index aa4291060d539..112cfabd6d434 100644
--- a/llvm/test/CodeGen/AMDGPU/frem.ll
+++ b/llvm/test/CodeGen/AMDGPU/frem.ll
@@ -5827,9 +5827,8 @@ define amdgpu_kernel void @frem_v2f16(ptr addrspace(1) %out, ptr addrspace(1) %i
; GFX9-NEXT: v_cmp_nge_f16_e64 s[4:5], |v3|, s4
; GFX9-NEXT: v_cndmask_b32_e32 v2, v1, v2, vcc
; GFX9-NEXT: s_and_b64 vcc, s[4:5], s[2:3]
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v4, vcc
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v2
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v1
+; GFX9-NEXT: v_cndmask_b32_sdwa v0, v1, v4, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: global_store_dword v5, v0, s[0:1]
; GFX9-NEXT: s_endpgm
;
@@ -6006,11 +6005,11 @@ define amdgpu_kernel void @frem_v2f16(ptr addrspace(1) %out, ptr addrspace(1) %i
; GFX10-NEXT: s_and_b32 vcc_lo, s2, vcc_lo
; GFX10-NEXT: v_cndmask_b32_e32 v1, 0x7e00, v2, vcc_lo
; GFX10-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX10-NEXT: v_cmp_lg_f16_sdwa s2, v0, v2 src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-NEXT: v_mov_b32_e32 v0, 0x7e00
; GFX10-NEXT: s_and_b32 vcc_lo, s3, s2
-; GFX10-NEXT: v_cndmask_b32_e32 v0, 0x7e00, v4, vcc_lo
-; GFX10-NEXT: v_lshl_or_b32 v0, v0, 16, v1
+; GFX10-NEXT: v_cndmask_b32_sdwa v0, v0, v4, vcc_lo dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: global_store_dword v2, v0, s[0:1]
; GFX10-NEXT: s_endpgm
;
@@ -8827,7 +8826,7 @@ define amdgpu_kernel void @frem_v4f16(ptr addrspace(1) %out, ptr addrspace(1) %i
; GFX9-NEXT: v_cmp_nge_f16_e64 s[4:5], |v5|, s6
; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v4, vcc
; GFX9-NEXT: s_and_b64 vcc, s[4:5], s[2:3]
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v6, vcc
+; GFX9-NEXT: v_cndmask_b32_sdwa v0, v2, v6, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_cmp_lg_f16_e32 vcc, 0, v1
; GFX9-NEXT: v_cmp_nge_f16_e64 s[2:3], |v3|, s6
; GFX9-NEXT: s_and_b64 vcc, s[2:3], vcc
@@ -8835,11 +8834,9 @@ define amdgpu_kernel void @frem_v4f16(ptr addrspace(1) %out, ptr addrspace(1) %i
; GFX9-NEXT: v_cmp_nge_f16_e64 s[4:5], |v8|, s6
; GFX9-NEXT: v_cndmask_b32_e32 v3, v2, v7, vcc
; GFX9-NEXT: s_and_b64 vcc, s[4:5], s[2:3]
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v9, vcc
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v3
-; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v2
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v4
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v2
+; GFX9-NEXT: v_cndmask_b32_sdwa v1, v2, v9, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: global_store_dwordx2 v10, v[0:1], s[0:1]
; GFX9-NEXT: s_endpgm
;
@@ -9171,21 +9168,20 @@ define amdgpu_kernel void @frem_v4f16(ptr addrspace(1) %out, ptr addrspace(1) %i
; GFX10-NEXT: s_and_b32 vcc_lo, s2, vcc_lo
; GFX10-NEXT: v_cndmask_b32_e32 v2, 0x7e00, v4, vcc_lo
; GFX10-NEXT: v_mov_b32_e32 v4, 0
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX10-NEXT: v_cmp_lg_f16_sdwa s2, v0, v4 src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-NEXT: v_mov_b32_e32 v0, 0x7e00
; GFX10-NEXT: s_and_b32 vcc_lo, s3, s2
; GFX10-NEXT: v_cmp_nle_f16_e64 s2, 0x7c00, |v3|
-; GFX10-NEXT: v_cndmask_b32_e32 v0, 0x7e00, v6, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_sdwa v5, v0, v6, vcc_lo dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX10-NEXT: v_cmp_lg_f16_e32 vcc_lo, 0, v1
; GFX10-NEXT: v_cmp_nle_f16_e64 s3, 0x7c00, |v8|
-; GFX10-NEXT: v_lshl_or_b32 v0, v0, 16, v2
; GFX10-NEXT: s_and_b32 vcc_lo, s2, vcc_lo
; GFX10-NEXT: v_cmp_lg_f16_sdwa s2, v1, v4 src0_sel:WORD_1 src1_sel:DWORD
; GFX10-NEXT: v_cndmask_b32_e32 v3, 0x7e00, v7, vcc_lo
; GFX10-NEXT: s_and_b32 vcc_lo, s3, s2
-; GFX10-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX10-NEXT: v_cndmask_b32_e32 v1, 0x7e00, v9, vcc_lo
-; GFX10-NEXT: v_lshl_or_b32 v1, v1, 16, v3
+; GFX10-NEXT: v_cndmask_b32_sdwa v0, v0, v9, vcc_lo dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v1, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v0, v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: global_store_dwordx2 v4, v[0:1], s[0:1]
; GFX10-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/idot4u.ll b/llvm/test/CodeGen/AMDGPU/idot4u.ll
index aa8e8289f5108..5e6c34c085aa4 100644
--- a/llvm/test/CodeGen/AMDGPU/idot4u.ll
+++ b/llvm/test/CodeGen/AMDGPU/idot4u.ll
@@ -2668,27 +2668,26 @@ define amdgpu_kernel void @udot4_acc8_vecMul(ptr addrspace(1) %src1,
; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX10-DL-NEXT: global_load_ubyte v3, v0, s[6:7]
; GFX10-DL-NEXT: s_waitcnt vmcnt(2)
-; GFX10-DL-NEXT: v_lshrrev_b16 v4, 8, v1
-; GFX10-DL-NEXT: v_lshrrev_b32_e32 v5, 24, v1
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v4, 24, v1
; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
-; GFX10-DL-NEXT: v_lshrrev_b32_e32 v6, 24, v2
-; GFX10-DL-NEXT: v_lshrrev_b16 v7, 8, v2
-; GFX10-DL-NEXT: v_lshrrev_b32_e32 v8, 16, v1
-; GFX10-DL-NEXT: v_lshrrev_b32_e32 v9, 16, v2
-; GFX10-DL-NEXT: v_mul_lo_u16 v5, v5, v6
-; GFX10-DL-NEXT: v_mul_lo_u16 v4, v4, v7
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v5, 24, v2
+; GFX10-DL-NEXT: v_lshrrev_b16 v6, 8, v1
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v7, 16, v1
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v8, 16, v2
+; GFX10-DL-NEXT: v_lshrrev_b16 v9, 8, v2
+; GFX10-DL-NEXT: v_mul_lo_u16 v4, v4, v5
; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
; GFX10-DL-NEXT: v_mad_u16 v1, v1, v2, v3
-; GFX10-DL-NEXT: v_mul_lo_u16 v6, v8, v9
-; GFX10-DL-NEXT: v_lshlrev_b16 v5, 8, v5
+; GFX10-DL-NEXT: v_mul_lo_u16 v5, v7, v8
+; GFX10-DL-NEXT: v_mul_lo_u16 v6, v6, v9
; GFX10-DL-NEXT: v_lshlrev_b16 v4, 8, v4
-; GFX10-DL-NEXT: v_or_b32_sdwa v6, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-DL-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX10-DL-NEXT: v_lshrrev_b32_e32 v2, 8, v5
-; GFX10-DL-NEXT: v_lshl_or_b32 v4, v6, 16, v4
-; GFX10-DL-NEXT: v_lshrrev_b32_e32 v4, 8, v4
-; GFX10-DL-NEXT: v_add_nc_u16 v1, v1, v4
-; GFX10-DL-NEXT: v_mad_u16 v1, v8, v9, v1
+; GFX10-DL-NEXT: v_lshlrev_b16 v6, 8, v6
+; GFX10-DL-NEXT: v_or_b32_sdwa v5, v5, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v2, 8, v4
+; GFX10-DL-NEXT: v_or_b32_sdwa v5, v5, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v5, 8, v5
+; GFX10-DL-NEXT: v_add_nc_u16 v1, v1, v5
+; GFX10-DL-NEXT: v_mad_u16 v1, v7, v8, v1
; GFX10-DL-NEXT: v_add_nc_u16 v1, v1, v2
; GFX10-DL-NEXT: global_store_byte v0, v1, s[6:7]
; GFX10-DL-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/idot8s.ll b/llvm/test/CodeGen/AMDGPU/idot8s.ll
index 684a21f7ac077..112c26aad518a 100644
--- a/llvm/test/CodeGen/AMDGPU/idot8s.ll
+++ b/llvm/test/CodeGen/AMDGPU/idot8s.ll
@@ -2729,53 +2729,52 @@ define amdgpu_kernel void @idot8_acc8_vecMul(ptr addrspace(1) %src1,
; GFX9-NEXT: v_lshlrev_b16_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
; GFX9-NEXT: v_lshlrev_b16_e32 v4, 12, v9
; GFX9-NEXT: v_ashrrev_i16_e32 v9, 12, v15
+; GFX9-NEXT: v_lshlrev_b16_e32 v8, 12, v8
+; GFX9-NEXT: v_lshlrev_b16_e32 v7, 12, v7
; GFX9-NEXT: v_ashrrev_i16_e32 v15, 12, v16
; GFX9-NEXT: v_lshlrev_b16_e32 v6, 12, v6
; GFX9-NEXT: v_ashrrev_i16_e32 v16, 12, v1
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 12, v5
; GFX9-NEXT: v_lshlrev_b16_e32 v5, 12, v14
; GFX9-NEXT: v_ashrrev_i16_e32 v14, 12, v17
+; GFX9-NEXT: v_lshlrev_b16_e32 v13, 12, v13
+; GFX9-NEXT: v_lshlrev_b16_e32 v12, 12, v12
; GFX9-NEXT: v_ashrrev_i16_e32 v17, 12, v18
; GFX9-NEXT: v_lshlrev_b16_e32 v11, 12, v11
; GFX9-NEXT: v_ashrrev_i16_e32 v18, 12, v2
; GFX9-NEXT: v_lshlrev_b16_e32 v2, 12, v10
-; GFX9-NEXT: v_lshlrev_b16_e32 v8, 12, v8
-; GFX9-NEXT: v_lshlrev_b16_e32 v7, 12, v7
-; GFX9-NEXT: v_lshlrev_b16_e32 v13, 12, v13
-; GFX9-NEXT: v_lshlrev_b16_e32 v12, 12, v12
-; GFX9-NEXT: v_ashrrev_i16_e32 v6, 12, v6
-; GFX9-NEXT: v_ashrrev_i16_e32 v1, 12, v1
-; GFX9-NEXT: v_ashrrev_i16_e32 v11, 12, v11
-; GFX9-NEXT: v_ashrrev_i16_e32 v2, 12, v2
; GFX9-NEXT: v_ashrrev_i16_e32 v8, 12, v8
; GFX9-NEXT: v_ashrrev_i16_e32 v7, 12, v7
+; GFX9-NEXT: v_ashrrev_i16_e32 v6, 12, v6
+; GFX9-NEXT: v_ashrrev_i16_e32 v1, 12, v1
; GFX9-NEXT: v_ashrrev_i16_e32 v10, 12, v13
; GFX9-NEXT: v_ashrrev_i16_e32 v12, 12, v12
-; GFX9-NEXT: v_mul_lo_u16_e32 v19, v15, v17
-; GFX9-NEXT: v_mul_lo_u16_sdwa v1, v1, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: v_mul_lo_u16_sdwa v2, v6, v11 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_ashrrev_i16_e32 v11, 12, v11
+; GFX9-NEXT: v_ashrrev_i16_e32 v2, 12, v2
; GFX9-NEXT: v_ashrrev_i16_e32 v4, 12, v4
; GFX9-NEXT: v_ashrrev_i16_e32 v5, 12, v5
; GFX9-NEXT: v_mul_lo_u16_e32 v13, v16, v18
+; GFX9-NEXT: v_mul_lo_u16_e32 v19, v15, v17
+; GFX9-NEXT: v_mul_lo_u16_sdwa v1, v1, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_mul_lo_u16_sdwa v2, v6, v11 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mul_lo_u16_sdwa v6, v7, v12 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mul_lo_u16_e32 v7, v8, v10
-; GFX9-NEXT: v_or_b32_sdwa v2, v19, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_mul_lo_u16_e32 v9, v9, v14
; GFX9-NEXT: v_mul_lo_u16_sdwa v4, v4, v5 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: v_or_b32_sdwa v5, v13, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v5, v13, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v2, v19, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v6, v7, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_or_b32_sdwa v7, v9, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_mul_lo_u16_e32 v9, v9, v14
; GFX9-NEXT: v_lshrrev_b32_e32 v8, 8, v1
+; GFX9-NEXT: v_or_b32_sdwa v2, v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v6
+; GFX9-NEXT: v_or_b32_sdwa v7, v9, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_lshl_or_b32 v4, v6, 16, v4
-; GFX9-NEXT: v_lshl_or_b32 v2, v5, 16, v2
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_add_u16_e32 v3, v7, v3
-; GFX9-NEXT: v_lshrrev_b32_e32 v4, 8, v4
; GFX9-NEXT: v_lshrrev_b32_e32 v5, 8, v2
; GFX9-NEXT: v_lshrrev_b64 v[1:2], 24, v[1:2]
-; GFX9-NEXT: v_add_u16_e32 v2, v3, v4
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 8, v4
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_add_u16_e32 v2, v7, v3
+; GFX9-NEXT: v_add_u16_e32 v2, v2, v4
; GFX9-NEXT: v_add_u16_e32 v2, v2, v6
; GFX9-NEXT: v_add_u16_e32 v1, v2, v1
; GFX9-NEXT: v_mad_legacy_u16 v1, v15, v17, v1
@@ -2822,53 +2821,52 @@ define amdgpu_kernel void @idot8_acc8_vecMul(ptr addrspace(1) %src1,
; GFX9-DL-NEXT: v_lshlrev_b16_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
; GFX9-DL-NEXT: v_lshlrev_b16_e32 v4, 12, v9
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v9, 12, v15
+; GFX9-DL-NEXT: v_lshlrev_b16_e32 v8, 12, v8
+; GFX9-DL-NEXT: v_lshlrev_b16_e32 v7, 12, v7
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v15, 12, v16
; GFX9-DL-NEXT: v_lshlrev_b16_e32 v6, 12, v6
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v16, 12, v1
; GFX9-DL-NEXT: v_lshlrev_b16_e32 v1, 12, v5
; GFX9-DL-NEXT: v_lshlrev_b16_e32 v5, 12, v14
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v14, 12, v17
+; GFX9-DL-NEXT: v_lshlrev_b16_e32 v13, 12, v13
+; GFX9-DL-NEXT: v_lshlrev_b16_e32 v12, 12, v12
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v17, 12, v18
; GFX9-DL-NEXT: v_lshlrev_b16_e32 v11, 12, v11
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v18, 12, v2
; GFX9-DL-NEXT: v_lshlrev_b16_e32 v2, 12, v10
-; GFX9-DL-NEXT: v_lshlrev_b16_e32 v8, 12, v8
-; GFX9-DL-NEXT: v_lshlrev_b16_e32 v7, 12, v7
-; GFX9-DL-NEXT: v_lshlrev_b16_e32 v13, 12, v13
-; GFX9-DL-NEXT: v_lshlrev_b16_e32 v12, 12, v12
-; GFX9-DL-NEXT: v_ashrrev_i16_e32 v6, 12, v6
-; GFX9-DL-NEXT: v_ashrrev_i16_e32 v1, 12, v1
-; GFX9-DL-NEXT: v_ashrrev_i16_e32 v11, 12, v11
-; GFX9-DL-NEXT: v_ashrrev_i16_e32 v2, 12, v2
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v8, 12, v8
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v7, 12, v7
+; GFX9-DL-NEXT: v_ashrrev_i16_e32 v6, 12, v6
+; GFX9-DL-NEXT: v_ashrrev_i16_e32 v1, 12, v1
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v10, 12, v13
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v12, 12, v12
-; GFX9-DL-NEXT: v_mul_lo_u16_e32 v19, v15, v17
-; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v1, v1, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v2, v6, v11 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-DL-NEXT: v_ashrrev_i16_e32 v11, 12, v11
+; GFX9-DL-NEXT: v_ashrrev_i16_e32 v2, 12, v2
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v4, 12, v4
; GFX9-DL-NEXT: v_ashrrev_i16_e32 v5, 12, v5
; GFX9-DL-NEXT: v_mul_lo_u16_e32 v13, v16, v18
+; GFX9-DL-NEXT: v_mul_lo_u16_e32 v19, v15, v17
+; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v1, v1, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v2, v6, v11 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v6, v7, v12 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-DL-NEXT: v_mul_lo_u16_e32 v7, v8, v10
-; GFX9-DL-NEXT: v_or_b32_sdwa v2, v19, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-DL-NEXT: v_mul_lo_u16_e32 v9, v9, v14
; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v4, v4, v5 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-DL-NEXT: v_or_b32_sdwa v5, v13, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-DL-NEXT: v_or_b32_sdwa v5, v13, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-DL-NEXT: v_or_b32_sdwa v2, v19, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-DL-NEXT: v_or_b32_sdwa v6, v7, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-DL-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-DL-NEXT: v_or_b32_sdwa v7, v9, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-DL-NEXT: v_mul_lo_u16_e32 v9, v9, v14
; GFX9-DL-NEXT: v_lshrrev_b32_e32 v8, 8, v1
+; GFX9-DL-NEXT: v_or_b32_sdwa v2, v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-DL-NEXT: v_lshlrev_b32_e32 v1, 16, v6
+; GFX9-DL-NEXT: v_or_b32_sdwa v7, v9, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-DL-NEXT: v_lshl_or_b32 v4, v6, 16, v4
-; GFX9-DL-NEXT: v_lshl_or_b32 v2, v5, 16, v2
-; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DL-NEXT: v_add_u16_e32 v3, v7, v3
-; GFX9-DL-NEXT: v_lshrrev_b32_e32 v4, 8, v4
; GFX9-DL-NEXT: v_lshrrev_b32_e32 v5, 8, v2
; GFX9-DL-NEXT: v_lshrrev_b64 v[1:2], 24, v[1:2]
-; GFX9-DL-NEXT: v_add_u16_e32 v2, v3, v4
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v4, 8, v4
+; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-DL-NEXT: v_add_u16_e32 v2, v7, v3
+; GFX9-DL-NEXT: v_add_u16_e32 v2, v2, v4
; GFX9-DL-NEXT: v_add_u16_e32 v2, v2, v6
; GFX9-DL-NEXT: v_add_u16_e32 v1, v2, v1
; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v15, v17, v1
@@ -2895,80 +2893,78 @@ define amdgpu_kernel void @idot8_acc8_vecMul(ptr addrspace(1) %src1,
; GFX10-DL-XNACK-NEXT: s_clause 0x1
; GFX10-DL-XNACK-NEXT: global_load_dword v1, v0, s[8:9]
; GFX10-DL-XNACK-NEXT: global_load_dword v2, v0, s[10:11]
-; GFX10-DL-XNACK-NEXT: global_load_ubyte v0, v4, s[0:1]
+; GFX10-DL-XNACK-NEXT: global_load_ubyte v3, v4, s[0:1]
; GFX10-DL-XNACK-NEXT: s_waitcnt vmcnt(2)
; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v8, 12, v1
-; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v10, 4, v1
; GFX10-DL-XNACK-NEXT: s_waitcnt vmcnt(1)
; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v15, 12, v2
-; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v17, 4, v2
-; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v6, 20, v1
-; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v13, 20, v2
; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v9, 8, v1
; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v16, 8, v2
-; GFX10-DL-XNACK-NEXT: v_lshlrev_b16 v10, 12, v10
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v10, 4, v1
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16 v8, 12, v8
-; GFX10-DL-XNACK-NEXT: v_lshlrev_b16 v17, 12, v17
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16 v15, 12, v15
-; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v3, 28, v1
-; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v7, 16, v1
-; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v11, 28, v2
-; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v14, 16, v2
-; GFX10-DL-XNACK-NEXT: v_lshlrev_b16 v6, 12, v6
-; GFX10-DL-XNACK-NEXT: v_lshlrev_b16 v13, 12, v13
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v17, 4, v2
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16 v9, 12, v9
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16 v16, 12, v16
-; GFX10-DL-XNACK-NEXT: v_ashrrev_i16 v10, 12, v10
; GFX10-DL-XNACK-NEXT: v_ashrrev_i16 v8, 12, v8
-; GFX10-DL-XNACK-NEXT: v_ashrrev_i16 v17, 12, v17
; GFX10-DL-XNACK-NEXT: v_ashrrev_i16 v15, 12, v15
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v0, 28, v1
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v6, 20, v1
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v11, 28, v2
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v13, 20, v2
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16 v10, 12, v10
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16 v17, 12, v17
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16 v9, 12, v9
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16 v16, 12, v16
+; GFX10-DL-XNACK-NEXT: v_mul_lo_u16 v8, v8, v15
; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v5, 24, v1
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v7, 16, v1
; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v12, 24, v2
-; GFX10-DL-XNACK-NEXT: v_lshlrev_b16 v7, 12, v7
-; GFX10-DL-XNACK-NEXT: v_lshlrev_b16 v3, 12, v3
-; GFX10-DL-XNACK-NEXT: v_lshlrev_b16 v14, 12, v14
-; GFX10-DL-XNACK-NEXT: v_lshlrev_b16 v11, 12, v11
-; GFX10-DL-XNACK-NEXT: v_ashrrev_i16 v6, 12, v6
-; GFX10-DL-XNACK-NEXT: v_ashrrev_i16 v13, 12, v13
+; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v14, 16, v2
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16 v1, 12, v1
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16 v2, 12, v2
-; GFX10-DL-XNACK-NEXT: v_ashrrev_i16 v9, 12, v9
-; GFX10-DL-XNACK-NEXT: v_ashrrev_i16 v16, 12, v16
-; GFX10-DL-XNACK-NEXT: v_mul_lo_u16 v8, v8, v15
-; GFX10-DL-XNACK-NEXT: v_mul_lo_u16 v10, v10, v17
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16 v6, 12, v6
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16 v0, 12, v0
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16 v13, 12, v13
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16 v11, 12, v11
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16 v10, 12, v10
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16 v15, 12, v17
+; GFX10-DL-XNACK-NEXT: v_mul_lo_u16 v9, v9, v16
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16 v8, 8, v8
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16 v1, 12, v1
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16 v7, 12, v7
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16 v5, 12, v5
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16 v2, 12, v2
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16 v14, 12, v14
; GFX10-DL-XNACK-NEXT: v_lshlrev_b16 v12, 12, v12
-; GFX10-DL-XNACK-NEXT: v_ashrrev_i16 v7, 12, v7
-; GFX10-DL-XNACK-NEXT: v_ashrrev_i16 v3, 12, v3
-; GFX10-DL-XNACK-NEXT: v_ashrrev_i16 v14, 12, v14
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16 v6, 12, v6
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16 v0, 12, v0
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16 v13, 12, v13
; GFX10-DL-XNACK-NEXT: v_ashrrev_i16 v11, 12, v11
-; GFX10-DL-XNACK-NEXT: v_mul_lo_u16 v6, v6, v13
-; GFX10-DL-XNACK-NEXT: v_ashrrev_i16 v1, 12, v1
-; GFX10-DL-XNACK-NEXT: v_ashrrev_i16 v2, 12, v2
-; GFX10-DL-XNACK-NEXT: v_mul_lo_u16 v9, v9, v16
-; GFX10-DL-XNACK-NEXT: v_lshlrev_b16 v8, 8, v8
-; GFX10-DL-XNACK-NEXT: v_lshlrev_b16 v10, 8, v10
+; GFX10-DL-XNACK-NEXT: v_mul_lo_u16 v10, v10, v15
+; GFX10-DL-XNACK-NEXT: v_or_b32_sdwa v8, v9, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16 v7, 12, v7
; GFX10-DL-XNACK-NEXT: v_ashrrev_i16 v5, 12, v5
+; GFX10-DL-XNACK-NEXT: v_ashrrev_i16 v14, 12, v14
; GFX10-DL-XNACK-NEXT: v_ashrrev_i16 v12, 12, v12
-; GFX10-DL-XNACK-NEXT: v_mul_lo_u16 v3, v3, v11
-; GFX10-DL-XNACK-NEXT: v_mul_lo_u16 v11, v7, v14
-; GFX10-DL-XNACK-NEXT: v_lshlrev_b16 v6, 8, v6
-; GFX10-DL-XNACK-NEXT: v_or_b32_sdwa v8, v9, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-DL-XNACK-NEXT: v_and_b32_e32 v9, 0xffff, v10
; GFX10-DL-XNACK-NEXT: v_mul_lo_u16 v1, v1, v2
-; GFX10-DL-XNACK-NEXT: v_mul_lo_u16 v2, v5, v12
-; GFX10-DL-XNACK-NEXT: v_lshlrev_b16 v13, 8, v3
-; GFX10-DL-XNACK-NEXT: v_or_b32_sdwa v3, v11, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-DL-XNACK-NEXT: v_lshl_or_b32 v6, v8, 16, v9
-; GFX10-DL-XNACK-NEXT: v_or_b32_sdwa v1, v1, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-DL-XNACK-NEXT: v_or_b32_sdwa v2, v2, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-DL-XNACK-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX10-DL-XNACK-NEXT: v_mul_lo_u16 v2, v0, v11
+; GFX10-DL-XNACK-NEXT: v_mul_lo_u16 v6, v6, v13
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16 v9, 8, v10
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b32_e32 v0, 16, v8
+; GFX10-DL-XNACK-NEXT: v_mul_lo_u16 v10, v5, v12
+; GFX10-DL-XNACK-NEXT: v_mul_lo_u16 v11, v7, v14
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16 v13, 8, v2
+; GFX10-DL-XNACK-NEXT: v_lshlrev_b16 v2, 8, v6
+; GFX10-DL-XNACK-NEXT: v_or_b32_sdwa v6, v0, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-DL-XNACK-NEXT: v_or_b32_sdwa v1, v1, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-DL-XNACK-NEXT: v_or_b32_sdwa v9, v10, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-DL-XNACK-NEXT: v_or_b32_sdwa v2, v11, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v6, 8, v6
; GFX10-DL-XNACK-NEXT: s_waitcnt vmcnt(0)
-; GFX10-DL-XNACK-NEXT: v_add_nc_u16 v9, v1, v0
-; GFX10-DL-XNACK-NEXT: v_lshlrev_b32_e32 v0, 16, v8
-; GFX10-DL-XNACK-NEXT: v_lshl_or_b32 v1, v2, 16, v3
-; GFX10-DL-XNACK-NEXT: v_add_nc_u16 v6, v9, v6
+; GFX10-DL-XNACK-NEXT: v_add_nc_u16 v3, v1, v3
+; GFX10-DL-XNACK-NEXT: v_or_b32_sdwa v1, v9, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-DL-XNACK-NEXT: v_add_nc_u16 v6, v3, v6
; GFX10-DL-XNACK-NEXT: v_lshrrev_b64 v[2:3], 24, v[0:1]
; GFX10-DL-XNACK-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; GFX10-DL-XNACK-NEXT: v_add_nc_u16 v0, v6, v8
@@ -3001,76 +2997,74 @@ define amdgpu_kernel void @idot8_acc8_vecMul(ptr addrspace(1) %src1,
; GFX10-DL-NOXNACK-NEXT: global_load_ubyte v2, v4, s[0:1]
; GFX10-DL-NOXNACK-NEXT: s_waitcnt vmcnt(2)
; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v8, 12, v1
-; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v10, 4, v1
; GFX10-DL-NOXNACK-NEXT: s_waitcnt vmcnt(1)
; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v15, 12, v0
-; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v17, 4, v0
-; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v6, 20, v1
-; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v13, 20, v0
; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v9, 8, v1
; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v16, 8, v0
-; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16 v10, 12, v10
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v10, 4, v1
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16 v8, 12, v8
-; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16 v17, 12, v17
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16 v15, 12, v15
-; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v3, 28, v1
-; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v7, 16, v1
-; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v11, 28, v0
-; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v14, 16, v0
-; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16 v6, 12, v6
-; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16 v13, 12, v13
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v17, 4, v0
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16 v9, 12, v9
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16 v16, 12, v16
-; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16 v10, 12, v10
; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16 v8, 12, v8
-; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16 v17, 12, v17
; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16 v15, 12, v15
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v3, 28, v1
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v6, 20, v1
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v11, 28, v0
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v13, 20, v0
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16 v10, 12, v10
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16 v17, 12, v17
+; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16 v9, 12, v9
+; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16 v16, 12, v16
+; GFX10-DL-NOXNACK-NEXT: v_mul_lo_u16 v8, v8, v15
; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v5, 24, v1
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v7, 16, v1
; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v12, 24, v0
-; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16 v7, 12, v7
+; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v14, 16, v0
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16 v6, 12, v6
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16 v3, 12, v3
-; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16 v14, 12, v14
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16 v13, 12, v13
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16 v11, 12, v11
-; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16 v6, 12, v6
-; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16 v13, 12, v13
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16 v1, 12, v1
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16 v0, 12, v0
-; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16 v9, 12, v9
-; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16 v16, 12, v16
-; GFX10-DL-NOXNACK-NEXT: v_mul_lo_u16 v8, v8, v15
-; GFX10-DL-NOXNACK-NEXT: v_mul_lo_u16 v10, v10, v17
+; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16 v10, 12, v10
+; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16 v15, 12, v17
+; GFX10-DL-NOXNACK-NEXT: v_mul_lo_u16 v9, v9, v16
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16 v8, 8, v8
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16 v7, 12, v7
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16 v5, 12, v5
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16 v14, 12, v14
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16 v12, 12, v12
-; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16 v7, 12, v7
+; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16 v6, 12, v6
; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16 v3, 12, v3
-; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16 v14, 12, v14
+; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16 v13, 12, v13
; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16 v11, 12, v11
-; GFX10-DL-NOXNACK-NEXT: v_mul_lo_u16 v6, v6, v13
; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16 v1, 12, v1
; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16 v0, 12, v0
-; GFX10-DL-NOXNACK-NEXT: v_mul_lo_u16 v9, v9, v16
-; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16 v8, 8, v8
-; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16 v10, 8, v10
+; GFX10-DL-NOXNACK-NEXT: v_mul_lo_u16 v10, v10, v15
+; GFX10-DL-NOXNACK-NEXT: v_or_b32_sdwa v8, v9, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16 v7, 12, v7
; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16 v5, 12, v5
+; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16 v14, 12, v14
; GFX10-DL-NOXNACK-NEXT: v_ashrrev_i16 v12, 12, v12
; GFX10-DL-NOXNACK-NEXT: v_mul_lo_u16 v3, v3, v11
+; GFX10-DL-NOXNACK-NEXT: v_mul_lo_u16 v6, v6, v13
+; GFX10-DL-NOXNACK-NEXT: v_mul_lo_u16 v1, v1, v0
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16 v9, 8, v10
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b32_e32 v0, 16, v8
+; GFX10-DL-NOXNACK-NEXT: v_mul_lo_u16 v10, v5, v12
; GFX10-DL-NOXNACK-NEXT: v_mul_lo_u16 v11, v7, v14
-; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16 v6, 8, v6
-; GFX10-DL-NOXNACK-NEXT: v_or_b32_sdwa v8, v9, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-DL-NOXNACK-NEXT: v_and_b32_e32 v9, 0xffff, v10
-; GFX10-DL-NOXNACK-NEXT: v_mul_lo_u16 v0, v1, v0
-; GFX10-DL-NOXNACK-NEXT: v_mul_lo_u16 v1, v5, v12
; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16 v13, 8, v3
-; GFX10-DL-NOXNACK-NEXT: v_or_b32_sdwa v3, v11, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-DL-NOXNACK-NEXT: v_lshl_or_b32 v6, v8, 16, v9
-; GFX10-DL-NOXNACK-NEXT: v_or_b32_sdwa v0, v0, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-DL-NOXNACK-NEXT: v_or_b32_sdwa v1, v1, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-DL-NOXNACK-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b16 v3, 8, v6
+; GFX10-DL-NOXNACK-NEXT: v_or_b32_sdwa v6, v0, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-DL-NOXNACK-NEXT: v_or_b32_sdwa v1, v1, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-DL-NOXNACK-NEXT: v_or_b32_sdwa v9, v10, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-DL-NOXNACK-NEXT: v_or_b32_sdwa v3, v11, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v6, 8, v6
; GFX10-DL-NOXNACK-NEXT: s_waitcnt vmcnt(0)
-; GFX10-DL-NOXNACK-NEXT: v_add_nc_u16 v2, v0, v2
-; GFX10-DL-NOXNACK-NEXT: v_lshlrev_b32_e32 v0, 16, v8
-; GFX10-DL-NOXNACK-NEXT: v_lshl_or_b32 v1, v1, 16, v3
+; GFX10-DL-NOXNACK-NEXT: v_add_nc_u16 v2, v1, v2
+; GFX10-DL-NOXNACK-NEXT: v_or_b32_sdwa v1, v9, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-DL-NOXNACK-NEXT: v_add_nc_u16 v6, v2, v6
; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b64 v[2:3], 24, v[0:1]
; GFX10-DL-NOXNACK-NEXT: v_lshrrev_b32_e32 v1, 8, v1
diff --git a/llvm/test/CodeGen/AMDGPU/idot8u.ll b/llvm/test/CodeGen/AMDGPU/idot8u.ll
index 646e9508164d4..5f1b3696c6663 100644
--- a/llvm/test/CodeGen/AMDGPU/idot8u.ll
+++ b/llvm/test/CodeGen/AMDGPU/idot8u.ll
@@ -2538,134 +2538,132 @@ define amdgpu_kernel void @udot8_acc8_vecMul(ptr addrspace(1) %src1,
;
; GFX9-LABEL: udot8_acc8_vecMul:
; GFX9: ; %bb.0: ; %entry
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NEXT: v_mov_b32_e32 v3, 0
; GFX9-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_load_dword v1, v0, s[0:1]
+; GFX9-NEXT: global_load_dword v2, v0, s[2:3]
+; GFX9-NEXT: global_load_ubyte v4, v3, s[6:7]
; GFX9-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX9-NEXT: s_mov_b32 s14, -1
; GFX9-NEXT: s_mov_b32 s15, 0xe00000
; GFX9-NEXT: s_add_u32 s12, s12, s11
-; GFX9-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v1, v0, s[8:9]
-; GFX9-NEXT: global_load_dword v2, v0, s[10:11]
-; GFX9-NEXT: global_load_ubyte v4, v3, s[0:1]
; GFX9-NEXT: s_addc_u32 s13, s13, 0
; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_bfe_u32 v8, v1, 20, 4
-; GFX9-NEXT: v_bfe_u32 v9, v1, 16, 4
-; GFX9-NEXT: v_lshrrev_b32_e32 v10, 28, v1
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_bfe_u32 v15, v2, 20, 4
-; GFX9-NEXT: v_bfe_u32 v16, v2, 16, 4
-; GFX9-NEXT: v_lshrrev_b32_e32 v17, 28, v2
; GFX9-NEXT: v_bfe_u32 v0, v1, 4, 4
; GFX9-NEXT: v_and_b32_e32 v5, 15, v1
; GFX9-NEXT: v_bfe_u32 v6, v1, 12, 4
; GFX9-NEXT: v_bfe_u32 v7, v1, 8, 4
+; GFX9-NEXT: v_bfe_u32 v8, v1, 20, 4
+; GFX9-NEXT: v_bfe_u32 v9, v1, 16, 4
+; GFX9-NEXT: v_lshrrev_b32_e32 v10, 28, v1
; GFX9-NEXT: v_bfe_u32 v11, v1, 24, 4
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_bfe_u32 v1, v2, 4, 4
; GFX9-NEXT: v_and_b32_e32 v12, 15, v2
; GFX9-NEXT: v_bfe_u32 v13, v2, 12, 4
; GFX9-NEXT: v_bfe_u32 v14, v2, 8, 4
+; GFX9-NEXT: v_bfe_u32 v15, v2, 20, 4
+; GFX9-NEXT: v_bfe_u32 v16, v2, 16, 4
+; GFX9-NEXT: v_lshrrev_b32_e32 v17, 28, v2
; GFX9-NEXT: v_bfe_u32 v2, v2, 24, 4
+; GFX9-NEXT: v_mul_lo_u16_e32 v18, v11, v2
; GFX9-NEXT: v_mul_lo_u16_sdwa v10, v10, v17 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_mul_lo_u16_e32 v17, v9, v16
; GFX9-NEXT: v_mul_lo_u16_sdwa v8, v8, v15 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: v_mul_lo_u16_e32 v18, v11, v2
; GFX9-NEXT: v_mul_lo_u16_e32 v7, v7, v14
; GFX9-NEXT: v_mul_lo_u16_sdwa v6, v6, v13 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: v_mul_lo_u16_sdwa v1, v0, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: v_or_b32_e32 v0, v17, v8
; GFX9-NEXT: v_mul_lo_u16_e32 v5, v5, v12
-; GFX9-NEXT: v_or_b32_e32 v12, v18, v10
+; GFX9-NEXT: v_mul_lo_u16_sdwa v12, v0, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v0, v18, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_e32 v1, v17, v8
; GFX9-NEXT: v_or_b32_e32 v6, v7, v6
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v0
-; GFX9-NEXT: v_or_b32_e32 v5, v5, v1
-; GFX9-NEXT: v_lshrrev_b32_e32 v7, 8, v10
+; GFX9-NEXT: v_or_b32_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v10, v6, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v1, v12, 16, v8
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_add_u16_e32 v4, v5, v4
-; GFX9-NEXT: v_lshrrev_b32_e32 v5, 8, v10
-; GFX9-NEXT: v_lshrrev_b32_e32 v8, 8, v1
+; GFX9-NEXT: v_or_b32_e32 v5, v5, v12
+; GFX9-NEXT: v_lshrrev_b32_e32 v7, 8, v10
+; GFX9-NEXT: v_lshl_or_b32 v8, v6, 16, v12
+; GFX9-NEXT: v_lshrrev_b32_e32 v10, 8, v1
; GFX9-NEXT: v_lshrrev_b64 v[0:1], 24, v[0:1]
-; GFX9-NEXT: v_add_u16_e32 v1, v4, v5
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 8, v8
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_add_u16_e32 v1, v5, v4
+; GFX9-NEXT: v_add_u16_e32 v1, v1, v8
; GFX9-NEXT: v_add_u16_e32 v1, v1, v6
; GFX9-NEXT: v_add_u16_e32 v0, v1, v0
; GFX9-NEXT: v_mad_legacy_u16 v0, v9, v16, v0
-; GFX9-NEXT: v_add_u16_e32 v0, v0, v8
+; GFX9-NEXT: v_add_u16_e32 v0, v0, v10
; GFX9-NEXT: v_mad_legacy_u16 v0, v11, v2, v0
; GFX9-NEXT: v_add_u16_e32 v0, v0, v7
-; GFX9-NEXT: global_store_byte v3, v0, s[0:1]
+; GFX9-NEXT: global_store_byte v3, v0, s[6:7]
; GFX9-NEXT: s_endpgm
;
; GFX9-DL-LABEL: udot8_acc8_vecMul:
; GFX9-DL: ; %bb.0: ; %entry
+; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-DL-NEXT: v_mov_b32_e32 v3, 0
; GFX9-DL-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-DL-NEXT: global_load_dword v1, v0, s[0:1]
+; GFX9-DL-NEXT: global_load_dword v2, v0, s[2:3]
+; GFX9-DL-NEXT: global_load_ubyte v4, v3, s[6:7]
; GFX9-DL-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX9-DL-NEXT: s_mov_b32 s14, -1
; GFX9-DL-NEXT: s_mov_b32 s15, 0xe00000
; GFX9-DL-NEXT: s_add_u32 s12, s12, s11
-; GFX9-DL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
-; GFX9-DL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX9-DL-NEXT: v_mov_b32_e32 v3, 0
-; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: global_load_dword v1, v0, s[8:9]
-; GFX9-DL-NEXT: global_load_dword v2, v0, s[10:11]
-; GFX9-DL-NEXT: global_load_ubyte v4, v3, s[0:1]
; GFX9-DL-NEXT: s_addc_u32 s13, s13, 0
; GFX9-DL-NEXT: s_waitcnt vmcnt(2)
-; GFX9-DL-NEXT: v_bfe_u32 v8, v1, 20, 4
-; GFX9-DL-NEXT: v_bfe_u32 v9, v1, 16, 4
-; GFX9-DL-NEXT: v_lshrrev_b32_e32 v10, 28, v1
-; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
-; GFX9-DL-NEXT: v_bfe_u32 v15, v2, 20, 4
-; GFX9-DL-NEXT: v_bfe_u32 v16, v2, 16, 4
-; GFX9-DL-NEXT: v_lshrrev_b32_e32 v17, 28, v2
; GFX9-DL-NEXT: v_bfe_u32 v0, v1, 4, 4
; GFX9-DL-NEXT: v_and_b32_e32 v5, 15, v1
; GFX9-DL-NEXT: v_bfe_u32 v6, v1, 12, 4
; GFX9-DL-NEXT: v_bfe_u32 v7, v1, 8, 4
+; GFX9-DL-NEXT: v_bfe_u32 v8, v1, 20, 4
+; GFX9-DL-NEXT: v_bfe_u32 v9, v1, 16, 4
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v10, 28, v1
; GFX9-DL-NEXT: v_bfe_u32 v11, v1, 24, 4
+; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
; GFX9-DL-NEXT: v_bfe_u32 v1, v2, 4, 4
; GFX9-DL-NEXT: v_and_b32_e32 v12, 15, v2
; GFX9-DL-NEXT: v_bfe_u32 v13, v2, 12, 4
; GFX9-DL-NEXT: v_bfe_u32 v14, v2, 8, 4
+; GFX9-DL-NEXT: v_bfe_u32 v15, v2, 20, 4
+; GFX9-DL-NEXT: v_bfe_u32 v16, v2, 16, 4
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v17, 28, v2
; GFX9-DL-NEXT: v_bfe_u32 v2, v2, 24, 4
+; GFX9-DL-NEXT: v_mul_lo_u16_e32 v18, v11, v2
; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v10, v10, v17 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-DL-NEXT: v_mul_lo_u16_e32 v17, v9, v16
; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v8, v8, v15 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-DL-NEXT: v_mul_lo_u16_e32 v18, v11, v2
; GFX9-DL-NEXT: v_mul_lo_u16_e32 v7, v7, v14
; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v6, v6, v13 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v1, v0, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-DL-NEXT: v_or_b32_e32 v0, v17, v8
; GFX9-DL-NEXT: v_mul_lo_u16_e32 v5, v5, v12
-; GFX9-DL-NEXT: v_or_b32_e32 v12, v18, v10
+; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v12, v0, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-DL-NEXT: v_or_b32_sdwa v0, v18, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-DL-NEXT: v_or_b32_e32 v1, v17, v8
; GFX9-DL-NEXT: v_or_b32_e32 v6, v7, v6
-; GFX9-DL-NEXT: v_and_b32_e32 v8, 0xffff, v0
-; GFX9-DL-NEXT: v_or_b32_e32 v5, v5, v1
-; GFX9-DL-NEXT: v_lshrrev_b32_e32 v7, 8, v10
+; GFX9-DL-NEXT: v_or_b32_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 16, v6
-; GFX9-DL-NEXT: v_lshl_or_b32 v10, v6, 16, v1
-; GFX9-DL-NEXT: v_lshl_or_b32 v1, v12, 16, v8
-; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DL-NEXT: v_add_u16_e32 v4, v5, v4
-; GFX9-DL-NEXT: v_lshrrev_b32_e32 v5, 8, v10
-; GFX9-DL-NEXT: v_lshrrev_b32_e32 v8, 8, v1
+; GFX9-DL-NEXT: v_or_b32_e32 v5, v5, v12
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v7, 8, v10
+; GFX9-DL-NEXT: v_lshl_or_b32 v8, v6, 16, v12
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v10, 8, v1
; GFX9-DL-NEXT: v_lshrrev_b64 v[0:1], 24, v[0:1]
-; GFX9-DL-NEXT: v_add_u16_e32 v1, v4, v5
+; GFX9-DL-NEXT: v_lshrrev_b32_e32 v8, 8, v8
+; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
+; GFX9-DL-NEXT: v_add_u16_e32 v1, v5, v4
+; GFX9-DL-NEXT: v_add_u16_e32 v1, v1, v8
; GFX9-DL-NEXT: v_add_u16_e32 v1, v1, v6
; GFX9-DL-NEXT: v_add_u16_e32 v0, v1, v0
; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v9, v16, v0
-; GFX9-DL-NEXT: v_add_u16_e32 v0, v0, v8
+; GFX9-DL-NEXT: v_add_u16_e32 v0, v0, v10
; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v11, v2, v0
; GFX9-DL-NEXT: v_add_u16_e32 v0, v0, v7
-; GFX9-DL-NEXT: global_store_byte v3, v0, s[0:1]
+; GFX9-DL-NEXT: global_store_byte v3, v0, s[6:7]
; GFX9-DL-NEXT: s_endpgm
;
; GFX10-DL-LABEL: udot8_acc8_vecMul:
@@ -2685,58 +2683,56 @@ define amdgpu_kernel void @udot8_acc8_vecMul(ptr addrspace(1) %src1,
; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: global_load_dword v1, v0, s[0:1]
; GFX10-DL-NEXT: global_load_dword v2, v0, s[2:3]
-; GFX10-DL-NEXT: global_load_ubyte v0, v4, s[6:7]
+; GFX10-DL-NEXT: global_load_ubyte v3, v4, s[6:7]
; GFX10-DL-NEXT: s_waitcnt vmcnt(2)
-; GFX10-DL-NEXT: v_bfe_u32 v3, v1, 4, 4
-; GFX10-DL-NEXT: v_and_b32_e32 v5, 15, v1
; GFX10-DL-NEXT: v_bfe_u32 v6, v1, 12, 4
+; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX10-DL-NEXT: v_bfe_u32 v9, v2, 12, 4
+; GFX10-DL-NEXT: v_bfe_u32 v0, v1, 4, 4
+; GFX10-DL-NEXT: v_and_b32_e32 v5, 15, v1
; GFX10-DL-NEXT: v_bfe_u32 v7, v1, 8, 4
; GFX10-DL-NEXT: v_bfe_u32 v8, v1, 20, 4
-; GFX10-DL-NEXT: v_bfe_u32 v9, v1, 16, 4
-; GFX10-DL-NEXT: v_lshrrev_b32_e32 v10, 28, v1
-; GFX10-DL-NEXT: v_bfe_u32 v11, v1, 24, 4
-; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
-; GFX10-DL-NEXT: v_bfe_u32 v1, v2, 4, 4
-; GFX10-DL-NEXT: v_bfe_u32 v13, v2, 12, 4
-; GFX10-DL-NEXT: v_bfe_u32 v15, v2, 20, 4
-; GFX10-DL-NEXT: v_bfe_u32 v14, v2, 8, 4
-; GFX10-DL-NEXT: v_and_b32_e32 v12, 15, v2
-; GFX10-DL-NEXT: v_mul_lo_u16 v1, v3, v1
-; GFX10-DL-NEXT: v_mul_lo_u16 v6, v6, v13
-; GFX10-DL-NEXT: v_bfe_u32 v16, v2, 16, 4
-; GFX10-DL-NEXT: v_lshrrev_b32_e32 v17, 28, v2
-; GFX10-DL-NEXT: v_bfe_u32 v13, v2, 24, 4
-; GFX10-DL-NEXT: v_mul_lo_u16 v2, v8, v15
-; GFX10-DL-NEXT: v_mul_lo_u16 v3, v7, v14
+; GFX10-DL-NEXT: v_bfe_u32 v10, v1, 16, 4
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v11, 28, v1
+; GFX10-DL-NEXT: v_bfe_u32 v12, v1, 24, 4
+; GFX10-DL-NEXT: v_bfe_u32 v1, v2, 8, 4
+; GFX10-DL-NEXT: v_mul_lo_u16 v6, v6, v9
+; GFX10-DL-NEXT: v_bfe_u32 v9, v2, 4, 4
+; GFX10-DL-NEXT: v_bfe_u32 v14, v2, 20, 4
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v15, 28, v2
+; GFX10-DL-NEXT: v_mul_lo_u16 v1, v7, v1
; GFX10-DL-NEXT: v_lshlrev_b16 v6, 8, v6
-; GFX10-DL-NEXT: v_lshlrev_b16 v1, 8, v1
-; GFX10-DL-NEXT: v_mul_lo_u16 v7, v10, v17
-; GFX10-DL-NEXT: v_mul_lo_u16 v8, v9, v16
-; GFX10-DL-NEXT: v_lshlrev_b16 v2, 8, v2
-; GFX10-DL-NEXT: v_or_b32_e32 v6, v3, v6
-; GFX10-DL-NEXT: v_and_b32_e32 v3, 0xffff, v1
-; GFX10-DL-NEXT: v_mul_lo_u16 v5, v5, v12
-; GFX10-DL-NEXT: v_mul_lo_u16 v10, v11, v13
-; GFX10-DL-NEXT: v_lshlrev_b16 v7, 8, v7
-; GFX10-DL-NEXT: v_or_b32_e32 v2, v8, v2
-; GFX10-DL-NEXT: v_lshl_or_b32 v3, v6, 16, v3
-; GFX10-DL-NEXT: v_or_b32_e32 v1, v5, v1
-; GFX10-DL-NEXT: v_or_b32_e32 v5, v10, v7
-; GFX10-DL-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10-DL-NEXT: v_lshrrev_b32_e32 v3, 8, v3
-; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
-; GFX10-DL-NEXT: v_add_nc_u16 v8, v1, v0
+; GFX10-DL-NEXT: v_and_b32_e32 v13, 15, v2
+; GFX10-DL-NEXT: v_mul_lo_u16 v0, v0, v9
+; GFX10-DL-NEXT: v_bfe_u32 v7, v2, 24, 4
+; GFX10-DL-NEXT: v_bfe_u32 v16, v2, 16, 4
+; GFX10-DL-NEXT: v_or_b32_e32 v6, v1, v6
+; GFX10-DL-NEXT: v_mul_lo_u16 v2, v11, v15
+; GFX10-DL-NEXT: v_mul_lo_u16 v8, v8, v14
+; GFX10-DL-NEXT: v_lshlrev_b16 v9, 8, v0
+; GFX10-DL-NEXT: v_mul_lo_u16 v5, v5, v13
; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 16, v6
-; GFX10-DL-NEXT: v_lshl_or_b32 v1, v5, 16, v2
-; GFX10-DL-NEXT: v_add_nc_u16 v5, v8, v3
+; GFX10-DL-NEXT: v_mul_lo_u16 v1, v12, v7
+; GFX10-DL-NEXT: v_mul_lo_u16 v11, v10, v16
+; GFX10-DL-NEXT: v_lshlrev_b16 v13, 8, v2
+; GFX10-DL-NEXT: v_lshlrev_b16 v2, 8, v8
+; GFX10-DL-NEXT: v_or_b32_sdwa v8, v0, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-DL-NEXT: v_or_b32_e32 v5, v5, v9
+; GFX10-DL-NEXT: v_or_b32_sdwa v1, v1, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-DL-NEXT: v_or_b32_e32 v2, v11, v2
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v8, 8, v8
+; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
+; GFX10-DL-NEXT: v_add_nc_u16 v3, v5, v3
+; GFX10-DL-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-DL-NEXT: v_add_nc_u16 v5, v3, v8
; GFX10-DL-NEXT: v_lshrrev_b64 v[2:3], 24, v[0:1]
; GFX10-DL-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; GFX10-DL-NEXT: v_add_nc_u16 v0, v5, v6
; GFX10-DL-NEXT: v_add_nc_u16 v0, v0, v2
-; GFX10-DL-NEXT: v_mad_u16 v0, v9, v16, v0
+; GFX10-DL-NEXT: v_mad_u16 v0, v10, v16, v0
; GFX10-DL-NEXT: v_add_nc_u16 v0, v0, v1
-; GFX10-DL-NEXT: v_lshrrev_b32_e32 v1, 8, v7
-; GFX10-DL-NEXT: v_mad_u16 v0, v11, v13, v0
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v1, 8, v13
+; GFX10-DL-NEXT: v_mad_u16 v0, v12, v7, v0
; GFX10-DL-NEXT: v_add_nc_u16 v0, v0, v1
; GFX10-DL-NEXT: global_store_byte v4, v0, s[6:7]
; GFX10-DL-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll b/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll
index 8c89db87ebd78..0a3874e6f70c0 100644
--- a/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll
+++ b/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll
@@ -11665,10 +11665,10 @@ define <2 x i16> @multi_use_mul_mad_i16_var(i16 %x, i16 %y, i16 %z0, i16 %z1) {
; GFX10-GISEL-LABEL: multi_use_mul_mad_i16_var:
; GFX10-GISEL: ; %bb.0: ; %entry
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mad_u16 v2, v0, v1, v2
-; GFX10-GISEL-NEXT: v_mad_u16 v0, v0, v1, v3
-; GFX10-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v2
-; GFX10-GISEL-NEXT: v_lshl_or_b32 v0, v0, 16, v1
+; GFX10-GISEL-NEXT: v_mad_u16 v3, v0, v1, v3
+; GFX10-GISEL-NEXT: v_mad_u16 v0, v0, v1, v2
+; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX10-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: multi_use_mul_mad_i16_var:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.a16.dim.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.a16.dim.ll
index 556fb9647309c..8920ba521b3b3 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.a16.dim.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.a16.dim.ll
@@ -13,8 +13,8 @@ define amdgpu_ps <4 x float> @sample_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX10GISEL-LABEL: sample_d_1d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10GISEL-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX10GISEL-NEXT: v_lshlrev_b32_e64 v3, 16, s0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_d v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -33,8 +33,8 @@ define amdgpu_ps <4 x float> @sample_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX10GISEL-LABEL: sample_d_2d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX10GISEL-NEXT: v_lshl_or_b32 v4, v5, 16, v4
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX10GISEL-NEXT: v_or_b32_sdwa v4, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_d v[0:3], v[0:4], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -60,10 +60,10 @@ define amdgpu_ps <4 x float> @sample_d_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX10GISEL-LABEL: sample_d_3d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX10GISEL-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX10GISEL-NEXT: v_lshl_or_b32 v6, v7, 16, v6
-; GFX10GISEL-NEXT: v_lshl_or_b32 v7, s0, 16, v8
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX10GISEL-NEXT: v_lshlrev_b32_e64 v9, 16, s0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v6, v7, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v7, v9, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_d v[0:3], v[0:7], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_3D a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -81,8 +81,8 @@ define amdgpu_ps <4 x float> @sample_c_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inr
;
; GFX10GISEL-LABEL: sample_c_d_1d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX10GISEL-NEXT: v_lshl_or_b32 v3, s0, 16, v3
+; GFX10GISEL-NEXT: v_lshlrev_b32_e64 v4, 16, s0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_c_d v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -101,8 +101,8 @@ define amdgpu_ps <4 x float> @sample_c_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
;
; GFX10GISEL-LABEL: sample_c_d_2d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX10GISEL-NEXT: v_lshl_or_b32 v5, v6, 16, v5
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX10GISEL-NEXT: v_or_b32_sdwa v5, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_c_d v[0:3], v[0:5], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -121,8 +121,8 @@ define amdgpu_ps <4 x float> @sample_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX10GISEL-LABEL: sample_d_cl_1d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10GISEL-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX10GISEL-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_d_cl v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -146,10 +146,10 @@ define amdgpu_ps <4 x float> @sample_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX10GISEL-LABEL: sample_d_cl_2d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX10GISEL-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX10GISEL-NEXT: v_lshl_or_b32 v4, v5, 16, v4
-; GFX10GISEL-NEXT: v_lshl_or_b32 v5, s0, 16, v6
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX10GISEL-NEXT: v_lshlrev_b32_e64 v7, 16, s0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v4, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v5, v7, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_d_cl v[0:3], v[0:5], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -168,8 +168,8 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX10GISEL-LABEL: sample_c_d_cl_1d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX10GISEL-NEXT: v_lshl_or_b32 v3, v4, 16, v3
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX10GISEL-NEXT: v_or_b32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_c_d_cl v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -194,10 +194,10 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX10GISEL-LABEL: sample_c_d_cl_2d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX10GISEL-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX10GISEL-NEXT: v_lshl_or_b32 v5, v6, 16, v5
-; GFX10GISEL-NEXT: v_lshl_or_b32 v6, s0, 16, v7
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX10GISEL-NEXT: v_lshlrev_b32_e64 v8, 16, s0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v5, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v6, v8, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_c_d_cl v[0:3], v[0:6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -215,8 +215,8 @@ define amdgpu_ps <4 x float> @sample_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> inre
;
; GFX10GISEL-LABEL: sample_cd_1d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10GISEL-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX10GISEL-NEXT: v_lshlrev_b32_e64 v3, 16, s0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_cd v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -235,8 +235,8 @@ define amdgpu_ps <4 x float> @sample_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
;
; GFX10GISEL-LABEL: sample_cd_2d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX10GISEL-NEXT: v_lshl_or_b32 v4, v5, 16, v4
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX10GISEL-NEXT: v_or_b32_sdwa v4, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_cd v[0:3], v[0:4], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -254,8 +254,8 @@ define amdgpu_ps <4 x float> @sample_c_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX10GISEL-LABEL: sample_c_cd_1d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX10GISEL-NEXT: v_lshl_or_b32 v3, s0, 16, v3
+; GFX10GISEL-NEXT: v_lshlrev_b32_e64 v4, 16, s0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_c_cd v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -274,8 +274,8 @@ define amdgpu_ps <4 x float> @sample_c_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX10GISEL-LABEL: sample_c_cd_2d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX10GISEL-NEXT: v_lshl_or_b32 v5, v6, 16, v5
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX10GISEL-NEXT: v_or_b32_sdwa v5, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_c_cd v[0:3], v[0:5], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -294,8 +294,8 @@ define amdgpu_ps <4 x float> @sample_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> i
;
; GFX10GISEL-LABEL: sample_cd_cl_1d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10GISEL-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX10GISEL-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_cd_cl v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -319,10 +319,10 @@ define amdgpu_ps <4 x float> @sample_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
;
; GFX10GISEL-LABEL: sample_cd_cl_2d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX10GISEL-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX10GISEL-NEXT: v_lshl_or_b32 v4, v5, 16, v4
-; GFX10GISEL-NEXT: v_lshl_or_b32 v5, s0, 16, v6
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX10GISEL-NEXT: v_lshlrev_b32_e64 v7, 16, s0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v4, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v5, v7, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_cd_cl v[0:3], v[0:5], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -341,8 +341,8 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX10GISEL-LABEL: sample_c_cd_cl_1d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX10GISEL-NEXT: v_lshl_or_b32 v3, v4, 16, v3
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX10GISEL-NEXT: v_or_b32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_c_cd_cl v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -367,10 +367,10 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX10GISEL-LABEL: sample_c_cd_cl_2d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX10GISEL-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX10GISEL-NEXT: v_lshl_or_b32 v5, v6, 16, v5
-; GFX10GISEL-NEXT: v_lshl_or_b32 v6, s0, 16, v7
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX10GISEL-NEXT: v_lshlrev_b32_e64 v8, 16, s0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v5, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v6, v8, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_c_cd_cl v[0:3], v[0:6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -396,10 +396,10 @@ define amdgpu_ps float @sample_c_d_o_2darray_V1(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX10GISEL-LABEL: sample_c_d_o_2darray_V1:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX10GISEL-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX10GISEL-NEXT: v_lshl_or_b32 v6, v7, 16, v6
-; GFX10GISEL-NEXT: v_lshl_or_b32 v7, s0, 16, v8
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX10GISEL-NEXT: v_lshlrev_b32_e64 v9, 16, s0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v6, v7, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v7, v9, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_c_d_o v0, v[0:7], s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_2D_ARRAY a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -425,10 +425,10 @@ define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4
;
; GFX10GISEL-LABEL: sample_c_d_o_2darray_V2:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX10GISEL-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX10GISEL-NEXT: v_lshl_or_b32 v6, v7, 16, v6
-; GFX10GISEL-NEXT: v_lshl_or_b32 v7, s0, 16, v8
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX10GISEL-NEXT: v_lshlrev_b32_e64 v9, 16, s0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v6, v7, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v7, v9, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_c_d_o v[0:1], v[0:7], s[0:7], s[8:11] dmask:0x6 dim:SQ_RSRC_IMG_2D_ARRAY a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -468,10 +468,9 @@ define amdgpu_ps <4 x float> @sample_g16_noa16_d_1d(<8 x i32> inreg %rsrc, <4 x
;
; GFX10GISEL-LABEL: sample_g16_noa16_d_1d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10GISEL-NEXT: v_lshl_or_b32 v0, s0, 16, v0
-; GFX10GISEL-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX10GISEL-NEXT: v_lshlrev_b32_e64 v3, 16, s0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_d_g16 v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -491,10 +490,10 @@ define amdgpu_ps <4 x float> @sample_g16_noa16_d_2d(<8 x i32> inreg %rsrc, <4 x
;
; GFX10GISEL-LABEL: sample_g16_noa16_d_2d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX10GISEL-NEXT: v_lshl_or_b32 v1, v3, 16, v2
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX10GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v1, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_d_g16 v[0:3], [v0, v1, v4, v5], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -516,14 +515,15 @@ define amdgpu_ps <4 x float> @sample_g16_noa16_d_3d(<8 x i32> inreg %rsrc, <4 x
;
; GFX10GISEL-LABEL: sample_g16_noa16_d_3d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10GISEL-NEXT: v_and_b32_e32 v9, 0xffff, v2
-; GFX10GISEL-NEXT: v_and_b32_e32 v10, 0xffff, v3
-; GFX10GISEL-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX10GISEL-NEXT: v_lshl_or_b32 v2, v1, 16, v0
-; GFX10GISEL-NEXT: v_lshl_or_b32 v3, s0, 16, v9
-; GFX10GISEL-NEXT: v_lshl_or_b32 v4, v4, 16, v10
-; GFX10GISEL-NEXT: v_lshl_or_b32 v5, s0, 16, v5
+; GFX10GISEL-NEXT: v_mov_b32_e32 v9, v2
+; GFX10GISEL-NEXT: v_mov_b32_e32 v10, v3
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX10GISEL-NEXT: v_lshlrev_b32_e64 v11, 16, s0
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX10GISEL-NEXT: v_or_b32_sdwa v2, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v3, v11, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v4, v4, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v5, v11, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_d_g16 v[0:3], v[2:8], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_3D
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -541,10 +541,9 @@ define amdgpu_ps <4 x float> @sample_g16_noa16_c_d_1d(<8 x i32> inreg %rsrc, <4
;
; GFX10GISEL-LABEL: sample_g16_noa16_c_d_1d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10GISEL-NEXT: v_lshl_or_b32 v1, s0, 16, v1
-; GFX10GISEL-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX10GISEL-NEXT: v_lshlrev_b32_e64 v4, 16, s0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_c_d_g16 v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -564,10 +563,10 @@ define amdgpu_ps <4 x float> @sample_g16_noa16_c_d_2d(<8 x i32> inreg %rsrc, <4
;
; GFX10GISEL-LABEL: sample_g16_noa16_c_d_2d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX10GISEL-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX10GISEL-NEXT: v_lshl_or_b32 v2, v4, 16, v3
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX10GISEL-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v2, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_c_d_g16 v[0:3], [v0, v1, v2, v5, v6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -585,10 +584,9 @@ define amdgpu_ps <4 x float> @sample_g16_noa16_d_cl_1d(<8 x i32> inreg %rsrc, <4
;
; GFX10GISEL-LABEL: sample_g16_noa16_d_cl_1d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10GISEL-NEXT: v_lshl_or_b32 v0, s0, 16, v0
-; GFX10GISEL-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX10GISEL-NEXT: v_lshlrev_b32_e64 v4, 16, s0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_d_cl_g16 v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -608,10 +606,10 @@ define amdgpu_ps <4 x float> @sample_g16_noa16_d_cl_2d(<8 x i32> inreg %rsrc, <4
;
; GFX10GISEL-LABEL: sample_g16_noa16_d_cl_2d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX10GISEL-NEXT: v_lshl_or_b32 v1, v3, 16, v2
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX10GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v1, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_d_cl_g16 v[0:3], [v0, v1, v4, v5, v6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -629,10 +627,9 @@ define amdgpu_ps <4 x float> @sample_g16_noa16_c_d_cl_1d(<8 x i32> inreg %rsrc,
;
; GFX10GISEL-LABEL: sample_g16_noa16_c_d_cl_1d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10GISEL-NEXT: v_lshl_or_b32 v1, s0, 16, v1
-; GFX10GISEL-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX10GISEL-NEXT: v_lshlrev_b32_e64 v5, 16, s0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v1, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v2, v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_c_d_cl_g16 v[0:3], v[0:4], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -655,11 +652,12 @@ define amdgpu_ps <4 x float> @sample_g16_noa16_c_d_cl_2d(<8 x i32> inreg %rsrc,
; GFX10GISEL-LABEL: sample_g16_noa16_c_d_cl_2d:
; GFX10GISEL: ; %bb.0: ; %main_body
; GFX10GISEL-NEXT: v_mov_b32_e32 v8, v2
+; GFX10GISEL-NEXT: v_mov_b32_e32 v9, v3
; GFX10GISEL-NEXT: v_mov_b32_e32 v2, v0
-; GFX10GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v1
-; GFX10GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v3
-; GFX10GISEL-NEXT: v_lshl_or_b32 v3, v8, 16, v0
-; GFX10GISEL-NEXT: v_lshl_or_b32 v4, v4, 16, v1
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v8
+; GFX10GISEL-NEXT: v_or_b32_sdwa v4, v4, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v3, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_c_d_cl_g16 v[0:3], v[2:7], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -677,10 +675,9 @@ define amdgpu_ps <4 x float> @sample_g16_noa16_cd_1d(<8 x i32> inreg %rsrc, <4 x
;
; GFX10GISEL-LABEL: sample_g16_noa16_cd_1d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10GISEL-NEXT: v_lshl_or_b32 v0, s0, 16, v0
-; GFX10GISEL-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX10GISEL-NEXT: v_lshlrev_b32_e64 v3, 16, s0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_cd_g16 v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -700,10 +697,10 @@ define amdgpu_ps <4 x float> @sample_g16_noa16_cd_2d(<8 x i32> inreg %rsrc, <4 x
;
; GFX10GISEL-LABEL: sample_g16_noa16_cd_2d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX10GISEL-NEXT: v_lshl_or_b32 v1, v3, 16, v2
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX10GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v1, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_cd_g16 v[0:3], [v0, v1, v4, v5], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -721,10 +718,9 @@ define amdgpu_ps <4 x float> @sample_g16_noa16_c_cd_1d(<8 x i32> inreg %rsrc, <4
;
; GFX10GISEL-LABEL: sample_g16_noa16_c_cd_1d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10GISEL-NEXT: v_lshl_or_b32 v1, s0, 16, v1
-; GFX10GISEL-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX10GISEL-NEXT: v_lshlrev_b32_e64 v4, 16, s0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_c_cd_g16 v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -744,10 +740,10 @@ define amdgpu_ps <4 x float> @sample_g16_noa16_c_cd_2d(<8 x i32> inreg %rsrc, <4
;
; GFX10GISEL-LABEL: sample_g16_noa16_c_cd_2d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX10GISEL-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX10GISEL-NEXT: v_lshl_or_b32 v2, v4, 16, v3
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX10GISEL-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v2, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_c_cd_g16 v[0:3], [v0, v1, v2, v5, v6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -765,10 +761,9 @@ define amdgpu_ps <4 x float> @sample_g16_noa16_cd_cl_1d(<8 x i32> inreg %rsrc, <
;
; GFX10GISEL-LABEL: sample_g16_noa16_cd_cl_1d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10GISEL-NEXT: v_lshl_or_b32 v0, s0, 16, v0
-; GFX10GISEL-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX10GISEL-NEXT: v_lshlrev_b32_e64 v4, 16, s0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_cd_cl_g16 v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -788,10 +783,10 @@ define amdgpu_ps <4 x float> @sample_g16_noa16_cd_cl_2d(<8 x i32> inreg %rsrc, <
;
; GFX10GISEL-LABEL: sample_g16_noa16_cd_cl_2d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX10GISEL-NEXT: v_lshl_or_b32 v1, v3, 16, v2
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX10GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v1, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_cd_cl_g16 v[0:3], [v0, v1, v4, v5, v6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -809,10 +804,9 @@ define amdgpu_ps <4 x float> @sample_g16_noa16_c_cd_cl_1d(<8 x i32> inreg %rsrc,
;
; GFX10GISEL-LABEL: sample_g16_noa16_c_cd_cl_1d:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10GISEL-NEXT: v_lshl_or_b32 v1, s0, 16, v1
-; GFX10GISEL-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX10GISEL-NEXT: v_lshlrev_b32_e64 v5, 16, s0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v1, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v2, v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_c_cd_cl_g16 v[0:3], v[0:4], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -835,11 +829,12 @@ define amdgpu_ps <4 x float> @sample_g16_noa16_c_cd_cl_2d(<8 x i32> inreg %rsrc,
; GFX10GISEL-LABEL: sample_g16_noa16_c_cd_cl_2d:
; GFX10GISEL: ; %bb.0: ; %main_body
; GFX10GISEL-NEXT: v_mov_b32_e32 v8, v2
+; GFX10GISEL-NEXT: v_mov_b32_e32 v9, v3
; GFX10GISEL-NEXT: v_mov_b32_e32 v2, v0
-; GFX10GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v1
-; GFX10GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v3
-; GFX10GISEL-NEXT: v_lshl_or_b32 v3, v8, 16, v0
-; GFX10GISEL-NEXT: v_lshl_or_b32 v4, v4, 16, v1
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v8
+; GFX10GISEL-NEXT: v_or_b32_sdwa v4, v4, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v3, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_c_cd_cl_g16 v[0:3], v[2:7], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -863,14 +858,15 @@ define amdgpu_ps float @sample_g16_noa16_c_d_o_2darray_V1(<8 x i32> inreg %rsrc,
;
; GFX10GISEL-LABEL: sample_g16_noa16_c_d_o_2darray_V1:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_mov_b32_e32 v9, v2
-; GFX10GISEL-NEXT: v_mov_b32_e32 v10, v3
+; GFX10GISEL-NEXT: v_mov_b32_e32 v9, v3
+; GFX10GISEL-NEXT: v_mov_b32_e32 v10, v2
+; GFX10GISEL-NEXT: v_mov_b32_e32 v11, v4
; GFX10GISEL-NEXT: v_mov_b32_e32 v2, v0
; GFX10GISEL-NEXT: v_mov_b32_e32 v3, v1
-; GFX10GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v4
-; GFX10GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v9
-; GFX10GISEL-NEXT: v_lshl_or_b32 v5, v5, 16, v1
-; GFX10GISEL-NEXT: v_lshl_or_b32 v4, v10, 16, v0
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v9
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v5
+; GFX10GISEL-NEXT: v_or_b32_sdwa v4, v0, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v5, v1, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_c_d_o_g16 v0, v[2:8], s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_2D_ARRAY
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -894,14 +890,15 @@ define amdgpu_ps <2 x float> @sample_g16_noa16_c_d_o_2darray_V2(<8 x i32> inreg
;
; GFX10GISEL-LABEL: sample_g16_noa16_c_d_o_2darray_V2:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_mov_b32_e32 v9, v2
-; GFX10GISEL-NEXT: v_mov_b32_e32 v10, v3
+; GFX10GISEL-NEXT: v_mov_b32_e32 v9, v3
+; GFX10GISEL-NEXT: v_mov_b32_e32 v10, v2
+; GFX10GISEL-NEXT: v_mov_b32_e32 v11, v4
; GFX10GISEL-NEXT: v_mov_b32_e32 v2, v0
; GFX10GISEL-NEXT: v_mov_b32_e32 v3, v1
-; GFX10GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v4
-; GFX10GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v9
-; GFX10GISEL-NEXT: v_lshl_or_b32 v5, v5, 16, v1
-; GFX10GISEL-NEXT: v_lshl_or_b32 v4, v10, 16, v0
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v9
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v5
+; GFX10GISEL-NEXT: v_or_b32_sdwa v4, v0, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v5, v1, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_c_d_o_g16 v[0:1], v[2:8], s[0:7], s[8:11] dmask:0x6 dim:SQ_RSRC_IMG_2D_ARRAY
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -941,12 +938,10 @@ define amdgpu_ps <4 x float> @sample_d_1d_g16_a16(<8 x i32> inreg %rsrc, <4 x i3
;
; GFX10GISEL-LABEL: sample_d_1d_g16_a16:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10GISEL-NEXT: v_lshl_or_b32 v0, s0, 16, v0
-; GFX10GISEL-NEXT: v_lshl_or_b32 v1, s0, 16, v1
-; GFX10GISEL-NEXT: v_lshl_or_b32 v2, s0, 16, v2
+; GFX10GISEL-NEXT: v_lshlrev_b32_e64 v3, 16, s0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_d_g16 v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -967,12 +962,12 @@ define amdgpu_ps <4 x float> @sample_d_2d_g16_a16(<8 x i32> inreg %rsrc, <4 x i3
;
; GFX10GISEL-LABEL: sample_d_2d_g16_a16:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX10GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX10GISEL-NEXT: v_lshl_or_b32 v1, v3, 16, v2
-; GFX10GISEL-NEXT: v_lshl_or_b32 v2, v5, 16, v4
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX10GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v1, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v2, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_d_g16 v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
@@ -996,18 +991,16 @@ define amdgpu_ps <4 x float> @sample_d_3d_g16_a16(<8 x i32> inreg %rsrc, <4 x i3
;
; GFX10GISEL-LABEL: sample_d_3d_g16_a16:
; GFX10GISEL: ; %bb.0: ; %main_body
-; GFX10GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX10GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v2
-; GFX10GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v3
-; GFX10GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v5
-; GFX10GISEL-NEXT: v_and_b32_e32 v5, 0xffff, v6
-; GFX10GISEL-NEXT: v_and_b32_e32 v6, 0xffff, v8
-; GFX10GISEL-NEXT: v_lshl_or_b32 v1, s0, 16, v1
-; GFX10GISEL-NEXT: v_lshl_or_b32 v2, v4, 16, v2
-; GFX10GISEL-NEXT: v_lshl_or_b32 v3, s0, 16, v3
-; GFX10GISEL-NEXT: v_lshl_or_b32 v4, v7, 16, v5
-; GFX10GISEL-NEXT: v_lshl_or_b32 v5, s0, 16, v6
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX10GISEL-NEXT: v_lshlrev_b32_e64 v9, 16, s0
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX10GISEL-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX10GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v1, v9, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v2, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v3, v9, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v4, v7, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10GISEL-NEXT: v_or_b32_sdwa v5, v9, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10GISEL-NEXT: image_sample_d_g16 v[0:3], v[0:5], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_3D a16
; GFX10GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10GISEL-NEXT: ; return to shader part epilog
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.noret.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.noret.ll
index b913a41094b48..2f0c53c9a5642 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.noret.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.noret.ll
@@ -444,10 +444,9 @@ define amdgpu_ps void @sample_d_1d_g16_nortn(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX10_11-GISEL-LABEL: sample_d_1d_g16_nortn:
; GFX10_11-GISEL: ; %bb.0: ; %main_body
-; GFX10_11-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10_11-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10_11-GISEL-NEXT: v_lshl_or_b32 v0, s0, 16, v0
-; GFX10_11-GISEL-NEXT: v_lshl_or_b32 v1, s0, 16, v1
+; GFX10_11-GISEL-NEXT: v_lshlrev_b32_e64 v3, 16, s0
+; GFX10_11-GISEL-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10_11-GISEL-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10_11-GISEL-NEXT: image_sample_d_g16 off, v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX10_11-GISEL-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.fptrunc.round.ll b/llvm/test/CodeGen/AMDGPU/llvm.fptrunc.round.ll
index a84942e06b5a0..2ab76c99095f8 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.fptrunc.round.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.fptrunc.round.ll
@@ -191,8 +191,8 @@ define amdgpu_gs <2 x half> @v_fptrunc_round_poison_to_v2f16_towardzero(float %a
; GISEL-LABEL: v_fptrunc_round_poison_to_v2f16_towardzero:
; GISEL: ; %bb.0:
; GISEL-NEXT: v_cvt_pkrtz_f16_f32_e32 v0, v0, v0
-; GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GISEL-NEXT: v_lshl_or_b32 v0, s0, 16, v0
+; GISEL-NEXT: v_lshlrev_b32_e64 v1, 16, s0
+; GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GISEL-NEXT: ; return to shader part epilog
;
; GFX12-SDAG-LABEL: v_fptrunc_round_poison_to_v2f16_towardzero:
@@ -1133,10 +1133,10 @@ define amdgpu_gs <2 x i32> @s_fptrunc_round_v2f32_to_v2f16_towardzero(<2 x float
define amdgpu_gs void @s_fptrunc_round_v2f32_to_v2f16_upward_multiple_calls(<2 x float> inreg %a, <2 x float> inreg %b, ptr addrspace(1) %out) {
; SDAG-LABEL: s_fptrunc_round_v2f32_to_v2f16_upward_multiple_calls:
; SDAG: ; %bb.0:
-; SDAG-NEXT: v_mov_b32_e32 v2, s0
-; SDAG-NEXT: v_mov_b32_e32 v3, s2
-; SDAG-NEXT: v_mov_b32_e32 v4, s1
-; SDAG-NEXT: v_mov_b32_e32 v5, s3
+; SDAG-NEXT: v_mov_b32_e32 v2, s1
+; SDAG-NEXT: v_mov_b32_e32 v3, s3
+; SDAG-NEXT: v_mov_b32_e32 v4, s0
+; SDAG-NEXT: v_mov_b32_e32 v5, s2
; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 1
; SDAG-NEXT: v_cvt_f16_f32_e32 v2, v2
; SDAG-NEXT: v_cvt_f16_f32_e32 v6, v3
@@ -1144,13 +1144,13 @@ define amdgpu_gs void @s_fptrunc_round_v2f32_to_v2f16_upward_multiple_calls(<2 x
; SDAG-NEXT: v_cvt_f16_f32_e32 v7, v5
; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 2
; SDAG-NEXT: v_cvt_f16_f32_e32 v3, v3
-; SDAG-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; SDAG-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; SDAG-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; SDAG-NEXT: v_lshl_or_b32 v2, v4, 16, v2
+; SDAG-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SDAG-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; SDAG-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SDAG-NEXT: v_or_b32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; SDAG-NEXT: v_cvt_f16_f32_e32 v4, v5
-; SDAG-NEXT: v_lshl_or_b32 v5, v7, 16, v6
-; SDAG-NEXT: v_lshl_or_b32 v3, v4, 16, v3
+; SDAG-NEXT: v_or_b32_sdwa v5, v6, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; SDAG-NEXT: v_or_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 3, 1), 0
; SDAG-NEXT: v_pk_add_f16 v2, v2, v5
; SDAG-NEXT: v_pk_add_f16 v2, v3, v2
@@ -1315,13 +1315,13 @@ define amdgpu_gs <3 x half> @v_fptrunc_round_v3f32_to_v3f16_upward(<3 x float> %
;
; GISEL-LABEL: v_fptrunc_round_v3f32_to_v3f16_upward:
; GISEL: ; %bb.0:
+; GISEL-NEXT: v_lshlrev_b32_e64 v3, 16, s0
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 1
-; GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
; GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
; GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
; GISEL-NEXT: v_pack_b32_f16 v0, v0, v1
-; GISEL-NEXT: v_lshl_or_b32 v1, s0, 16, v2
+; GISEL-NEXT: v_or_b32_sdwa v1, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GISEL-NEXT: ; return to shader part epilog
;
; GFX12-SDAG-LABEL: v_fptrunc_round_v3f32_to_v3f16_upward:
@@ -1375,13 +1375,13 @@ define amdgpu_gs <3 x half> @v_fptrunc_round_v3f32_to_v3f16_downward(<3 x float>
;
; GISEL-LABEL: v_fptrunc_round_v3f32_to_v3f16_downward:
; GISEL: ; %bb.0:
+; GISEL-NEXT: v_lshlrev_b32_e64 v3, 16, s0
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 3, 1), 1
-; GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
; GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
; GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
; GISEL-NEXT: v_pack_b32_f16 v0, v0, v1
-; GISEL-NEXT: v_lshl_or_b32 v1, s0, 16, v2
+; GISEL-NEXT: v_or_b32_sdwa v1, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GISEL-NEXT: ; return to shader part epilog
;
; GFX12-SDAG-LABEL: v_fptrunc_round_v3f32_to_v3f16_downward:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.maximum.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.maximum.f16.ll
index c8149ca804226..50430cac9ee3b 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.maximum.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.maximum.f16.ll
@@ -1151,9 +1151,8 @@ define void @s_maximum_v2f16(<2 x half> inreg %src0, <2 x half> inreg %src1) {
; GFX900-NEXT: v_mov_b32_e32 v3, s4
; GFX900-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
; GFX900-NEXT: v_cmp_o_f16_e32 vcc, s5, v3
-; GFX900-NEXT: v_cndmask_b32_sdwa v1, v2, v1, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX900-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX900-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX900-NEXT: v_cndmask_b32_sdwa v1, v2, v1, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX900-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX900-NEXT: ;;#ASMSTART
; GFX900-NEXT: ; use v0
; GFX900-NEXT: ;;#ASMEND
@@ -1177,12 +1176,11 @@ define void @s_maximum_v2f16(<2 x half> inreg %src0, <2 x half> inreg %src1) {
; GFX10-NEXT: s_lshr_b32 s5, s16, 16
; GFX10-NEXT: v_pk_max_f16 v0, s16, s17
; GFX10-NEXT: v_cmp_o_f16_e64 vcc_lo, s5, s4
-; GFX10-NEXT: v_cmp_o_f16_e64 s4, s16, s17
; GFX10-NEXT: v_mov_b32_e32 v1, 0x7e00
+; GFX10-NEXT: v_cmp_o_f16_e64 s4, s16, s17
; GFX10-NEXT: v_cndmask_b32_e64 v2, 0x7e00, v0, s4
-; GFX10-NEXT: v_cndmask_b32_sdwa v0, v1, v0, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v2
-; GFX10-NEXT: v_lshl_or_b32 v0, v0, 16, v1
+; GFX10-NEXT: v_cndmask_b32_sdwa v0, v1, v0, vcc_lo dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: ;;#ASMSTART
; GFX10-NEXT: ; use v0
; GFX10-NEXT: ;;#ASMEND
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.minimum.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.minimum.f16.ll
index 3fe8bebc5474f..963d4fc819a2a 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.minimum.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.minimum.f16.ll
@@ -961,9 +961,8 @@ define void @s_minimum_v2f16(<2 x half> inreg %src0, <2 x half> inreg %src1) {
; GFX900-NEXT: v_mov_b32_e32 v3, s4
; GFX900-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
; GFX900-NEXT: v_cmp_o_f16_e32 vcc, s5, v3
-; GFX900-NEXT: v_cndmask_b32_sdwa v1, v2, v1, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX900-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX900-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX900-NEXT: v_cndmask_b32_sdwa v1, v2, v1, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX900-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX900-NEXT: ;;#ASMSTART
; GFX900-NEXT: ; use v0
; GFX900-NEXT: ;;#ASMEND
@@ -987,12 +986,11 @@ define void @s_minimum_v2f16(<2 x half> inreg %src0, <2 x half> inreg %src1) {
; GFX10-NEXT: s_lshr_b32 s5, s16, 16
; GFX10-NEXT: v_pk_min_f16 v0, s16, s17
; GFX10-NEXT: v_cmp_o_f16_e64 vcc_lo, s5, s4
-; GFX10-NEXT: v_cmp_o_f16_e64 s4, s16, s17
; GFX10-NEXT: v_mov_b32_e32 v1, 0x7e00
+; GFX10-NEXT: v_cmp_o_f16_e64 s4, s16, s17
; GFX10-NEXT: v_cndmask_b32_e64 v2, 0x7e00, v0, s4
-; GFX10-NEXT: v_cndmask_b32_sdwa v0, v1, v0, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v2
-; GFX10-NEXT: v_lshl_or_b32 v0, v0, 16, v1
+; GFX10-NEXT: v_cndmask_b32_sdwa v0, v1, v0, vcc_lo dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: ;;#ASMSTART
; GFX10-NEXT: ; use v0
; GFX10-NEXT: ;;#ASMEND
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.modf.ll b/llvm/test/CodeGen/AMDGPU/llvm.modf.ll
index 64154d25aa552..e79726614c629 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.modf.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.modf.ll
@@ -120,9 +120,8 @@ define { <2 x half>, <2 x half> } @test_modf_v2f16(<2 x half> %x) {
; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v3, v6, 0, s[4:5]
; GFX9-GISEL-NEXT: v_and_b32_e32 v3, 0x7fff, v3
; GFX9-GISEL-NEXT: v_and_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-GISEL-NEXT: v_or_b32_e32 v0, v3, v0
-; GFX9-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v0, 16, v2
+; GFX9-GISEL-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-GISEL-NEXT: v_pack_b32_f16 v1, v1, v5
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
%result = call { <2 x half>, <2 x half> } @llvm.modf.v2f16(<2 x half> %x)
@@ -167,9 +166,8 @@ define <2 x half> @test_modf_v2f16_only_use_fract(<2 x half> %x) {
; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v2, v4, 0, s[4:5]
; GFX9-GISEL-NEXT: v_and_b32_e32 v2, 0x7fff, v2
; GFX9-GISEL-NEXT: v_and_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-GISEL-NEXT: v_or_b32_e32 v0, v2, v0
-; GFX9-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v0, 16, v1
+; GFX9-GISEL-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
%result = call { <2 x half>, <2 x half> } @llvm.modf.v2f16(<2 x half> %x)
%result.0 = extractvalue { <2 x half>, <2 x half> } %result, 0
diff --git a/llvm/test/CodeGen/AMDGPU/mad-mix-lo.ll b/llvm/test/CodeGen/AMDGPU/mad-mix-lo.ll
index 88f482a0ac889..db0e6fb66dc67 100644
--- a/llvm/test/CodeGen/AMDGPU/mad-mix-lo.ll
+++ b/llvm/test/CodeGen/AMDGPU/mad-mix-lo.ll
@@ -646,8 +646,8 @@ define <3 x half> @v_mad_mix_v3f32(<3 x half> %src0, <3 x half> %src1, <3 x half
; GISEL-GFX900-NEXT: v_mad_mixlo_f16 v6, v0, v2, v4 op_sel_hi:[1,1,1]
; GISEL-GFX900-NEXT: v_mad_mixlo_f16 v1, v1, v3, v5 op_sel_hi:[1,1,1]
; GISEL-GFX900-NEXT: v_mad_mixhi_f16 v6, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1]
-; GISEL-GFX900-NEXT: v_and_b32_e32 v0, 0xffff, v1
-; GISEL-GFX900-NEXT: v_lshl_or_b32 v1, s4, 16, v0
+; GISEL-GFX900-NEXT: v_lshlrev_b32_e64 v0, 16, s4
+; GISEL-GFX900-NEXT: v_or_b32_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GISEL-GFX900-NEXT: v_mov_b32_e32 v0, v6
; GISEL-GFX900-NEXT: s_setpc_b64 s[30:31]
;
@@ -657,8 +657,8 @@ define <3 x half> @v_mad_mix_v3f32(<3 x half> %src0, <3 x half> %src1, <3 x half
; GISEL-GFX906-NEXT: v_fma_mixlo_f16 v6, v0, v2, v4 op_sel_hi:[1,1,1]
; GISEL-GFX906-NEXT: v_fma_mixlo_f16 v1, v1, v3, v5 op_sel_hi:[1,1,1]
; GISEL-GFX906-NEXT: v_fma_mixhi_f16 v6, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1]
-; GISEL-GFX906-NEXT: v_and_b32_e32 v0, 0xffff, v1
-; GISEL-GFX906-NEXT: v_lshl_or_b32 v1, s4, 16, v0
+; GISEL-GFX906-NEXT: v_lshlrev_b32_e64 v0, 16, s4
+; GISEL-GFX906-NEXT: v_or_b32_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GISEL-GFX906-NEXT: v_mov_b32_e32 v0, v6
; GISEL-GFX906-NEXT: s_setpc_b64 s[30:31]
;
@@ -1229,9 +1229,9 @@ define <3 x half> @v_mad_mix_v3f32_clamp_postcvt(<3 x half> %src0, <3 x half> %s
; GISEL-GFX900: ; %bb.0:
; GISEL-GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GISEL-GFX900-NEXT: v_mad_mixlo_f16 v1, v1, v3, v5 op_sel_hi:[1,1,1]
-; GISEL-GFX900-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GISEL-GFX900-NEXT: v_lshlrev_b32_e64 v3, 16, s4
+; GISEL-GFX900-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GISEL-GFX900-NEXT: v_mad_mixlo_f16 v3, v0, v2, v4 op_sel_hi:[1,1,1] clamp
-; GISEL-GFX900-NEXT: v_lshl_or_b32 v1, s4, 16, v1
; GISEL-GFX900-NEXT: v_mad_mixhi_f16 v3, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp
; GISEL-GFX900-NEXT: v_pk_max_f16 v1, v1, v1 clamp
; GISEL-GFX900-NEXT: v_mov_b32_e32 v0, v3
@@ -1241,9 +1241,9 @@ define <3 x half> @v_mad_mix_v3f32_clamp_postcvt(<3 x half> %src0, <3 x half> %s
; GISEL-GFX906: ; %bb.0:
; GISEL-GFX906-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GISEL-GFX906-NEXT: v_fma_mixlo_f16 v1, v1, v3, v5 op_sel_hi:[1,1,1]
-; GISEL-GFX906-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GISEL-GFX906-NEXT: v_lshlrev_b32_e64 v3, 16, s4
+; GISEL-GFX906-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GISEL-GFX906-NEXT: v_fma_mixlo_f16 v3, v0, v2, v4 op_sel_hi:[1,1,1] clamp
-; GISEL-GFX906-NEXT: v_lshl_or_b32 v1, s4, 16, v1
; GISEL-GFX906-NEXT: v_fma_mixhi_f16 v3, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp
; GISEL-GFX906-NEXT: v_pk_max_f16 v1, v1, v1 clamp
; GISEL-GFX906-NEXT: v_mov_b32_e32 v0, v3
diff --git a/llvm/test/CodeGen/AMDGPU/packed-op-sel.ll b/llvm/test/CodeGen/AMDGPU/packed-op-sel.ll
index 0ee380f690492..84ad5a592dd4a 100644
--- a/llvm/test/CodeGen/AMDGPU/packed-op-sel.ll
+++ b/llvm/test/CodeGen/AMDGPU/packed-op-sel.ll
@@ -223,6 +223,7 @@ define amdgpu_kernel void @fma_vector_vector_scalar_lo_neg_scalar_hi(ptr addrspa
; GCN-LABEL: fma_vector_vector_scalar_lo_neg_scalar_hi:
; GCN: ; %bb.0: ; %bb
; GCN-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GCN-NEXT: s_mov_b32 s0, 0x8000
; GCN-NEXT: v_mov_b32_e32 v4, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: v_mov_b32_e32 v0, s6
@@ -231,11 +232,9 @@ define amdgpu_kernel void @fma_vector_vector_scalar_lo_neg_scalar_hi(ptr addrspa
; GCN-NEXT: ds_read_b32 v0, v0 offset:4
; GCN-NEXT: ds_read_u16 v3, v1
; GCN-NEXT: ds_read_u16 v1, v1 offset:4
-; GCN-NEXT: s_waitcnt lgkmcnt(1)
-; GCN-NEXT: v_and_b32_e32 v3, 0xffff, v3
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_xor_b32_e32 v1, 0x8000, v1
-; GCN-NEXT: v_lshl_or_b32 v1, v1, 16, v3
+; GCN-NEXT: v_xor_b32_sdwa v1, v1, s0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GCN-NEXT: v_or_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GCN-NEXT: v_pk_fma_f16 v0, v2, v0, v1
; GCN-NEXT: global_store_dword v4, v0, s[4:5]
; GCN-NEXT: s_endpgm
@@ -688,7 +687,6 @@ define amdgpu_kernel void @extract_from_i64(ptr addrspace(1) %out, ptr addrspace
; GCN-LABEL: extract_from_i64:
; GCN: ; %bb.0: ; %bb
; GCN-NEXT: s_load_dword s2, s[0:1], 0x2c
-; GCN-NEXT: v_mov_b32_e32 v3, 0xffff
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: v_mov_b32_e32 v0, s2
; GCN-NEXT: ds_read_b32 v2, v0
@@ -696,8 +694,8 @@ define amdgpu_kernel void @extract_from_i64(ptr addrspace(1) %out, ptr addrspace
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24
; GCN-NEXT: v_mov_b32_e32 v1, 0
-; GCN-NEXT: v_and_b32_sdwa v3, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GCN-NEXT: v_lshl_or_b32 v0, v0, 16, v3
+; GCN-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GCN-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: v_pk_add_u16 v0, v2, v0
; GCN-NEXT: global_store_dword v1, v0, s[2:3]
diff --git a/llvm/test/CodeGen/AMDGPU/permute_i8.ll b/llvm/test/CodeGen/AMDGPU/permute_i8.ll
index e098fd19dd437..a7aadc64d750d 100644
--- a/llvm/test/CodeGen/AMDGPU/permute_i8.ll
+++ b/llvm/test/CodeGen/AMDGPU/permute_i8.ll
@@ -567,17 +567,16 @@ define hidden void @addUsesOr(ptr addrspace(1) %in0, ptr addrspace(1) %in1, i8 %
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 24, v4
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_e32 v1, 24, v7
-; GFX10-NEXT: v_lshrrev_b16 v2, 8, v7
-; GFX10-NEXT: v_lshrrev_b32_e32 v3, 16, v4
-; GFX10-NEXT: v_lshrrev_b32_e32 v7, 16, v7
+; GFX10-NEXT: v_lshrrev_b32_e32 v2, 16, v4
+; GFX10-NEXT: v_lshrrev_b32_e32 v3, 16, v7
; GFX10-NEXT: v_add_nc_u16 v0, v0, v1
-; GFX10-NEXT: v_add_nc_u16 v1, v4, v2
-; GFX10-NEXT: v_add_nc_u16 v2, v3, v7
+; GFX10-NEXT: v_lshrrev_b16 v1, 8, v7
+; GFX10-NEXT: v_add_nc_u16 v2, v2, v3
; GFX10-NEXT: v_lshlrev_b16 v0, 8, v0
+; GFX10-NEXT: v_add_nc_u16 v1, v4, v1
+; GFX10-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX10-NEXT: v_lshlrev_b16 v1, 8, v1
-; GFX10-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_lshl_or_b32 v0, v0, 16, v1
+; GFX10-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: global_store_dword v[5:6], v0, off
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -689,20 +688,19 @@ define hidden void @add(ptr addrspace(1) %in0, ptr addrspace(1) %in1, i8 %elt, p
; GFX10-NEXT: s_waitcnt vmcnt(1)
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 24, v4
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshrrev_b16 v1, 8, v7
-; GFX10-NEXT: v_lshrrev_b32_e32 v2, 24, v7
-; GFX10-NEXT: v_lshrrev_b16 v3, 8, v4
-; GFX10-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; GFX10-NEXT: v_add_nc_u16 v0, v0, v1
-; GFX10-NEXT: v_add_nc_u16 v2, v7, v2
-; GFX10-NEXT: v_add_nc_u16 v3, v3, v7
-; GFX10-NEXT: v_add_nc_u16 v1, v1, v4
+; GFX10-NEXT: v_lshrrev_b32_e32 v1, 24, v7
+; GFX10-NEXT: v_lshrrev_b16 v2, 8, v7
+; GFX10-NEXT: v_lshrrev_b32_e32 v3, 16, v7
+; GFX10-NEXT: v_lshrrev_b16 v4, 8, v4
+; GFX10-NEXT: v_add_nc_u16 v1, v7, v1
+; GFX10-NEXT: v_add_nc_u16 v0, v0, v2
+; GFX10-NEXT: v_add_nc_u16 v2, v2, v3
+; GFX10-NEXT: v_add_nc_u16 v3, v4, v7
+; GFX10-NEXT: v_lshlrev_b16 v1, 8, v1
; GFX10-NEXT: v_lshlrev_b16 v0, 8, v0
-; GFX10-NEXT: v_lshlrev_b16 v2, 8, v2
+; GFX10-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX10-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: global_store_dword v[5:6], v0, off
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -717,10 +715,9 @@ define hidden void @add(ptr addrspace(1) %in0, ptr addrspace(1) %in1, i8 %elt, p
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_add_u16_sdwa v2, v7, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_1
; GFX9-NEXT: v_add_u16_sdwa v3, v7, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
-; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: global_store_dword v[5:6], v0, off
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -925,20 +922,19 @@ define hidden void @add_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1,
; GFX10-NEXT: s_waitcnt vmcnt(1)
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 24, v4
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshrrev_b16 v1, 8, v9
-; GFX10-NEXT: v_lshrrev_b32_e32 v2, 24, v9
-; GFX10-NEXT: v_lshrrev_b16 v3, 8, v4
-; GFX10-NEXT: v_lshrrev_b32_e32 v10, 16, v9
-; GFX10-NEXT: v_add_nc_u16 v0, v0, v1
-; GFX10-NEXT: v_add_nc_u16 v2, v9, v2
-; GFX10-NEXT: v_add_nc_u16 v3, v3, v9
-; GFX10-NEXT: v_add_nc_u16 v1, v1, v10
+; GFX10-NEXT: v_lshrrev_b32_e32 v1, 24, v9
+; GFX10-NEXT: v_lshrrev_b16 v2, 8, v9
+; GFX10-NEXT: v_lshrrev_b32_e32 v3, 16, v9
+; GFX10-NEXT: v_lshrrev_b16 v10, 8, v4
+; GFX10-NEXT: v_add_nc_u16 v1, v9, v1
+; GFX10-NEXT: v_add_nc_u16 v0, v0, v2
+; GFX10-NEXT: v_add_nc_u16 v2, v2, v3
+; GFX10-NEXT: v_add_nc_u16 v3, v10, v9
+; GFX10-NEXT: v_lshlrev_b16 v1, 8, v1
; GFX10-NEXT: v_lshlrev_b16 v0, 8, v0
-; GFX10-NEXT: v_lshlrev_b16 v2, 8, v2
+; GFX10-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX10-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_perm_b32 v1, v4, v9, 0x10705
; GFX10-NEXT: global_store_dword v[5:6], v0, off
; GFX10-NEXT: global_store_dword v[7:8], v1, off
@@ -962,10 +958,9 @@ define hidden void @add_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1,
; GFX9-NEXT: v_add_u16_sdwa v2, v9, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:WORD_1
; GFX9-NEXT: v_add_u16_sdwa v3, v4, v9 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_1
; GFX9-NEXT: v_add_u16_sdwa v4, v4, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
-; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v2, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: global_store_dword v[5:6], v1, off
; GFX9-NEXT: global_store_dword v[7:8], v0, off
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -988,25 +983,24 @@ define hidden void @and_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1,
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_and_b32_e32 v4, 0x3ff, v31
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 2, v4
-; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, v2, v4
-; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
-; GFX10-NEXT: global_load_dword v4, v[2:3], off
-; GFX10-NEXT: global_load_dword v9, v[0:1], off
+; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, v2, v4
+; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
+; GFX10-NEXT: global_load_dword v4, v[0:1], off
+; GFX10-NEXT: global_load_dword v9, v[2:3], off
; GFX10-NEXT: v_mov_b32_e32 v0, 2
; GFX10-NEXT: v_mov_b32_e32 v1, 1
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_and_b32_sdwa v2, v4, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_and_b32_e32 v2, 0x100, v4
+; GFX10-NEXT: v_and_b32_sdwa v3, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_and_b32_sdwa v1, v9, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-NEXT: v_and_b32_e32 v3, 0x100, v9
-; GFX10-NEXT: v_and_b32_sdwa v0, v9, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
-; GFX10-NEXT: v_or_b32_e32 v1, v1, v2
-; GFX10-NEXT: v_or_b32_e32 v0, v0, v3
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_lshl_or_b32 v0, v0, 16, v1
-; GFX10-NEXT: v_perm_b32 v1, v9, v4, 0x5070006
+; GFX10-NEXT: v_and_b32_sdwa v0, v9, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_and_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX10-NEXT: v_perm_b32 v1, v4, v9, 0x5070006
+; GFX10-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: global_store_dword v[5:6], v0, off
; GFX10-NEXT: global_store_dword v[7:8], v1, off
; GFX10-NEXT: s_setpc_b64 s[30:31]
@@ -1026,15 +1020,14 @@ define hidden void @and_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1,
; GFX9-NEXT: v_mov_b32_e32 v1, 1
; GFX9-NEXT: s_mov_b32 s4, 0x5070006
; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_and_b32_e32 v3, 0x100, v4
; GFX9-NEXT: v_and_b32_sdwa v10, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_and_b32_sdwa v0, v9, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_and_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v3, 0x100, v4
+; GFX9-NEXT: v_or_b32_sdwa v3, v10, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX9-NEXT: v_or_b32_e32 v3, v10, v3
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v0, v3, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_perm_b32 v2, v4, v9, s4
; GFX9-NEXT: global_store_dword v[5:6], v0, off
; GFX9-NEXT: global_store_dword v[7:8], v2, off
@@ -1063,21 +1056,20 @@ define hidden void @ashr_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1,
; GFX10-NEXT: global_load_dword v9, v[0:1], off
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v2, v4
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v3, vcc_lo
+; GFX10-NEXT: v_mov_b32_e32 v2, 26
; GFX10-NEXT: global_load_dword v0, v[0:1], off
-; GFX10-NEXT: v_mov_b32_e32 v1, 26
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_bfe_i32 v2, v9, 0, 8
-; GFX10-NEXT: v_ashrrev_i32_sdwa v1, v1, v9 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_bfe_i32 v1, v9, 0, 8
+; GFX10-NEXT: v_ashrrev_i32_sdwa v2, v2, v9 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX10-NEXT: v_ashrrev_i32_e32 v3, 25, v9
-; GFX10-NEXT: v_lshlrev_b16 v2, 7, v2
-; GFX10-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT: v_lshlrev_b16 v1, 7, v1
+; GFX10-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_ashrrev_i16 v4, 10, v0
; GFX10-NEXT: v_perm_b32 v0, v9, v0, 0x4010707
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffffff00, v2
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_or_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffffff00, v1
+; GFX10-NEXT: v_or_b32_sdwa v1, v4, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: global_store_dword v[5:6], v1, off
; GFX10-NEXT: global_store_dword v[7:8], v0, off
; GFX10-NEXT: s_setpc_b64 s[30:31]
@@ -1093,21 +1085,20 @@ define hidden void @ashr_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1,
; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
; GFX9-NEXT: global_load_dword v4, v[0:1], off
; GFX9-NEXT: global_load_dword v9, v[2:3], off
-; GFX9-NEXT: v_mov_b32_e32 v0, 26
; GFX9-NEXT: v_mov_b32_e32 v1, 7
; GFX9-NEXT: s_mov_b32 s4, 0x4010707
+; GFX9-NEXT: v_mov_b32_e32 v0, 26
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_ashrrev_i32_sdwa v0, v0, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: v_ashrrev_i32_e32 v3, 25, v4
; GFX9-NEXT: v_lshlrev_b16_sdwa v1, v1, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v2, v4, v9, s4
+; GFX9-NEXT: v_ashrrev_i32_sdwa v0, v0, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_ashrrev_i32_e32 v3, 25, v4
; GFX9-NEXT: v_ashrrev_i16_e32 v9, 10, v9
-; GFX9-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_and_b32_e32 v1, 0xffffff00, v1
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_or_b32_sdwa v1, v9, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v1, v9, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: global_store_dword v[5:6], v0, off
; GFX9-NEXT: global_store_dword v[7:8], v2, off
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -1303,15 +1294,14 @@ define hidden void @lhsr_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1,
; GFX10-NEXT: v_mov_b32_e32 v0, 26
; GFX10-NEXT: s_waitcnt vmcnt(1)
; GFX10-NEXT: v_lshrrev_b16 v1, 1, v4
+; GFX10-NEXT: v_lshrrev_b32_e32 v2, 26, v4
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_sdwa v0, v0, v9 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT: v_lshrrev_b32_e32 v2, 25, v9
-; GFX10-NEXT: v_lshrrev_b32_e32 v3, 26, v4
+; GFX10-NEXT: v_lshrrev_b32_e32 v3, 25, v9
; GFX10-NEXT: v_and_b32_e32 v1, 0x7f00, v1
-; GFX10-NEXT: v_or_b32_e32 v0, v2, v0
-; GFX10-NEXT: v_or_b32_e32 v1, v3, v1
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10-NEXT: v_or_b32_e32 v0, v3, v0
+; GFX10-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_perm_b32 v1, v9, v4, 0x1030707
; GFX10-NEXT: global_store_dword v[5:6], v0, off
; GFX10-NEXT: global_store_dword v[7:8], v1, off
@@ -1339,9 +1329,8 @@ define hidden void @lhsr_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1,
; GFX9-NEXT: v_lshrrev_b32_e32 v4, 25, v9
; GFX9-NEXT: v_and_b32_e32 v2, 0x7f00, v2
; GFX9-NEXT: v_or_b32_e32 v0, v4, v0
-; GFX9-NEXT: v_or_b32_e32 v2, v3, v2
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v0, v2, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: global_store_dword v[5:6], v0, off
; GFX9-NEXT: global_store_dword v[7:8], v1, off
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -1374,19 +1363,18 @@ define hidden void @mul_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1,
; GFX10-NEXT: s_waitcnt vmcnt(1)
; GFX10-NEXT: v_lshrrev_b16 v0, 8, v4
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshrrev_b16 v1, 8, v9
-; GFX10-NEXT: v_lshrrev_b32_e32 v2, 24, v9
-; GFX10-NEXT: v_lshrrev_b32_e32 v3, 16, v9
-; GFX10-NEXT: v_mul_lo_u16 v0, v0, v1
-; GFX10-NEXT: v_mul_lo_u16 v1, v4, v9
-; GFX10-NEXT: v_mul_lo_u16 v2, v3, v2
-; GFX10-NEXT: v_mul_lo_u16 v3, v9, v3
+; GFX10-NEXT: v_lshrrev_b32_e32 v1, 24, v9
+; GFX10-NEXT: v_lshrrev_b32_e32 v2, 16, v9
+; GFX10-NEXT: v_lshrrev_b16 v3, 8, v9
+; GFX10-NEXT: v_mul_lo_u16 v1, v2, v1
+; GFX10-NEXT: v_mul_lo_u16 v0, v0, v3
+; GFX10-NEXT: v_mul_lo_u16 v3, v4, v9
+; GFX10-NEXT: v_mul_lo_u16 v2, v9, v2
+; GFX10-NEXT: v_lshlrev_b16 v1, 8, v1
; GFX10-NEXT: v_lshlrev_b16 v0, 8, v0
-; GFX10-NEXT: v_lshlrev_b16 v2, 8, v2
-; GFX10-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_or_b32_sdwa v1, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_perm_b32 v1, v4, v9, 0x2000504
; GFX10-NEXT: global_store_dword v[5:6], v0, off
; GFX10-NEXT: global_store_dword v[7:8], v1, off
@@ -1410,10 +1398,9 @@ define hidden void @mul_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1,
; GFX9-NEXT: v_mul_lo_u16_sdwa v2, v9, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_mul_lo_u16_e32 v3, v4, v9
; GFX9-NEXT: v_mul_lo_u16_sdwa v4, v4, v9 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
-; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v2, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: global_store_dword v[5:6], v1, off
; GFX9-NEXT: global_store_dword v[7:8], v0, off
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -1444,6 +1431,7 @@ define hidden void @or_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1, i
; GFX10-NEXT: global_load_dword v4, v[2:3], off
; GFX10-NEXT: global_load_dword v9, v[0:1], off
; GFX10-NEXT: v_mov_b32_e32 v0, 16
+; GFX10-NEXT: v_bfrev_b32_e32 v2, 4.0
; GFX10-NEXT: s_waitcnt vmcnt(1)
; GFX10-NEXT: v_lshlrev_b16 v1, 8, v4
; GFX10-NEXT: v_lshrrev_b32_sdwa v0, v0, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -1451,9 +1439,8 @@ define hidden void @or_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1, i
; GFX10-NEXT: v_or_b32_sdwa v1, v9, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
; GFX10-NEXT: v_or_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
; GFX10-NEXT: v_or_b32_e32 v1, 0x201, v1
-; GFX10-NEXT: v_or_b32_e32 v0, 0x102, v0
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_lshl_or_b32 v0, v0, 16, v1
+; GFX10-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_perm_b32 v1, v9, v4, 0x2010005
; GFX10-NEXT: global_store_dword v[5:6], v0, off
; GFX10-NEXT: global_store_dword v[7:8], v1, off
@@ -1466,23 +1453,23 @@ define hidden void @or_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1, i
; GFX9-NEXT: v_lshlrev_b32_e32 v4, 2, v4
; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v2, v4
; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v4
; GFX9-NEXT: global_load_dword v2, v[2:3], off
+; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v4
; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX9-NEXT: global_load_dword v0, v[0:1], off
; GFX9-NEXT: s_mov_b32 s4, 0x2010005
+; GFX9-NEXT: s_movk_i32 s5, 0x102
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; GFX9-NEXT: v_lshlrev_b16_e32 v3, 8, v2
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v4, v0, v2, s4
; GFX9-NEXT: v_or_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
-; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_e32 v0, 0x201, v0
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_or_b32_e32 v1, 0x102, v1
-; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: global_store_dword v[5:6], v0, off
; GFX9-NEXT: global_store_dword v[7:8], v4, off
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -1513,35 +1500,34 @@ define hidden void @sdiv_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1,
; GFX10-NEXT: global_load_dword v9, v[0:1], off
; GFX10-NEXT: s_waitcnt vmcnt(1)
; GFX10-NEXT: v_cvt_f32_i32_sdwa v0, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v2, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v2, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v3, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v10, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cvt_f32_i32_sdwa v1, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v3, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v10, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v11, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v11, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
; GFX10-NEXT: v_rcp_f32_e32 v12, v0
; GFX10-NEXT: v_rcp_f32_e32 v2, v2
-; GFX10-NEXT: v_add_nc_u32_e32 v1, 1, v1
+; GFX10-NEXT: v_rcp_f32_e32 v3, v3
; GFX10-NEXT: v_rcp_f32_e32 v10, v10
-; GFX10-NEXT: v_rcp_f32_e32 v11, v11
-; GFX10-NEXT: v_add_nc_u32_e32 v3, 1, v3
+; GFX10-NEXT: v_add_nc_u32_e32 v1, 1, v1
; GFX10-NEXT: v_add_nc_u32_e32 v0, 1, v0
+; GFX10-NEXT: v_add_nc_u32_e32 v11, 1, v11
; GFX10-NEXT: v_mul_f32_e32 v12, v1, v12
-; GFX10-NEXT: v_mul_f32_e32 v2, v3, v2
-; GFX10-NEXT: v_mul_f32_e32 v1, v1, v10
-; GFX10-NEXT: v_mul_f32_e32 v0, v0, v11
+; GFX10-NEXT: v_mul_f32_e32 v1, v1, v2
+; GFX10-NEXT: v_mul_f32_e32 v0, v0, v3
+; GFX10-NEXT: v_mul_f32_e32 v2, v11, v10
; GFX10-NEXT: v_trunc_f32_e32 v3, v12
-; GFX10-NEXT: v_trunc_f32_e32 v2, v2
; GFX10-NEXT: v_trunc_f32_e32 v1, v1
; GFX10-NEXT: v_trunc_f32_e32 v0, v0
+; GFX10-NEXT: v_trunc_f32_e32 v2, v2
; GFX10-NEXT: v_cvt_i32_f32_e32 v3, v3
-; GFX10-NEXT: v_cvt_i32_f32_sdwa v2, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD
; GFX10-NEXT: v_cvt_i32_f32_e32 v1, v1
; GFX10-NEXT: v_cvt_i32_f32_sdwa v0, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD
-; GFX10-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v2
-; GFX10-NEXT: v_lshl_or_b32 v0, v0, 16, v1
+; GFX10-NEXT: v_cvt_i32_f32_sdwa v2, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v1, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_perm_b32 v1, v9, v4, 0x60706
; GFX10-NEXT: global_store_dword v[5:6], v0, off
; GFX10-NEXT: global_store_dword v[7:8], v1, off
@@ -1587,10 +1573,9 @@ define hidden void @sdiv_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1,
; GFX9-NEXT: v_cvt_i32_f32_sdwa v0, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD
; GFX9-NEXT: v_cvt_i32_f32_sdwa v2, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD
; GFX9-NEXT: v_perm_b32 v4, v9, v4, s4
-; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: global_store_dword v[5:6], v0, off
; GFX9-NEXT: global_store_dword v[7:8], v4, off
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -1678,25 +1663,24 @@ define hidden void @shl_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1,
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_and_b32_e32 v4, 0x3ff, v31
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 2, v4
-; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
-; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, v2, v4
; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
-; GFX10-NEXT: global_load_dword v4, v[0:1], off
-; GFX10-NEXT: global_load_dword v9, v[2:3], off
+; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
+; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT: global_load_dword v4, v[2:3], off
+; GFX10-NEXT: global_load_dword v9, v[0:1], off
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_lshlrev_b16 v0, 2, v4
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshlrev_b16 v1, 1, v9
-; GFX10-NEXT: v_and_b32_e32 v2, 0xfffffc00, v0
-; GFX10-NEXT: v_and_b32_e32 v3, 0xfe, v1
-; GFX10-NEXT: v_and_b32_e32 v1, 0xfffffe00, v1
-; GFX10-NEXT: v_and_b32_e32 v0, 0xfc, v0
-; GFX10-NEXT: v_or_b32_e32 v2, v3, v2
+; GFX10-NEXT: v_lshlrev_b16 v0, 1, v4
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_lshlrev_b16 v1, 2, v9
+; GFX10-NEXT: v_and_b32_e32 v2, 0xfffffe00, v0
+; GFX10-NEXT: v_and_b32_e32 v3, 0xfc, v1
+; GFX10-NEXT: v_and_b32_e32 v1, 0xfffffc00, v1
+; GFX10-NEXT: v_and_b32_e32 v0, 0xfe, v0
+; GFX10-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX10-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v2
-; GFX10-NEXT: v_lshl_or_b32 v0, v0, 16, v1
-; GFX10-NEXT: v_perm_b32 v1, v9, v4, 0x5000104
+; GFX10-NEXT: v_perm_b32 v1, v4, v9, 0x5000104
+; GFX10-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: global_store_dword v[5:6], v0, off
; GFX10-NEXT: global_store_dword v[7:8], v1, off
; GFX10-NEXT: s_setpc_b64 s[30:31]
@@ -1722,10 +1706,9 @@ define hidden void @shl_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1,
; GFX9-NEXT: v_and_b32_e32 v4, 0xfc, v2
; GFX9-NEXT: v_and_b32_e32 v2, 0xfffffc00, v2
; GFX9-NEXT: v_and_b32_e32 v1, 0xfe, v1
+; GFX9-NEXT: v_or_b32_sdwa v3, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_or_b32_e32 v1, v1, v2
-; GFX9-NEXT: v_or_b32_e32 v3, v4, v3
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_lshl_or_b32 v1, v3, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: global_store_dword v[5:6], v1, off
; GFX9-NEXT: global_store_dword v[7:8], v0, off
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -1820,47 +1803,46 @@ define hidden void @srem_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1,
; GFX10-NEXT: global_load_dword v9, v[0:1], off
; GFX10-NEXT: s_waitcnt vmcnt(1)
; GFX10-NEXT: v_cvt_f32_i32_sdwa v0, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v1, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v2, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v1, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v3, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v10, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v3, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v10, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
-; GFX10-NEXT: v_rcp_f32_e32 v0, v0
-; GFX10-NEXT: v_rcp_f32_e32 v1, v1
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v2, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
; GFX10-NEXT: v_cvt_f32_i32_sdwa v11, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
-; GFX10-NEXT: v_rcp_f32_e32 v12, v2
-; GFX10-NEXT: v_rcp_f32_e32 v13, v10
-; GFX10-NEXT: v_add_nc_u32_e32 v2, 1, v2
+; GFX10-NEXT: v_rcp_f32_e32 v0, v0
+; GFX10-NEXT: v_rcp_f32_e32 v12, v1
+; GFX10-NEXT: v_rcp_f32_e32 v13, v3
+; GFX10-NEXT: v_rcp_f32_e32 v10, v10
+; GFX10-NEXT: v_add_nc_u32_e32 v1, 1, v1
; GFX10-NEXT: v_add_nc_u32_e32 v3, 1, v3
-; GFX10-NEXT: v_add_nc_u32_e32 v10, 1, v10
; GFX10-NEXT: v_add_nc_u32_e32 v11, 1, v11
-; GFX10-NEXT: v_mul_f32_e32 v0, v2, v0
-; GFX10-NEXT: v_mul_f32_e32 v1, v3, v1
-; GFX10-NEXT: v_lshrrev_b32_e32 v2, 16, v4
-; GFX10-NEXT: v_mul_f32_e32 v3, v10, v12
-; GFX10-NEXT: v_mul_f32_e32 v10, v11, v13
+; GFX10-NEXT: v_add_nc_u32_e32 v2, 1, v2
+; GFX10-NEXT: v_lshrrev_b32_e32 v14, 16, v4
+; GFX10-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX10-NEXT: v_mul_f32_e32 v1, v3, v12
+; GFX10-NEXT: v_mul_f32_e32 v3, v11, v13
+; GFX10-NEXT: v_mul_f32_e32 v2, v2, v10
+; GFX10-NEXT: v_lshrrev_b32_e32 v10, 8, v4
; GFX10-NEXT: v_trunc_f32_e32 v0, v0
; GFX10-NEXT: v_trunc_f32_e32 v1, v1
-; GFX10-NEXT: v_lshrrev_b32_e32 v11, 8, v4
; GFX10-NEXT: v_trunc_f32_e32 v3, v3
-; GFX10-NEXT: v_trunc_f32_e32 v10, v10
+; GFX10-NEXT: v_trunc_f32_e32 v2, v2
+; GFX10-NEXT: v_lshrrev_b32_e32 v11, 24, v4
; GFX10-NEXT: v_cvt_i32_f32_e32 v0, v0
; GFX10-NEXT: v_cvt_i32_f32_e32 v1, v1
-; GFX10-NEXT: v_lshrrev_b32_e32 v12, 24, v4
; GFX10-NEXT: v_cvt_i32_f32_e32 v3, v3
-; GFX10-NEXT: v_cvt_i32_f32_e32 v10, v10
+; GFX10-NEXT: v_cvt_i32_f32_e32 v2, v2
; GFX10-NEXT: v_mul_lo_u32 v0, v0, v4
-; GFX10-NEXT: v_mul_lo_u32 v1, v1, v11
-; GFX10-NEXT: v_mul_lo_u32 v3, v3, v2
-; GFX10-NEXT: v_mul_lo_u32 v10, v10, v12
-; GFX10-NEXT: v_sub_nc_u32_e32 v0, v2, v0
-; GFX10-NEXT: v_sub_nc_u32_sdwa v1, v9, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
-; GFX10-NEXT: v_sub_nc_u32_e32 v2, v12, v3
-; GFX10-NEXT: v_sub_nc_u32_sdwa v3, v9, v10 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_or_b32_sdwa v1, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10-NEXT: v_mul_lo_u32 v1, v1, v14
+; GFX10-NEXT: v_mul_lo_u32 v3, v3, v11
+; GFX10-NEXT: v_mul_lo_u32 v2, v2, v10
+; GFX10-NEXT: v_sub_nc_u32_e32 v0, v14, v0
+; GFX10-NEXT: v_sub_nc_u32_e32 v1, v11, v1
+; GFX10-NEXT: v_sub_nc_u32_sdwa v3, v9, v3 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-NEXT: v_sub_nc_u32_sdwa v2, v9, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v1, v1, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_perm_b32 v1, v4, v9, 0x2070306
; GFX10-NEXT: global_store_dword v[5:6], v0, off
; GFX10-NEXT: global_store_dword v[7:8], v1, off
@@ -1880,15 +1862,15 @@ define hidden void @srem_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1,
; GFX9-NEXT: s_mov_b32 s4, 0x2070306
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_cvt_f32_i32_sdwa v2, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0
-; GFX9-NEXT: v_cvt_f32_i32_sdwa v12, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1
; GFX9-NEXT: v_cvt_f32_i32_sdwa v3, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
+; GFX9-NEXT: v_cvt_f32_i32_sdwa v12, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1
; GFX9-NEXT: v_cvt_f32_i32_sdwa v13, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cvt_f32_i32_sdwa v11, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
; GFX9-NEXT: v_cvt_f32_i32_sdwa v15, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
; GFX9-NEXT: v_rcp_f32_e32 v2, v2
-; GFX9-NEXT: v_rcp_f32_e32 v12, v12
; GFX9-NEXT: v_add_u32_e32 v16, 1, v3
+; GFX9-NEXT: v_rcp_f32_e32 v12, v12
; GFX9-NEXT: v_rcp_f32_e32 v3, v3
; GFX9-NEXT: v_add_u32_e32 v17, 1, v13
; GFX9-NEXT: v_rcp_f32_e32 v13, v13
@@ -1906,10 +1888,10 @@ define hidden void @srem_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1,
; GFX9-NEXT: v_cvt_i32_f32_e32 v11, v11
; GFX9-NEXT: v_cvt_i32_f32_e32 v3, v3
; GFX9-NEXT: v_cvt_i32_f32_e32 v12, v12
-; GFX9-NEXT: v_lshrrev_b32_e32 v10, 8, v4
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v4
-; GFX9-NEXT: v_perm_b32 v1, v4, v9, s4
+; GFX9-NEXT: v_lshrrev_b32_e32 v10, 8, v4
; GFX9-NEXT: v_lshrrev_b32_e32 v14, 24, v4
+; GFX9-NEXT: v_perm_b32 v1, v4, v9, s4
; GFX9-NEXT: v_mul_lo_u32 v2, v2, v4
; GFX9-NEXT: v_mul_lo_u32 v4, v11, v10
; GFX9-NEXT: v_mul_lo_u32 v3, v3, v0
@@ -1918,10 +1900,9 @@ define hidden void @srem_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1,
; GFX9-NEXT: v_sub_u32_sdwa v2, v9, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
; GFX9-NEXT: v_sub_u32_e32 v3, v14, v3
; GFX9-NEXT: v_sub_u32_sdwa v4, v9, v10 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v3, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_or_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v0, v3, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: global_store_dword v[5:6], v0, off
; GFX9-NEXT: global_store_dword v[7:8], v1, off
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -1949,26 +1930,25 @@ define hidden void @sub_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1,
; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
-; GFX10-NEXT: global_load_dword v4, v[2:3], off
-; GFX10-NEXT: global_load_dword v9, v[0:1], off
+; GFX10-NEXT: global_load_dword v2, v[2:3], off
+; GFX10-NEXT: global_load_dword v0, v[0:1], off
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_lshrrev_b16 v0, 8, v4
-; GFX10-NEXT: v_lshrrev_b32_e32 v1, 24, v4
-; GFX10-NEXT: v_lshrrev_b32_e32 v2, 16, v4
+; GFX10-NEXT: v_lshrrev_b32_e32 v1, 24, v2
+; GFX10-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; GFX10-NEXT: v_lshrrev_b16 v4, 8, v2
+; GFX10-NEXT: v_sub_nc_u16 v9, v3, v1
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_sub_nc_u16 v0, v9, v0
-; GFX10-NEXT: v_sub_nc_u16 v10, v1, v4
-; GFX10-NEXT: v_sub_nc_u16 v3, v2, v1
+; GFX10-NEXT: v_sub_nc_u16 v4, v0, v4
+; GFX10-NEXT: v_sub_nc_u16 v3, v1, v3
; GFX10-NEXT: v_sub_nc_u16 v1, v1, v2
-; GFX10-NEXT: v_lshlrev_b16 v0, 8, v0
-; GFX10-NEXT: v_lshlrev_b16 v2, 8, v3
-; GFX10-NEXT: v_or_b32_sdwa v0, v10, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX10-NEXT: v_perm_b32 v1, v4, v9, 0x6070007
-; GFX10-NEXT: global_store_dword v[5:6], v0, off
-; GFX10-NEXT: global_store_dword v[7:8], v1, off
+; GFX10-NEXT: v_perm_b32 v0, v2, v0, 0x6070007
+; GFX10-NEXT: v_lshlrev_b16 v9, 8, v9
+; GFX10-NEXT: v_lshlrev_b16 v4, 8, v4
+; GFX10-NEXT: v_or_b32_sdwa v3, v3, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: global_store_dword v[5:6], v1, off
+; GFX10-NEXT: global_store_dword v[7:8], v0, off
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: sub_store_div:
@@ -1989,10 +1969,9 @@ define hidden void @sub_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1,
; GFX9-NEXT: v_sub_u16_sdwa v2, v9, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:WORD_1
; GFX9-NEXT: v_sub_u16_sdwa v3, v4, v9 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_1
; GFX9-NEXT: v_sub_u16_sdwa v4, v9, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
-; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v2, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: global_store_dword v[5:6], v1, off
; GFX9-NEXT: global_store_dword v[7:8], v0, off
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -2133,45 +2112,44 @@ define hidden void @udiv(ptr addrspace(1) %in0, ptr addrspace(1) %in1, i8 %elt,
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_and_b32_e32 v4, 0x3ff, v31
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 2, v4
-; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, v2, v4
-; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
-; GFX10-NEXT: global_load_dword v4, v[2:3], off
-; GFX10-NEXT: global_load_dword v9, v[0:1], off
+; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, v2, v4
+; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
+; GFX10-NEXT: global_load_dword v4, v[0:1], off
+; GFX10-NEXT: global_load_dword v9, v[2:3], off
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, v4
-; GFX10-NEXT: v_cvt_f32_ubyte1_e32 v1, v4
+; GFX10-NEXT: v_cvt_f32_ubyte3_e32 v0, v4
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_cvt_f32_ubyte3_e32 v2, v9
-; GFX10-NEXT: v_cvt_f32_ubyte2_e32 v3, v4
-; GFX10-NEXT: v_cvt_f32_ubyte3_e32 v10, v4
-; GFX10-NEXT: v_rcp_f32_e32 v11, v0
-; GFX10-NEXT: v_rcp_f32_e32 v1, v1
-; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v12, v9
-; GFX10-NEXT: v_rcp_f32_e32 v13, v3
+; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v1, v9
+; GFX10-NEXT: v_cvt_f32_ubyte2_e32 v2, v9
+; GFX10-NEXT: v_cvt_f32_ubyte3_e32 v3, v9
+; GFX10-NEXT: v_cvt_f32_ubyte1_e32 v10, v9
+; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v11, v4
+; GFX10-NEXT: v_rcp_f32_e32 v12, v1
+; GFX10-NEXT: v_rcp_f32_e32 v13, v2
+; GFX10-NEXT: v_rcp_f32_e32 v3, v3
; GFX10-NEXT: v_rcp_f32_e32 v10, v10
-; GFX10-NEXT: v_add_nc_u32_e32 v2, 1, v2
-; GFX10-NEXT: v_add_nc_u32_e32 v3, 1, v3
-; GFX10-NEXT: v_add_nc_u32_e32 v12, 1, v12
; GFX10-NEXT: v_add_nc_u32_e32 v0, 1, v0
-; GFX10-NEXT: v_mul_f32_e32 v2, v2, v11
-; GFX10-NEXT: v_mul_f32_e32 v1, v3, v1
-; GFX10-NEXT: v_mul_f32_e32 v3, v12, v13
-; GFX10-NEXT: v_mul_f32_e32 v0, v0, v10
-; GFX10-NEXT: v_trunc_f32_e32 v2, v2
-; GFX10-NEXT: v_trunc_f32_e32 v1, v1
-; GFX10-NEXT: v_trunc_f32_e32 v3, v3
+; GFX10-NEXT: v_add_nc_u32_e32 v11, 1, v11
+; GFX10-NEXT: v_add_nc_u32_e32 v1, 1, v1
+; GFX10-NEXT: v_add_nc_u32_e32 v2, 1, v2
+; GFX10-NEXT: v_mul_f32_e32 v0, v0, v12
+; GFX10-NEXT: v_mul_f32_e32 v11, v11, v13
+; GFX10-NEXT: v_mul_f32_e32 v1, v1, v3
+; GFX10-NEXT: v_mul_f32_e32 v2, v2, v10
; GFX10-NEXT: v_trunc_f32_e32 v0, v0
-; GFX10-NEXT: v_cvt_u32_f32_e32 v2, v2
-; GFX10-NEXT: v_cvt_u32_f32_sdwa v1, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; GFX10-NEXT: v_trunc_f32_e32 v3, v11
+; GFX10-NEXT: v_trunc_f32_e32 v1, v1
+; GFX10-NEXT: v_trunc_f32_e32 v2, v2
+; GFX10-NEXT: v_cvt_u32_f32_e32 v0, v0
; GFX10-NEXT: v_cvt_u32_f32_e32 v3, v3
-; GFX10-NEXT: v_cvt_u32_f32_sdwa v0, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD
-; GFX10-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_lshl_or_b32 v0, v0, 16, v1
-; GFX10-NEXT: v_perm_b32 v1, v9, v4, 0x40207
+; GFX10-NEXT: v_cvt_u32_f32_sdwa v1, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; GFX10-NEXT: v_cvt_u32_f32_sdwa v2, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_perm_b32 v1, v4, v9, 0x40207
; GFX10-NEXT: global_store_dword v[5:6], v0, off
; GFX10-NEXT: global_store_dword v[7:8], v1, off
; GFX10-NEXT: s_setpc_b64 s[30:31]
@@ -2213,14 +2191,13 @@ define hidden void @udiv(ptr addrspace(1) %in0, ptr addrspace(1) %in1, i8 %elt,
; GFX9-NEXT: v_trunc_f32_e32 v3, v3
; GFX9-NEXT: v_trunc_f32_e32 v1, v1
; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX9-NEXT: v_cvt_u32_f32_sdwa v2, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD
; GFX9-NEXT: v_cvt_u32_f32_e32 v3, v3
; GFX9-NEXT: v_cvt_u32_f32_sdwa v1, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; GFX9-NEXT: v_cvt_u32_f32_sdwa v2, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD
; GFX9-NEXT: v_perm_b32 v4, v4, v9, s4
+; GFX9-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: global_store_dword v[5:6], v0, off
; GFX9-NEXT: global_store_dword v[7:8], v4, off
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -2307,51 +2284,50 @@ define hidden void @urem_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1,
; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
-; GFX10-NEXT: global_load_dword v2, v[2:3], off
-; GFX10-NEXT: global_load_dword v0, v[0:1], off
+; GFX10-NEXT: global_load_dword v4, v[2:3], off
+; GFX10-NEXT: global_load_dword v9, v[0:1], off
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v1, v2
-; GFX10-NEXT: v_cvt_f32_ubyte1_e32 v3, v2
-; GFX10-NEXT: v_cvt_f32_ubyte2_e32 v4, v2
-; GFX10-NEXT: v_cvt_f32_ubyte3_e32 v9, v2
+; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, v4
+; GFX10-NEXT: v_cvt_f32_ubyte2_e32 v1, v4
+; GFX10-NEXT: v_cvt_f32_ubyte3_e32 v2, v4
+; GFX10-NEXT: v_cvt_f32_ubyte1_e32 v3, v4
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_cvt_f32_ubyte2_e32 v11, v0
+; GFX10-NEXT: v_cvt_f32_ubyte2_e32 v10, v9
+; GFX10-NEXT: v_rcp_f32_e32 v0, v0
; GFX10-NEXT: v_rcp_f32_e32 v1, v1
-; GFX10-NEXT: v_rcp_f32_e32 v10, v3
-; GFX10-NEXT: v_rcp_f32_e32 v4, v4
-; GFX10-NEXT: v_rcp_f32_e32 v9, v9
+; GFX10-NEXT: v_rcp_f32_e32 v2, v2
+; GFX10-NEXT: v_rcp_f32_e32 v11, v3
; GFX10-NEXT: v_add_nc_u32_e32 v3, 1, v3
-; GFX10-NEXT: v_add_nc_u32_e32 v11, 1, v11
-; GFX10-NEXT: v_lshrrev_b32_e32 v12, 16, v2
+; GFX10-NEXT: v_add_nc_u32_e32 v10, 1, v10
+; GFX10-NEXT: v_lshrrev_b32_e32 v12, 16, v4
+; GFX10-NEXT: v_mul_f32_e32 v0, v3, v0
; GFX10-NEXT: v_mul_f32_e32 v1, v3, v1
-; GFX10-NEXT: v_mul_f32_e32 v10, v3, v10
-; GFX10-NEXT: v_mul_f32_e32 v3, v3, v4
-; GFX10-NEXT: v_mul_f32_e32 v4, v11, v9
-; GFX10-NEXT: v_lshrrev_b32_e32 v11, 24, v2
+; GFX10-NEXT: v_mul_f32_e32 v2, v10, v2
+; GFX10-NEXT: v_mul_f32_e32 v3, v3, v11
+; GFX10-NEXT: v_lshrrev_b32_e32 v10, 8, v4
+; GFX10-NEXT: v_trunc_f32_e32 v0, v0
; GFX10-NEXT: v_trunc_f32_e32 v1, v1
-; GFX10-NEXT: v_trunc_f32_e32 v9, v10
-; GFX10-NEXT: v_lshrrev_b32_e32 v10, 8, v2
+; GFX10-NEXT: v_trunc_f32_e32 v2, v2
; GFX10-NEXT: v_trunc_f32_e32 v3, v3
-; GFX10-NEXT: v_trunc_f32_e32 v4, v4
+; GFX10-NEXT: v_lshrrev_b32_e32 v11, 24, v4
+; GFX10-NEXT: v_cvt_u32_f32_e32 v0, v0
; GFX10-NEXT: v_cvt_u32_f32_e32 v1, v1
-; GFX10-NEXT: v_cvt_u32_f32_e32 v9, v9
+; GFX10-NEXT: v_cvt_u32_f32_e32 v2, v2
; GFX10-NEXT: v_cvt_u32_f32_e32 v3, v3
-; GFX10-NEXT: v_cvt_u32_f32_e32 v4, v4
-; GFX10-NEXT: v_mul_lo_u32 v1, v1, v2
-; GFX10-NEXT: v_mul_lo_u32 v9, v9, v10
-; GFX10-NEXT: v_mul_lo_u32 v3, v3, v12
-; GFX10-NEXT: v_mul_lo_u32 v4, v4, v11
+; GFX10-NEXT: v_mul_lo_u32 v0, v0, v4
+; GFX10-NEXT: v_mul_lo_u32 v1, v1, v12
+; GFX10-NEXT: v_mul_lo_u32 v2, v2, v11
+; GFX10-NEXT: v_mul_lo_u32 v3, v3, v10
+; GFX10-NEXT: v_sub_nc_u32_e32 v0, v10, v0
; GFX10-NEXT: v_sub_nc_u32_e32 v1, v10, v1
-; GFX10-NEXT: v_sub_nc_u32_sdwa v9, v10, v9 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT: v_sub_nc_u32_e32 v3, v10, v3
-; GFX10-NEXT: v_sub_nc_u32_sdwa v4, v0, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-NEXT: v_perm_b32 v0, v2, v0, 0x2050505
-; GFX10-NEXT: v_or_b32_sdwa v1, v1, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_or_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_lshl_or_b32 v1, v3, 16, v1
-; GFX10-NEXT: global_store_dword v[5:6], v1, off
-; GFX10-NEXT: global_store_dword v[7:8], v0, off
+; GFX10-NEXT: v_sub_nc_u32_sdwa v2, v9, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-NEXT: v_sub_nc_u32_sdwa v3, v10, v3 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_perm_b32 v1, v4, v9, 0x2050505
+; GFX10-NEXT: global_store_dword v[5:6], v0, off
+; GFX10-NEXT: global_store_dword v[7:8], v1, off
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: urem_store_div:
@@ -2391,22 +2367,21 @@ define hidden void @urem_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1,
; GFX9-NEXT: v_cvt_u32_f32_e32 v3, v3
; GFX9-NEXT: v_cvt_u32_f32_e32 v11, v11
; GFX9-NEXT: v_cvt_u32_f32_e32 v13, v13
-; GFX9-NEXT: v_lshrrev_b32_e32 v10, 8, v4
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v4
+; GFX9-NEXT: v_lshrrev_b32_e32 v10, 8, v4
; GFX9-NEXT: v_lshrrev_b32_e32 v12, 24, v4
+; GFX9-NEXT: v_perm_b32 v1, v4, v9, s4
; GFX9-NEXT: v_mul_lo_u32 v2, v2, v4
; GFX9-NEXT: v_mul_lo_u32 v3, v3, v10
-; GFX9-NEXT: v_perm_b32 v1, v4, v9, s4
; GFX9-NEXT: v_mul_lo_u32 v0, v11, v0
; GFX9-NEXT: v_mul_lo_u32 v4, v13, v12
; GFX9-NEXT: v_sub_u32_e32 v2, v10, v2
; GFX9-NEXT: v_sub_u32_sdwa v3, v10, v3 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_sub_u32_e32 v0, v10, v0
; GFX9-NEXT: v_sub_u32_sdwa v4, v9, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: global_store_dword v[5:6], v0, off
; GFX9-NEXT: global_store_dword v[7:8], v1, off
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -2430,28 +2405,27 @@ define hidden void @xor_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1,
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_and_b32_e32 v4, 0x3ff, v31
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 2, v4
-; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
-; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, v2, v4
; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
-; GFX10-NEXT: global_load_dword v4, v[0:1], off
-; GFX10-NEXT: global_load_dword v9, v[2:3], off
+; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
+; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT: global_load_dword v4, v[2:3], off
+; GFX10-NEXT: global_load_dword v9, v[0:1], off
; GFX10-NEXT: v_mov_b32_e32 v0, 0xffffff00
-; GFX10-NEXT: v_mov_b32_e32 v1, 1
-; GFX10-NEXT: v_mov_b32_e32 v2, 2
+; GFX10-NEXT: v_mov_b32_e32 v1, 2
+; GFX10-NEXT: v_mov_b32_e32 v2, 1
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_and_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-NEXT: v_and_b32_e32 v3, 0xffffff00, v4
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_and_b32_e32 v3, 0xffffff00, v9
-; GFX10-NEXT: v_xor_b32_sdwa v1, v9, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
-; GFX10-NEXT: v_xor_b32_sdwa v2, v9, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-NEXT: v_xor_b32_e32 v0, 0x200, v0
+; GFX10-NEXT: v_and_b32_sdwa v0, v9, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-NEXT: v_xor_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-NEXT: v_xor_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
; GFX10-NEXT: v_xor_b32_e32 v3, 0x100, v3
-; GFX10-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX10-NEXT: v_or_b32_sdwa v1, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX10-NEXT: v_perm_b32 v1, v9, v4, 0x5060307
+; GFX10-NEXT: v_xor_b32_e32 v0, 0x200, v0
+; GFX10-NEXT: v_or_b32_sdwa v1, v1, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX10-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_perm_b32 v1, v4, v9, 0x5060307
; GFX10-NEXT: global_store_dword v[5:6], v0, off
; GFX10-NEXT: global_store_dword v[7:8], v1, off
; GFX10-NEXT: s_setpc_b64 s[30:31]
@@ -2468,21 +2442,20 @@ define hidden void @xor_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1,
; GFX9-NEXT: global_load_dword v4, v[0:1], off
; GFX9-NEXT: global_load_dword v9, v[2:3], off
; GFX9-NEXT: s_movk_i32 s4, 0xff00
-; GFX9-NEXT: v_mov_b32_e32 v1, 1
; GFX9-NEXT: v_mov_b32_e32 v0, 2
+; GFX9-NEXT: v_mov_b32_e32 v1, 1
; GFX9-NEXT: s_mov_b32 s5, 0x5060307
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_and_b32_sdwa v2, v4, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_and_b32_e32 v3, 0xffffff00, v9
-; GFX9-NEXT: v_xor_b32_sdwa v1, v9, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
-; GFX9-NEXT: v_xor_b32_e32 v2, 0x200, v2
; GFX9-NEXT: v_xor_b32_sdwa v0, v9, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT: v_xor_b32_sdwa v1, v9, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
; GFX9-NEXT: v_xor_b32_e32 v3, 0x100, v3
+; GFX9-NEXT: v_xor_b32_e32 v2, 0x200, v2
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_e32 v1, v1, v2
-; GFX9-NEXT: v_or_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_perm_b32 v4, v9, v4, s5
; GFX9-NEXT: global_store_dword v[5:6], v0, off
; GFX9-NEXT: global_store_dword v[7:8], v4, off
diff --git a/llvm/test/CodeGen/AMDGPU/scalar_to_vector.ll b/llvm/test/CodeGen/AMDGPU/scalar_to_vector.ll
index 6b1a46d0fe10c..3bbc5b9113672 100644
--- a/llvm/test/CodeGen/AMDGPU/scalar_to_vector.ll
+++ b/llvm/test/CodeGen/AMDGPU/scalar_to_vector.ll
@@ -458,11 +458,10 @@ define i64 @bitcast_combine_scalar_to_vector_v4i16(i16 %arg) {
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_and_b32_e32 v1, 0xffffff00, v0
-; GFX9-NEXT: v_or_b32_sdwa v2, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v2
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v0, v2, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: bitcast_combine_scalar_to_vector_v4i16:
diff --git a/llvm/test/CodeGen/AMDGPU/sdwa-peephole-lshl-or-hi16pack.ll b/llvm/test/CodeGen/AMDGPU/sdwa-peephole-lshl-or-hi16pack.ll
new file mode 100644
index 0000000000000..65fd104a4d505
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/sdwa-peephole-lshl-or-hi16pack.ll
@@ -0,0 +1,20 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=amdgpu9.00 < %s | FileCheck -check-prefix=GFX9 %s
+
+; End-to-end companion to sdwa-peephole-lshl-or-hi16pack.mir: the divergent
+; high-half pack (hi << 16) | (z & 0xffff) is selected to V_AND_B32 0xffff +
+; V_LSHL_OR_B32 (#206058). SIPeepholeSDWA rewrites the pair back into a
+; V_LSHLREV_B32 + V_OR_B32_sdwa with src1_sel:WORD_0, so the mask is free.
+
+define i32 @pack_hi16_masked(i32 %hi, i32 %z) {
+; GFX9-LABEL: pack_hi16_masked:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+ %s = shl i32 %hi, 16
+ %m = and i32 %z, 65535
+ %o = or i32 %s, %m
+ ret i32 %o
+}
diff --git a/llvm/test/CodeGen/AMDGPU/sdwa-peephole-lshl-or-hi16pack.mir b/llvm/test/CodeGen/AMDGPU/sdwa-peephole-lshl-or-hi16pack.mir
new file mode 100644
index 0000000000000..07e20257238a0
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/sdwa-peephole-lshl-or-hi16pack.mir
@@ -0,0 +1,156 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgpu9.00 -run-pass=si-peephole-sdwa -o - %s | FileCheck -check-prefix=GFX9 %s
+
+# The (hi << 16) | (z & 0xffff) high-half pack is selected to a fused
+# V_LSHL_OR_B32 preceded by an explicit V_AND_B32 0xffff mask (V_LSHL_OR_B32
+# has no SDWA form). This peephole rewrites the pair back into a
+# V_LSHLREV_B32 + V_OR_B32_sdwa with src1_sel:WORD_0 (the WORD_0 select does
+# the masking for free) and erases the dead V_AND, restoring the SDWA fold.
+
+---
+name: pack_mask_on_and_src0
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr1
+ ; GFX9-LABEL: name: pack_mask_on_and_src0
+ ; GFX9: liveins: $vgpr0, $vgpr1
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX9-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX9-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 65535, implicit $exec
+ ; GFX9-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 16, [[COPY]], implicit $exec
+ ; GFX9-NEXT: [[V_OR_B32_sdwa:%[0-9]+]]:vgpr_32 = V_OR_B32_sdwa 0, [[V_LSHLREV_B32_e64_]], 0, [[COPY1]], 0, 6, 0, 6, 4, implicit $exec
+ ; GFX9-NEXT: $vgpr0 = COPY [[V_OR_B32_sdwa]]
+ ; GFX9-NEXT: SI_RETURN implicit $vgpr0
+ %0:vgpr_32 = COPY $vgpr0
+ %1:vgpr_32 = COPY $vgpr1
+ %2:vgpr_32 = V_MOV_B32_e32 65535, implicit $exec
+ %3:vgpr_32 = V_AND_B32_e64 %2, %1, implicit $exec
+ %4:vgpr_32 = V_LSHL_OR_B32_e64 %0, 16, %3, implicit $exec
+ $vgpr0 = COPY %4
+ SI_RETURN implicit $vgpr0
+...
+
+---
+name: pack_mask_on_and_src1
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr1
+ ; GFX9-LABEL: name: pack_mask_on_and_src1
+ ; GFX9: liveins: $vgpr0, $vgpr1
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX9-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX9-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 65535, implicit $exec
+ ; GFX9-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 16, [[COPY]], implicit $exec
+ ; GFX9-NEXT: [[V_OR_B32_sdwa:%[0-9]+]]:vgpr_32 = V_OR_B32_sdwa 0, [[V_LSHLREV_B32_e64_]], 0, [[COPY1]], 0, 6, 0, 6, 4, implicit $exec
+ ; GFX9-NEXT: $vgpr0 = COPY [[V_OR_B32_sdwa]]
+ ; GFX9-NEXT: SI_RETURN implicit $vgpr0
+ %0:vgpr_32 = COPY $vgpr0
+ %1:vgpr_32 = COPY $vgpr1
+ %2:vgpr_32 = V_MOV_B32_e32 65535, implicit $exec
+ %3:vgpr_32 = V_AND_B32_e64 %1, %2, implicit $exec
+ %4:vgpr_32 = V_LSHL_OR_B32_e64 %0, 16, %3, implicit $exec
+ $vgpr0 = COPY %4
+ SI_RETURN implicit $vgpr0
+...
+
+---
+name: pack_mask_and_e32
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr1
+ ; GFX9-LABEL: name: pack_mask_and_e32
+ ; GFX9: liveins: $vgpr0, $vgpr1
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX9-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX9-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 16, [[COPY]], implicit $exec
+ ; GFX9-NEXT: [[V_OR_B32_sdwa:%[0-9]+]]:vgpr_32 = V_OR_B32_sdwa 0, [[V_LSHLREV_B32_e64_]], 0, [[COPY1]], 0, 6, 0, 6, 4, implicit $exec
+ ; GFX9-NEXT: $vgpr0 = COPY [[V_OR_B32_sdwa]]
+ ; GFX9-NEXT: SI_RETURN implicit $vgpr0
+ %0:vgpr_32 = COPY $vgpr0
+ %1:vgpr_32 = COPY $vgpr1
+ %2:vgpr_32 = V_AND_B32_e32 65535, %1, implicit $exec
+ %3:vgpr_32 = V_LSHL_OR_B32_e64 %0, 16, %2, implicit $exec
+ $vgpr0 = COPY %3
+ SI_RETURN implicit $vgpr0
+...
+
+# Negative: the V_AND has a second use, so it cannot be removed and the
+# V_LSHL_OR_B32 must be left untouched.
+---
+name: no_rewrite_multiuse_and
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr1
+ ; GFX9-LABEL: name: no_rewrite_multiuse_and
+ ; GFX9: liveins: $vgpr0, $vgpr1
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX9-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX9-NEXT: [[V_AND_B32_e32_:%[0-9]+]]:vgpr_32 = V_AND_B32_e32 65535, [[COPY1]], implicit $exec
+ ; GFX9-NEXT: [[V_LSHL_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHL_OR_B32_e64 [[COPY]], 16, [[V_AND_B32_e32_]], implicit $exec
+ ; GFX9-NEXT: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[V_AND_B32_e32_]], [[V_LSHL_OR_B32_e64_]], 0, implicit $exec
+ ; GFX9-NEXT: $vgpr0 = COPY [[V_ADD_U32_e64_]]
+ ; GFX9-NEXT: SI_RETURN implicit $vgpr0
+ %0:vgpr_32 = COPY $vgpr0
+ %1:vgpr_32 = COPY $vgpr1
+ %2:vgpr_32 = V_AND_B32_e32 65535, %1, implicit $exec
+ %3:vgpr_32 = V_LSHL_OR_B32_e64 %0, 16, %2, implicit $exec
+ %4:vgpr_32 = V_ADD_U32_e64 %2, %3, 0, implicit $exec
+ $vgpr0 = COPY %4
+ SI_RETURN implicit $vgpr0
+...
+
+# Negative: the mask is not 0xffff, so WORD_0 would not be equivalent.
+---
+name: no_rewrite_wrong_mask
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr1
+ ; GFX9-LABEL: name: no_rewrite_wrong_mask
+ ; GFX9: liveins: $vgpr0, $vgpr1
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX9-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX9-NEXT: [[V_AND_B32_e32_:%[0-9]+]]:vgpr_32 = V_AND_B32_e32 32767, [[COPY1]], implicit $exec
+ ; GFX9-NEXT: [[V_LSHL_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHL_OR_B32_e64 [[COPY]], 16, [[V_AND_B32_e32_]], implicit $exec
+ ; GFX9-NEXT: $vgpr0 = COPY [[V_LSHL_OR_B32_e64_]]
+ ; GFX9-NEXT: SI_RETURN implicit $vgpr0
+ %0:vgpr_32 = COPY $vgpr0
+ %1:vgpr_32 = COPY $vgpr1
+ %2:vgpr_32 = V_AND_B32_e32 32767, %1, implicit $exec
+ %3:vgpr_32 = V_LSHL_OR_B32_e64 %0, 16, %2, implicit $exec
+ $vgpr0 = COPY %3
+ SI_RETURN implicit $vgpr0
+...
+
+# Negative: the shift amount is not 16.
+---
+name: no_rewrite_wrong_shift
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr1
+ ; GFX9-LABEL: name: no_rewrite_wrong_shift
+ ; GFX9: liveins: $vgpr0, $vgpr1
+ ; GFX9-NEXT: {{ $}}
+ ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX9-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX9-NEXT: [[V_AND_B32_e32_:%[0-9]+]]:vgpr_32 = V_AND_B32_e32 65535, [[COPY1]], implicit $exec
+ ; GFX9-NEXT: [[V_LSHL_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHL_OR_B32_e64 [[COPY]], 8, [[V_AND_B32_e32_]], implicit $exec
+ ; GFX9-NEXT: $vgpr0 = COPY [[V_LSHL_OR_B32_e64_]]
+ ; GFX9-NEXT: SI_RETURN implicit $vgpr0
+ %0:vgpr_32 = COPY $vgpr0
+ %1:vgpr_32 = COPY $vgpr1
+ %2:vgpr_32 = V_AND_B32_e32 65535, %1, implicit $exec
+ %3:vgpr_32 = V_LSHL_OR_B32_e64 %0, 8, %2, implicit $exec
+ $vgpr0 = COPY %3
+ SI_RETURN implicit $vgpr0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/sdwa-peephole.ll b/llvm/test/CodeGen/AMDGPU/sdwa-peephole.ll
index b596d9d4ad6e6..992f2db2ce48c 100644
--- a/llvm/test/CodeGen/AMDGPU/sdwa-peephole.ll
+++ b/llvm/test/CodeGen/AMDGPU/sdwa-peephole.ll
@@ -1221,10 +1221,9 @@ define amdgpu_kernel void @mul_v4i8(ptr addrspace(1) %out, ptr addrspace(1) %ina
; GFX9-NEXT: v_mul_lo_u16_sdwa v4, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX9-NEXT: v_mul_lo_u16_e32 v5, v1, v2
; GFX9-NEXT: v_mul_lo_u16_sdwa v1, v1, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
+; GFX9-NEXT: v_or_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_or_b32_sdwa v2, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: global_store_dword v0, v1, s[0:1]
; GFX9-NEXT: s_endpgm
;
@@ -1240,24 +1239,23 @@ define amdgpu_kernel void @mul_v4i8(ptr addrspace(1) %out, ptr addrspace(1) %ina
; GFX10-NEXT: global_load_dword v2, v0, s[6:7]
; GFX10-NEXT: s_waitcnt vmcnt(1)
; GFX10-NEXT: v_lshrrev_b16 v0, 8, v1
+; GFX10-NEXT: v_lshrrev_b32_e32 v3, 24, v1
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshrrev_b16 v3, 8, v2
-; GFX10-NEXT: v_lshrrev_b32_e32 v4, 24, v1
-; GFX10-NEXT: v_lshrrev_b32_e32 v5, 24, v2
-; GFX10-NEXT: v_lshrrev_b32_e32 v6, 16, v2
-; GFX10-NEXT: v_mul_lo_u16 v0, v0, v3
-; GFX10-NEXT: v_lshrrev_b32_e32 v3, 16, v1
-; GFX10-NEXT: v_mul_lo_u16 v4, v4, v5
+; GFX10-NEXT: v_lshrrev_b32_e32 v4, 24, v2
+; GFX10-NEXT: v_lshrrev_b16 v5, 8, v2
+; GFX10-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX10-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX10-NEXT: v_mul_lo_u16 v1, v1, v2
+; GFX10-NEXT: v_mul_lo_u16 v3, v3, v4
+; GFX10-NEXT: v_mul_lo_u16 v0, v0, v5
+; GFX10-NEXT: v_mul_lo_u16 v2, v6, v7
+; GFX10-NEXT: v_lshlrev_b16 v3, 8, v3
; GFX10-NEXT: v_lshlrev_b16 v0, 8, v0
-; GFX10-NEXT: v_mul_lo_u16 v3, v3, v6
-; GFX10-NEXT: v_lshlrev_b16 v2, 8, v4
+; GFX10-NEXT: v_or_b32_sdwa v2, v2, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX10-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_or_b32_sdwa v1, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX10-NEXT: global_store_dword v2, v0, s[0:1]
+; GFX10-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: global_store_dword v1, v0, s[0:1]
; GFX10-NEXT: s_endpgm
entry:
%idx = call i32 @llvm.amdgcn.workitem.id.x()
@@ -1383,13 +1381,11 @@ define amdgpu_kernel void @mul_v8i8(ptr addrspace(1) %out, ptr addrspace(1) %ina
; GFX9-NEXT: v_mul_lo_u16_e32 v9, v1, v3
; GFX9-NEXT: v_mul_lo_u16_sdwa v1, v1, v3 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
; GFX9-NEXT: v_or_b32_sdwa v3, v5, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v2, v8, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v9, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_or_b32_sdwa v0, v0, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_or_b32_sdwa v2, v8, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: global_store_dwordx2 v4, v[0:1], s[0:1]
; GFX9-NEXT: s_endpgm
;
@@ -1405,39 +1401,37 @@ define amdgpu_kernel void @mul_v8i8(ptr addrspace(1) %out, ptr addrspace(1) %ina
; GFX10-NEXT: global_load_dwordx2 v[2:3], v4, s[6:7]
; GFX10-NEXT: s_waitcnt vmcnt(1)
; GFX10-NEXT: v_lshrrev_b16 v4, 8, v1
+; GFX10-NEXT: v_lshrrev_b32_e32 v6, 24, v1
+; GFX10-NEXT: v_lshrrev_b32_e32 v8, 24, v0
; GFX10-NEXT: v_lshrrev_b16 v9, 8, v0
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshrrev_b16 v10, 8, v3
+; GFX10-NEXT: v_lshrrev_b32_e32 v12, 24, v3
; GFX10-NEXT: v_lshrrev_b16 v13, 8, v2
+; GFX10-NEXT: v_lshrrev_b32_e32 v14, 24, v2
; GFX10-NEXT: v_lshrrev_b32_e32 v5, 16, v1
-; GFX10-NEXT: v_lshrrev_b32_e32 v6, 24, v1
; GFX10-NEXT: v_lshrrev_b32_e32 v7, 16, v0
-; GFX10-NEXT: v_lshrrev_b32_e32 v8, 24, v0
; GFX10-NEXT: v_lshrrev_b32_e32 v11, 16, v3
-; GFX10-NEXT: v_lshrrev_b32_e32 v12, 24, v3
-; GFX10-NEXT: v_lshrrev_b32_e32 v14, 16, v2
-; GFX10-NEXT: v_lshrrev_b32_e32 v15, 24, v2
+; GFX10-NEXT: v_lshrrev_b32_e32 v15, 16, v2
; GFX10-NEXT: v_mul_lo_u16 v0, v0, v2
; GFX10-NEXT: v_mul_lo_u16 v1, v1, v3
; GFX10-NEXT: v_mul_lo_u16 v2, v9, v13
-; GFX10-NEXT: v_mul_lo_u16 v3, v4, v10
-; GFX10-NEXT: v_mul_lo_u16 v4, v8, v15
+; GFX10-NEXT: v_mul_lo_u16 v3, v8, v14
; GFX10-NEXT: v_mul_lo_u16 v6, v6, v12
-; GFX10-NEXT: v_mul_lo_u16 v7, v7, v14
+; GFX10-NEXT: v_mul_lo_u16 v4, v4, v10
+; GFX10-NEXT: v_mul_lo_u16 v7, v7, v15
+; GFX10-NEXT: v_mul_lo_u16 v5, v5, v11
; GFX10-NEXT: v_lshlrev_b16 v2, 8, v2
; GFX10-NEXT: v_lshlrev_b16 v3, 8, v3
-; GFX10-NEXT: v_mul_lo_u16 v5, v5, v11
-; GFX10-NEXT: v_lshlrev_b16 v4, 8, v4
; GFX10-NEXT: v_lshlrev_b16 v6, 8, v6
+; GFX10-NEXT: v_lshlrev_b16 v4, 8, v4
; GFX10-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_or_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_or_b32_sdwa v2, v7, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_or_b32_sdwa v3, v5, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX10-NEXT: v_or_b32_sdwa v2, v7, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v3, v5, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX10-NEXT: v_mov_b32_e32 v4, 0
-; GFX10-NEXT: v_lshl_or_b32 v0, v2, 16, v0
-; GFX10-NEXT: v_lshl_or_b32 v1, v3, 16, v1
+; GFX10-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: global_store_dwordx2 v4, v[0:1], s[0:1]
; GFX10-NEXT: s_endpgm
entry:
@@ -1508,9 +1502,8 @@ define amdgpu_kernel void @sitofp_v2i16_to_v2f16(
; GFX10-NEXT: global_load_dword v1, v0, s[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cvt_f16_i16_e32 v2, v1
-; GFX10-NEXT: v_cvt_f16_i16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10-NEXT: v_lshl_or_b32 v1, v1, 16, v2
+; GFX10-NEXT: v_cvt_f16_i16_sdwa v1, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX10-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: global_store_dword v0, v1, s[0:1]
; GFX10-NEXT: s_endpgm
ptr addrspace(1) %r,
diff --git a/llvm/test/CodeGen/AMDGPU/sminmax.v2i16.ll b/llvm/test/CodeGen/AMDGPU/sminmax.v2i16.ll
index ea91318c97abd..c432a815070a1 100644
--- a/llvm/test/CodeGen/AMDGPU/sminmax.v2i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/sminmax.v2i16.ll
@@ -861,20 +861,20 @@ define amdgpu_kernel void @v_min_max_v2i16_user(ptr addrspace(1) %out0, ptr addr
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v2
-; GFX9-NEXT: v_cmp_gt_i32_sdwa vcc, sext(v1), sext(v2) src0_sel:WORD_0 src1_sel:WORD_0
; GFX9-NEXT: v_cmp_gt_i32_sdwa s[0:1], sext(v1), sext(v2) src0_sel:WORD_1 src1_sel:WORD_1
-; GFX9-NEXT: v_cndmask_b32_e32 v5, v2, v1, vcc
+; GFX9-NEXT: v_cmp_gt_i32_sdwa vcc, sext(v1), sext(v2) src0_sel:WORD_0 src1_sel:WORD_0
; GFX9-NEXT: v_cndmask_b32_e64 v6, v4, v3, s[0:1]
; GFX9-NEXT: v_cndmask_b32_e64 v3, v3, v4, s[0:1]
; GFX9-NEXT: v_cndmask_b32_e64 v4, 0, 1, s[0:1]
+; GFX9-NEXT: v_cndmask_b32_e32 v5, v2, v1, vcc
; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
; GFX9-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v4, 1, v4
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_lshl_or_b32 v5, v6, 16, v5
-; GFX9-NEXT: v_lshl_or_b32 v1, v3, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v5, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_or_b32_e32 v2, v2, v4
+; GFX9-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: global_store_dword v0, v5, s[8:9]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: global_store_dword v0, v1, s[10:11]
diff --git a/llvm/test/CodeGen/AMDGPU/strict_fsub.f16.ll b/llvm/test/CodeGen/AMDGPU/strict_fsub.f16.ll
index 1726a913dba21..b78a0871784cb 100644
--- a/llvm/test/CodeGen/AMDGPU/strict_fsub.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/strict_fsub.f16.ll
@@ -525,9 +525,9 @@ define <3 x half> @v_constained_fsub_v3f16_fpexcept_strict(<3 x half> %x, <3 x h
; GFX10-GISEL: ; %bb.0:
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-GISEL-NEXT: v_sub_f16_e32 v1, v1, v3
+; GFX10-GISEL-NEXT: v_lshlrev_b32_e64 v3, 16, s4
; GFX10-GISEL-NEXT: v_pk_add_f16 v0, v0, v2 neg_lo:[0,1] neg_hi:[0,1]
-; GFX10-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-GISEL-NEXT: v_lshl_or_b32 v1, s4, 16, v1
+; GFX10-GISEL-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: v_constained_fsub_v3f16_fpexcept_strict:
@@ -823,12 +823,12 @@ define amdgpu_ps <2 x half> @s_constained_fsub_v2f16_fpexcept_strict(<2 x half>
;
; GFX10-SDAG-LABEL: s_constained_fsub_v2f16_fpexcept_strict:
; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: v_sub_f16_e64 v0, s2, s3
; GFX10-SDAG-NEXT: s_lshr_b32 s0, s3, 16
-; GFX10-SDAG-NEXT: s_lshr_b32 s1, s2, 16
-; GFX10-SDAG-NEXT: v_sub_f16_e64 v1, s1, s0
-; GFX10-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-SDAG-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10-SDAG-NEXT: v_sub_f16_e64 v1, s2, s3
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s0
+; GFX10-SDAG-NEXT: s_lshr_b32 s0, s2, 16
+; GFX10-SDAG-NEXT: v_sub_f16_sdwa v0, s0, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-SDAG-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-SDAG-NEXT: ; return to shader part epilog
;
; GFX10-GISEL-LABEL: s_constained_fsub_v2f16_fpexcept_strict:
diff --git a/llvm/test/CodeGen/AMDGPU/v_sat_pk_u8_i16.ll b/llvm/test/CodeGen/AMDGPU/v_sat_pk_u8_i16.ll
index 9dfb53da4a2ea..58ccba8b84b56 100644
--- a/llvm/test/CodeGen/AMDGPU/v_sat_pk_u8_i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/v_sat_pk_u8_i16.ll
@@ -98,10 +98,10 @@ define <2 x i16> @basic_smax_smin(i16 %src0, i16 %src1) {
; GISEL-GFX9: ; %bb.0:
; GISEL-GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GISEL-GFX9-NEXT: v_mov_b32_e32 v2, 0xff
-; GISEL-GFX9-NEXT: v_med3_i16 v0, v0, 0, v2
; GISEL-GFX9-NEXT: v_med3_i16 v1, v1, 0, v2
-; GISEL-GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GISEL-GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GISEL-GFX9-NEXT: v_med3_i16 v0, v0, 0, v2
+; GISEL-GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GISEL-GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GISEL-GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GISEL-GFX11-TRUE16-LABEL: basic_smax_smin:
@@ -181,8 +181,8 @@ define amdgpu_kernel void @basic_smax_smin_sgpr(ptr addrspace(1) %out, i32 inreg
; SDAG-GFX9-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX9-NEXT: v_med3_i16 v2, s2, 0, v1
; SDAG-GFX9-NEXT: v_med3_i16 v1, s3, 0, v1
-; SDAG-GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; SDAG-GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v2
+; SDAG-GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SDAG-GFX9-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; SDAG-GFX9-NEXT: global_store_dword v0, v1, s[0:1]
; SDAG-GFX9-NEXT: s_endpgm
;
@@ -399,10 +399,10 @@ define <2 x i16> @basic_smin_smax(i16 %src0, i16 %src1) {
; GISEL-GFX9: ; %bb.0:
; GISEL-GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GISEL-GFX9-NEXT: v_mov_b32_e32 v2, 0xff
-; GISEL-GFX9-NEXT: v_med3_i16 v0, v0, 0, v2
; GISEL-GFX9-NEXT: v_med3_i16 v1, v1, 0, v2
-; GISEL-GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GISEL-GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GISEL-GFX9-NEXT: v_med3_i16 v0, v0, 0, v2
+; GISEL-GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GISEL-GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GISEL-GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GISEL-GFX11-TRUE16-LABEL: basic_smin_smax:
@@ -533,10 +533,10 @@ define <2 x i16> @basic_smin_smax_combined(i16 %src0, i16 %src1) {
; GISEL-GFX9: ; %bb.0:
; GISEL-GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GISEL-GFX9-NEXT: v_mov_b32_e32 v2, 0xff
-; GISEL-GFX9-NEXT: v_med3_i16 v0, v0, 0, v2
; GISEL-GFX9-NEXT: v_med3_i16 v1, v1, 0, v2
-; GISEL-GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GISEL-GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GISEL-GFX9-NEXT: v_med3_i16 v0, v0, 0, v2
+; GISEL-GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GISEL-GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GISEL-GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GISEL-GFX11-TRUE16-LABEL: basic_smin_smax_combined:
More information about the llvm-commits
mailing list