[llvm-branch-commits] [llvm] [AMDGPU] Extract byte lanes of a split vector from the 32-bit source (PR #228424)
Dmitry Sidorov via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Sat Oct 10 03:34:03 PDT 2026
https://github.com/MrSidims updated https://github.com/llvm/llvm-project/pull/228424
>From 4866939e28643493f2b28de1d41aa98ed1e92a5d Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Fri, 2 Oct 2026 12:53:19 +0200
Subject: [PATCH 1/3] [AMDGPU] Extract byte lanes of a split vector from the
32-bit source
After a <4 x i8> is split into i16 halves, a byte lane is extended
from an i16 shift of a truncate. Rewrite it on the 32-bit source as an
and of srl or a sign_extend_inreg of srl, which select to a single bit
field extract.
Uniform zero and any extends are left alone, their i16 shift is already
promoted to i32.
---
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 64 +++
llvm/lib/Target/AMDGPU/SIISelLowering.h | 1 +
llvm/test/CodeGen/AMDGPU/idot4-test.ll | 109 +++--
llvm/test/CodeGen/AMDGPU/idot4s.ll | 79 ++--
llvm/test/CodeGen/AMDGPU/idot4u.ll | 240 +++++------
llvm/test/CodeGen/AMDGPU/lshr.v2i16.ll | 9 +-
llvm/test/CodeGen/AMDGPU/min.ll | 162 ++++----
llvm/test/CodeGen/AMDGPU/permute_i8.ll | 36 +-
llvm/test/CodeGen/AMDGPU/sad.ll | 18 +-
.../test/CodeGen/AMDGPU/v4i8-byte-lane-ext.ll | 379 +++++++-----------
10 files changed, 511 insertions(+), 586 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 7be36bf8f9174..49888927e8a9c 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -1165,6 +1165,7 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
ISD::FCANONICALIZE,
ISD::SCALAR_TO_VECTOR,
ISD::ZERO_EXTEND,
+ ISD::SIGN_EXTEND,
ISD::SIGN_EXTEND_INREG,
ISD::ANY_EXTEND,
ISD::EXTRACT_VECTOR_ELT,
@@ -15056,6 +15057,22 @@ SDValue SITargetLowering::performAndCombine(SDNode *N,
}
}
+ // and (trunc (srl x, c)), mask => srl (trunc x), c
+ // if mask keeps exactly the bits that srl leaves in the low half
+ if (CRHS && VT == MVT::i16 && LHS.getOpcode() == ISD::TRUNCATE &&
+ LHS.hasOneUse() && LHS.getOperand(0).getOpcode() == ISD::SRL) {
+ auto *Shift = dyn_cast<ConstantSDNode>(LHS.getOperand(0).getOperand(1));
+ if (Shift && Shift->getZExtValue() > 0 && Shift->getZExtValue() < 16 &&
+ CRHS->getAPIntValue().isMask(16 - Shift->getZExtValue())) {
+ SDLoc SL(N);
+ SDValue Trunc =
+ DAG.getNode(ISD::TRUNCATE, SL, VT, LHS.getOperand(0).getOperand(0));
+ return DAG.getNode(
+ ISD::SRL, SL, VT, Trunc,
+ DAG.getShiftAmountConstant(Shift->getZExtValue(), VT, SL));
+ }
+ }
+
// (and (fcmp ord x, x), (fcmp une (fabs x), inf)) ->
// fp_class x, ~(s_nan | q_nan | n_infinity | p_infinity)
if (LHS.getOpcode() == ISD::SETCC && RHS.getOpcode() == ISD::SETCC) {
@@ -16056,6 +16073,9 @@ SITargetLowering::performZeroOrAnyExtendCombine(SDNode *N,
if (!Src->hasOneUse())
return SDValue();
+ if (SDValue Ext = performExtendTruncShiftCombine(N, DCI))
+ return Ext;
+
// TODO: We bail out below if SrcOffset is not in the first dword (>= 4). It's
// possible we're missing out on some combine opportunities, but we'd need to
// weigh the cost of extracting the byte from the upper dwords.
@@ -16093,6 +16113,48 @@ SITargetLowering::performZeroOrAnyExtendCombine(SDNode *N,
DAG.getConstant(PermMask, DL, MVT::i32));
}
+// zext (srl (trunc x), c) => and (srl x, c), lowbits
+// aext (srl (trunc x), c) => and (srl x, c), lowbits
+// sext (sra (trunc x), 8) => sext_inreg (srl x, 8), i8
+SDValue
+SITargetLowering::performExtendTruncShiftCombine(SDNode *N,
+ DAGCombinerInfo &DCI) const {
+ if (DCI.getDAGCombineLevel() < AfterLegalizeTypes)
+ return SDValue();
+
+ const unsigned Opc = N->getOpcode();
+ const bool IsSigned = Opc == ISD::SIGN_EXTEND;
+ SDValue Src = N->getOperand(0);
+ if (N->getValueType(0) != MVT::i32 || Src.getValueType() != MVT::i16 ||
+ Src.getOpcode() != (IsSigned ? ISD::SRA : ISD::SRL) || !Src.hasOneUse() ||
+ Src.getOperand(0).getOpcode() != ISD::TRUNCATE)
+ return SDValue();
+
+ // A uniform i16 srl is promoted to i32 later and yields the same nodes.
+ // Skip it so that the node order, and with it the schedule, stays the same.
+ if (!IsSigned && !Src->isDivergent())
+ return SDValue();
+
+ SDValue X = Src.getOperand(0).getOperand(0);
+ auto *Shift = dyn_cast<ConstantSDNode>(Src.getOperand(1));
+ if (X.getValueType() != MVT::i32 || !Shift || Shift->getZExtValue() == 0 ||
+ Shift->getZExtValue() >= 16 || (IsSigned && Shift->getZExtValue() != 8))
+ return SDValue();
+
+ SelectionDAG &DAG = DCI.DAG;
+ SDLoc SL(N);
+ const unsigned Width = 16 - Shift->getZExtValue();
+ SDValue Srl = DAG.getNode(
+ ISD::SRL, SL, MVT::i32, X,
+ DAG.getShiftAmountConstant(Shift->getZExtValue(), MVT::i32, SL));
+ if (IsSigned)
+ return DAG.getNode(ISD::SIGN_EXTEND_INREG, SL, MVT::i32, Srl,
+ DAG.getValueType(MVT::i8));
+ return DAG.getNode(
+ ISD::AND, SL, MVT::i32, Srl,
+ DAG.getConstant(maskTrailingOnes<uint32_t>(Width), SL, MVT::i32));
+}
+
SDValue
SITargetLowering::performSignExtendInRegCombine(SDNode *N,
DAGCombinerInfo &DCI) const {
@@ -19331,6 +19393,8 @@ SDValue SITargetLowering::PerformDAGCombine(SDNode *N,
case ISD::ANY_EXTEND:
case ISD::ZERO_EXTEND:
return performZeroOrAnyExtendCombine(N, DCI);
+ case ISD::SIGN_EXTEND:
+ return performExtendTruncShiftCombine(N, DCI);
case ISD::SIGN_EXTEND_INREG:
return performSignExtendInRegCombine(N, DCI);
case AMDGPUISD::FP_CLASS:
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h
index 30e58f560a232..4fffb7bd2d120 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h
@@ -229,6 +229,7 @@ class SITargetLowering final : public AMDGPUTargetLowering {
SDValue performOrCombine(SDNode *N, DAGCombinerInfo &DCI) const;
SDValue performXorCombine(SDNode *N, DAGCombinerInfo &DCI) const;
SDValue performZeroOrAnyExtendCombine(SDNode *N, DAGCombinerInfo &DCI) const;
+ SDValue performExtendTruncShiftCombine(SDNode *N, DAGCombinerInfo &DCI) const;
SDValue performSignExtendInRegCombine(SDNode *N, DAGCombinerInfo &DCI) const;
SDValue performClassCombine(SDNode *N, DAGCombinerInfo &DCI) const;
SDValue getCanonicalConstantFP(SelectionDAG &DAG, const SDLoc &SL, EVT VT,
diff --git a/llvm/test/CodeGen/AMDGPU/idot4-test.ll b/llvm/test/CodeGen/AMDGPU/idot4-test.ll
index 79017f3ea3cd7..99f518c7f256f 100644
--- a/llvm/test/CodeGen/AMDGPU/idot4-test.ll
+++ b/llvm/test/CodeGen/AMDGPU/idot4-test.ll
@@ -297,13 +297,12 @@ define i32 @udot4_unsat_bitcast(i32 %a_packed, i32 %b_packed, i32 %c) {
; GFX9-NODL-LABEL: udot4_unsat_bitcast:
; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v3, 8, v0
-; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v4, 8, v1
-; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v5, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v6, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
+; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v3, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v4, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
+; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v5, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
-; GFX9-NODL-NEXT: v_mad_u32_u24 v0, v3, v4, v0
-; GFX9-NODL-NEXT: v_add_u32_e32 v1, v5, v6
+; GFX9-NODL-NEXT: v_add_u32_e32 v0, v4, v0
+; GFX9-NODL-NEXT: v_add_u32_e32 v1, v3, v5
; GFX9-NODL-NEXT: v_add3_u32 v0, v1, v0, v2
; GFX9-NODL-NEXT: s_setpc_b64 s[30:31]
entry:
@@ -355,14 +354,12 @@ define i32 @sdot4_unsat_bitcast(i32 %a_packed, i32 %b_packed, i32 %c) {
; GFX9-NODL-LABEL: sdot4_unsat_bitcast:
; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v3, 8, v1
-; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v4, 8, v0
-; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v5, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, sext(v4), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v4, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
+; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v4, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
+; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v5, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v0, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
-; GFX9-NODL-NEXT: v_add_u32_e32 v0, v3, v0
-; GFX9-NODL-NEXT: v_add_u32_e32 v1, v5, v4
+; GFX9-NODL-NEXT: v_add_u32_e32 v0, v4, v0
+; GFX9-NODL-NEXT: v_add_u32_e32 v1, v3, v5
; GFX9-NODL-NEXT: v_add3_u32 v0, v1, v0, v2
; GFX9-NODL-NEXT: s_setpc_b64 s[30:31]
entry:
@@ -407,13 +404,12 @@ define i32 @udot4_sat_bitcast(i32 %a_packed, i32 %b_packed, i32 %c) {
; GFX9-NODL-LABEL: udot4_sat_bitcast:
; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v3, 8, v0
-; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v4, 8, v1
-; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v5, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v6, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
+; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v3, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v4, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
+; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v5, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
-; GFX9-NODL-NEXT: v_mad_u32_u24 v0, v3, v4, v0
-; GFX9-NODL-NEXT: v_add3_u32 v0, v5, v6, v0
+; GFX9-NODL-NEXT: v_add_u32_e32 v0, v4, v0
+; GFX9-NODL-NEXT: v_add3_u32 v0, v3, v5, v0
; GFX9-NODL-NEXT: v_add_u32_e64 v0, v0, v2 clamp
; GFX9-NODL-NEXT: s_setpc_b64 s[30:31]
entry:
@@ -458,14 +454,12 @@ define i32 @sdot4_sat_bitcast(i32 %a_packed, i32 %b_packed, i32 %c) {
; GFX9-NODL-LABEL: sdot4_sat_bitcast:
; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v3, 8, v1
-; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v4, 8, v0
-; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v5, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, sext(v4), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v4, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
+; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v4, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
+; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v5, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v0, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
-; GFX9-NODL-NEXT: v_add_u32_e32 v0, v3, v0
-; GFX9-NODL-NEXT: v_add3_u32 v0, v5, v4, v0
+; GFX9-NODL-NEXT: v_add_u32_e32 v0, v4, v0
+; GFX9-NODL-NEXT: v_add3_u32 v0, v3, v5, v0
; GFX9-NODL-NEXT: v_add_i32 v0, v0, v2 clamp
; GFX9-NODL-NEXT: s_setpc_b64 s[30:31]
entry:
@@ -803,13 +797,12 @@ define i32 @dot4_mismatched_sign_zext_sadd(i32 %a_packed, i32 %b_packed, i32 %c)
; GFX9-NODL-LABEL: dot4_mismatched_sign_zext_sadd:
; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v3, 8, v0
-; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v4, 8, v1
-; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v5, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v6, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
+; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v3, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v4, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
+; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v5, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
-; GFX9-NODL-NEXT: v_mad_u32_u24 v0, v3, v4, v0
-; GFX9-NODL-NEXT: v_add3_u32 v0, v5, v6, v0
+; GFX9-NODL-NEXT: v_add_u32_e32 v0, v4, v0
+; GFX9-NODL-NEXT: v_add3_u32 v0, v3, v5, v0
; GFX9-NODL-NEXT: v_add_i32 v0, v0, v2 clamp
; GFX9-NODL-NEXT: s_setpc_b64 s[30:31]
entry:
@@ -852,14 +845,12 @@ define i32 @dot4_mismatched_sign_sext_uadd(i32 %a_packed, i32 %b_packed, i32 %c)
; GFX9-NODL-LABEL: dot4_mismatched_sign_sext_uadd:
; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v3, 8, v1
-; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v4, 8, v0
-; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v5, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, sext(v4), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v4, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
+; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v4, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
+; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v5, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v0, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
-; GFX9-NODL-NEXT: v_add_u32_e32 v0, v3, v0
-; GFX9-NODL-NEXT: v_add3_u32 v0, v5, v4, v0
+; GFX9-NODL-NEXT: v_add_u32_e32 v0, v4, v0
+; GFX9-NODL-NEXT: v_add3_u32 v0, v3, v5, v0
; GFX9-NODL-NEXT: v_add_u32_e64 v0, v0, v2 clamp
; GFX9-NODL-NEXT: s_setpc_b64 s[30:31]
entry:
@@ -878,54 +869,48 @@ define i32 @dot4_mixed_ext(i32 %a_packed, i32 %b_packed, i32 %c) {
; GFX9-DL-LABEL: dot4_mixed_ext:
; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-DL-NEXT: v_lshrrev_b16_e32 v3, 8, v1
-; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v4, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v3, v0, sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_0
+; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v3, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v4, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v5, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v0, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
-; GFX9-DL-NEXT: v_add_u32_e32 v0, v3, v0
-; GFX9-DL-NEXT: v_add_u32_e32 v1, v4, v5
+; GFX9-DL-NEXT: v_add_u32_e32 v0, v4, v0
+; GFX9-DL-NEXT: v_add_u32_e32 v1, v3, v5
; GFX9-DL-NEXT: v_add3_u32 v0, v1, v0, v2
; GFX9-DL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-DL-LABEL: dot4_mixed_ext:
; GFX10-DL: ; %bb.0: ; %entry
; GFX10-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-DL-NEXT: v_mov_b32_e32 v3, 0xffff
-; GFX10-DL-NEXT: v_lshrrev_b16 v4, 8, v1
-; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v5, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v6, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
-; GFX10-DL-NEXT: v_and_b32_sdwa v3, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_1
+; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v3, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v4, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
+; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v5, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v0, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
-; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v3, v3, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX10-DL-NEXT: v_add_nc_u32_e32 v0, v5, v0
-; GFX10-DL-NEXT: v_add_nc_u32_e32 v1, v3, v6
+; GFX10-DL-NEXT: v_add_nc_u32_e32 v1, v4, v5
+; GFX10-DL-NEXT: v_add_nc_u32_e32 v0, v3, v0
; GFX10-DL-NEXT: v_add3_u32 v0, v0, v1, v2
; GFX10-DL-NEXT: s_setpc_b64 s[30:31]
;
; GFX950-LABEL: dot4_mixed_ext:
; GFX950: ; %bb.0: ; %entry
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_lshrrev_b16_e32 v3, 8, v1
-; GFX950-NEXT: v_mul_i32_i24_sdwa v4, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX950-NEXT: v_mul_i32_i24_sdwa v3, v0, sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_0
+; GFX950-NEXT: v_mul_i32_i24_sdwa v3, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX950-NEXT: v_mul_i32_i24_sdwa v4, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
; GFX950-NEXT: v_mul_i32_i24_sdwa v5, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
; GFX950-NEXT: v_mul_i32_i24_sdwa v0, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
-; GFX950-NEXT: v_add_u32_e32 v0, v3, v0
-; GFX950-NEXT: v_add_u32_e32 v1, v4, v5
+; GFX950-NEXT: v_add_u32_e32 v0, v4, v0
+; GFX950-NEXT: v_add_u32_e32 v1, v3, v5
; GFX950-NEXT: v_add3_u32 v0, v1, v0, v2
; GFX950-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-NODL-LABEL: dot4_mixed_ext:
; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v3, 8, v1
-; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v4, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, v0, sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_0
+; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v4, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v5, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v0, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
-; GFX9-NODL-NEXT: v_add_u32_e32 v0, v3, v0
-; GFX9-NODL-NEXT: v_add_u32_e32 v1, v4, v5
+; GFX9-NODL-NEXT: v_add_u32_e32 v0, v4, v0
+; GFX9-NODL-NEXT: v_add_u32_e32 v1, v3, v5
; GFX9-NODL-NEXT: v_add3_u32 v0, v1, v0, v2
; GFX9-NODL-NEXT: s_setpc_b64 s[30:31]
entry:
diff --git a/llvm/test/CodeGen/AMDGPU/idot4s.ll b/llvm/test/CodeGen/AMDGPU/idot4s.ll
index 86cf5682af484..37ef74e7c5b4e 100644
--- a/llvm/test/CodeGen/AMDGPU/idot4s.ll
+++ b/llvm/test/CodeGen/AMDGPU/idot4s.ll
@@ -819,18 +819,15 @@ define amdgpu_kernel void @idot4_acc32_vecMul(ptr addrspace(1) %src1,
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_dword v0, v[0:1]
; GFX8-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX8-NEXT: s_waitcnt vmcnt(1)
-; GFX8-NEXT: v_lshrrev_b16_e32 v1, 8, v3
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_lshrrev_b16_e32 v2, 8, v0
-; GFX8-NEXT: v_mul_i32_i24_sdwa v4, sext(v3), sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT: v_mul_i32_i24_sdwa v1, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-NEXT: v_mul_i32_i24_sdwa v1, sext(v3), sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-NEXT: v_mul_i32_i24_sdwa v2, sext(v3), sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v2, vcc, s0, v4
-; GFX8-NEXT: v_mul_i32_i24_sdwa v5, sext(v3), sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
-; GFX8-NEXT: v_add_u32_e32 v1, vcc, v2, v1
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, s0, v1
+; GFX8-NEXT: v_mul_i32_i24_sdwa v4, sext(v3), sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v2
; GFX8-NEXT: v_mul_i32_i24_sdwa v0, sext(v3), sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
-; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v5
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v4
; GFX8-NEXT: v_add_u32_e32 v2, vcc, v1, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s4
; GFX8-NEXT: v_mov_b32_e32 v1, s5
@@ -847,17 +844,14 @@ define amdgpu_kernel void @idot4_acc32_vecMul(ptr addrspace(1) %src1,
; GFX9-NODL-NEXT: global_load_dword v2, v0, s[2:3]
; GFX9-NODL-NEXT: s_load_dword s0, s[6:7], 0x0
; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v3, 8, v1
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v4, 8, v2
-; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v5, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, sext(v3), sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v6, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
+; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v4, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
+; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v5, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v1, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: v_add3_u32 v2, v5, s0, v3
-; GFX9-NODL-NEXT: v_add3_u32 v1, v2, v6, v1
+; GFX9-NODL-NEXT: v_add3_u32 v2, v3, s0, v4
+; GFX9-NODL-NEXT: v_add3_u32 v1, v2, v5, v1
; GFX9-NODL-NEXT: global_store_dword v0, v1, s[6:7]
; GFX9-NODL-NEXT: s_endpgm
;
@@ -3247,9 +3241,8 @@ define amdgpu_kernel void @idot4_nonstandard_signed(ptr addrspace(1) %src1,
; GFX8-NEXT: v_bfe_i32 v5, v5, 0, 8
; GFX8-NEXT: v_bfe_i32 v3, v3, 0, 8
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_lshrrev_b32_e32 v9, 8, v2
; GFX8-NEXT: v_and_b32_e32 v7, 0xff, v2
-; GFX8-NEXT: v_mul_lo_u16_sdwa v8, v9, sext(v8) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-NEXT: v_mul_lo_u16_sdwa v8, v2, sext(v8) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_0
; GFX8-NEXT: v_and_b32_sdwa v4, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX8-NEXT: v_mad_u16 v6, v6, v7, v8
; GFX8-NEXT: v_mad_u16 v4, v4, v5, v6
@@ -3271,18 +3264,17 @@ define amdgpu_kernel void @idot4_nonstandard_signed(ptr addrspace(1) %src1,
; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v6, 8, v1
-; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v7, 8, v2
; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; GFX9-NODL-NEXT: v_bfe_i32 v4, v1, 0, 8
+; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
; GFX9-NODL-NEXT: v_and_b32_e32 v5, 0xff, v2
-; GFX9-NODL-NEXT: v_mul_lo_u16_sdwa v6, v7, sext(v6) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NODL-NEXT: v_and_b32_sdwa v8, v2, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NODL-NEXT: v_mul_lo_u16_sdwa v6, v2, sext(v6) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_0
+; GFX9-NODL-NEXT: v_and_b32_sdwa v7, v2, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v1, 24, v1
; GFX9-NODL-NEXT: v_bfe_i32 v3, v3, 0, 8
; GFX9-NODL-NEXT: v_mad_legacy_u16 v4, v4, v5, v6
; GFX9-NODL-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX9-NODL-NEXT: v_mad_legacy_u16 v3, v8, v3, v4
+; GFX9-NODL-NEXT: v_mad_legacy_u16 v3, v7, v3, v4
; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v2, 24, v2
; GFX9-NODL-NEXT: v_mad_legacy_u16 v1, v1, v2, v3
; GFX9-NODL-NEXT: v_bfe_i32 v1, v1, 0, 16
@@ -3301,18 +3293,17 @@ define amdgpu_kernel void @idot4_nonstandard_signed(ptr addrspace(1) %src1,
; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
; GFX9-DL-NEXT: v_lshrrev_b32_e32 v6, 8, v1
-; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DL-NEXT: v_lshrrev_b32_e32 v7, 8, v2
; GFX9-DL-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; GFX9-DL-NEXT: v_bfe_i32 v4, v1, 0, 8
+; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
; GFX9-DL-NEXT: v_and_b32_e32 v5, 0xff, v2
-; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v6, v7, sext(v6) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-DL-NEXT: v_and_b32_sdwa v8, v2, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v6, v2, sext(v6) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_0
+; GFX9-DL-NEXT: v_and_b32_sdwa v7, v2, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX9-DL-NEXT: v_lshrrev_b32_e32 v1, 24, v1
; GFX9-DL-NEXT: v_bfe_i32 v3, v3, 0, 8
; GFX9-DL-NEXT: v_mad_legacy_u16 v4, v4, v5, v6
; GFX9-DL-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX9-DL-NEXT: v_mad_legacy_u16 v3, v8, v3, v4
+; GFX9-DL-NEXT: v_mad_legacy_u16 v3, v7, v3, v4
; GFX9-DL-NEXT: v_lshrrev_b32_e32 v2, 24, v2
; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v1, v2, v3
; GFX9-DL-NEXT: v_bfe_i32 v1, v1, 0, 16
@@ -3333,12 +3324,11 @@ define amdgpu_kernel void @idot4_nonstandard_signed(ptr addrspace(1) %src1,
; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
; GFX10-DL-NEXT: v_lshrrev_b32_e32 v0, 8, v1
; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
-; GFX10-DL-NEXT: v_lshrrev_b32_e32 v3, 8, v2
+; GFX10-DL-NEXT: v_lshrrev_b16 v3, 8, v2
; GFX10-DL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
; GFX10-DL-NEXT: v_bfe_i32 v6, v1, 0, 8
; GFX10-DL-NEXT: v_and_b32_e32 v7, 0xff, v2
; GFX10-DL-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX10-DL-NEXT: v_and_b32_e32 v3, 0xff, v3
; GFX10-DL-NEXT: v_lshrrev_b32_e32 v1, 24, v1
; GFX10-DL-NEXT: v_mul_lo_u16 v0, v3, v0
; GFX10-DL-NEXT: v_and_b32_sdwa v3, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
@@ -3366,32 +3356,28 @@ define amdgpu_kernel void @idot4_nonstandard_signed(ptr addrspace(1) %src1,
; GFX11-DL-TRUE16-NEXT: global_load_b32 v4, v0, s[2:3]
; GFX11-DL-TRUE16-NEXT: s_waitcnt vmcnt(1)
; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 8, v3
+; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v2, v3, 0, 8
+; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 24, v3
+; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_3)
+; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v1, v0, 0, 8
; GFX11-DL-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v4
-; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v5, v3, 0, 8
-; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.h
-; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v2, v0, 0, 8
-; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-DL-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v1.l
+; GFX11-DL-TRUE16-NEXT: v_lshrrev_b16 v0.l, 8, v4.l
; GFX11-DL-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v4.l
-; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v2.l, v5.l
-; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v5, v6, 0, 8
-; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 24, v3
-; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v5, v5, 0, 8
; GFX11-DL-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v1.l
; GFX11-DL-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v4.h
+; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 24, v4
-; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-DL-TRUE16-NEXT: v_mad_u16 v0.l, v2.l, v0.h, v0.l
; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v2, v6, 0, 8
+; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-DL-TRUE16-NEXT: v_mad_u16 v0.l, v1.l, v3.l, v0.l
-; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
+; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-DL-TRUE16-NEXT: v_mad_u16 v0.l, v1.l, v4.l, v0.l
; GFX11-DL-TRUE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 16
; GFX11-DL-TRUE16-NEXT: global_store_b32 v1, v0, s[4:5]
; GFX11-DL-TRUE16-NEXT: s_endpgm
@@ -3410,12 +3396,11 @@ define amdgpu_kernel void @idot4_nonstandard_signed(ptr addrspace(1) %src1,
; GFX11-DL-FAKE16-NEXT: s_waitcnt vmcnt(1)
; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 8, v1
; GFX11-DL-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 8, v0
+; GFX11-DL-FAKE16-NEXT: v_lshrrev_b16 v3, 8, v0
; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 16, v1
; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v0
; GFX11-DL-FAKE16-NEXT: v_bfe_i32 v6, v1, 0, 8
; GFX11-DL-FAKE16-NEXT: v_bfe_i32 v2, v2, 0, 8
-; GFX11-DL-FAKE16-NEXT: v_and_b32_e32 v3, 0xff, v3
; GFX11-DL-FAKE16-NEXT: v_and_b32_e32 v7, 0xff, v0
; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 24, v1
; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 24, v0
diff --git a/llvm/test/CodeGen/AMDGPU/idot4u.ll b/llvm/test/CodeGen/AMDGPU/idot4u.ll
index b83694e9d89d7..aa3da5a0315db 100644
--- a/llvm/test/CodeGen/AMDGPU/idot4u.ll
+++ b/llvm/test/CodeGen/AMDGPU/idot4u.ll
@@ -242,17 +242,15 @@ define amdgpu_kernel void @udot4_acc16(ptr addrspace(1) %src1,
; GFX8-NEXT: flat_load_ushort v4, v[0:1]
; GFX8-NEXT: s_waitcnt vmcnt(2)
; GFX8-NEXT: v_and_b32_e32 v6, 0xff, v3
-; GFX8-NEXT: v_lshrrev_b32_e32 v8, 8, v3
-; GFX8-NEXT: v_and_b32_e32 v8, 0xff, v8
+; GFX8-NEXT: v_lshrrev_b16_e32 v8, 8, v3
; GFX8-NEXT: v_and_b32_sdwa v10, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX8-NEXT: v_lshrrev_b32_e32 v3, 24, v3
; GFX8-NEXT: s_waitcnt vmcnt(1)
; GFX8-NEXT: v_and_b32_e32 v7, 0xff, v2
-; GFX8-NEXT: v_lshrrev_b32_e32 v9, 8, v2
-; GFX8-NEXT: v_and_b32_e32 v9, 0xff, v9
+; GFX8-NEXT: v_lshrrev_b16_e32 v9, 8, v2
+; GFX8-NEXT: v_and_b32_sdwa v5, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mad_u16 v4, v6, v7, v4
-; GFX8-NEXT: v_and_b32_sdwa v5, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX8-NEXT: v_mad_u16 v4, v8, v9, v4
; GFX8-NEXT: v_lshrrev_b32_e32 v2, 24, v2
; GFX8-NEXT: v_mad_u16 v4, v10, v5, v4
@@ -275,10 +273,8 @@ define amdgpu_kernel void @udot4_acc16(ptr addrspace(1) %src1,
; GFX9-NODL-NEXT: v_and_b32_e32 v4, 0xff, v1
; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
; GFX9-NODL-NEXT: v_and_b32_e32 v5, 0xff, v2
-; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v6, 8, v1
-; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v7, 8, v2
-; GFX9-NODL-NEXT: v_and_b32_e32 v6, 0xff, v6
-; GFX9-NODL-NEXT: v_and_b32_e32 v7, 0xff, v7
+; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v6, 8, v1
+; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v7, 8, v2
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
; GFX9-NODL-NEXT: v_mad_legacy_u16 v3, v4, v5, v3
; GFX9-NODL-NEXT: v_and_b32_sdwa v8, v1, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
@@ -1534,18 +1530,16 @@ define amdgpu_kernel void @notdot4_mixedtypes(ptr addrspace(1) %src1,
; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: flat_load_ushort v4, v[0:1]
; GFX8-NEXT: s_waitcnt vmcnt(2)
-; GFX8-NEXT: v_lshrrev_b32_e32 v8, 8, v3
-; GFX8-NEXT: v_and_b32_e32 v8, 0xff, v8
+; GFX8-NEXT: v_lshrrev_b16_e32 v8, 8, v3
; GFX8-NEXT: v_bfe_i32 v6, v3, 0, 8
; GFX8-NEXT: v_and_b32_sdwa v10, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX8-NEXT: v_lshrrev_b32_e32 v3, 24, v3
; GFX8-NEXT: s_waitcnt vmcnt(1)
-; GFX8-NEXT: v_lshrrev_b32_e32 v9, 8, v2
-; GFX8-NEXT: v_and_b32_e32 v9, 0xff, v9
+; GFX8-NEXT: v_lshrrev_b16_e32 v9, 8, v2
; GFX8-NEXT: v_bfe_i32 v7, v2, 0, 8
+; GFX8-NEXT: v_and_b32_sdwa v5, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mad_u16 v4, v8, v9, v4
-; GFX8-NEXT: v_and_b32_sdwa v5, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX8-NEXT: v_mad_u16 v4, v6, v7, v4
; GFX8-NEXT: v_lshrrev_b32_e32 v2, 24, v2
; GFX8-NEXT: v_mad_u16 v4, v10, v5, v4
@@ -1565,11 +1559,9 @@ define amdgpu_kernel void @notdot4_mixedtypes(ptr addrspace(1) %src1,
; GFX9-NODL-NEXT: global_load_ushort v3, v0, s[6:7]
; GFX9-NODL-NEXT: s_movk_i32 s0, 0xff
; GFX9-NODL-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v6, 8, v1
+; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v6, 8, v1
; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v7, 8, v2
-; GFX9-NODL-NEXT: v_and_b32_e32 v6, 0xff, v6
-; GFX9-NODL-NEXT: v_and_b32_e32 v7, 0xff, v7
+; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v7, 8, v2
; GFX9-NODL-NEXT: v_bfe_i32 v4, v1, 0, 8
; GFX9-NODL-NEXT: v_bfe_i32 v5, v2, 0, 8
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
@@ -1596,11 +1588,9 @@ define amdgpu_kernel void @notdot4_mixedtypes(ptr addrspace(1) %src1,
; GFX9-DL-NEXT: global_load_ushort v3, v0, s[6:7]
; GFX9-DL-NEXT: s_mov_b32 s0, 0xc0c0302
; GFX9-DL-NEXT: s_waitcnt vmcnt(2)
-; GFX9-DL-NEXT: v_lshrrev_b32_e32 v6, 8, v1
+; GFX9-DL-NEXT: v_lshrrev_b16_e32 v6, 8, v1
; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
-; GFX9-DL-NEXT: v_lshrrev_b32_e32 v7, 8, v2
-; GFX9-DL-NEXT: v_and_b32_e32 v6, 0xff, v6
-; GFX9-DL-NEXT: v_and_b32_e32 v7, 0xff, v7
+; GFX9-DL-NEXT: v_lshrrev_b16_e32 v7, 8, v2
; GFX9-DL-NEXT: v_bfe_i32 v4, v1, 0, 8
; GFX9-DL-NEXT: v_bfe_i32 v5, v2, 0, 8
; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
@@ -1618,96 +1608,89 @@ define amdgpu_kernel void @notdot4_mixedtypes(ptr addrspace(1) %src1,
; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: s_clause 0x1
-; GFX10-DL-NEXT: global_load_dword v1, v0, s[0:1]
-; GFX10-DL-NEXT: global_load_dword v2, v0, s[2:3]
-; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-DL-NEXT: global_load_ushort v3, v0, s[6:7]
+; GFX10-DL-NEXT: global_load_dword v2, v0, s[0:1]
+; GFX10-DL-NEXT: global_load_dword v3, v0, s[2:3]
+; GFX10-DL-NEXT: global_load_ushort v4, v1, s[6:7]
; GFX10-DL-NEXT: s_waitcnt vmcnt(2)
-; GFX10-DL-NEXT: v_lshrrev_b32_e32 v4, 8, v1
+; GFX10-DL-NEXT: v_lshrrev_b16 v0, 8, v2
; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
-; GFX10-DL-NEXT: v_lshrrev_b32_e32 v5, 8, v2
-; GFX10-DL-NEXT: v_bfe_i32 v6, v1, 0, 8
-; GFX10-DL-NEXT: v_bfe_i32 v7, v2, 0, 8
+; GFX10-DL-NEXT: v_lshrrev_b16 v5, 8, v3
+; GFX10-DL-NEXT: v_bfe_i32 v6, v2, 0, 8
+; GFX10-DL-NEXT: v_bfe_i32 v7, v3, 0, 8
+; GFX10-DL-NEXT: v_perm_b32 v3, v3, v3, 0xc0c0302
; GFX10-DL-NEXT: v_perm_b32 v2, v2, v2, 0xc0c0302
-; GFX10-DL-NEXT: v_and_b32_e32 v4, 0xff, v4
-; GFX10-DL-NEXT: v_and_b32_e32 v5, 0xff, v5
-; GFX10-DL-NEXT: v_perm_b32 v1, v1, v1, 0xc0c0302
; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
-; GFX10-DL-NEXT: v_mad_u16 v3, v4, v5, v3
-; GFX10-DL-NEXT: v_mad_u16 v3, v6, v7, v3
-; GFX10-DL-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX10-DL-NEXT: v_dot4_u32_u8 v1, v1, v2, v3
-; GFX10-DL-NEXT: global_store_short v0, v1, s[6:7]
+; GFX10-DL-NEXT: v_mad_u16 v0, v0, v5, v4
+; GFX10-DL-NEXT: v_mad_u16 v0, v6, v7, v0
+; GFX10-DL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX10-DL-NEXT: v_dot4_u32_u8 v0, v2, v3, v0
+; GFX10-DL-NEXT: global_store_short v1, v0, s[6:7]
; GFX10-DL-NEXT: s_endpgm
;
; GFX11-DL-TRUE16-LABEL: notdot4_mixedtypes:
; GFX11-DL-TRUE16: ; %bb.0: ; %entry
+; GFX11-DL-TRUE16-NEXT: s_clause 0x1
; GFX11-DL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-DL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX11-DL-TRUE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
-; GFX11-DL-TRUE16-NEXT: v_mov_b32_e32 v6, 0
-; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-DL-TRUE16-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-DL-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX11-DL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-DL-TRUE16-NEXT: s_clause 0x1
; GFX11-DL-TRUE16-NEXT: global_load_b32 v4, v0, s[0:1]
; GFX11-DL-TRUE16-NEXT: global_load_b32 v5, v0, s[2:3]
-; GFX11-DL-TRUE16-NEXT: global_load_d16_b16 v0, v6, s[4:5]
-; GFX11-DL-TRUE16-NEXT: s_waitcnt vmcnt(2)
-; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v4
-; GFX11-DL-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 8, v5
-; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v3, v4, 0, 8
-; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v7, v5, 0, 8
+; GFX11-DL-TRUE16-NEXT: global_load_d16_b16 v0, v3, s[4:5]
; GFX11-DL-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-DL-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.l
-; GFX11-DL-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v2.l, v3.l
-; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v3.l, v7.l
-; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-DL-TRUE16-NEXT: v_lshrrev_b16 v0.h, 8, v4.l
+; GFX11-DL-TRUE16-NEXT: v_lshrrev_b16 v1.l, 8, v5.l
+; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v2, v4, 0, 8
+; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v6, v5, 0, 8
+; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-DL-TRUE16-NEXT: v_mad_u16 v0.l, v0.h, v1.l, v0.l
+; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
+; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v2.l, v6.l
+; GFX11-DL-TRUE16-NEXT: v_mad_u16 v0.l, v1.l, v2.l, v0.l
; GFX11-DL-TRUE16-NEXT: v_perm_b32 v1, v5, v5, 0xc0c0302
-; GFX11-DL-TRUE16-NEXT: v_mad_u16 v0.l, v2.l, v3.l, v0.l
; GFX11-DL-TRUE16-NEXT: v_perm_b32 v2, v4, v4, 0xc0c0302
-; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-DL-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX11-DL-TRUE16-NEXT: v_dot4_u32_u8 v0, v2, v1, v0
-; GFX11-DL-TRUE16-NEXT: global_store_b16 v6, v0, s[4:5]
+; GFX11-DL-TRUE16-NEXT: global_store_b16 v3, v0, s[4:5]
; GFX11-DL-TRUE16-NEXT: s_endpgm
;
; GFX11-DL-FAKE16-LABEL: notdot4_mixedtypes:
; GFX11-DL-FAKE16: ; %bb.0: ; %entry
+; GFX11-DL-FAKE16-NEXT: s_clause 0x1
; GFX11-DL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-DL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX11-DL-FAKE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
-; GFX11-DL-FAKE16-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-DL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-DL-FAKE16-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-DL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-DL-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX11-DL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-DL-FAKE16-NEXT: s_clause 0x1
-; GFX11-DL-FAKE16-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX11-DL-FAKE16-NEXT: global_load_b32 v2, v0, s[0:1]
; GFX11-DL-FAKE16-NEXT: global_load_b32 v0, v0, s[2:3]
-; GFX11-DL-FAKE16-NEXT: global_load_u16 v3, v2, s[4:5]
+; GFX11-DL-FAKE16-NEXT: global_load_u16 v3, v1, s[4:5]
; GFX11-DL-FAKE16-NEXT: s_waitcnt vmcnt(2)
-; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 8, v1
+; GFX11-DL-FAKE16-NEXT: v_lshrrev_b16 v4, 8, v2
; GFX11-DL-FAKE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 8, v0
-; GFX11-DL-FAKE16-NEXT: v_bfe_i32 v6, v1, 0, 8
+; GFX11-DL-FAKE16-NEXT: v_lshrrev_b16 v5, 8, v0
+; GFX11-DL-FAKE16-NEXT: v_bfe_i32 v6, v2, 0, 8
; GFX11-DL-FAKE16-NEXT: v_bfe_i32 v7, v0, 0, 8
; GFX11-DL-FAKE16-NEXT: v_perm_b32 v0, v0, v0, 0xc0c0302
-; GFX11-DL-FAKE16-NEXT: v_and_b32_e32 v4, 0xff, v4
-; GFX11-DL-FAKE16-NEXT: v_and_b32_e32 v5, 0xff, v5
-; GFX11-DL-FAKE16-NEXT: v_perm_b32 v1, v1, v1, 0xc0c0302
+; GFX11-DL-FAKE16-NEXT: v_perm_b32 v2, v2, v2, 0xc0c0302
; GFX11-DL-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-DL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-DL-FAKE16-NEXT: v_mad_u16 v3, v4, v5, v3
-; GFX11-DL-FAKE16-NEXT: v_mad_u16 v3, v6, v7, v3
; GFX11-DL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-DL-FAKE16-NEXT: v_mad_u16 v3, v6, v7, v3
; GFX11-DL-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX11-DL-FAKE16-NEXT: v_dot4_u32_u8 v0, v1, v0, v3
-; GFX11-DL-FAKE16-NEXT: global_store_b16 v2, v0, s[4:5]
+; GFX11-DL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-DL-FAKE16-NEXT: v_dot4_u32_u8 v0, v2, v0, v3
+; GFX11-DL-FAKE16-NEXT: global_store_b16 v1, v0, s[4:5]
; GFX11-DL-FAKE16-NEXT: s_endpgm
ptr addrspace(1) %src2,
ptr addrspace(1) nocapture %dst) {
@@ -1811,8 +1794,7 @@ define amdgpu_kernel void @notdot4_mixedtypes2(ptr addrspace(1) %src1,
; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: flat_load_ushort v4, v[0:1]
; GFX8-NEXT: s_waitcnt vmcnt(2)
-; GFX8-NEXT: v_lshrrev_b32_e32 v9, 8, v3
-; GFX8-NEXT: v_and_b32_e32 v9, 0xff, v9
+; GFX8-NEXT: v_lshrrev_b16_e32 v9, 8, v3
; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v3
; GFX8-NEXT: v_bfe_i32 v7, v3, 0, 8
; GFX8-NEXT: v_bfe_i32 v6, v6, 0, 8
@@ -1844,10 +1826,9 @@ define amdgpu_kernel void @notdot4_mixedtypes2(ptr addrspace(1) %src1,
; GFX9-NODL-NEXT: global_load_ushort v3, v0, s[6:7]
; GFX9-NODL-NEXT: s_movk_i32 s0, 0xff
; GFX9-NODL-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v7, 8, v1
+; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v7, 8, v1
; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v8, 8, v2
-; GFX9-NODL-NEXT: v_and_b32_e32 v7, 0xff, v7
; GFX9-NODL-NEXT: v_bfe_i32 v8, v8, 0, 8
; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v4, 16, v1
; GFX9-NODL-NEXT: v_bfe_i32 v5, v1, 0, 8
@@ -1877,10 +1858,9 @@ define amdgpu_kernel void @notdot4_mixedtypes2(ptr addrspace(1) %src1,
; GFX9-DL-NEXT: global_load_ushort v3, v0, s[6:7]
; GFX9-DL-NEXT: s_movk_i32 s0, 0xff
; GFX9-DL-NEXT: s_waitcnt vmcnt(2)
-; GFX9-DL-NEXT: v_lshrrev_b32_e32 v7, 8, v1
+; GFX9-DL-NEXT: v_lshrrev_b16_e32 v7, 8, v1
; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
; GFX9-DL-NEXT: v_lshrrev_b32_e32 v8, 8, v2
-; GFX9-DL-NEXT: v_and_b32_e32 v7, 0xff, v7
; GFX9-DL-NEXT: v_bfe_i32 v8, v8, 0, 8
; GFX9-DL-NEXT: v_lshrrev_b32_e32 v4, 16, v1
; GFX9-DL-NEXT: v_bfe_i32 v5, v1, 0, 8
@@ -1907,37 +1887,37 @@ define amdgpu_kernel void @notdot4_mixedtypes2(ptr addrspace(1) %src1,
; GFX10-DL-NEXT: v_mov_b32_e32 v8, 0xff
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: s_clause 0x1
-; GFX10-DL-NEXT: global_load_dword v1, v0, s[0:1]
-; GFX10-DL-NEXT: global_load_dword v2, v0, s[2:3]
+; GFX10-DL-NEXT: global_load_dword v1, v0, s[2:3]
+; GFX10-DL-NEXT: global_load_dword v2, v0, s[0:1]
; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX10-DL-NEXT: global_load_ushort v3, v0, s[6:7]
; GFX10-DL-NEXT: s_waitcnt vmcnt(2)
; GFX10-DL-NEXT: v_lshrrev_b32_e32 v4, 8, v1
; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
-; GFX10-DL-NEXT: v_lshrrev_b32_e32 v5, 8, v2
-; GFX10-DL-NEXT: v_lshrrev_b32_e32 v6, 16, v1
-; GFX10-DL-NEXT: v_bfe_i32 v7, v1, 0, 8
-; GFX10-DL-NEXT: v_and_b32_e32 v9, 0xff, v2
-; GFX10-DL-NEXT: v_and_b32_e32 v4, 0xff, v4
-; GFX10-DL-NEXT: v_bfe_i32 v5, v5, 0, 8
-; GFX10-DL-NEXT: v_lshrrev_b32_e32 v1, 24, v1
+; GFX10-DL-NEXT: v_lshrrev_b16 v5, 8, v2
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v6, 16, v2
+; GFX10-DL-NEXT: v_bfe_i32 v7, v2, 0, 8
+; GFX10-DL-NEXT: v_and_b32_e32 v9, 0xff, v1
+; GFX10-DL-NEXT: v_bfe_i32 v4, v4, 0, 8
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v2, 24, v2
; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
-; GFX10-DL-NEXT: v_mad_u16 v3, v4, v5, v3
+; GFX10-DL-NEXT: v_mad_u16 v3, v5, v4, v3
; GFX10-DL-NEXT: v_bfe_i32 v4, v6, 0, 8
-; GFX10-DL-NEXT: v_and_b32_sdwa v5, v2, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-DL-NEXT: v_lshrrev_b32_e32 v2, 24, v2
+; GFX10-DL-NEXT: v_and_b32_sdwa v5, v1, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v1, 24, v1
; GFX10-DL-NEXT: v_mad_u16 v3, v7, v9, v3
-; GFX10-DL-NEXT: v_bfe_i32 v2, v2, 0, 8
+; GFX10-DL-NEXT: v_bfe_i32 v1, v1, 0, 8
; GFX10-DL-NEXT: v_mad_u16 v3, v4, v5, v3
-; GFX10-DL-NEXT: v_mad_u16 v1, v1, v2, v3
+; GFX10-DL-NEXT: v_mad_u16 v1, v2, v1, v3
; GFX10-DL-NEXT: global_store_short v0, v1, s[6:7]
; GFX10-DL-NEXT: s_endpgm
;
; GFX11-DL-TRUE16-LABEL: notdot4_mixedtypes2:
; GFX11-DL-TRUE16: ; %bb.0: ; %entry
+; GFX11-DL-TRUE16-NEXT: s_clause 0x1
; GFX11-DL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-DL-TRUE16-NEXT: v_dual_mov_b32 v5, 0 :: v_dual_and_b32 v0, 0x3ff, v0
; GFX11-DL-TRUE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-DL-TRUE16-NEXT: v_dual_mov_b32 v5, 0 :: v_dual_and_b32 v0, 0x3ff, v0
; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-DL-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX11-DL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
@@ -1948,23 +1928,22 @@ define amdgpu_kernel void @notdot4_mixedtypes2(ptr addrspace(1) %src1,
; GFX11-DL-TRUE16-NEXT: s_waitcnt vmcnt(2)
; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
; GFX11-DL-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 8, v4
-; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v6, v4, 0, 8
-; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.h
-; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.h
; GFX11-DL-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-DL-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v2.l
+; GFX11-DL-TRUE16-NEXT: v_lshrrev_b16 v0.h, 8, v4.l
+; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v2, v4, 0, 8
+; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v3
+; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8
; GFX11-DL-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v3.l
-; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v7, v7, 0, 8
-; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v2.l, v6.l
-; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 24, v3
+; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v6, v6, 0, 8
+; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-DL-TRUE16-NEXT: v_mad_u16 v0.l, v0.h, v1.l, v0.l
; GFX11-DL-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v3.h
-; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v1.l, v7.l
+; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v1.l, v6.l
; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v4
; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-DL-TRUE16-NEXT: v_mad_u16 v0.l, v2.l, v1.h, v0.l
-; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v2, v6, 0, 8
+; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v2, v7, 0, 8
; GFX11-DL-TRUE16-NEXT: v_mad_u16 v0.l, v1.l, v0.h, v0.l
; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
@@ -1974,39 +1953,39 @@ define amdgpu_kernel void @notdot4_mixedtypes2(ptr addrspace(1) %src1,
;
; GFX11-DL-FAKE16-LABEL: notdot4_mixedtypes2:
; GFX11-DL-FAKE16: ; %bb.0: ; %entry
+; GFX11-DL-FAKE16-NEXT: s_clause 0x1
; GFX11-DL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-DL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX11-DL-FAKE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-DL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX11-DL-FAKE16-NEXT: v_mov_b32_e32 v2, 0
; GFX11-DL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-DL-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX11-DL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-DL-FAKE16-NEXT: s_clause 0x1
-; GFX11-DL-FAKE16-NEXT: global_load_b32 v1, v0, s[0:1]
-; GFX11-DL-FAKE16-NEXT: global_load_b32 v0, v0, s[2:3]
+; GFX11-DL-FAKE16-NEXT: global_load_b32 v1, v0, s[2:3]
+; GFX11-DL-FAKE16-NEXT: global_load_b32 v0, v0, s[0:1]
; GFX11-DL-FAKE16-NEXT: global_load_u16 v3, v2, s[4:5]
; GFX11-DL-FAKE16-NEXT: s_waitcnt vmcnt(2)
; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 8, v1
; GFX11-DL-FAKE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 8, v0
-; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 16, v1
-; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v7, 16, v0
-; GFX11-DL-FAKE16-NEXT: v_bfe_i32 v8, v1, 0, 8
-; GFX11-DL-FAKE16-NEXT: v_and_b32_e32 v4, 0xff, v4
-; GFX11-DL-FAKE16-NEXT: v_bfe_i32 v5, v5, 0, 8
-; GFX11-DL-FAKE16-NEXT: v_and_b32_e32 v9, 0xff, v0
-; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 24, v0
+; GFX11-DL-FAKE16-NEXT: v_lshrrev_b16 v5, 8, v0
+; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v7, 16, v1
+; GFX11-DL-FAKE16-NEXT: v_bfe_i32 v8, v0, 0, 8
+; GFX11-DL-FAKE16-NEXT: v_bfe_i32 v4, v4, 0, 8
+; GFX11-DL-FAKE16-NEXT: v_and_b32_e32 v9, 0xff, v1
; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 24, v1
+; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 24, v0
; GFX11-DL-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-DL-FAKE16-NEXT: v_mad_u16 v3, v4, v5, v3
+; GFX11-DL-FAKE16-NEXT: v_mad_u16 v3, v5, v4, v3
; GFX11-DL-FAKE16-NEXT: v_bfe_i32 v4, v6, 0, 8
; GFX11-DL-FAKE16-NEXT: v_and_b32_e32 v5, 0xff, v7
-; GFX11-DL-FAKE16-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX11-DL-FAKE16-NEXT: v_bfe_i32 v1, v1, 0, 8
; GFX11-DL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-DL-FAKE16-NEXT: v_mad_u16 v3, v8, v9, v3
; GFX11-DL-FAKE16-NEXT: v_mad_u16 v3, v4, v5, v3
; GFX11-DL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-DL-FAKE16-NEXT: v_mad_u16 v0, v1, v0, v3
+; GFX11-DL-FAKE16-NEXT: v_mad_u16 v0, v0, v1, v3
; GFX11-DL-FAKE16-NEXT: global_store_b16 v2, v0, s[4:5]
; GFX11-DL-FAKE16-NEXT: s_endpgm
ptr addrspace(1) %src2,
@@ -2103,17 +2082,15 @@ define amdgpu_kernel void @udot4_acc32_vecMul(ptr addrspace(1) %src1,
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_dword v0, v[0:1]
; GFX8-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX8-NEXT: s_waitcnt vmcnt(1)
-; GFX8-NEXT: v_lshrrev_b16_e32 v1, 8, v3
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mul_u32_u24_sdwa v4, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT: v_lshrrev_b16_e32 v2, 8, v0
-; GFX8-NEXT: v_mul_u32_u24_sdwa v5, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
-; GFX8-NEXT: v_mul_u32_u24_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
+; GFX8-NEXT: v_mul_u32_u24_sdwa v1, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-NEXT: v_mul_u32_u24_sdwa v2, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v3, vcc, s0, v4
-; GFX8-NEXT: v_mad_u32_u24 v1, v1, v2, v3
-; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v5
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, s0, v1
+; GFX8-NEXT: v_mul_u32_u24_sdwa v4, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v2
+; GFX8-NEXT: v_mul_u32_u24_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v4
; GFX8-NEXT: v_add_u32_e32 v2, vcc, v1, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s4
; GFX8-NEXT: v_mov_b32_e32 v1, s5
@@ -2130,17 +2107,14 @@ define amdgpu_kernel void @udot4_acc32_vecMul(ptr addrspace(1) %src1,
; GFX9-NODL-NEXT: global_load_dword v2, v0, s[2:3]
; GFX9-NODL-NEXT: s_load_dword s0, s[6:7], 0x0
; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v3, 8, v1
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v4, 8, v2
-; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v6, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
+; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v3, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v4, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
+; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: v_add_u32_e32 v2, s0, v5
-; GFX9-NODL-NEXT: v_mad_u32_u24 v2, v3, v4, v2
-; GFX9-NODL-NEXT: v_add3_u32 v1, v2, v6, v1
+; GFX9-NODL-NEXT: v_add3_u32 v2, v3, s0, v4
+; GFX9-NODL-NEXT: v_add3_u32 v1, v2, v5, v1
; GFX9-NODL-NEXT: global_store_dword v0, v1, s[6:7]
; GFX9-NODL-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/lshr.v2i16.ll b/llvm/test/CodeGen/AMDGPU/lshr.v2i16.ll
index 8252a4422febb..214486cd7ab12 100644
--- a/llvm/test/CodeGen/AMDGPU/lshr.v2i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/lshr.v2i16.ll
@@ -453,13 +453,14 @@ define amdgpu_kernel void @lshr_v_imm_v2i16(ptr addrspace(1) %out, ptr addrspace
; VI-NEXT: v_add_u32_e32 v0, vcc, s2, v2
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-NEXT: flat_load_dword v3, v[0:1]
-; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v2
; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v2
+; VI-NEXT: s_mov_b32 s0, 0x5040c00
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_lshrrev_b32_e32 v2, 24, v3
-; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; VI-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; VI-NEXT: v_lshrrev_b32_e32 v2, 8, v3
+; VI-NEXT: v_lshrrev_b32_e32 v3, 24, v3
+; VI-NEXT: v_perm_b32 v2, v3, v2, s0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/min.ll b/llvm/test/CodeGen/AMDGPU/min.ll
index b6ca816550cc7..414165dbc3108 100644
--- a/llvm/test/CodeGen/AMDGPU/min.ll
+++ b/llvm/test/CodeGen/AMDGPU/min.ll
@@ -706,28 +706,26 @@ define amdgpu_kernel void @s_test_imin_sle_v4i8(ptr addrspace(1) %out, [8 x i32]
;
; VI-LABEL: s_test_imin_sle_v4i8:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[8:9], 0x28
-; VI-NEXT: s_load_dword s3, s[8:9], 0x4c
+; VI-NEXT: s_load_dword s2, s[8:9], 0x4c
+; VI-NEXT: s_load_dword s3, s[8:9], 0x28
; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
; VI-NEXT: s_add_i32 s12, s12, s17
; VI-NEXT: v_mov_b32_e32 v1, 0xc0c0400
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_sext_i32_i16 s5, s2
-; VI-NEXT: s_sext_i32_i16 s7, s3
-; VI-NEXT: s_ashr_i32 s7, s7, 8
-; VI-NEXT: s_ashr_i32 s5, s5, 8
-; VI-NEXT: s_min_i32 s5, s5, s7
-; VI-NEXT: s_sext_i32_i8 s7, s3
-; VI-NEXT: s_sext_i32_i8 s8, s2
+; VI-NEXT: s_sext_i32_i8 s4, s2
+; VI-NEXT: s_sext_i32_i8 s5, s3
+; VI-NEXT: s_bfe_i32 s6, s2, 0x80008
+; VI-NEXT: s_min_i32 s4, s5, s4
+; VI-NEXT: s_bfe_i32 s5, s3, 0x80008
+; VI-NEXT: s_min_i32 s5, s5, s6
+; VI-NEXT: v_mov_b32_e32 v0, s4
+; VI-NEXT: v_perm_b32 v0, s5, v0, v1
; VI-NEXT: s_ashr_i32 s4, s2, 24
-; VI-NEXT: s_ashr_i32 s6, s3, 24
-; VI-NEXT: s_min_i32 s7, s8, s7
-; VI-NEXT: s_bfe_i32 s3, s3, 0x80010
+; VI-NEXT: s_ashr_i32 s5, s3, 24
; VI-NEXT: s_bfe_i32 s2, s2, 0x80010
-; VI-NEXT: v_mov_b32_e32 v0, s7
-; VI-NEXT: s_min_i32 s2, s2, s3
-; VI-NEXT: v_perm_b32 v0, s5, v0, v1
-; VI-NEXT: s_min_i32 s4, s4, s6
+; VI-NEXT: s_bfe_i32 s3, s3, 0x80010
+; VI-NEXT: s_min_i32 s2, s3, s2
+; VI-NEXT: s_min_i32 s4, s5, s4
; VI-NEXT: v_mov_b32_e32 v1, s2
; VI-NEXT: v_mov_b32_e32 v2, 0x6050400
; VI-NEXT: v_perm_b32 v1, s4, v1, v2
@@ -742,28 +740,26 @@ define amdgpu_kernel void @s_test_imin_sle_v4i8(ptr addrspace(1) %out, [8 x i32]
;
; GFX9-LABEL: s_test_imin_sle_v4i8:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x28
-; GFX9-NEXT: s_load_dword s3, s[8:9], 0x4c
+; GFX9-NEXT: s_load_dword s2, s[8:9], 0x4c
+; GFX9-NEXT: s_load_dword s3, s[8:9], 0x28
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v2, 0xc0c0400
; GFX9-NEXT: v_mov_b32_e32 v3, 0x6050400
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_sext_i32_i16 s5, s2
-; GFX9-NEXT: s_sext_i32_i16 s7, s3
-; GFX9-NEXT: s_ashr_i32 s7, s7, 8
-; GFX9-NEXT: s_ashr_i32 s5, s5, 8
-; GFX9-NEXT: s_min_i32 s5, s5, s7
-; GFX9-NEXT: s_sext_i32_i8 s7, s3
-; GFX9-NEXT: s_sext_i32_i8 s8, s2
+; GFX9-NEXT: s_sext_i32_i8 s4, s2
+; GFX9-NEXT: s_sext_i32_i8 s5, s3
+; GFX9-NEXT: s_min_i32 s4, s5, s4
+; GFX9-NEXT: s_bfe_i32 s5, s2, 0x80008
+; GFX9-NEXT: s_bfe_i32 s6, s3, 0x80008
+; GFX9-NEXT: s_min_i32 s5, s6, s5
+; GFX9-NEXT: v_mov_b32_e32 v1, s4
+; GFX9-NEXT: v_perm_b32 v1, s5, v1, v2
; GFX9-NEXT: s_ashr_i32 s4, s2, 24
-; GFX9-NEXT: s_ashr_i32 s6, s3, 24
-; GFX9-NEXT: s_min_i32 s7, s8, s7
-; GFX9-NEXT: s_bfe_i32 s3, s3, 0x80010
+; GFX9-NEXT: s_ashr_i32 s5, s3, 24
; GFX9-NEXT: s_bfe_i32 s2, s2, 0x80010
-; GFX9-NEXT: v_mov_b32_e32 v1, s7
-; GFX9-NEXT: s_min_i32 s2, s2, s3
-; GFX9-NEXT: v_perm_b32 v1, s5, v1, v2
-; GFX9-NEXT: s_min_i32 s4, s4, s6
+; GFX9-NEXT: s_bfe_i32 s3, s3, 0x80010
+; GFX9-NEXT: s_min_i32 s2, s3, s2
+; GFX9-NEXT: s_min_i32 s4, s5, s4
; GFX9-NEXT: v_mov_b32_e32 v2, s2
; GFX9-NEXT: v_perm_b32 v2, s4, v2, v3
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
@@ -775,29 +771,27 @@ define amdgpu_kernel void @s_test_imin_sle_v4i8(ptr addrspace(1) %out, [8 x i32]
; GFX10-LABEL: s_test_imin_sle_v4i8:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_clause 0x2
-; GFX10-NEXT: s_load_dword s2, s[8:9], 0x28
-; GFX10-NEXT: s_load_dword s3, s[8:9], 0x4c
+; GFX10-NEXT: s_load_dword s2, s[8:9], 0x4c
+; GFX10-NEXT: s_load_dword s3, s[8:9], 0x28
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX10-NEXT: v_mov_b32_e32 v0, 0x6050400
; GFX10-NEXT: v_mov_b32_e32 v1, 0xc0c0400
; GFX10-NEXT: v_mov_b32_e32 v2, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_ashr_i32 s4, s2, 24
-; GFX10-NEXT: s_sext_i32_i16 s5, s2
-; GFX10-NEXT: s_ashr_i32 s6, s3, 24
-; GFX10-NEXT: s_sext_i32_i16 s7, s3
-; GFX10-NEXT: s_sext_i32_i8 s8, s3
-; GFX10-NEXT: s_sext_i32_i8 s9, s2
-; GFX10-NEXT: s_bfe_i32 s3, s3, 0x80010
+; GFX10-NEXT: s_sext_i32_i8 s4, s2
+; GFX10-NEXT: s_sext_i32_i8 s5, s3
+; GFX10-NEXT: s_bfe_i32 s6, s2, 0x80008
+; GFX10-NEXT: s_bfe_i32 s7, s3, 0x80008
+; GFX10-NEXT: s_ashr_i32 s8, s2, 24
+; GFX10-NEXT: s_ashr_i32 s9, s3, 24
; GFX10-NEXT: s_bfe_i32 s2, s2, 0x80010
-; GFX10-NEXT: s_min_i32 s4, s4, s6
-; GFX10-NEXT: s_min_i32 s2, s2, s3
-; GFX10-NEXT: s_ashr_i32 s7, s7, 8
-; GFX10-NEXT: v_perm_b32 v0, s4, s2, v0
-; GFX10-NEXT: s_ashr_i32 s5, s5, 8
-; GFX10-NEXT: s_min_i32 s3, s9, s8
-; GFX10-NEXT: s_min_i32 s2, s5, s7
-; GFX10-NEXT: v_perm_b32 v1, s2, s3, v1
+; GFX10-NEXT: s_bfe_i32 s3, s3, 0x80010
+; GFX10-NEXT: s_min_i32 s8, s9, s8
+; GFX10-NEXT: s_min_i32 s2, s3, s2
+; GFX10-NEXT: s_min_i32 s3, s7, s6
+; GFX10-NEXT: v_perm_b32 v0, s8, s2, v0
+; GFX10-NEXT: s_min_i32 s2, s5, s4
+; GFX10-NEXT: v_perm_b32 v1, s3, s2, v1
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX10-NEXT: v_or_b32_e32 v0, v1, v0
; GFX10-NEXT: global_store_dword v2, v0, s[0:1]
@@ -806,31 +800,30 @@ define amdgpu_kernel void @s_test_imin_sle_v4i8(ptr addrspace(1) %out, [8 x i32]
; GFX11-LABEL: s_test_imin_sle_v4i8:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_clause 0x2
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x28
-; GFX11-NEXT: s_load_b32 s3, s[4:5], 0x4c
+; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x4c
+; GFX11-NEXT: s_load_b32 s3, s[4:5], 0x28
; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
; GFX11-NEXT: v_mov_b32_e32 v0, 0x6050400
; GFX11-NEXT: v_mov_b32_e32 v1, 0xc0c0400
; GFX11-NEXT: v_mov_b32_e32 v2, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_ashr_i32 s4, s2, 24
-; GFX11-NEXT: s_sext_i32_i16 s5, s2
-; GFX11-NEXT: s_ashr_i32 s6, s3, 24
-; GFX11-NEXT: s_sext_i32_i16 s7, s3
-; GFX11-NEXT: s_sext_i32_i8 s8, s3
-; GFX11-NEXT: s_sext_i32_i8 s9, s2
-; GFX11-NEXT: s_bfe_i32 s3, s3, 0x80010
+; GFX11-NEXT: s_sext_i32_i8 s4, s2
+; GFX11-NEXT: s_sext_i32_i8 s5, s3
+; GFX11-NEXT: s_bfe_i32 s6, s2, 0x80008
+; GFX11-NEXT: s_bfe_i32 s7, s3, 0x80008
+; GFX11-NEXT: s_ashr_i32 s8, s2, 24
+; GFX11-NEXT: s_ashr_i32 s9, s3, 24
; GFX11-NEXT: s_bfe_i32 s2, s2, 0x80010
-; GFX11-NEXT: s_min_i32 s4, s4, s6
-; GFX11-NEXT: s_min_i32 s2, s2, s3
-; GFX11-NEXT: s_ashr_i32 s7, s7, 8
-; GFX11-NEXT: v_perm_b32 v0, s4, s2, v0
-; GFX11-NEXT: s_ashr_i32 s5, s5, 8
-; GFX11-NEXT: s_min_i32 s3, s9, s8
-; GFX11-NEXT: s_min_i32 s2, s5, s7
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_perm_b32 v1, s2, s3, v1
+; GFX11-NEXT: s_bfe_i32 s3, s3, 0x80010
+; GFX11-NEXT: s_min_i32 s8, s9, s8
+; GFX11-NEXT: s_min_i32 s2, s3, s2
+; GFX11-NEXT: s_min_i32 s3, s7, s6
+; GFX11-NEXT: v_perm_b32 v0, s8, s2, v0
+; GFX11-NEXT: s_min_i32 s2, s5, s4
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_perm_b32 v1, s3, s2, v1
; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_or_b32_e32 v0, v1, v0
; GFX11-NEXT: global_store_b32 v2, v0, s[0:1]
; GFX11-NEXT: s_endpgm
@@ -842,31 +835,30 @@ define amdgpu_kernel void @s_test_imin_sle_v4i8(ptr addrspace(1) %out, [8 x i32]
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NEXT: s_clause 0x2
-; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x28 nv
-; GFX1250-NEXT: s_load_b32 s3, s[4:5], 0x4c nv
+; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x4c nv
+; GFX1250-NEXT: s_load_b32 s3, s[4:5], 0x28 nv
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0x6050400
; GFX1250-NEXT: v_mov_b32_e32 v1, 0xc0c0400
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: s_ashr_i32 s4, s2, 24
-; GFX1250-NEXT: s_sext_i32_i16 s5, s2
-; GFX1250-NEXT: s_ashr_i32 s6, s3, 24
-; GFX1250-NEXT: s_sext_i32_i16 s7, s3
-; GFX1250-NEXT: s_sext_i32_i8 s8, s3
-; GFX1250-NEXT: s_sext_i32_i8 s9, s2
-; GFX1250-NEXT: s_bfe_i32 s3, s3, 0x80010
+; GFX1250-NEXT: s_sext_i32_i8 s4, s2
+; GFX1250-NEXT: s_sext_i32_i8 s5, s3
+; GFX1250-NEXT: s_bfe_i32 s6, s2, 0x80008
+; GFX1250-NEXT: s_bfe_i32 s7, s3, 0x80008
+; GFX1250-NEXT: s_ashr_i32 s8, s2, 24
+; GFX1250-NEXT: s_ashr_i32 s9, s3, 24
; GFX1250-NEXT: s_bfe_i32 s2, s2, 0x80010
-; GFX1250-NEXT: s_min_i32 s4, s4, s6
-; GFX1250-NEXT: s_min_i32 s2, s2, s3
-; GFX1250-NEXT: s_ashr_i32 s7, s7, 8
-; GFX1250-NEXT: v_perm_b32 v0, s4, s2, v0
-; GFX1250-NEXT: s_ashr_i32 s5, s5, 8
-; GFX1250-NEXT: s_min_i32 s3, s9, s8
-; GFX1250-NEXT: s_min_i32 s2, s5, s7
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_perm_b32 v1, s2, s3, v1
+; GFX1250-NEXT: s_bfe_i32 s3, s3, 0x80010
+; GFX1250-NEXT: s_min_i32 s8, s9, s8
+; GFX1250-NEXT: s_min_i32 s2, s3, s2
+; GFX1250-NEXT: s_min_i32 s3, s7, s6
+; GFX1250-NEXT: v_perm_b32 v0, s8, s2, v0
+; GFX1250-NEXT: s_min_i32 s2, s5, s4
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-NEXT: v_perm_b32 v1, s3, s2, v1
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-NEXT: v_or_b32_e32 v0, v1, v0
; GFX1250-NEXT: global_store_b32 v2, v0, s[0:1]
; GFX1250-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/permute_i8.ll b/llvm/test/CodeGen/AMDGPU/permute_i8.ll
index f5928d99d4350..77e7cb9a5f6c0 100644
--- a/llvm/test/CodeGen/AMDGPU/permute_i8.ll
+++ b/llvm/test/CodeGen/AMDGPU/permute_i8.ll
@@ -1716,21 +1716,19 @@ define hidden void @sitofp_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_and_b32_e32 v4, 0x3ff, v31
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 2, v4
-; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
-; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, v2, v4
; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
-; GFX10-NEXT: global_load_dword v4, v[0:1], off
-; GFX10-NEXT: global_load_dword v9, v[2:3], off
+; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
+; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT: global_load_dword v4, v[2:3], off
+; GFX10-NEXT: global_load_dword v9, v[0:1], off
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_ashrrev_i16 v0, 8, v4
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v3, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_ashrrev_i16 v10, 8, v9
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v3, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v1, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
-; GFX10-NEXT: v_perm_b32 v4, v9, v4, 0x6010205
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v2, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v0, sext(v10) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v2, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v1, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v0, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1
+; GFX10-NEXT: v_perm_b32 v4, v4, v9, 0x6010205
; GFX10-NEXT: global_store_dwordx4 v[7:8], v[0:3], off
; GFX10-NEXT: global_store_dword v[5:6], v4, off
; GFX10-NEXT: s_setpc_b64 s[30:31]
@@ -1744,18 +1742,16 @@ define hidden void @sitofp_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in
; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v2, v4
; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX9-NEXT: global_load_dword v4, v[0:1], off
-; GFX9-NEXT: global_load_dword v9, v[2:3], off
+; GFX9-NEXT: global_load_dword v4, v[2:3], off
+; GFX9-NEXT: global_load_dword v9, v[0:1], off
; GFX9-NEXT: s_mov_b32 s4, 0x6010205
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_ashrrev_i16_e32 v0, 8, v4
+; GFX9-NEXT: v_cvt_f32_i32_sdwa v3, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_ashrrev_i16_e32 v10, 8, v9
-; GFX9-NEXT: v_cvt_f32_i32_sdwa v3, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
-; GFX9-NEXT: v_cvt_f32_i32_sdwa v1, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
-; GFX9-NEXT: v_cvt_f32_i32_sdwa v2, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
-; GFX9-NEXT: v_cvt_f32_i32_sdwa v0, sext(v10) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
-; GFX9-NEXT: v_perm_b32 v4, v9, v4, s4
+; GFX9-NEXT: v_cvt_f32_i32_sdwa v2, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1
+; GFX9-NEXT: v_cvt_f32_i32_sdwa v1, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
+; GFX9-NEXT: v_cvt_f32_i32_sdwa v0, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1
+; GFX9-NEXT: v_perm_b32 v4, v4, v9, s4
; GFX9-NEXT: global_store_dwordx4 v[7:8], v[0:3], off
; GFX9-NEXT: global_store_dword v[5:6], v4, off
; GFX9-NEXT: s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/sad.ll b/llvm/test/CodeGen/AMDGPU/sad.ll
index c73e60022b623..2233008b446b1 100644
--- a/llvm/test/CodeGen/AMDGPU/sad.ll
+++ b/llvm/test/CodeGen/AMDGPU/sad.ll
@@ -1425,12 +1425,11 @@ define amdgpu_kernel void @v_sad_u32_i8_pat1(ptr addrspace(1) %out, i8 %a, i8 %b
; GFX12-5-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX12-5-SDAG-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
; GFX12-5-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-5-SDAG-NEXT: s_lshr_b32 s3, s2, 8
-; GFX12-5-SDAG-NEXT: s_and_b32 s4, s2, 0xff
-; GFX12-5-SDAG-NEXT: s_and_b32 s3, s3, 0xff
+; GFX12-5-SDAG-NEXT: s_and_b32 s3, s2, 0xff
+; GFX12-5-SDAG-NEXT: s_bfe_u32 s4, s2, 0x80008
; GFX12-5-SDAG-NEXT: s_lshr_b32 s2, s2, 16
-; GFX12-5-SDAG-NEXT: s_min_u32 s5, s4, s3
-; GFX12-5-SDAG-NEXT: s_max_u32 s3, s4, s3
+; GFX12-5-SDAG-NEXT: s_min_u32 s5, s3, s4
+; GFX12-5-SDAG-NEXT: s_max_u32 s3, s3, s4
; GFX12-5-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-5-SDAG-NEXT: s_sub_co_i32 s3, s3, s5
; GFX12-5-SDAG-NEXT: s_add_co_i32 s2, s3, s2
@@ -1635,12 +1634,11 @@ define amdgpu_kernel void @s_sad_u32_i8_pat2(ptr addrspace(1) %out, i8 zeroext %
; GFX12-5-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX12-5-SDAG-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
; GFX12-5-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-5-SDAG-NEXT: s_lshr_b32 s3, s2, 8
-; GFX12-5-SDAG-NEXT: s_and_b32 s4, s2, 0xff
-; GFX12-5-SDAG-NEXT: s_and_b32 s3, s3, 0xff
+; GFX12-5-SDAG-NEXT: s_and_b32 s3, s2, 0xff
+; GFX12-5-SDAG-NEXT: s_bfe_u32 s4, s2, 0x80008
; GFX12-5-SDAG-NEXT: s_lshr_b32 s2, s2, 16
-; GFX12-5-SDAG-NEXT: s_min_u32 s5, s4, s3
-; GFX12-5-SDAG-NEXT: s_max_u32 s3, s4, s3
+; GFX12-5-SDAG-NEXT: s_min_u32 s5, s3, s4
+; GFX12-5-SDAG-NEXT: s_max_u32 s3, s3, s4
; GFX12-5-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-5-SDAG-NEXT: s_sub_co_i32 s3, s3, s5
; GFX12-5-SDAG-NEXT: s_add_co_i32 s2, s3, s2
diff --git a/llvm/test/CodeGen/AMDGPU/v4i8-byte-lane-ext.ll b/llvm/test/CodeGen/AMDGPU/v4i8-byte-lane-ext.ll
index 7bb37146fa77c..089dd79b78c36 100644
--- a/llvm/test/CodeGen/AMDGPU/v4i8-byte-lane-ext.ll
+++ b/llvm/test/CodeGen/AMDGPU/v4i8-byte-lane-ext.ll
@@ -12,13 +12,13 @@ define void @uitofp_v4i8_to_v4f64(ptr addrspace(1) %p, ptr addrspace(1) %o) {
; GFX9-NEXT: global_load_dword v0, v[0:1], off
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 24, v0
-; GFX9-NEXT: v_and_b32_e32 v4, 0xff, v0
-; GFX9-NEXT: v_lshrrev_b16_e32 v6, 8, v0
+; GFX9-NEXT: v_bfe_u32 v4, v0, 8, 8
+; GFX9-NEXT: v_and_b32_e32 v5, 0xff, v0
; GFX9-NEXT: v_bfe_u32 v0, v0, 16, 8
; GFX9-NEXT: v_cvt_f64_u32_e32 v[10:11], v1
; GFX9-NEXT: v_cvt_f64_u32_e32 v[8:9], v0
-; GFX9-NEXT: v_cvt_f64_u32_e32 v[4:5], v4
-; GFX9-NEXT: v_cvt_f64_u32_e32 v[6:7], v6
+; GFX9-NEXT: v_cvt_f64_u32_e32 v[6:7], v4
+; GFX9-NEXT: v_cvt_f64_u32_e32 v[4:5], v5
; GFX9-NEXT: global_store_dwordx4 v[2:3], v[8:11], off offset:16
; GFX9-NEXT: global_store_dwordx4 v[2:3], v[4:7], off
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -28,16 +28,15 @@ define void @uitofp_v4i8_to_v4f64(ptr addrspace(1) %p, ptr addrspace(1) %o) {
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v0, v[0:1], off
-; GFX10-NEXT: v_mov_b32_e32 v1, 0xffff
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_e32 v6, 24, v0
-; GFX10-NEXT: v_bfe_u32 v7, v0, 16, 8
-; GFX10-NEXT: v_and_b32_e32 v4, 0xff, v0
-; GFX10-NEXT: v_and_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_1
-; GFX10-NEXT: v_cvt_f64_u32_e32 v[10:11], v6
-; GFX10-NEXT: v_cvt_f64_u32_e32 v[8:9], v7
-; GFX10-NEXT: v_cvt_f64_u32_e32 v[4:5], v4
-; GFX10-NEXT: v_cvt_f64_u32_e32 v[6:7], v0
+; GFX10-NEXT: v_lshrrev_b32_e32 v4, 24, v0
+; GFX10-NEXT: v_bfe_u32 v5, v0, 16, 8
+; GFX10-NEXT: v_bfe_u32 v1, v0, 8, 8
+; GFX10-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX10-NEXT: v_cvt_f64_u32_e32 v[10:11], v4
+; GFX10-NEXT: v_cvt_f64_u32_e32 v[8:9], v5
+; GFX10-NEXT: v_cvt_f64_u32_e32 v[6:7], v1
+; GFX10-NEXT: v_cvt_f64_u32_e32 v[4:5], v0
; GFX10-NEXT: global_store_dwordx4 v[2:3], v[8:11], off offset:16
; GFX10-NEXT: global_store_dwordx4 v[2:3], v[4:7], off
; GFX10-NEXT: s_setpc_b64 s[30:31]
@@ -45,20 +44,18 @@ define void @uitofp_v4i8_to_v4f64(ptr addrspace(1) %p, ptr addrspace(1) %o) {
; GFX11-TRUE16-LABEL: uitofp_v4i8_to_v4f64:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b32 v1, v[0:1], off
+; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.l, 8, v1.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xff, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 24, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v1, 16, 8
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 24, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v0, 16, 8
+; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v0, 8, 8
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
-; GFX11-TRUE16-NEXT: v_cvt_f64_u32_e32 v[4:5], v4
+; GFX11-TRUE16-NEXT: v_cvt_f64_u32_e32 v[10:11], v4
+; GFX11-TRUE16-NEXT: v_cvt_f64_u32_e32 v[8:9], v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cvt_f64_u32_e32 v[10:11], v6
-; GFX11-TRUE16-NEXT: v_cvt_f64_u32_e32 v[8:9], v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cvt_f64_u32_e32 v[6:7], v0
+; GFX11-TRUE16-NEXT: v_cvt_f64_u32_e32 v[6:7], v1
+; GFX11-TRUE16-NEXT: v_cvt_f64_u32_e32 v[4:5], v0
; GFX11-TRUE16-NEXT: s_clause 0x1
; GFX11-TRUE16-NEXT: global_store_b128 v[2:3], v[8:11], off offset:16
; GFX11-TRUE16-NEXT: global_store_b128 v[2:3], v[4:7], off
@@ -69,18 +66,16 @@ define void @uitofp_v4i8_to_v4f64(ptr addrspace(1) %p, ptr addrspace(1) %o) {
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: global_load_b32 v0, v[0:1], off
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b16 v1, 8, v0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 0xff, v0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 24, v0
-; GFX11-FAKE16-NEXT: v_bfe_u32 v0, v0, 16, 8
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 24, v0
+; GFX11-FAKE16-NEXT: v_bfe_u32 v5, v0, 16, 8
+; GFX11-FAKE16-NEXT: v_bfe_u32 v1, v0, 8, 8
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-FAKE16-NEXT: v_cvt_f64_u32_e32 v[4:5], v4
+; GFX11-FAKE16-NEXT: v_cvt_f64_u32_e32 v[10:11], v4
+; GFX11-FAKE16-NEXT: v_cvt_f64_u32_e32 v[8:9], v5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_cvt_f64_u32_e32 v[10:11], v6
-; GFX11-FAKE16-NEXT: v_cvt_f64_u32_e32 v[8:9], v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-FAKE16-NEXT: v_cvt_f64_u32_e32 v[6:7], v1
+; GFX11-FAKE16-NEXT: v_cvt_f64_u32_e32 v[4:5], v0
; GFX11-FAKE16-NEXT: s_clause 0x1
; GFX11-FAKE16-NEXT: global_store_b128 v[2:3], v[8:11], off offset:16
; GFX11-FAKE16-NEXT: global_store_b128 v[2:3], v[4:7], off
@@ -90,21 +85,18 @@ define void @uitofp_v4i8_to_v4f64(ptr addrspace(1) %p, ptr addrspace(1) %o) {
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_load_b32 v1, v[0:1], off
+; GFX1250-NEXT: global_load_b32 v0, v[0:1], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_lshrrev_b32_e32 v4, 24, v0
+; GFX1250-NEXT: v_bfe_u32 v5, v0, 16, 8
; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_lshrrev_b16 v0.l, 8, v1.l
-; GFX1250-NEXT: s_mov_b32 s0, 0xff
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1250-NEXT: v_dual_lshrrev_b32 v6, 24, v1 :: v_dual_bitop2_b32 v4, s0, v1 bitop3:0x40
-; GFX1250-NEXT: v_bfe_u32 v1, v1, 16, 8
-; GFX1250-NEXT: v_cvt_u32_u16_e32 v0, v0.l
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1250-NEXT: v_cvt_f64_u32_e32 v[4:5], v4
-; GFX1250-NEXT: v_cvt_f64_u32_e32 v[10:11], v6
+; GFX1250-NEXT: v_bfe_u32 v1, v0, 8, 8
+; GFX1250-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX1250-NEXT: v_cvt_f64_u32_e32 v[10:11], v4
+; GFX1250-NEXT: v_cvt_f64_u32_e32 v[8:9], v5
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1250-NEXT: v_cvt_f64_u32_e32 v[8:9], v1
-; GFX1250-NEXT: v_cvt_f64_u32_e32 v[6:7], v0
+; GFX1250-NEXT: v_cvt_f64_u32_e32 v[6:7], v1
+; GFX1250-NEXT: v_cvt_f64_u32_e32 v[4:5], v0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: global_store_b128 v[2:3], v[8:11], off offset:16
; GFX1250-NEXT: global_store_b128 v[2:3], v[4:7], off
@@ -121,11 +113,10 @@ define void @sitofp_v4i8_to_v4f32(ptr addrspace(1) %p, ptr addrspace(1) %o) {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_load_dword v0, v[0:1], off
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_ashrrev_i16_e32 v1, 8, v0
; GFX9-NEXT: v_cvt_f32_i32_sdwa v7, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
; GFX9-NEXT: v_cvt_f32_i32_sdwa v6, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
+; GFX9-NEXT: v_cvt_f32_i32_sdwa v5, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1
; GFX9-NEXT: v_cvt_f32_i32_sdwa v4, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0
-; GFX9-NEXT: v_cvt_f32_i32_sdwa v5, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
; GFX9-NEXT: global_store_dwordx4 v[2:3], v[4:7], off
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -135,11 +126,10 @@ define void @sitofp_v4i8_to_v4f32(ptr addrspace(1) %p, ptr addrspace(1) %o) {
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v0, v[0:1], off
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_ashrrev_i16 v1, 8, v0
; GFX10-NEXT: v_cvt_f32_i32_sdwa v7, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
; GFX10-NEXT: v_cvt_f32_i32_sdwa v6, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v5, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1
; GFX10-NEXT: v_cvt_f32_i32_sdwa v4, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v5, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
; GFX10-NEXT: global_store_dwordx4 v[2:3], v[4:7], off
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -148,18 +138,16 @@ define void @sitofp_v4i8_to_v4f32(ptr addrspace(1) %p, ptr addrspace(1) %o) {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_ashrrev_i16 v1.l, 8, v0.l
-; GFX11-TRUE16-NEXT: v_ashrrev_i32_e32 v4, 24, v0
-; GFX11-TRUE16-NEXT: v_bfe_i32 v5, v0, 16, 8
+; GFX11-TRUE16-NEXT: v_ashrrev_i32_e32 v1, 24, v0
+; GFX11-TRUE16-NEXT: v_bfe_i32 v4, v0, 16, 8
+; GFX11-TRUE16-NEXT: v_bfe_i32 v5, v0, 8, 8
; GFX11-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 8
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 16
-; GFX11-TRUE16-NEXT: v_cvt_f32_i32_e32 v7, v4
+; GFX11-TRUE16-NEXT: v_cvt_f32_i32_e32 v7, v1
+; GFX11-TRUE16-NEXT: v_cvt_f32_i32_e32 v6, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cvt_f32_i32_e32 v6, v5
+; GFX11-TRUE16-NEXT: v_cvt_f32_i32_e32 v5, v5
; GFX11-TRUE16-NEXT: v_cvt_f32_i32_e32 v4, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cvt_f32_i32_e32 v5, v1
; GFX11-TRUE16-NEXT: global_store_b128 v[2:3], v[4:7], off
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -168,18 +156,16 @@ define void @sitofp_v4i8_to_v4f32(ptr addrspace(1) %p, ptr addrspace(1) %o) {
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: global_load_b32 v0, v[0:1], off
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_ashrrev_i16 v1, 8, v0
-; GFX11-FAKE16-NEXT: v_ashrrev_i32_e32 v4, 24, v0
-; GFX11-FAKE16-NEXT: v_bfe_i32 v5, v0, 16, 8
+; GFX11-FAKE16-NEXT: v_ashrrev_i32_e32 v1, 24, v0
+; GFX11-FAKE16-NEXT: v_bfe_i32 v4, v0, 16, 8
+; GFX11-FAKE16-NEXT: v_bfe_i32 v5, v0, 8, 8
; GFX11-FAKE16-NEXT: v_bfe_i32 v0, v0, 0, 8
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_bfe_i32 v1, v1, 0, 16
-; GFX11-FAKE16-NEXT: v_cvt_f32_i32_e32 v7, v4
+; GFX11-FAKE16-NEXT: v_cvt_f32_i32_e32 v7, v1
+; GFX11-FAKE16-NEXT: v_cvt_f32_i32_e32 v6, v4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_cvt_f32_i32_e32 v6, v5
+; GFX11-FAKE16-NEXT: v_cvt_f32_i32_e32 v5, v5
; GFX11-FAKE16-NEXT: v_cvt_f32_i32_e32 v4, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_cvt_f32_i32_e32 v5, v1
; GFX11-FAKE16-NEXT: global_store_b128 v[2:3], v[4:7], off
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -190,18 +176,16 @@ define void @sitofp_v4i8_to_v4f32(ptr addrspace(1) %p, ptr addrspace(1) %o) {
; GFX1250-NEXT: global_load_b32 v0, v[0:1], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_ashrrev_i16 v1.l, 8, v0.l
-; GFX1250-NEXT: v_ashrrev_i32_e32 v4, 24, v0
-; GFX1250-NEXT: v_bfe_i32 v5, v0, 16, 8
+; GFX1250-NEXT: v_ashrrev_i32_e32 v1, 24, v0
+; GFX1250-NEXT: v_bfe_i32 v4, v0, 16, 8
+; GFX1250-NEXT: v_bfe_i32 v5, v0, 8, 8
; GFX1250-NEXT: v_bfe_i32 v0, v0, 0, 8
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1250-NEXT: v_bfe_i32 v1, v1, 0, 16
-; GFX1250-NEXT: v_cvt_f32_i32_e32 v7, v4
+; GFX1250-NEXT: v_cvt_f32_i32_e32 v7, v1
+; GFX1250-NEXT: v_cvt_f32_i32_e32 v6, v4
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1250-NEXT: v_cvt_f32_i32_e32 v6, v5
+; GFX1250-NEXT: v_cvt_f32_i32_e32 v5, v5
; GFX1250-NEXT: v_cvt_f32_i32_e32 v4, v0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX1250-NEXT: v_cvt_f32_i32_e32 v5, v1
; GFX1250-NEXT: global_store_b128 v[2:3], v[4:7], off
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%v = load <4 x i8>, ptr addrspace(1) %p, align 4
@@ -215,20 +199,15 @@ define void @sitofp_2x_v4i8_to_v4f32(ptr addrspace(1) noalias %p, ptr addrspace(
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
-; GFX9-NEXT: v_mov_b32_e32 v4, 8
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_ashrrev_i16_sdwa v7, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_ashrrev_i16_e32 v5, 8, v0
-; GFX9-NEXT: v_ashrrev_i16_sdwa v11, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_ashrrev_i16_e32 v9, 8, v1
+; GFX9-NEXT: v_cvt_f32_i32_sdwa v7, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
; GFX9-NEXT: v_cvt_f32_i32_sdwa v6, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
+; GFX9-NEXT: v_cvt_f32_i32_sdwa v5, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1
; GFX9-NEXT: v_cvt_f32_i32_sdwa v4, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0
-; GFX9-NEXT: v_cvt_f32_i32_sdwa v5, sext(v5) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
-; GFX9-NEXT: v_cvt_f32_i32_sdwa v7, sext(v7) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX9-NEXT: v_cvt_f32_i32_sdwa v11, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
; GFX9-NEXT: v_cvt_f32_i32_sdwa v10, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
+; GFX9-NEXT: v_cvt_f32_i32_sdwa v9, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1
; GFX9-NEXT: v_cvt_f32_i32_sdwa v8, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0
-; GFX9-NEXT: v_cvt_f32_i32_sdwa v9, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
-; GFX9-NEXT: v_cvt_f32_i32_sdwa v11, sext(v11) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
; GFX9-NEXT: global_store_dwordx4 v[2:3], v[4:7], off
; GFX9-NEXT: global_store_dwordx4 v[2:3], v[8:11], off offset:16
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -239,18 +218,14 @@ define void @sitofp_2x_v4i8_to_v4f32(ptr addrspace(1) noalias %p, ptr addrspace(
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_ashrrev_i16 v5, 8, v0
-; GFX10-NEXT: v_ashrrev_i16 v7, 8, v0 op_sel:[0,1,0]
-; GFX10-NEXT: v_ashrrev_i16 v9, 8, v1
-; GFX10-NEXT: v_ashrrev_i16 v11, 8, v1 op_sel:[0,1,0]
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v7, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
; GFX10-NEXT: v_cvt_f32_i32_sdwa v6, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v5, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1
; GFX10-NEXT: v_cvt_f32_i32_sdwa v4, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v7, sext(v7) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v5, sext(v5) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v11, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
; GFX10-NEXT: v_cvt_f32_i32_sdwa v10, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v9, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1
; GFX10-NEXT: v_cvt_f32_i32_sdwa v8, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v11, sext(v11) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v9, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
; GFX10-NEXT: global_store_dwordx4 v[2:3], v[4:7], off
; GFX10-NEXT: global_store_dwordx4 v[2:3], v[8:11], off offset:16
; GFX10-NEXT: s_setpc_b64 s[30:31]
@@ -260,28 +235,22 @@ define void @sitofp_2x_v4i8_to_v4f32(ptr addrspace(1) noalias %p, ptr addrspace(
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b64 v[0:1], v[0:1], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v1
-; GFX11-TRUE16-NEXT: v_ashrrev_i16 v5.l, 8, v0.l
-; GFX11-TRUE16-NEXT: v_ashrrev_i16 v7.l, 8, v1.l
-; GFX11-TRUE16-NEXT: v_bfe_i32 v8, v0, 16, 8
+; GFX11-TRUE16-NEXT: v_ashrrev_i32_e32 v4, 24, v0
+; GFX11-TRUE16-NEXT: v_bfe_i32 v5, v0, 16, 8
+; GFX11-TRUE16-NEXT: v_bfe_i32 v8, v0, 8, 8
; GFX11-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX11-TRUE16-NEXT: v_ashrrev_i16 v11.l, 8, v4.l
-; GFX11-TRUE16-NEXT: v_ashrrev_i16 v12.l, 8, v6.l
-; GFX11-TRUE16-NEXT: v_bfe_i32 v9, v1, 16, 8
-; GFX11-TRUE16-NEXT: v_bfe_i32 v7, v7, 0, 16
-; GFX11-TRUE16-NEXT: v_cvt_f32_i32_e32 v4, v0
-; GFX11-TRUE16-NEXT: v_bfe_i32 v0, v5, 0, 16
-; GFX11-TRUE16-NEXT: v_bfe_i32 v11, v11, 0, 16
+; GFX11-TRUE16-NEXT: v_ashrrev_i32_e32 v9, 24, v1
+; GFX11-TRUE16-NEXT: v_bfe_i32 v10, v1, 16, 8
+; GFX11-TRUE16-NEXT: v_bfe_i32 v12, v1, 8, 8
; GFX11-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX11-TRUE16-NEXT: v_bfe_i32 v12, v12, 0, 16
-; GFX11-TRUE16-NEXT: v_cvt_f32_i32_e32 v6, v8
-; GFX11-TRUE16-NEXT: v_cvt_f32_i32_e32 v10, v9
-; GFX11-TRUE16-NEXT: v_cvt_f32_i32_e32 v5, v0
-; GFX11-TRUE16-NEXT: v_cvt_f32_i32_e32 v9, v7
-; GFX11-TRUE16-NEXT: v_cvt_f32_i32_e32 v7, v11
+; GFX11-TRUE16-NEXT: v_cvt_f32_i32_e32 v7, v4
+; GFX11-TRUE16-NEXT: v_cvt_f32_i32_e32 v6, v5
+; GFX11-TRUE16-NEXT: v_cvt_f32_i32_e32 v5, v8
+; GFX11-TRUE16-NEXT: v_cvt_f32_i32_e32 v4, v0
+; GFX11-TRUE16-NEXT: v_cvt_f32_i32_e32 v11, v9
+; GFX11-TRUE16-NEXT: v_cvt_f32_i32_e32 v10, v10
+; GFX11-TRUE16-NEXT: v_cvt_f32_i32_e32 v9, v12
; GFX11-TRUE16-NEXT: v_cvt_f32_i32_e32 v8, v1
-; GFX11-TRUE16-NEXT: v_cvt_f32_i32_e32 v11, v12
; GFX11-TRUE16-NEXT: s_clause 0x1
; GFX11-TRUE16-NEXT: global_store_b128 v[2:3], v[4:7], off
; GFX11-TRUE16-NEXT: global_store_b128 v[2:3], v[8:11], off offset:16
@@ -292,28 +261,22 @@ define void @sitofp_2x_v4i8_to_v4f32(ptr addrspace(1) noalias %p, ptr addrspace(
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: global_load_b64 v[0:1], v[0:1], off
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 16, v0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 16, v1
-; GFX11-FAKE16-NEXT: v_ashrrev_i16 v5, 8, v0
-; GFX11-FAKE16-NEXT: v_ashrrev_i16 v7, 8, v1
-; GFX11-FAKE16-NEXT: v_bfe_i32 v8, v0, 16, 8
+; GFX11-FAKE16-NEXT: v_ashrrev_i32_e32 v4, 24, v0
+; GFX11-FAKE16-NEXT: v_bfe_i32 v5, v0, 16, 8
+; GFX11-FAKE16-NEXT: v_bfe_i32 v8, v0, 8, 8
; GFX11-FAKE16-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX11-FAKE16-NEXT: v_ashrrev_i16 v11, 8, v4
-; GFX11-FAKE16-NEXT: v_ashrrev_i16 v12, 8, v6
-; GFX11-FAKE16-NEXT: v_bfe_i32 v9, v1, 16, 8
-; GFX11-FAKE16-NEXT: v_bfe_i32 v7, v7, 0, 16
-; GFX11-FAKE16-NEXT: v_cvt_f32_i32_e32 v4, v0
-; GFX11-FAKE16-NEXT: v_bfe_i32 v0, v5, 0, 16
-; GFX11-FAKE16-NEXT: v_bfe_i32 v11, v11, 0, 16
+; GFX11-FAKE16-NEXT: v_ashrrev_i32_e32 v9, 24, v1
+; GFX11-FAKE16-NEXT: v_bfe_i32 v10, v1, 16, 8
+; GFX11-FAKE16-NEXT: v_bfe_i32 v12, v1, 8, 8
; GFX11-FAKE16-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX11-FAKE16-NEXT: v_bfe_i32 v12, v12, 0, 16
-; GFX11-FAKE16-NEXT: v_cvt_f32_i32_e32 v6, v8
-; GFX11-FAKE16-NEXT: v_cvt_f32_i32_e32 v10, v9
-; GFX11-FAKE16-NEXT: v_cvt_f32_i32_e32 v5, v0
-; GFX11-FAKE16-NEXT: v_cvt_f32_i32_e32 v9, v7
-; GFX11-FAKE16-NEXT: v_cvt_f32_i32_e32 v7, v11
+; GFX11-FAKE16-NEXT: v_cvt_f32_i32_e32 v7, v4
+; GFX11-FAKE16-NEXT: v_cvt_f32_i32_e32 v6, v5
+; GFX11-FAKE16-NEXT: v_cvt_f32_i32_e32 v5, v8
+; GFX11-FAKE16-NEXT: v_cvt_f32_i32_e32 v4, v0
+; GFX11-FAKE16-NEXT: v_cvt_f32_i32_e32 v11, v9
+; GFX11-FAKE16-NEXT: v_cvt_f32_i32_e32 v10, v10
+; GFX11-FAKE16-NEXT: v_cvt_f32_i32_e32 v9, v12
; GFX11-FAKE16-NEXT: v_cvt_f32_i32_e32 v8, v1
-; GFX11-FAKE16-NEXT: v_cvt_f32_i32_e32 v11, v12
; GFX11-FAKE16-NEXT: s_clause 0x1
; GFX11-FAKE16-NEXT: global_store_b128 v[2:3], v[4:7], off
; GFX11-FAKE16-NEXT: global_store_b128 v[2:3], v[8:11], off offset:16
@@ -325,28 +288,22 @@ define void @sitofp_2x_v4i8_to_v4f32(ptr addrspace(1) noalias %p, ptr addrspace(
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: global_load_b64 v[0:1], v[0:1], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_dual_lshrrev_b32 v4, 16, v0 :: v_dual_lshrrev_b32 v6, 16, v1
-; GFX1250-NEXT: v_ashrrev_i16 v5.l, 8, v0.l
-; GFX1250-NEXT: v_ashrrev_i16 v7.l, 8, v1.l
-; GFX1250-NEXT: v_bfe_i32 v8, v0, 16, 8
+; GFX1250-NEXT: v_dual_ashrrev_i32 v4, 24, v0 :: v_dual_ashrrev_i32 v9, 24, v1
+; GFX1250-NEXT: v_bfe_i32 v5, v0, 16, 8
+; GFX1250-NEXT: v_bfe_i32 v8, v0, 8, 8
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX1250-NEXT: v_ashrrev_i16 v11.l, 8, v4.l
-; GFX1250-NEXT: v_ashrrev_i16 v12.l, 8, v6.l
-; GFX1250-NEXT: v_bfe_i32 v9, v1, 16, 8
-; GFX1250-NEXT: v_bfe_i32 v7, v7, 0, 16
-; GFX1250-NEXT: v_cvt_f32_i32_e32 v4, v0
-; GFX1250-NEXT: v_bfe_i32 v0, v5, 0, 16
-; GFX1250-NEXT: v_bfe_i32 v11, v11, 0, 16
+; GFX1250-NEXT: v_bfe_i32 v10, v1, 16, 8
+; GFX1250-NEXT: v_bfe_i32 v12, v1, 8, 8
; GFX1250-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX1250-NEXT: v_bfe_i32 v12, v12, 0, 16
-; GFX1250-NEXT: v_cvt_f32_i32_e32 v6, v8
-; GFX1250-NEXT: v_cvt_f32_i32_e32 v10, v9
-; GFX1250-NEXT: v_cvt_f32_i32_e32 v5, v0
-; GFX1250-NEXT: v_cvt_f32_i32_e32 v9, v7
-; GFX1250-NEXT: v_cvt_f32_i32_e32 v7, v11
+; GFX1250-NEXT: v_cvt_f32_i32_e32 v7, v4
+; GFX1250-NEXT: v_cvt_f32_i32_e32 v6, v5
+; GFX1250-NEXT: v_cvt_f32_i32_e32 v5, v8
+; GFX1250-NEXT: v_cvt_f32_i32_e32 v4, v0
+; GFX1250-NEXT: v_cvt_f32_i32_e32 v11, v9
+; GFX1250-NEXT: v_cvt_f32_i32_e32 v10, v10
+; GFX1250-NEXT: v_cvt_f32_i32_e32 v9, v12
; GFX1250-NEXT: v_cvt_f32_i32_e32 v8, v1
-; GFX1250-NEXT: v_cvt_f32_i32_e32 v11, v12
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: global_store_b128 v[2:3], v[4:7], off
; GFX1250-NEXT: global_store_b128 v[2:3], v[8:11], off offset:16
@@ -369,14 +326,13 @@ define void @uitofp_2x_v2i8_to_v2f16(ptr addrspace(1) noalias %p, ptr addrspace(
; GFX9-NEXT: global_load_dword v0, v[0:1], off
; GFX9-NEXT: s_movk_i32 s4, 0xff
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 8, v0
-; GFX9-NEXT: v_and_b32_sdwa v4, v0, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT: v_and_b32_sdwa v1, v0, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT: v_cvt_f16_u16_sdwa v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1
; GFX9-NEXT: v_cvt_f16_u16_sdwa v5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0
-; GFX9-NEXT: v_cvt_f16_u16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
-; GFX9-NEXT: v_cvt_f16_u16_sdwa v6, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0
-; GFX9-NEXT: v_cvt_f16_u16_e32 v1, v4
-; GFX9-NEXT: v_pack_b32_f16 v1, v1, v0
-; GFX9-NEXT: v_pack_b32_f16 v0, v5, v6
+; GFX9-NEXT: v_cvt_f16_u16_sdwa v6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
+; GFX9-NEXT: v_cvt_f16_u16_e32 v1, v1
+; GFX9-NEXT: v_pack_b32_f16 v0, v5, v4
+; GFX9-NEXT: v_pack_b32_f16 v1, v1, v6
; GFX9-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -388,13 +344,12 @@ define void @uitofp_2x_v2i8_to_v2f16(ptr addrspace(1) noalias %p, ptr addrspace(
; GFX10-NEXT: v_mov_b32_e32 v1, 0xff
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_and_b32_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-NEXT: v_lshrrev_b32_e32 v4, 8, v0
+; GFX10-NEXT: v_cvt_f16_u16_sdwa v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1
; GFX10-NEXT: v_cvt_f16_u16_sdwa v5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0
-; GFX10-NEXT: v_cvt_f16_u16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
+; GFX10-NEXT: v_cvt_f16_u16_sdwa v6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
; GFX10-NEXT: v_cvt_f16_u16_e32 v1, v1
-; GFX10-NEXT: v_cvt_f16_u16_sdwa v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0
-; GFX10-NEXT: v_pack_b32_f16 v1, v1, v0
; GFX10-NEXT: v_pack_b32_f16 v0, v5, v4
+; GFX10-NEXT: v_pack_b32_f16 v1, v1, v6
; GFX10-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -403,19 +358,17 @@ define void @uitofp_2x_v2i8_to_v2f16(ptr addrspace(1) noalias %p, ptr addrspace(
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v1, v[0:1], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 8, v1
-; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v1.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.l, 8, v1.l
+; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.l
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 24, v1
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v4.l
-; GFX11-TRUE16-NEXT: v_cvt_f16_u16_e32 v0.l, v0.l
+; GFX11-TRUE16-NEXT: v_cvt_f16_u16_e32 v4.h, v0.l
+; GFX11-TRUE16-NEXT: v_cvt_f16_u16_e32 v4.l, v0.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cvt_f16_u16_e32 v1.h, v5.l
-; GFX11-TRUE16-NEXT: v_cvt_f16_u16_e32 v1.l, v1.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cvt_f16_u16_e32 v0.h, v0.h
-; GFX11-TRUE16-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GFX11-TRUE16-NEXT: v_cvt_f16_u16_e32 v5.h, v5.l
+; GFX11-TRUE16-NEXT: v_cvt_f16_u16_e32 v5.l, v1.l
+; GFX11-TRUE16-NEXT: global_store_b64 v[2:3], v[4:5], off
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: uitofp_2x_v2i8_to_v2f16:
@@ -424,21 +377,20 @@ define void @uitofp_2x_v2i8_to_v2f16(ptr addrspace(1) noalias %p, ptr addrspace(
; GFX11-FAKE16-NEXT: global_load_b32 v0, v[0:1], off
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 8, v0
+; GFX11-FAKE16-NEXT: v_lshrrev_b16 v4, 8, v0
; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xff, v0
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 24, v0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 0xff, v4
+; GFX11-FAKE16-NEXT: v_cvt_f16_u16_e32 v4, v4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-FAKE16-NEXT: v_cvt_f16_u16_e32 v5, v5
-; GFX11-FAKE16-NEXT: v_cvt_f16_u16_e32 v0, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_cvt_f16_u16_e32 v6, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-FAKE16-NEXT: v_cvt_f16_u16_e32 v1, v1
-; GFX11-FAKE16-NEXT: v_cvt_f16_u16_e32 v4, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_pack_b32_f16 v1, v1, v0
; GFX11-FAKE16-NEXT: v_pack_b32_f16 v0, v5, v4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_pack_b32_f16 v1, v1, v6
; GFX11-FAKE16-NEXT: global_store_b64 v[2:3], v[0:1], off
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -448,18 +400,18 @@ define void @uitofp_2x_v2i8_to_v2f16(ptr addrspace(1) noalias %p, ptr addrspace(
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: global_load_b32 v1, v[0:1], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_dual_lshrrev_b32 v4, 8, v1 :: v_dual_lshrrev_b32 v5, 24, v1
; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_and_b16 v0.l, 0xff, v1.l
+; GFX1250-NEXT: v_lshrrev_b16 v0.l, 8, v1.l
+; GFX1250-NEXT: v_and_b16 v0.h, 0xff, v1.l
+; GFX1250-NEXT: v_lshrrev_b32_e32 v5, 24, v1
; GFX1250-NEXT: v_and_b16 v1.l, 0xff, v1.h
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX1250-NEXT: v_and_b16 v0.h, 0xff, v4.l
-; GFX1250-NEXT: v_cvt_f16_u16_e32 v1.h, v5.l
-; GFX1250-NEXT: v_cvt_f16_u16_e32 v0.l, v0.l
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1250-NEXT: v_cvt_f16_u16_e32 v1.l, v1.l
-; GFX1250-NEXT: v_cvt_f16_u16_e32 v0.h, v0.h
-; GFX1250-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GFX1250-NEXT: v_cvt_f16_u16_e32 v4.h, v0.l
+; GFX1250-NEXT: v_cvt_f16_u16_e32 v4.l, v0.h
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1250-NEXT: v_cvt_f16_u16_e32 v5.h, v5.l
+; GFX1250-NEXT: v_cvt_f16_u16_e32 v5.l, v1.l
+; GFX1250-NEXT: global_store_b64 v[2:3], v[4:5], off
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%v0 = load <2 x i8>, ptr addrspace(1) %p, align 4
%c0 = uitofp <2 x i8> %v0 to <2 x half>
@@ -476,39 +428,32 @@ define i32 @zext_lshr_trunc_i16(i32 %x) {
; GFX9-LABEL: zext_lshr_trunc_i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_lshrrev_b16_e32 v0, 4, v0
+; GFX9-NEXT: v_bfe_u32 v0, v0, 4, 12
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: zext_lshr_trunc_i16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshrrev_b16 v0, 4, v0
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX10-NEXT: v_bfe_u32 v0, v0, 4, 12
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: zext_lshr_trunc_i16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.l, 4, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
+; GFX11-TRUE16-NEXT: v_bfe_u32 v0, v0, 4, 12
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: zext_lshr_trunc_i16:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b16 v0, 4, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-FAKE16-NEXT: v_bfe_u32 v0, v0, 4, 12
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: zext_lshr_trunc_i16:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_lshrrev_b16 v0.l, 4, v0.l
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cvt_u32_u16_e32 v0, v0.l
+; GFX1250-NEXT: v_bfe_u32 v0, v0, 4, 12
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%t = trunc i32 %x to i16
%s = lshr i16 %t, 4
@@ -520,40 +465,32 @@ define i32 @sext_ashr_trunc_i16(i32 %x) {
; GFX9-LABEL: sext_ashr_trunc_i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_ashrrev_i16_e32 v0, 8, v0
-; GFX9-NEXT: v_bfe_i32 v0, v0, 0, 16
+; GFX9-NEXT: v_bfe_i32 v0, v0, 8, 8
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: sext_ashr_trunc_i16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_ashrrev_i16 v0, 8, v0
-; GFX10-NEXT: v_bfe_i32 v0, v0, 0, 16
+; GFX10-NEXT: v_bfe_i32 v0, v0, 8, 8
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: sext_ashr_trunc_i16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_ashrrev_i16 v0.l, 8, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 16
+; GFX11-TRUE16-NEXT: v_bfe_i32 v0, v0, 8, 8
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: sext_ashr_trunc_i16:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_ashrrev_i16 v0, 8, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_bfe_i32 v0, v0, 0, 16
+; GFX11-FAKE16-NEXT: v_bfe_i32 v0, v0, 8, 8
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: sext_ashr_trunc_i16:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_ashrrev_i16 v0.l, 8, v0.l
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_bfe_i32 v0, v0, 0, 16
+; GFX1250-NEXT: v_bfe_i32 v0, v0, 8, 8
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%t = trunc i32 %x to i16
%s = ashr i16 %t, 8
@@ -564,28 +501,22 @@ define i32 @sext_ashr_trunc_i16(i32 %x) {
define amdgpu_ps i32 @s_sext_ashr_trunc_i16(i32 inreg %x) {
; GFX9-LABEL: s_sext_ashr_trunc_i16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_sext_i32_i16 s0, s0
-; GFX9-NEXT: s_ashr_i32 s0, s0, 8
+; GFX9-NEXT: s_bfe_i32 s0, s0, 0x80008
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: s_sext_ashr_trunc_i16:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_sext_i32_i16 s0, s0
-; GFX10-NEXT: s_ashr_i32 s0, s0, 8
+; GFX10-NEXT: s_bfe_i32 s0, s0, 0x80008
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-TRUE16-LABEL: s_sext_ashr_trunc_i16:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_sext_i32_i16 s0, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_ashr_i32 s0, s0, 8
+; GFX11-TRUE16-NEXT: s_bfe_i32 s0, s0, 0x80008
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
; GFX11-FAKE16-LABEL: s_sext_ashr_trunc_i16:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_sext_i32_i16 s0, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_ashr_i32 s0, s0, 8
+; GFX11-FAKE16-NEXT: s_bfe_i32 s0, s0, 0x80008
; GFX11-FAKE16-NEXT: ; return to shader part epilog
;
; GFX1250-LABEL: s_sext_ashr_trunc_i16:
@@ -594,9 +525,7 @@ define amdgpu_ps i32 @s_sext_ashr_trunc_i16(i32 inreg %x) {
; GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_sext_i32_i16 s0, s0
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: s_ashr_i32 s0, s0, 8
+; GFX1250-NEXT: s_bfe_i32 s0, s0, 0x80008
; GFX1250-NEXT: ; return to shader part epilog
%t = trunc i32 %x to i16
%s = ashr i16 %t, 8
@@ -659,25 +588,25 @@ define i32 @zext_lshr_trunc_i16_multi_use(i32 %x, ptr addrspace(1) %o) {
define amdgpu_ps void @anyext_lshr_trunc_i16(<4 x i32> inreg %rsrc, i32 %x) {
; GFX9-LABEL: anyext_lshr_trunc_i16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: v_lshrrev_b16_e32 v0, 8, v0
+; GFX9-NEXT: v_bfe_u32 v0, v0, 8, 8
; GFX9-NEXT: buffer_store_short v0, off, s[0:3], 0
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: anyext_lshr_trunc_i16:
; GFX10: ; %bb.0:
-; GFX10-NEXT: v_lshrrev_b16 v0, 8, v0
+; GFX10-NEXT: v_bfe_u32 v0, v0, 8, 8
; GFX10-NEXT: buffer_store_short v0, off, s[0:3], 0
; GFX10-NEXT: s_endpgm
;
; GFX11-TRUE16-LABEL: anyext_lshr_trunc_i16:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.l, 8, v0.l
+; GFX11-TRUE16-NEXT: v_bfe_u32 v0, v0, 8, 8
; GFX11-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0
; GFX11-TRUE16-NEXT: s_endpgm
;
; GFX11-FAKE16-LABEL: anyext_lshr_trunc_i16:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: v_lshrrev_b16 v0, 8, v0
+; GFX11-FAKE16-NEXT: v_bfe_u32 v0, v0, 8, 8
; GFX11-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0
; GFX11-FAKE16-NEXT: s_endpgm
;
@@ -687,7 +616,7 @@ define amdgpu_ps void @anyext_lshr_trunc_i16(<4 x i32> inreg %rsrc, i32 %x) {
; GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: v_lshrrev_b16 v0.l, 8, v0.l
+; GFX1250-NEXT: v_bfe_u32 v0, v0, 8, 8
; GFX1250-NEXT: buffer_store_b16 v0, off, s[0:3], null
; GFX1250-NEXT: s_endpgm
%t = trunc i32 %x to i16
>From f7d8315be4882a56e373422d94f11fa4f88ec951 Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Sat, 10 Oct 2026 11:25:24 +0200
Subject: [PATCH 2/3] Regenerate load-global-i8.ll and sign_extend.ll after
rebase
---
llvm/test/CodeGen/AMDGPU/load-global-i8.ll | 282 ++++++++++-----------
llvm/test/CodeGen/AMDGPU/sign_extend.ll | 11 +-
2 files changed, 146 insertions(+), 147 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/load-global-i8.ll b/llvm/test/CodeGen/AMDGPU/load-global-i8.ll
index 6b729701ffb7d..3866f0fd54ba3 100644
--- a/llvm/test/CodeGen/AMDGPU/load-global-i8.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-global-i8.ll
@@ -1314,13 +1314,13 @@ define amdgpu_kernel void @global_zextload_v3i8_to_v3i32(ptr addrspace(1) %out,
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v1, s3
; GCN-NOHSA-VI-NEXT: v_add_u32_e32 v0, vcc, s2, v0
; GCN-NOHSA-VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GCN-NOHSA-VI-NEXT: flat_load_dword v2, v[0:1]
+; GCN-NOHSA-VI-NEXT: flat_load_dword v1, v[0:1]
; GCN-NOHSA-VI-NEXT: s_mov_b32 s3, 0xf000
; GCN-NOHSA-VI-NEXT: s_mov_b32 s2, -1
; GCN-NOHSA-VI-NEXT: s_waitcnt vmcnt(0)
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v0, 0xff, v2
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b16_e32 v1, 8, v2
-; GCN-NOHSA-VI-NEXT: v_bfe_u32 v2, v2, 16, 8
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v0, 0xff, v1
+; GCN-NOHSA-VI-NEXT: v_bfe_u32 v2, v1, 16, 8
+; GCN-NOHSA-VI-NEXT: v_bfe_u32 v1, v1, 8, 8
; GCN-NOHSA-VI-NEXT: buffer_store_dwordx3 v[0:2], off, s[0:3], 0
; GCN-NOHSA-VI-NEXT: s_endpgm
;
@@ -1443,10 +1443,9 @@ define amdgpu_kernel void @global_sextload_v3i8_to_v3i32(ptr addrspace(1) %out,
; GCN-NOHSA-VI-NEXT: s_mov_b32 s3, 0xf000
; GCN-NOHSA-VI-NEXT: s_mov_b32 s2, -1
; GCN-NOHSA-VI-NEXT: s_waitcnt vmcnt(0)
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b16_e32 v1, 8, v0
; GCN-NOHSA-VI-NEXT: v_bfe_i32 v2, v0, 16, 8
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v1, v0, 8, 8
; GCN-NOHSA-VI-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v1, v1, 0, 8
; GCN-NOHSA-VI-NEXT: buffer_store_dwordx3 v[0:2], off, s[0:3], 0
; GCN-NOHSA-VI-NEXT: s_endpgm
;
@@ -1568,14 +1567,14 @@ define amdgpu_kernel void @global_zextload_v4i8_to_v4i32(ptr addrspace(1) %out,
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v1, s3
; GCN-NOHSA-VI-NEXT: v_add_u32_e32 v0, vcc, s2, v0
; GCN-NOHSA-VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GCN-NOHSA-VI-NEXT: flat_load_dword v2, v[0:1]
+; GCN-NOHSA-VI-NEXT: flat_load_dword v1, v[0:1]
; GCN-NOHSA-VI-NEXT: s_mov_b32 s3, 0xf000
; GCN-NOHSA-VI-NEXT: s_mov_b32 s2, -1
; GCN-NOHSA-VI-NEXT: s_waitcnt vmcnt(0)
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v3, 24, v2
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v0, 0xff, v2
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b16_e32 v1, 8, v2
-; GCN-NOHSA-VI-NEXT: v_bfe_u32 v2, v2, 16, 8
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v3, 24, v1
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v0, 0xff, v1
+; GCN-NOHSA-VI-NEXT: v_bfe_u32 v2, v1, 16, 8
+; GCN-NOHSA-VI-NEXT: v_bfe_u32 v1, v1, 8, 8
; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
; GCN-NOHSA-VI-NEXT: s_endpgm
;
@@ -1697,11 +1696,10 @@ define amdgpu_kernel void @global_sextload_v4i8_to_v4i32(ptr addrspace(1) %out,
; GCN-NOHSA-VI-NEXT: s_mov_b32 s3, 0xf000
; GCN-NOHSA-VI-NEXT: s_mov_b32 s2, -1
; GCN-NOHSA-VI-NEXT: s_waitcnt vmcnt(0)
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b16_e32 v1, 8, v0
; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v3, 24, v0
; GCN-NOHSA-VI-NEXT: v_bfe_i32 v2, v0, 16, 8
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v1, v0, 8, 8
; GCN-NOHSA-VI-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v1, v1, 0, 8
; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
; GCN-NOHSA-VI-NEXT: s_endpgm
;
@@ -1836,18 +1834,18 @@ define amdgpu_kernel void @global_zextload_v8i8_to_v8i32(ptr addrspace(1) %out,
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v1, s3
; GCN-NOHSA-VI-NEXT: v_add_u32_e32 v0, vcc, s2, v0
; GCN-NOHSA-VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GCN-NOHSA-VI-NEXT: flat_load_dwordx2 v[8:9], v[0:1]
+; GCN-NOHSA-VI-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
; GCN-NOHSA-VI-NEXT: s_mov_b32 s3, 0xf000
; GCN-NOHSA-VI-NEXT: s_mov_b32 s2, -1
; GCN-NOHSA-VI-NEXT: s_waitcnt vmcnt(0)
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v7, 24, v8
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b16_e32 v5, 8, v8
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v4, 0xff, v8
-; GCN-NOHSA-VI-NEXT: v_bfe_u32 v6, v8, 16, 8
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v3, 24, v9
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v0, 0xff, v9
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b16_e32 v1, 8, v9
-; GCN-NOHSA-VI-NEXT: v_bfe_u32 v2, v9, 16, 8
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v3, 24, v7
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v0, 0xff, v7
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b16_e32 v1, 8, v7
+; GCN-NOHSA-VI-NEXT: v_bfe_u32 v2, v7, 16, 8
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v7, 24, v6
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b16_e32 v5, 8, v6
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v4, 0xff, v6
+; GCN-NOHSA-VI-NEXT: v_bfe_u32 v6, v6, 16, 8
; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0
; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:16
; GCN-NOHSA-VI-NEXT: s_endpgm
@@ -2201,14 +2199,14 @@ define amdgpu_kernel void @global_zextload_v16i8_to_v16i32(ptr addrspace(1) %out
; GCN-NOHSA-VI-NEXT: v_lshrrev_b16_e32 v17, 8, v3
; GCN-NOHSA-VI-NEXT: v_bfe_u32 v18, v3, 16, 8
; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v7, 24, v0
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v11, 24, v1
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v15, 24, v2
; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v4, 0xff, v0
; GCN-NOHSA-VI-NEXT: v_lshrrev_b16_e32 v5, 8, v0
; GCN-NOHSA-VI-NEXT: v_bfe_u32 v6, v0, 16, 8
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v11, 24, v1
; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v8, 0xff, v1
; GCN-NOHSA-VI-NEXT: v_lshrrev_b16_e32 v9, 8, v1
; GCN-NOHSA-VI-NEXT: v_bfe_u32 v10, v1, 16, 8
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v15, 24, v2
; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v12, 0xff, v2
; GCN-NOHSA-VI-NEXT: v_lshrrev_b16_e32 v13, 8, v2
; GCN-NOHSA-VI-NEXT: v_bfe_u32 v14, v2, 16, 8
@@ -2740,55 +2738,55 @@ define amdgpu_kernel void @global_zextload_v32i8_to_v32i32(ptr addrspace(1) %out
; GCN-NOHSA-VI-NEXT: v_lshlrev_b32_e32 v0, 5, v0
; GCN-NOHSA-VI-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v1, s3
-; GCN-NOHSA-VI-NEXT: v_add_u32_e32 v0, vcc, s2, v0
-; GCN-NOHSA-VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GCN-NOHSA-VI-NEXT: v_add_u32_e32 v2, vcc, 16, v0
-; GCN-NOHSA-VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc
-; GCN-NOHSA-VI-NEXT: flat_load_dwordx4 v[12:15], v[2:3]
-; GCN-NOHSA-VI-NEXT: flat_load_dwordx4 v[16:19], v[0:1]
+; GCN-NOHSA-VI-NEXT: v_add_u32_e32 v4, vcc, s2, v0
+; GCN-NOHSA-VI-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GCN-NOHSA-VI-NEXT: flat_load_dwordx4 v[0:3], v[4:5]
+; GCN-NOHSA-VI-NEXT: v_add_u32_e32 v4, vcc, 16, v4
+; GCN-NOHSA-VI-NEXT: v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; GCN-NOHSA-VI-NEXT: flat_load_dwordx4 v[4:7], v[4:5]
; GCN-NOHSA-VI-NEXT: s_mov_b32 s3, 0xf000
; GCN-NOHSA-VI-NEXT: s_mov_b32 s2, -1
; GCN-NOHSA-VI-NEXT: s_waitcnt vmcnt(1)
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v3, 24, v12
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v23, 24, v3
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v20, 0xff, v3
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b16_e32 v21, 8, v3
+; GCN-NOHSA-VI-NEXT: v_bfe_u32 v22, v3, 16, 8
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v11, 24, v0
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v8, 0xff, v0
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b16_e32 v9, 8, v0
+; GCN-NOHSA-VI-NEXT: v_bfe_u32 v10, v0, 16, 8
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v15, 24, v1
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v12, 0xff, v1
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b16_e32 v13, 8, v1
+; GCN-NOHSA-VI-NEXT: v_bfe_u32 v14, v1, 16, 8
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v19, 24, v2
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v16, 0xff, v2
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b16_e32 v17, 8, v2
+; GCN-NOHSA-VI-NEXT: v_bfe_u32 v18, v2, 16, 8
; GCN-NOHSA-VI-NEXT: s_waitcnt vmcnt(0)
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v35, 24, v19
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v32, 0xff, v19
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b16_e32 v33, 8, v19
-; GCN-NOHSA-VI-NEXT: v_bfe_u32 v34, v19, 16, 8
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v7, 24, v13
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v11, 24, v14
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v0, 0xff, v12
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b16_e32 v1, 8, v12
-; GCN-NOHSA-VI-NEXT: v_bfe_u32 v2, v12, 16, 8
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v4, 0xff, v13
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b16_e32 v5, 8, v13
-; GCN-NOHSA-VI-NEXT: v_bfe_u32 v6, v13, 16, 8
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v8, 0xff, v14
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b16_e32 v9, 8, v14
-; GCN-NOHSA-VI-NEXT: v_bfe_u32 v10, v14, 16, 8
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v23, 24, v15
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v20, 0xff, v15
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b16_e32 v21, 8, v15
-; GCN-NOHSA-VI-NEXT: v_bfe_u32 v22, v15, 16, 8
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v15, 24, v16
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v12, 0xff, v16
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b16_e32 v13, 8, v16
-; GCN-NOHSA-VI-NEXT: v_bfe_u32 v14, v16, 16, 8
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v27, 24, v17
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v24, 0xff, v17
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b16_e32 v25, 8, v17
-; GCN-NOHSA-VI-NEXT: v_bfe_u32 v26, v17, 16, 8
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v31, 24, v18
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v28, 0xff, v18
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b16_e32 v29, 8, v18
-; GCN-NOHSA-VI-NEXT: v_bfe_u32 v30, v18, 16, 8
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[32:35], off, s[0:3], 0 offset:48
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[28:31], off, s[0:3], 0 offset:32
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[24:27], off, s[0:3], 0 offset:16
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[12:15], off, s[0:3], 0
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[20:23], off, s[0:3], 0 offset:112
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:96
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:80
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v3, 24, v4
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v0, 0xff, v4
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b16_e32 v1, 8, v4
+; GCN-NOHSA-VI-NEXT: v_bfe_u32 v2, v4, 16, 8
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v27, 24, v5
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v24, 0xff, v5
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b16_e32 v25, 8, v5
+; GCN-NOHSA-VI-NEXT: v_bfe_u32 v26, v5, 16, 8
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v31, 24, v6
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v28, 0xff, v6
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b16_e32 v29, 8, v6
+; GCN-NOHSA-VI-NEXT: v_bfe_u32 v30, v6, 16, 8
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v35, 24, v7
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v32, 0xff, v7
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b16_e32 v33, 8, v7
+; GCN-NOHSA-VI-NEXT: v_bfe_u32 v34, v7, 16, 8
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[20:23], off, s[0:3], 0 offset:48
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[16:19], off, s[0:3], 0 offset:32
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[12:15], off, s[0:3], 0 offset:16
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[8:11], off, s[0:3], 0
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[32:35], off, s[0:3], 0 offset:112
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[28:31], off, s[0:3], 0 offset:96
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[24:27], off, s[0:3], 0 offset:80
; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:64
; GCN-NOHSA-VI-NEXT: s_endpgm
;
@@ -3714,96 +3712,98 @@ define amdgpu_kernel void @global_zextload_v64i8_to_v64i32(ptr addrspace(1) %out
; GCN-NOHSA-VI-NEXT: v_lshlrev_b32_e32 v0, 6, v0
; GCN-NOHSA-VI-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v1, s3
-; GCN-NOHSA-VI-NEXT: v_add_u32_e32 v0, vcc, s2, v0
-; GCN-NOHSA-VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GCN-NOHSA-VI-NEXT: v_add_u32_e32 v2, vcc, 16, v0
-; GCN-NOHSA-VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc
-; GCN-NOHSA-VI-NEXT: flat_load_dwordx4 v[12:15], v[2:3]
-; GCN-NOHSA-VI-NEXT: v_add_u32_e32 v2, vcc, 32, v0
-; GCN-NOHSA-VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc
-; GCN-NOHSA-VI-NEXT: flat_load_dwordx4 v[28:31], v[2:3]
-; GCN-NOHSA-VI-NEXT: v_add_u32_e32 v2, vcc, 48, v0
-; GCN-NOHSA-VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc
-; GCN-NOHSA-VI-NEXT: flat_load_dwordx4 v[44:47], v[2:3]
-; GCN-NOHSA-VI-NEXT: flat_load_dwordx4 v[48:51], v[0:1]
+; GCN-NOHSA-VI-NEXT: v_add_u32_e32 v4, vcc, s2, v0
+; GCN-NOHSA-VI-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GCN-NOHSA-VI-NEXT: v_add_u32_e32 v6, vcc, 16, v4
+; GCN-NOHSA-VI-NEXT: v_addc_u32_e32 v7, vcc, 0, v5, vcc
+; GCN-NOHSA-VI-NEXT: flat_load_dwordx4 v[0:3], v[4:5]
+; GCN-NOHSA-VI-NEXT: flat_load_dwordx4 v[12:15], v[6:7]
+; GCN-NOHSA-VI-NEXT: v_add_u32_e32 v6, vcc, 32, v4
+; GCN-NOHSA-VI-NEXT: v_addc_u32_e32 v7, vcc, 0, v5, vcc
+; GCN-NOHSA-VI-NEXT: v_add_u32_e32 v4, vcc, 48, v4
+; GCN-NOHSA-VI-NEXT: v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; GCN-NOHSA-VI-NEXT: flat_load_dwordx4 v[36:39], v[6:7]
+; GCN-NOHSA-VI-NEXT: flat_load_dwordx4 v[40:43], v[4:5]
; GCN-NOHSA-VI-NEXT: s_mov_b32 s3, 0xf000
; GCN-NOHSA-VI-NEXT: s_mov_b32 s2, -1
; GCN-NOHSA-VI-NEXT: s_waitcnt vmcnt(3)
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v51, 24, v2
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v48, 0xff, v2
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b16_e32 v49, 8, v2
+; GCN-NOHSA-VI-NEXT: v_bfe_u32 v50, v2, 16, 8
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v27, 24, v1
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v24, 0xff, v1
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b16_e32 v25, 8, v1
+; GCN-NOHSA-VI-NEXT: v_bfe_u32 v26, v1, 16, 8
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v35, 24, v0
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v32, 0xff, v0
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b16_e32 v33, 8, v0
+; GCN-NOHSA-VI-NEXT: v_bfe_u32 v34, v0, 16, 8
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v47, 24, v3
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v44, 0xff, v3
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b16_e32 v45, 8, v3
+; GCN-NOHSA-VI-NEXT: v_bfe_u32 v46, v3, 16, 8
+; GCN-NOHSA-VI-NEXT: s_waitcnt vmcnt(2)
; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v3, 24, v13
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v7, 24, v12
; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v0, 0xff, v13
; GCN-NOHSA-VI-NEXT: v_lshrrev_b16_e32 v1, 8, v13
; GCN-NOHSA-VI-NEXT: v_bfe_u32 v2, v13, 16, 8
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v7, 24, v12
; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v4, 0xff, v12
; GCN-NOHSA-VI-NEXT: v_lshrrev_b16_e32 v5, 8, v12
; GCN-NOHSA-VI-NEXT: v_bfe_u32 v6, v12, 16, 8
; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v11, 24, v15
; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v8, 0xff, v15
-; GCN-NOHSA-VI-NEXT: s_waitcnt vmcnt(0)
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v55, 24, v49
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v52, 0xff, v49
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b16_e32 v53, 8, v49
-; GCN-NOHSA-VI-NEXT: v_bfe_u32 v54, v49, 16, 8
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v59, 24, v48
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v56, 0xff, v48
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b16_e32 v57, 8, v48
-; GCN-NOHSA-VI-NEXT: v_bfe_u32 v58, v48, 16, 8
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v63, 24, v51
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v60, 0xff, v51
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b16_e32 v61, 8, v51
-; GCN-NOHSA-VI-NEXT: v_bfe_u32 v62, v51, 16, 8
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v51, 24, v50
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v48, 0xff, v50
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b16_e32 v49, 8, v50
-; GCN-NOHSA-VI-NEXT: v_bfe_u32 v50, v50, 16, 8
; GCN-NOHSA-VI-NEXT: v_lshrrev_b16_e32 v9, 8, v15
; GCN-NOHSA-VI-NEXT: v_bfe_u32 v10, v15, 16, 8
; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v15, 24, v14
; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v12, 0xff, v14
; GCN-NOHSA-VI-NEXT: v_lshrrev_b16_e32 v13, 8, v14
; GCN-NOHSA-VI-NEXT: v_bfe_u32 v14, v14, 16, 8
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v19, 24, v29
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v16, 0xff, v29
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b16_e32 v17, 8, v29
-; GCN-NOHSA-VI-NEXT: v_bfe_u32 v18, v29, 16, 8
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v23, 24, v28
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v20, 0xff, v28
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b16_e32 v21, 8, v28
-; GCN-NOHSA-VI-NEXT: v_bfe_u32 v22, v28, 16, 8
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v27, 24, v31
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v24, 0xff, v31
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b16_e32 v25, 8, v31
-; GCN-NOHSA-VI-NEXT: v_bfe_u32 v26, v31, 16, 8
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v31, 24, v30
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v28, 0xff, v30
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b16_e32 v29, 8, v30
-; GCN-NOHSA-VI-NEXT: v_bfe_u32 v30, v30, 16, 8
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v35, 24, v45
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v32, 0xff, v45
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b16_e32 v33, 8, v45
-; GCN-NOHSA-VI-NEXT: v_bfe_u32 v34, v45, 16, 8
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v39, 24, v44
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v36, 0xff, v44
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b16_e32 v37, 8, v44
-; GCN-NOHSA-VI-NEXT: v_bfe_u32 v38, v44, 16, 8
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v43, 24, v47
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v40, 0xff, v47
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b16_e32 v41, 8, v47
-; GCN-NOHSA-VI-NEXT: v_bfe_u32 v42, v47, 16, 8
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v47, 24, v46
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v44, 0xff, v46
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b16_e32 v45, 8, v46
-; GCN-NOHSA-VI-NEXT: v_bfe_u32 v46, v46, 16, 8
+; GCN-NOHSA-VI-NEXT: s_waitcnt vmcnt(1)
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v19, 24, v37
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v16, 0xff, v37
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b16_e32 v17, 8, v37
+; GCN-NOHSA-VI-NEXT: v_bfe_u32 v18, v37, 16, 8
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v23, 24, v36
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v20, 0xff, v36
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b16_e32 v21, 8, v36
+; GCN-NOHSA-VI-NEXT: v_bfe_u32 v22, v36, 16, 8
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v31, 24, v39
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v28, 0xff, v39
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b16_e32 v29, 8, v39
+; GCN-NOHSA-VI-NEXT: v_bfe_u32 v30, v39, 16, 8
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v39, 24, v38
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v36, 0xff, v38
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b16_e32 v37, 8, v38
+; GCN-NOHSA-VI-NEXT: v_bfe_u32 v38, v38, 16, 8
+; GCN-NOHSA-VI-NEXT: s_waitcnt vmcnt(0)
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v55, 24, v41
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v52, 0xff, v41
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b16_e32 v53, 8, v41
+; GCN-NOHSA-VI-NEXT: v_bfe_u32 v54, v41, 16, 8
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v59, 24, v40
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v56, 0xff, v40
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b16_e32 v57, 8, v40
+; GCN-NOHSA-VI-NEXT: v_bfe_u32 v58, v40, 16, 8
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v63, 24, v43
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v60, 0xff, v43
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b16_e32 v61, 8, v43
+; GCN-NOHSA-VI-NEXT: v_bfe_u32 v62, v43, 16, 8
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v43, 24, v42
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v40, 0xff, v42
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b16_e32 v41, 8, v42
+; GCN-NOHSA-VI-NEXT: v_bfe_u32 v42, v42, 16, 8
; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[48:51], off, s[0:3], 0 offset:32
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[60:63], off, s[0:3], 0 offset:48
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[56:59], off, s[0:3], 0
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[52:55], off, s[0:3], 0 offset:16
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[44:47], off, s[0:3], 0 offset:224
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[40:43], off, s[0:3], 0 offset:240
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[36:39], off, s[0:3], 0 offset:192
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[32:35], off, s[0:3], 0 offset:208
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[28:31], off, s[0:3], 0 offset:160
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[24:27], off, s[0:3], 0 offset:176
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[44:47], off, s[0:3], 0 offset:48
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[32:35], off, s[0:3], 0
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[24:27], off, s[0:3], 0 offset:16
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[40:43], off, s[0:3], 0 offset:224
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[60:63], off, s[0:3], 0 offset:240
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[56:59], off, s[0:3], 0 offset:192
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[52:55], off, s[0:3], 0 offset:208
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[36:39], off, s[0:3], 0 offset:160
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[28:31], off, s[0:3], 0 offset:176
; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[20:23], off, s[0:3], 0 offset:128
; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[16:19], off, s[0:3], 0 offset:144
; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[12:15], off, s[0:3], 0 offset:96
@@ -9416,7 +9416,7 @@ define amdgpu_kernel void @global_zextload_v4i8_to_v4i16(ptr addrspace(1) %out,
; GCN-NOHSA-VI-NEXT: s_mov_b32 s2, -1
; GCN-NOHSA-VI-NEXT: s_waitcnt vmcnt(0)
; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v2, 24, v1
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b16_e32 v0, 8, v1
+; GCN-NOHSA-VI-NEXT: v_bfe_u32 v0, v1, 8, 8
; GCN-NOHSA-VI-NEXT: v_perm_b32 v0, v0, v1, s4
; GCN-NOHSA-VI-NEXT: v_alignbit_b32 v1, v2, v1, 16
; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v1, 0xff00ff, v1
diff --git a/llvm/test/CodeGen/AMDGPU/sign_extend.ll b/llvm/test/CodeGen/AMDGPU/sign_extend.ll
index 59db5e42d9b1e..f00bc046c6298 100644
--- a/llvm/test/CodeGen/AMDGPU/sign_extend.ll
+++ b/llvm/test/CodeGen/AMDGPU/sign_extend.ll
@@ -418,19 +418,18 @@ define amdgpu_kernel void @v_sext_v4i8_to_v4i32(ptr addrspace(1) %out, ptr addrs
; VI-NEXT: s_mov_b32 s3, 0xf000
; VI-NEXT: s_mov_b32 s2, -1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_lshrrev_b16_e32 v1, 8, v0
-; VI-NEXT: v_ashrrev_i32_e32 v2, 24, v0
-; VI-NEXT: v_bfe_i32 v3, v0, 16, 8
+; VI-NEXT: v_ashrrev_i32_e32 v1, 24, v0
+; VI-NEXT: v_bfe_i32 v2, v0, 16, 8
+; VI-NEXT: v_bfe_i32 v3, v0, 8, 8
; VI-NEXT: v_bfe_i32 v0, v0, 0, 8
-; VI-NEXT: v_bfe_i32 v1, v1, 0, 8
; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], 0
-; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_store_dword v3, off, s[0:3], 0
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_store_dword v2, off, s[0:3], 0
; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], 0
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%in.gep = getelementptr inbounds i32, ptr addrspace(1) %in, i32 %tid
>From f3ee96e58f525faf53617063311f48815e1e12f6 Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Sat, 10 Oct 2026 11:49:09 +0200
Subject: [PATCH 3/3] Drop the zero shift check in the and fold
---
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 49888927e8a9c..3704596a4c07b 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -15062,7 +15062,7 @@ SDValue SITargetLowering::performAndCombine(SDNode *N,
if (CRHS && VT == MVT::i16 && LHS.getOpcode() == ISD::TRUNCATE &&
LHS.hasOneUse() && LHS.getOperand(0).getOpcode() == ISD::SRL) {
auto *Shift = dyn_cast<ConstantSDNode>(LHS.getOperand(0).getOperand(1));
- if (Shift && Shift->getZExtValue() > 0 && Shift->getZExtValue() < 16 &&
+ if (Shift && Shift->getZExtValue() < 16 &&
CRHS->getAPIntValue().isMask(16 - Shift->getZExtValue())) {
SDLoc SL(N);
SDValue Trunc =
More information about the llvm-branch-commits
mailing list