[llvm-branch-commits] [llvm] [AMDGPU] Extract byte lanes of a split vector from the 32-bit source (PR #228424)
Dmitry Sidorov via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Mon Oct 5 02:08:21 PDT 2026
https://github.com/MrSidims updated https://github.com/llvm/llvm-project/pull/228424
>From e1cdff9a042c4be525e5e5c9922a4b086e9b4783 Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Fri, 2 Oct 2026 12:53:19 +0200
Subject: [PATCH] [AMDGPU] Extract byte lanes of a split vector from the 32-bit
source
After a <4 x i8> is split into i16 halves, a byte lane is extended
from an i16 shift of a truncate. Rewrite it on the 32-bit source as an
and of srl or a sign_extend_inreg of srl, which select to a single bit
field extract.
Uniform zero and any extends are left alone, their i16 shift is already
promoted to i32.
---
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 64 +++
llvm/lib/Target/AMDGPU/SIISelLowering.h | 1 +
llvm/test/CodeGen/AMDGPU/idot4-test.ll | 109 +++--
llvm/test/CodeGen/AMDGPU/idot4s.ll | 79 ++--
llvm/test/CodeGen/AMDGPU/idot4u.ll | 240 +++++------
llvm/test/CodeGen/AMDGPU/lshr.v2i16.ll | 9 +-
llvm/test/CodeGen/AMDGPU/min.ll | 162 ++++----
llvm/test/CodeGen/AMDGPU/permute_i8.ll | 36 +-
llvm/test/CodeGen/AMDGPU/sad.ll | 18 +-
.../test/CodeGen/AMDGPU/v4i8-byte-lane-ext.ll | 381 +++++++-----------
10 files changed, 511 insertions(+), 588 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 2a2d4c5f91988..916290c6030e0 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -1181,6 +1181,7 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
ISD::FCANONICALIZE,
ISD::SCALAR_TO_VECTOR,
ISD::ZERO_EXTEND,
+ ISD::SIGN_EXTEND,
ISD::SIGN_EXTEND_INREG,
ISD::ANY_EXTEND,
ISD::EXTRACT_VECTOR_ELT,
@@ -15127,6 +15128,22 @@ SDValue SITargetLowering::performAndCombine(SDNode *N,
}
}
+ // and (trunc (srl x, c)), mask => srl (trunc x), c
+ // if mask keeps exactly the bits that srl leaves in the low half
+ if (CRHS && VT == MVT::i16 && LHS.getOpcode() == ISD::TRUNCATE &&
+ LHS.hasOneUse() && LHS.getOperand(0).getOpcode() == ISD::SRL) {
+ auto *Shift = dyn_cast<ConstantSDNode>(LHS.getOperand(0).getOperand(1));
+ if (Shift && Shift->getZExtValue() > 0 && Shift->getZExtValue() < 16 &&
+ CRHS->getAPIntValue().isMask(16 - Shift->getZExtValue())) {
+ SDLoc SL(N);
+ SDValue Trunc =
+ DAG.getNode(ISD::TRUNCATE, SL, VT, LHS.getOperand(0).getOperand(0));
+ return DAG.getNode(
+ ISD::SRL, SL, VT, Trunc,
+ DAG.getShiftAmountConstant(Shift->getZExtValue(), VT, SL));
+ }
+ }
+
// (and (fcmp ord x, x), (fcmp une (fabs x), inf)) ->
// fp_class x, ~(s_nan | q_nan | n_infinity | p_infinity)
if (LHS.getOpcode() == ISD::SETCC && RHS.getOpcode() == ISD::SETCC) {
@@ -16127,6 +16144,9 @@ SITargetLowering::performZeroOrAnyExtendCombine(SDNode *N,
if (!Src->hasOneUse())
return SDValue();
+ if (SDValue Ext = performExtendTruncShiftCombine(N, DCI))
+ return Ext;
+
// TODO: We bail out below if SrcOffset is not in the first dword (>= 4). It's
// possible we're missing out on some combine opportunities, but we'd need to
// weigh the cost of extracting the byte from the upper dwords.
@@ -16164,6 +16184,48 @@ SITargetLowering::performZeroOrAnyExtendCombine(SDNode *N,
DAG.getConstant(PermMask, DL, MVT::i32));
}
+// zext (srl (trunc x), c) => and (srl x, c), lowbits
+// aext (srl (trunc x), c) => and (srl x, c), lowbits
+// sext (sra (trunc x), 8) => sext_inreg (srl x, 8), i8
+SDValue
+SITargetLowering::performExtendTruncShiftCombine(SDNode *N,
+ DAGCombinerInfo &DCI) const {
+ if (DCI.getDAGCombineLevel() < AfterLegalizeTypes)
+ return SDValue();
+
+ const unsigned Opc = N->getOpcode();
+ const bool IsSigned = Opc == ISD::SIGN_EXTEND;
+ SDValue Src = N->getOperand(0);
+ if (N->getValueType(0) != MVT::i32 || Src.getValueType() != MVT::i16 ||
+ Src.getOpcode() != (IsSigned ? ISD::SRA : ISD::SRL) || !Src.hasOneUse() ||
+ Src.getOperand(0).getOpcode() != ISD::TRUNCATE)
+ return SDValue();
+
+ // A uniform i16 srl is promoted to i32 later and yields the same nodes.
+ // Skip it so that the node order, and with it the schedule, stays the same.
+ if (!IsSigned && !Src->isDivergent())
+ return SDValue();
+
+ SDValue X = Src.getOperand(0).getOperand(0);
+ auto *Shift = dyn_cast<ConstantSDNode>(Src.getOperand(1));
+ if (X.getValueType() != MVT::i32 || !Shift || Shift->getZExtValue() == 0 ||
+ Shift->getZExtValue() >= 16 || (IsSigned && Shift->getZExtValue() != 8))
+ return SDValue();
+
+ SelectionDAG &DAG = DCI.DAG;
+ SDLoc SL(N);
+ const unsigned Width = 16 - Shift->getZExtValue();
+ SDValue Srl = DAG.getNode(
+ ISD::SRL, SL, MVT::i32, X,
+ DAG.getShiftAmountConstant(Shift->getZExtValue(), MVT::i32, SL));
+ if (IsSigned)
+ return DAG.getNode(ISD::SIGN_EXTEND_INREG, SL, MVT::i32, Srl,
+ DAG.getValueType(MVT::i8));
+ return DAG.getNode(
+ ISD::AND, SL, MVT::i32, Srl,
+ DAG.getConstant(maskTrailingOnes<uint32_t>(Width), SL, MVT::i32));
+}
+
SDValue
SITargetLowering::performSignExtendInRegCombine(SDNode *N,
DAGCombinerInfo &DCI) const {
@@ -19402,6 +19464,8 @@ SDValue SITargetLowering::PerformDAGCombine(SDNode *N,
case ISD::ANY_EXTEND:
case ISD::ZERO_EXTEND:
return performZeroOrAnyExtendCombine(N, DCI);
+ case ISD::SIGN_EXTEND:
+ return performExtendTruncShiftCombine(N, DCI);
case ISD::SIGN_EXTEND_INREG:
return performSignExtendInRegCombine(N, DCI);
case AMDGPUISD::FP_CLASS:
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h
index d1200ca7ce825..177f1ad9711b9 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h
@@ -229,6 +229,7 @@ class SITargetLowering final : public AMDGPUTargetLowering {
SDValue performOrCombine(SDNode *N, DAGCombinerInfo &DCI) const;
SDValue performXorCombine(SDNode *N, DAGCombinerInfo &DCI) const;
SDValue performZeroOrAnyExtendCombine(SDNode *N, DAGCombinerInfo &DCI) const;
+ SDValue performExtendTruncShiftCombine(SDNode *N, DAGCombinerInfo &DCI) const;
SDValue performSignExtendInRegCombine(SDNode *N, DAGCombinerInfo &DCI) const;
SDValue performClassCombine(SDNode *N, DAGCombinerInfo &DCI) const;
SDValue getCanonicalConstantFP(SelectionDAG &DAG, const SDLoc &SL, EVT VT,
diff --git a/llvm/test/CodeGen/AMDGPU/idot4-test.ll b/llvm/test/CodeGen/AMDGPU/idot4-test.ll
index 79017f3ea3cd7..99f518c7f256f 100644
--- a/llvm/test/CodeGen/AMDGPU/idot4-test.ll
+++ b/llvm/test/CodeGen/AMDGPU/idot4-test.ll
@@ -297,13 +297,12 @@ define i32 @udot4_unsat_bitcast(i32 %a_packed, i32 %b_packed, i32 %c) {
; GFX9-NODL-LABEL: udot4_unsat_bitcast:
; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v3, 8, v0
-; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v4, 8, v1
-; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v5, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v6, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
+; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v3, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v4, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
+; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v5, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
-; GFX9-NODL-NEXT: v_mad_u32_u24 v0, v3, v4, v0
-; GFX9-NODL-NEXT: v_add_u32_e32 v1, v5, v6
+; GFX9-NODL-NEXT: v_add_u32_e32 v0, v4, v0
+; GFX9-NODL-NEXT: v_add_u32_e32 v1, v3, v5
; GFX9-NODL-NEXT: v_add3_u32 v0, v1, v0, v2
; GFX9-NODL-NEXT: s_setpc_b64 s[30:31]
entry:
@@ -355,14 +354,12 @@ define i32 @sdot4_unsat_bitcast(i32 %a_packed, i32 %b_packed, i32 %c) {
; GFX9-NODL-LABEL: sdot4_unsat_bitcast:
; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v3, 8, v1
-; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v4, 8, v0
-; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v5, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, sext(v4), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v4, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
+; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v4, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
+; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v5, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v0, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
-; GFX9-NODL-NEXT: v_add_u32_e32 v0, v3, v0
-; GFX9-NODL-NEXT: v_add_u32_e32 v1, v5, v4
+; GFX9-NODL-NEXT: v_add_u32_e32 v0, v4, v0
+; GFX9-NODL-NEXT: v_add_u32_e32 v1, v3, v5
; GFX9-NODL-NEXT: v_add3_u32 v0, v1, v0, v2
; GFX9-NODL-NEXT: s_setpc_b64 s[30:31]
entry:
@@ -407,13 +404,12 @@ define i32 @udot4_sat_bitcast(i32 %a_packed, i32 %b_packed, i32 %c) {
; GFX9-NODL-LABEL: udot4_sat_bitcast:
; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v3, 8, v0
-; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v4, 8, v1
-; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v5, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v6, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
+; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v3, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v4, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
+; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v5, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
-; GFX9-NODL-NEXT: v_mad_u32_u24 v0, v3, v4, v0
-; GFX9-NODL-NEXT: v_add3_u32 v0, v5, v6, v0
+; GFX9-NODL-NEXT: v_add_u32_e32 v0, v4, v0
+; GFX9-NODL-NEXT: v_add3_u32 v0, v3, v5, v0
; GFX9-NODL-NEXT: v_add_u32_e64 v0, v0, v2 clamp
; GFX9-NODL-NEXT: s_setpc_b64 s[30:31]
entry:
@@ -458,14 +454,12 @@ define i32 @sdot4_sat_bitcast(i32 %a_packed, i32 %b_packed, i32 %c) {
; GFX9-NODL-LABEL: sdot4_sat_bitcast:
; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v3, 8, v1
-; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v4, 8, v0
-; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v5, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, sext(v4), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v4, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
+; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v4, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
+; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v5, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v0, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
-; GFX9-NODL-NEXT: v_add_u32_e32 v0, v3, v0
-; GFX9-NODL-NEXT: v_add3_u32 v0, v5, v4, v0
+; GFX9-NODL-NEXT: v_add_u32_e32 v0, v4, v0
+; GFX9-NODL-NEXT: v_add3_u32 v0, v3, v5, v0
; GFX9-NODL-NEXT: v_add_i32 v0, v0, v2 clamp
; GFX9-NODL-NEXT: s_setpc_b64 s[30:31]
entry:
@@ -803,13 +797,12 @@ define i32 @dot4_mismatched_sign_zext_sadd(i32 %a_packed, i32 %b_packed, i32 %c)
; GFX9-NODL-LABEL: dot4_mismatched_sign_zext_sadd:
; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v3, 8, v0
-; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v4, 8, v1
-; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v5, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v6, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
+; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v3, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v4, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
+; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v5, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
-; GFX9-NODL-NEXT: v_mad_u32_u24 v0, v3, v4, v0
-; GFX9-NODL-NEXT: v_add3_u32 v0, v5, v6, v0
+; GFX9-NODL-NEXT: v_add_u32_e32 v0, v4, v0
+; GFX9-NODL-NEXT: v_add3_u32 v0, v3, v5, v0
; GFX9-NODL-NEXT: v_add_i32 v0, v0, v2 clamp
; GFX9-NODL-NEXT: s_setpc_b64 s[30:31]
entry:
@@ -852,14 +845,12 @@ define i32 @dot4_mismatched_sign_sext_uadd(i32 %a_packed, i32 %b_packed, i32 %c)
; GFX9-NODL-LABEL: dot4_mismatched_sign_sext_uadd:
; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v3, 8, v1
-; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v4, 8, v0
-; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v5, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, sext(v4), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v4, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
+; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v4, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
+; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v5, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v0, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
-; GFX9-NODL-NEXT: v_add_u32_e32 v0, v3, v0
-; GFX9-NODL-NEXT: v_add3_u32 v0, v5, v4, v0
+; GFX9-NODL-NEXT: v_add_u32_e32 v0, v4, v0
+; GFX9-NODL-NEXT: v_add3_u32 v0, v3, v5, v0
; GFX9-NODL-NEXT: v_add_u32_e64 v0, v0, v2 clamp
; GFX9-NODL-NEXT: s_setpc_b64 s[30:31]
entry:
@@ -878,54 +869,48 @@ define i32 @dot4_mixed_ext(i32 %a_packed, i32 %b_packed, i32 %c) {
; GFX9-DL-LABEL: dot4_mixed_ext:
; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-DL-NEXT: v_lshrrev_b16_e32 v3, 8, v1
-; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v4, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v3, v0, sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_0
+; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v3, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v4, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v5, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v0, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
-; GFX9-DL-NEXT: v_add_u32_e32 v0, v3, v0
-; GFX9-DL-NEXT: v_add_u32_e32 v1, v4, v5
+; GFX9-DL-NEXT: v_add_u32_e32 v0, v4, v0
+; GFX9-DL-NEXT: v_add_u32_e32 v1, v3, v5
; GFX9-DL-NEXT: v_add3_u32 v0, v1, v0, v2
; GFX9-DL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-DL-LABEL: dot4_mixed_ext:
; GFX10-DL: ; %bb.0: ; %entry
; GFX10-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-DL-NEXT: v_mov_b32_e32 v3, 0xffff
-; GFX10-DL-NEXT: v_lshrrev_b16 v4, 8, v1
-; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v5, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v6, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
-; GFX10-DL-NEXT: v_and_b32_sdwa v3, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_1
+; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v3, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v4, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
+; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v5, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v0, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
-; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v3, v3, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX10-DL-NEXT: v_add_nc_u32_e32 v0, v5, v0
-; GFX10-DL-NEXT: v_add_nc_u32_e32 v1, v3, v6
+; GFX10-DL-NEXT: v_add_nc_u32_e32 v1, v4, v5
+; GFX10-DL-NEXT: v_add_nc_u32_e32 v0, v3, v0
; GFX10-DL-NEXT: v_add3_u32 v0, v0, v1, v2
; GFX10-DL-NEXT: s_setpc_b64 s[30:31]
;
; GFX950-LABEL: dot4_mixed_ext:
; GFX950: ; %bb.0: ; %entry
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_lshrrev_b16_e32 v3, 8, v1
-; GFX950-NEXT: v_mul_i32_i24_sdwa v4, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX950-NEXT: v_mul_i32_i24_sdwa v3, v0, sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_0
+; GFX950-NEXT: v_mul_i32_i24_sdwa v3, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX950-NEXT: v_mul_i32_i24_sdwa v4, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
; GFX950-NEXT: v_mul_i32_i24_sdwa v5, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
; GFX950-NEXT: v_mul_i32_i24_sdwa v0, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
-; GFX950-NEXT: v_add_u32_e32 v0, v3, v0
-; GFX950-NEXT: v_add_u32_e32 v1, v4, v5
+; GFX950-NEXT: v_add_u32_e32 v0, v4, v0
+; GFX950-NEXT: v_add_u32_e32 v1, v3, v5
; GFX950-NEXT: v_add3_u32 v0, v1, v0, v2
; GFX950-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-NODL-LABEL: dot4_mixed_ext:
; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v3, 8, v1
-; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v4, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, v0, sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_0
+; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v4, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v5, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v0, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
-; GFX9-NODL-NEXT: v_add_u32_e32 v0, v3, v0
-; GFX9-NODL-NEXT: v_add_u32_e32 v1, v4, v5
+; GFX9-NODL-NEXT: v_add_u32_e32 v0, v4, v0
+; GFX9-NODL-NEXT: v_add_u32_e32 v1, v3, v5
; GFX9-NODL-NEXT: v_add3_u32 v0, v1, v0, v2
; GFX9-NODL-NEXT: s_setpc_b64 s[30:31]
entry:
diff --git a/llvm/test/CodeGen/AMDGPU/idot4s.ll b/llvm/test/CodeGen/AMDGPU/idot4s.ll
index 53b522fa4d84b..c7cc5847636be 100644
--- a/llvm/test/CodeGen/AMDGPU/idot4s.ll
+++ b/llvm/test/CodeGen/AMDGPU/idot4s.ll
@@ -819,18 +819,15 @@ define amdgpu_kernel void @idot4_acc32_vecMul(ptr addrspace(1) %src1,
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_dword v0, v[0:1]
; GFX8-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX8-NEXT: s_waitcnt vmcnt(1)
-; GFX8-NEXT: v_lshrrev_b16_e32 v1, 8, v3
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_lshrrev_b16_e32 v2, 8, v0
-; GFX8-NEXT: v_mul_i32_i24_sdwa v4, sext(v3), sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT: v_mul_i32_i24_sdwa v1, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-NEXT: v_mul_i32_i24_sdwa v1, sext(v3), sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-NEXT: v_mul_i32_i24_sdwa v2, sext(v3), sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v2, vcc, s0, v4
-; GFX8-NEXT: v_mul_i32_i24_sdwa v5, sext(v3), sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
-; GFX8-NEXT: v_add_u32_e32 v1, vcc, v2, v1
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, s0, v1
+; GFX8-NEXT: v_mul_i32_i24_sdwa v4, sext(v3), sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v2
; GFX8-NEXT: v_mul_i32_i24_sdwa v0, sext(v3), sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
-; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v5
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v4
; GFX8-NEXT: v_add_u32_e32 v2, vcc, v1, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s4
; GFX8-NEXT: v_mov_b32_e32 v1, s5
@@ -847,17 +844,14 @@ define amdgpu_kernel void @idot4_acc32_vecMul(ptr addrspace(1) %src1,
; GFX9-NODL-NEXT: global_load_dword v2, v0, s[2:3]
; GFX9-NODL-NEXT: s_load_dword s0, s[6:7], 0x0
; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v3, 8, v1
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v4, 8, v2
-; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v5, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, sext(v3), sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v6, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
+; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v4, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
+; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v5, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v1, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: v_add3_u32 v2, v5, s0, v3
-; GFX9-NODL-NEXT: v_add3_u32 v1, v2, v6, v1
+; GFX9-NODL-NEXT: v_add3_u32 v2, v3, s0, v4
+; GFX9-NODL-NEXT: v_add3_u32 v1, v2, v5, v1
; GFX9-NODL-NEXT: global_store_dword v0, v1, s[6:7]
; GFX9-NODL-NEXT: s_endpgm
;
@@ -3251,9 +3245,8 @@ define amdgpu_kernel void @idot4_nonstandard_signed(ptr addrspace(1) %src1,
; GFX8-NEXT: v_bfe_i32 v5, v5, 0, 8
; GFX8-NEXT: v_bfe_i32 v3, v3, 0, 8
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_lshrrev_b32_e32 v9, 8, v2
; GFX8-NEXT: v_and_b32_e32 v7, 0xff, v2
-; GFX8-NEXT: v_mul_lo_u16_sdwa v8, v9, sext(v8) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-NEXT: v_mul_lo_u16_sdwa v8, v2, sext(v8) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_0
; GFX8-NEXT: v_and_b32_sdwa v4, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX8-NEXT: v_mad_u16 v6, v6, v7, v8
; GFX8-NEXT: v_mad_u16 v4, v4, v5, v6
@@ -3275,18 +3268,17 @@ define amdgpu_kernel void @idot4_nonstandard_signed(ptr addrspace(1) %src1,
; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v6, 8, v1
-; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v7, 8, v2
; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; GFX9-NODL-NEXT: v_bfe_i32 v4, v1, 0, 8
+; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
; GFX9-NODL-NEXT: v_and_b32_e32 v5, 0xff, v2
-; GFX9-NODL-NEXT: v_mul_lo_u16_sdwa v6, v7, sext(v6) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NODL-NEXT: v_and_b32_sdwa v8, v2, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NODL-NEXT: v_mul_lo_u16_sdwa v6, v2, sext(v6) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_0
+; GFX9-NODL-NEXT: v_and_b32_sdwa v7, v2, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v1, 24, v1
; GFX9-NODL-NEXT: v_bfe_i32 v3, v3, 0, 8
; GFX9-NODL-NEXT: v_mad_legacy_u16 v4, v4, v5, v6
; GFX9-NODL-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX9-NODL-NEXT: v_mad_legacy_u16 v3, v8, v3, v4
+; GFX9-NODL-NEXT: v_mad_legacy_u16 v3, v7, v3, v4
; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v2, 24, v2
; GFX9-NODL-NEXT: v_mad_legacy_u16 v1, v1, v2, v3
; GFX9-NODL-NEXT: v_bfe_i32 v1, v1, 0, 16
@@ -3305,18 +3297,17 @@ define amdgpu_kernel void @idot4_nonstandard_signed(ptr addrspace(1) %src1,
; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
; GFX9-DL-NEXT: v_lshrrev_b32_e32 v6, 8, v1
-; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DL-NEXT: v_lshrrev_b32_e32 v7, 8, v2
; GFX9-DL-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; GFX9-DL-NEXT: v_bfe_i32 v4, v1, 0, 8
+; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
; GFX9-DL-NEXT: v_and_b32_e32 v5, 0xff, v2
-; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v6, v7, sext(v6) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-DL-NEXT: v_and_b32_sdwa v8, v2, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v6, v2, sext(v6) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_0
+; GFX9-DL-NEXT: v_and_b32_sdwa v7, v2, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX9-DL-NEXT: v_lshrrev_b32_e32 v1, 24, v1
; GFX9-DL-NEXT: v_bfe_i32 v3, v3, 0, 8
; GFX9-DL-NEXT: v_mad_legacy_u16 v4, v4, v5, v6
; GFX9-DL-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX9-DL-NEXT: v_mad_legacy_u16 v3, v8, v3, v4
+; GFX9-DL-NEXT: v_mad_legacy_u16 v3, v7, v3, v4
; GFX9-DL-NEXT: v_lshrrev_b32_e32 v2, 24, v2
; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v1, v2, v3
; GFX9-DL-NEXT: v_bfe_i32 v1, v1, 0, 16
@@ -3337,12 +3328,11 @@ define amdgpu_kernel void @idot4_nonstandard_signed(ptr addrspace(1) %src1,
; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
; GFX10-DL-NEXT: v_lshrrev_b32_e32 v0, 8, v1
; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
-; GFX10-DL-NEXT: v_lshrrev_b32_e32 v3, 8, v2
+; GFX10-DL-NEXT: v_lshrrev_b16 v3, 8, v2
; GFX10-DL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
; GFX10-DL-NEXT: v_bfe_i32 v6, v1, 0, 8
; GFX10-DL-NEXT: v_and_b32_e32 v7, 0xff, v2
; GFX10-DL-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX10-DL-NEXT: v_and_b32_e32 v3, 0xff, v3
; GFX10-DL-NEXT: v_lshrrev_b32_e32 v1, 24, v1
; GFX10-DL-NEXT: v_mul_lo_u16 v0, v3, v0
; GFX10-DL-NEXT: v_and_b32_sdwa v3, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
@@ -3370,32 +3360,28 @@ define amdgpu_kernel void @idot4_nonstandard_signed(ptr addrspace(1) %src1,
; GFX11-DL-TRUE16-NEXT: global_load_b32 v4, v0, s[2:3]
; GFX11-DL-TRUE16-NEXT: s_waitcnt vmcnt(1)
; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 8, v3
+; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v2, v3, 0, 8
+; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 24, v3
+; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_3)
+; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v1, v0, 0, 8
; GFX11-DL-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v4
-; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v5, v3, 0, 8
-; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.h
-; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v2, v0, 0, 8
-; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-DL-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v1.l
+; GFX11-DL-TRUE16-NEXT: v_lshrrev_b16 v0.l, 8, v4.l
; GFX11-DL-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v4.l
-; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v2.l, v5.l
-; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v5, v6, 0, 8
-; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 24, v3
-; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v5, v5, 0, 8
; GFX11-DL-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v1.l
; GFX11-DL-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v4.h
+; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 24, v4
-; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-DL-TRUE16-NEXT: v_mad_u16 v0.l, v2.l, v0.h, v0.l
; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v2, v6, 0, 8
+; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-DL-TRUE16-NEXT: v_mad_u16 v0.l, v1.l, v3.l, v0.l
-; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
+; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-DL-TRUE16-NEXT: v_mad_u16 v0.l, v1.l, v4.l, v0.l
; GFX11-DL-TRUE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 16
; GFX11-DL-TRUE16-NEXT: global_store_b32 v1, v0, s[4:5]
; GFX11-DL-TRUE16-NEXT: s_endpgm
@@ -3414,12 +3400,11 @@ define amdgpu_kernel void @idot4_nonstandard_signed(ptr addrspace(1) %src1,
; GFX11-DL-FAKE16-NEXT: s_waitcnt vmcnt(1)
; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 8, v1
; GFX11-DL-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 8, v0
+; GFX11-DL-FAKE16-NEXT: v_lshrrev_b16 v3, 8, v0
; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 16, v1
; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v0
; GFX11-DL-FAKE16-NEXT: v_bfe_i32 v6, v1, 0, 8
; GFX11-DL-FAKE16-NEXT: v_bfe_i32 v2, v2, 0, 8
-; GFX11-DL-FAKE16-NEXT: v_and_b32_e32 v3, 0xff, v3
; GFX11-DL-FAKE16-NEXT: v_and_b32_e32 v7, 0xff, v0
; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 24, v1
; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 24, v0
diff --git a/llvm/test/CodeGen/AMDGPU/idot4u.ll b/llvm/test/CodeGen/AMDGPU/idot4u.ll
index b982cfc451c56..283b739f83b27 100644
--- a/llvm/test/CodeGen/AMDGPU/idot4u.ll
+++ b/llvm/test/CodeGen/AMDGPU/idot4u.ll
@@ -242,17 +242,15 @@ define amdgpu_kernel void @udot4_acc16(ptr addrspace(1) %src1,
; GFX8-NEXT: flat_load_ushort v4, v[0:1]
; GFX8-NEXT: s_waitcnt vmcnt(2)
; GFX8-NEXT: v_and_b32_e32 v6, 0xff, v3
-; GFX8-NEXT: v_lshrrev_b32_e32 v8, 8, v3
-; GFX8-NEXT: v_and_b32_e32 v8, 0xff, v8
+; GFX8-NEXT: v_lshrrev_b16_e32 v8, 8, v3
; GFX8-NEXT: v_and_b32_sdwa v10, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX8-NEXT: v_lshrrev_b32_e32 v3, 24, v3
; GFX8-NEXT: s_waitcnt vmcnt(1)
; GFX8-NEXT: v_and_b32_e32 v7, 0xff, v2
-; GFX8-NEXT: v_lshrrev_b32_e32 v9, 8, v2
-; GFX8-NEXT: v_and_b32_e32 v9, 0xff, v9
+; GFX8-NEXT: v_lshrrev_b16_e32 v9, 8, v2
+; GFX8-NEXT: v_and_b32_sdwa v5, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mad_u16 v4, v6, v7, v4
-; GFX8-NEXT: v_and_b32_sdwa v5, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX8-NEXT: v_mad_u16 v4, v8, v9, v4
; GFX8-NEXT: v_lshrrev_b32_e32 v2, 24, v2
; GFX8-NEXT: v_mad_u16 v4, v10, v5, v4
@@ -275,10 +273,8 @@ define amdgpu_kernel void @udot4_acc16(ptr addrspace(1) %src1,
; GFX9-NODL-NEXT: v_and_b32_e32 v4, 0xff, v1
; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
; GFX9-NODL-NEXT: v_and_b32_e32 v5, 0xff, v2
-; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v6, 8, v1
-; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v7, 8, v2
-; GFX9-NODL-NEXT: v_and_b32_e32 v6, 0xff, v6
-; GFX9-NODL-NEXT: v_and_b32_e32 v7, 0xff, v7
+; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v6, 8, v1
+; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v7, 8, v2
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
; GFX9-NODL-NEXT: v_mad_legacy_u16 v3, v4, v5, v3
; GFX9-NODL-NEXT: v_and_b32_sdwa v8, v1, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
@@ -1534,18 +1530,16 @@ define amdgpu_kernel void @notdot4_mixedtypes(ptr addrspace(1) %src1,
; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: flat_load_ushort v4, v[0:1]
; GFX8-NEXT: s_waitcnt vmcnt(2)
-; GFX8-NEXT: v_lshrrev_b32_e32 v8, 8, v3
-; GFX8-NEXT: v_and_b32_e32 v8, 0xff, v8
+; GFX8-NEXT: v_lshrrev_b16_e32 v8, 8, v3
; GFX8-NEXT: v_bfe_i32 v6, v3, 0, 8
; GFX8-NEXT: v_and_b32_sdwa v10, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX8-NEXT: v_lshrrev_b32_e32 v3, 24, v3
; GFX8-NEXT: s_waitcnt vmcnt(1)
-; GFX8-NEXT: v_lshrrev_b32_e32 v9, 8, v2
-; GFX8-NEXT: v_and_b32_e32 v9, 0xff, v9
+; GFX8-NEXT: v_lshrrev_b16_e32 v9, 8, v2
; GFX8-NEXT: v_bfe_i32 v7, v2, 0, 8
+; GFX8-NEXT: v_and_b32_sdwa v5, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mad_u16 v4, v8, v9, v4
-; GFX8-NEXT: v_and_b32_sdwa v5, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX8-NEXT: v_mad_u16 v4, v6, v7, v4
; GFX8-NEXT: v_lshrrev_b32_e32 v2, 24, v2
; GFX8-NEXT: v_mad_u16 v4, v10, v5, v4
@@ -1565,11 +1559,9 @@ define amdgpu_kernel void @notdot4_mixedtypes(ptr addrspace(1) %src1,
; GFX9-NODL-NEXT: global_load_ushort v3, v0, s[6:7]
; GFX9-NODL-NEXT: s_movk_i32 s0, 0xff
; GFX9-NODL-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v6, 8, v1
+; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v6, 8, v1
; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v7, 8, v2
-; GFX9-NODL-NEXT: v_and_b32_e32 v6, 0xff, v6
-; GFX9-NODL-NEXT: v_and_b32_e32 v7, 0xff, v7
+; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v7, 8, v2
; GFX9-NODL-NEXT: v_bfe_i32 v4, v1, 0, 8
; GFX9-NODL-NEXT: v_bfe_i32 v5, v2, 0, 8
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
@@ -1596,11 +1588,9 @@ define amdgpu_kernel void @notdot4_mixedtypes(ptr addrspace(1) %src1,
; GFX9-DL-NEXT: global_load_ushort v3, v0, s[6:7]
; GFX9-DL-NEXT: s_mov_b32 s0, 0xc0c0302
; GFX9-DL-NEXT: s_waitcnt vmcnt(2)
-; GFX9-DL-NEXT: v_lshrrev_b32_e32 v6, 8, v1
+; GFX9-DL-NEXT: v_lshrrev_b16_e32 v6, 8, v1
; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
-; GFX9-DL-NEXT: v_lshrrev_b32_e32 v7, 8, v2
-; GFX9-DL-NEXT: v_and_b32_e32 v6, 0xff, v6
-; GFX9-DL-NEXT: v_and_b32_e32 v7, 0xff, v7
+; GFX9-DL-NEXT: v_lshrrev_b16_e32 v7, 8, v2
; GFX9-DL-NEXT: v_bfe_i32 v4, v1, 0, 8
; GFX9-DL-NEXT: v_bfe_i32 v5, v2, 0, 8
; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
@@ -1618,96 +1608,89 @@ define amdgpu_kernel void @notdot4_mixedtypes(ptr addrspace(1) %src1,
; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: s_clause 0x1
-; GFX10-DL-NEXT: global_load_dword v1, v0, s[0:1]
-; GFX10-DL-NEXT: global_load_dword v2, v0, s[2:3]
-; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-DL-NEXT: global_load_ushort v3, v0, s[6:7]
+; GFX10-DL-NEXT: global_load_dword v2, v0, s[0:1]
+; GFX10-DL-NEXT: global_load_dword v3, v0, s[2:3]
+; GFX10-DL-NEXT: global_load_ushort v4, v1, s[6:7]
; GFX10-DL-NEXT: s_waitcnt vmcnt(2)
-; GFX10-DL-NEXT: v_lshrrev_b32_e32 v4, 8, v1
+; GFX10-DL-NEXT: v_lshrrev_b16 v0, 8, v2
; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
-; GFX10-DL-NEXT: v_lshrrev_b32_e32 v5, 8, v2
-; GFX10-DL-NEXT: v_bfe_i32 v6, v1, 0, 8
-; GFX10-DL-NEXT: v_bfe_i32 v7, v2, 0, 8
+; GFX10-DL-NEXT: v_lshrrev_b16 v5, 8, v3
+; GFX10-DL-NEXT: v_bfe_i32 v6, v2, 0, 8
+; GFX10-DL-NEXT: v_bfe_i32 v7, v3, 0, 8
+; GFX10-DL-NEXT: v_perm_b32 v3, v3, v3, 0xc0c0302
; GFX10-DL-NEXT: v_perm_b32 v2, v2, v2, 0xc0c0302
-; GFX10-DL-NEXT: v_and_b32_e32 v4, 0xff, v4
-; GFX10-DL-NEXT: v_and_b32_e32 v5, 0xff, v5
-; GFX10-DL-NEXT: v_perm_b32 v1, v1, v1, 0xc0c0302
; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
-; GFX10-DL-NEXT: v_mad_u16 v3, v4, v5, v3
-; GFX10-DL-NEXT: v_mad_u16 v3, v6, v7, v3
-; GFX10-DL-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX10-DL-NEXT: v_dot4_u32_u8 v1, v1, v2, v3
-; GFX10-DL-NEXT: global_store_short v0, v1, s[6:7]
+; GFX10-DL-NEXT: v_mad_u16 v0, v0, v5, v4
+; GFX10-DL-NEXT: v_mad_u16 v0, v6, v7, v0
+; GFX10-DL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX10-DL-NEXT: v_dot4_u32_u8 v0, v2, v3, v0
+; GFX10-DL-NEXT: global_store_short v1, v0, s[6:7]
; GFX10-DL-NEXT: s_endpgm
;
; GFX11-DL-TRUE16-LABEL: notdot4_mixedtypes:
; GFX11-DL-TRUE16: ; %bb.0: ; %entry
+; GFX11-DL-TRUE16-NEXT: s_clause 0x1
; GFX11-DL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-DL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX11-DL-TRUE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
-; GFX11-DL-TRUE16-NEXT: v_mov_b32_e32 v6, 0
-; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-DL-TRUE16-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-DL-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX11-DL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-DL-TRUE16-NEXT: s_clause 0x1
; GFX11-DL-TRUE16-NEXT: global_load_b32 v4, v0, s[0:1]
; GFX11-DL-TRUE16-NEXT: global_load_b32 v5, v0, s[2:3]
-; GFX11-DL-TRUE16-NEXT: global_load_d16_b16 v0, v6, s[4:5]
-; GFX11-DL-TRUE16-NEXT: s_waitcnt vmcnt(2)
-; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v4
-; GFX11-DL-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 8, v5
-; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v3, v4, 0, 8
-; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v7, v5, 0, 8
+; GFX11-DL-TRUE16-NEXT: global_load_d16_b16 v0, v3, s[4:5]
; GFX11-DL-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-DL-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.l
-; GFX11-DL-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v2.l, v3.l
-; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v3.l, v7.l
-; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-DL-TRUE16-NEXT: v_lshrrev_b16 v0.h, 8, v4.l
+; GFX11-DL-TRUE16-NEXT: v_lshrrev_b16 v1.l, 8, v5.l
+; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v2, v4, 0, 8
+; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v6, v5, 0, 8
+; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-DL-TRUE16-NEXT: v_mad_u16 v0.l, v0.h, v1.l, v0.l
+; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
+; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v2.l, v6.l
+; GFX11-DL-TRUE16-NEXT: v_mad_u16 v0.l, v1.l, v2.l, v0.l
; GFX11-DL-TRUE16-NEXT: v_perm_b32 v1, v5, v5, 0xc0c0302
-; GFX11-DL-TRUE16-NEXT: v_mad_u16 v0.l, v2.l, v3.l, v0.l
; GFX11-DL-TRUE16-NEXT: v_perm_b32 v2, v4, v4, 0xc0c0302
-; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-DL-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX11-DL-TRUE16-NEXT: v_dot4_u32_u8 v0, v2, v1, v0
-; GFX11-DL-TRUE16-NEXT: global_store_b16 v6, v0, s[4:5]
+; GFX11-DL-TRUE16-NEXT: global_store_b16 v3, v0, s[4:5]
; GFX11-DL-TRUE16-NEXT: s_endpgm
;
; GFX11-DL-FAKE16-LABEL: notdot4_mixedtypes:
; GFX11-DL-FAKE16: ; %bb.0: ; %entry
+; GFX11-DL-FAKE16-NEXT: s_clause 0x1
; GFX11-DL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-DL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX11-DL-FAKE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
-; GFX11-DL-FAKE16-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-DL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-DL-FAKE16-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-DL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-DL-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX11-DL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-DL-FAKE16-NEXT: s_clause 0x1
-; GFX11-DL-FAKE16-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX11-DL-FAKE16-NEXT: global_load_b32 v2, v0, s[0:1]
; GFX11-DL-FAKE16-NEXT: global_load_b32 v0, v0, s[2:3]
-; GFX11-DL-FAKE16-NEXT: global_load_u16 v3, v2, s[4:5]
+; GFX11-DL-FAKE16-NEXT: global_load_u16 v3, v1, s[4:5]
; GFX11-DL-FAKE16-NEXT: s_waitcnt vmcnt(2)
-; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 8, v1
+; GFX11-DL-FAKE16-NEXT: v_lshrrev_b16 v4, 8, v2
; GFX11-DL-FAKE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 8, v0
-; GFX11-DL-FAKE16-NEXT: v_bfe_i32 v6, v1, 0, 8
+; GFX11-DL-FAKE16-NEXT: v_lshrrev_b16 v5, 8, v0
+; GFX11-DL-FAKE16-NEXT: v_bfe_i32 v6, v2, 0, 8
; GFX11-DL-FAKE16-NEXT: v_bfe_i32 v7, v0, 0, 8
; GFX11-DL-FAKE16-NEXT: v_perm_b32 v0, v0, v0, 0xc0c0302
-; GFX11-DL-FAKE16-NEXT: v_and_b32_e32 v4, 0xff, v4
-; GFX11-DL-FAKE16-NEXT: v_and_b32_e32 v5, 0xff, v5
-; GFX11-DL-FAKE16-NEXT: v_perm_b32 v1, v1, v1, 0xc0c0302
+; GFX11-DL-FAKE16-NEXT: v_perm_b32 v2, v2, v2, 0xc0c0302
; GFX11-DL-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-DL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-DL-FAKE16-NEXT: v_mad_u16 v3, v4, v5, v3
-; GFX11-DL-FAKE16-NEXT: v_mad_u16 v3, v6, v7, v3
; GFX11-DL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-DL-FAKE16-NEXT: v_mad_u16 v3, v6, v7, v3
; GFX11-DL-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX11-DL-FAKE16-NEXT: v_dot4_u32_u8 v0, v1, v0, v3
-; GFX11-DL-FAKE16-NEXT: global_store_b16 v2, v0, s[4:5]
+; GFX11-DL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-DL-FAKE16-NEXT: v_dot4_u32_u8 v0, v2, v0, v3
+; GFX11-DL-FAKE16-NEXT: global_store_b16 v1, v0, s[4:5]
; GFX11-DL-FAKE16-NEXT: s_endpgm
ptr addrspace(1) %src2,
ptr addrspace(1) nocapture %dst) {
@@ -1811,8 +1794,7 @@ define amdgpu_kernel void @notdot4_mixedtypes2(ptr addrspace(1) %src1,
; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: flat_load_ushort v4, v[0:1]
; GFX8-NEXT: s_waitcnt vmcnt(2)
-; GFX8-NEXT: v_lshrrev_b32_e32 v9, 8, v3
-; GFX8-NEXT: v_and_b32_e32 v9, 0xff, v9
+; GFX8-NEXT: v_lshrrev_b16_e32 v9, 8, v3
; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v3
; GFX8-NEXT: v_bfe_i32 v7, v3, 0, 8
; GFX8-NEXT: v_bfe_i32 v6, v6, 0, 8
@@ -1844,10 +1826,9 @@ define amdgpu_kernel void @notdot4_mixedtypes2(ptr addrspace(1) %src1,
; GFX9-NODL-NEXT: global_load_ushort v3, v0, s[6:7]
; GFX9-NODL-NEXT: s_movk_i32 s0, 0xff
; GFX9-NODL-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v7, 8, v1
+; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v7, 8, v1
; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v8, 8, v2
-; GFX9-NODL-NEXT: v_and_b32_e32 v7, 0xff, v7
; GFX9-NODL-NEXT: v_bfe_i32 v8, v8, 0, 8
; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v4, 16, v1
; GFX9-NODL-NEXT: v_bfe_i32 v5, v1, 0, 8
@@ -1877,10 +1858,9 @@ define amdgpu_kernel void @notdot4_mixedtypes2(ptr addrspace(1) %src1,
; GFX9-DL-NEXT: global_load_ushort v3, v0, s[6:7]
; GFX9-DL-NEXT: s_movk_i32 s0, 0xff
; GFX9-DL-NEXT: s_waitcnt vmcnt(2)
-; GFX9-DL-NEXT: v_lshrrev_b32_e32 v7, 8, v1
+; GFX9-DL-NEXT: v_lshrrev_b16_e32 v7, 8, v1
; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
; GFX9-DL-NEXT: v_lshrrev_b32_e32 v8, 8, v2
-; GFX9-DL-NEXT: v_and_b32_e32 v7, 0xff, v7
; GFX9-DL-NEXT: v_bfe_i32 v8, v8, 0, 8
; GFX9-DL-NEXT: v_lshrrev_b32_e32 v4, 16, v1
; GFX9-DL-NEXT: v_bfe_i32 v5, v1, 0, 8
@@ -1907,37 +1887,37 @@ define amdgpu_kernel void @notdot4_mixedtypes2(ptr addrspace(1) %src1,
; GFX10-DL-NEXT: v_mov_b32_e32 v8, 0xff
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: s_clause 0x1
-; GFX10-DL-NEXT: global_load_dword v1, v0, s[0:1]
-; GFX10-DL-NEXT: global_load_dword v2, v0, s[2:3]
+; GFX10-DL-NEXT: global_load_dword v1, v0, s[2:3]
+; GFX10-DL-NEXT: global_load_dword v2, v0, s[0:1]
; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX10-DL-NEXT: global_load_ushort v3, v0, s[6:7]
; GFX10-DL-NEXT: s_waitcnt vmcnt(2)
; GFX10-DL-NEXT: v_lshrrev_b32_e32 v4, 8, v1
; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
-; GFX10-DL-NEXT: v_lshrrev_b32_e32 v5, 8, v2
-; GFX10-DL-NEXT: v_lshrrev_b32_e32 v6, 16, v1
-; GFX10-DL-NEXT: v_bfe_i32 v7, v1, 0, 8
-; GFX10-DL-NEXT: v_and_b32_e32 v9, 0xff, v2
-; GFX10-DL-NEXT: v_and_b32_e32 v4, 0xff, v4
-; GFX10-DL-NEXT: v_bfe_i32 v5, v5, 0, 8
-; GFX10-DL-NEXT: v_lshrrev_b32_e32 v1, 24, v1
+; GFX10-DL-NEXT: v_lshrrev_b16 v5, 8, v2
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v6, 16, v2
+; GFX10-DL-NEXT: v_bfe_i32 v7, v2, 0, 8
+; GFX10-DL-NEXT: v_and_b32_e32 v9, 0xff, v1
+; GFX10-DL-NEXT: v_bfe_i32 v4, v4, 0, 8
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v2, 24, v2
; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
-; GFX10-DL-NEXT: v_mad_u16 v3, v4, v5, v3
+; GFX10-DL-NEXT: v_mad_u16 v3, v5, v4, v3
; GFX10-DL-NEXT: v_bfe_i32 v4, v6, 0, 8
-; GFX10-DL-NEXT: v_and_b32_sdwa v5, v2, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-DL-NEXT: v_lshrrev_b32_e32 v2, 24, v2
+; GFX10-DL-NEXT: v_and_b32_sdwa v5, v1, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-DL-NEXT: v_lshrrev_b32_e32 v1, 24, v1
; GFX10-DL-NEXT: v_mad_u16 v3, v7, v9, v3
-; GFX10-DL-NEXT: v_bfe_i32 v2, v2, 0, 8
+; GFX10-DL-NEXT: v_bfe_i32 v1, v1, 0, 8
; GFX10-DL-NEXT: v_mad_u16 v3, v4, v5, v3
-; GFX10-DL-NEXT: v_mad_u16 v1, v1, v2, v3
+; GFX10-DL-NEXT: v_mad_u16 v1, v2, v1, v3
; GFX10-DL-NEXT: global_store_short v0, v1, s[6:7]
; GFX10-DL-NEXT: s_endpgm
;
; GFX11-DL-TRUE16-LABEL: notdot4_mixedtypes2:
; GFX11-DL-TRUE16: ; %bb.0: ; %entry
+; GFX11-DL-TRUE16-NEXT: s_clause 0x1
; GFX11-DL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-DL-TRUE16-NEXT: v_dual_mov_b32 v5, 0 :: v_dual_and_b32 v0, 0x3ff, v0
; GFX11-DL-TRUE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-DL-TRUE16-NEXT: v_dual_mov_b32 v5, 0 :: v_dual_and_b32 v0, 0x3ff, v0
; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-DL-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX11-DL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
@@ -1948,23 +1928,22 @@ define amdgpu_kernel void @notdot4_mixedtypes2(ptr addrspace(1) %src1,
; GFX11-DL-TRUE16-NEXT: s_waitcnt vmcnt(2)
; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
; GFX11-DL-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 8, v4
-; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v6, v4, 0, 8
-; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.h
-; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.h
; GFX11-DL-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-DL-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v2.l
+; GFX11-DL-TRUE16-NEXT: v_lshrrev_b16 v0.h, 8, v4.l
+; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v2, v4, 0, 8
+; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v3
+; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8
; GFX11-DL-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v3.l
-; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v7, v7, 0, 8
-; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v2.l, v6.l
-; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 24, v3
+; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v6, v6, 0, 8
+; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-DL-TRUE16-NEXT: v_mad_u16 v0.l, v0.h, v1.l, v0.l
; GFX11-DL-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v3.h
-; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v1.l, v7.l
+; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v1.l, v6.l
; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v4
; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-DL-TRUE16-NEXT: v_mad_u16 v0.l, v2.l, v1.h, v0.l
-; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v2, v6, 0, 8
+; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v2, v7, 0, 8
; GFX11-DL-TRUE16-NEXT: v_mad_u16 v0.l, v1.l, v0.h, v0.l
; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
@@ -1974,39 +1953,39 @@ define amdgpu_kernel void @notdot4_mixedtypes2(ptr addrspace(1) %src1,
;
; GFX11-DL-FAKE16-LABEL: notdot4_mixedtypes2:
; GFX11-DL-FAKE16: ; %bb.0: ; %entry
+; GFX11-DL-FAKE16-NEXT: s_clause 0x1
; GFX11-DL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-DL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX11-DL-FAKE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-DL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX11-DL-FAKE16-NEXT: v_mov_b32_e32 v2, 0
; GFX11-DL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-DL-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX11-DL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-DL-FAKE16-NEXT: s_clause 0x1
-; GFX11-DL-FAKE16-NEXT: global_load_b32 v1, v0, s[0:1]
-; GFX11-DL-FAKE16-NEXT: global_load_b32 v0, v0, s[2:3]
+; GFX11-DL-FAKE16-NEXT: global_load_b32 v1, v0, s[2:3]
+; GFX11-DL-FAKE16-NEXT: global_load_b32 v0, v0, s[0:1]
; GFX11-DL-FAKE16-NEXT: global_load_u16 v3, v2, s[4:5]
; GFX11-DL-FAKE16-NEXT: s_waitcnt vmcnt(2)
; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 8, v1
; GFX11-DL-FAKE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 8, v0
-; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 16, v1
-; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v7, 16, v0
-; GFX11-DL-FAKE16-NEXT: v_bfe_i32 v8, v1, 0, 8
-; GFX11-DL-FAKE16-NEXT: v_and_b32_e32 v4, 0xff, v4
-; GFX11-DL-FAKE16-NEXT: v_bfe_i32 v5, v5, 0, 8
-; GFX11-DL-FAKE16-NEXT: v_and_b32_e32 v9, 0xff, v0
-; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 24, v0
+; GFX11-DL-FAKE16-NEXT: v_lshrrev_b16 v5, 8, v0
+; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v7, 16, v1
+; GFX11-DL-FAKE16-NEXT: v_bfe_i32 v8, v0, 0, 8
+; GFX11-DL-FAKE16-NEXT: v_bfe_i32 v4, v4, 0, 8
+; GFX11-DL-FAKE16-NEXT: v_and_b32_e32 v9, 0xff, v1
; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 24, v1
+; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 24, v0
; GFX11-DL-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-DL-FAKE16-NEXT: v_mad_u16 v3, v4, v5, v3
+; GFX11-DL-FAKE16-NEXT: v_mad_u16 v3, v5, v4, v3
; GFX11-DL-FAKE16-NEXT: v_bfe_i32 v4, v6, 0, 8
; GFX11-DL-FAKE16-NEXT: v_and_b32_e32 v5, 0xff, v7
-; GFX11-DL-FAKE16-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX11-DL-FAKE16-NEXT: v_bfe_i32 v1, v1, 0, 8
; GFX11-DL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-DL-FAKE16-NEXT: v_mad_u16 v3, v8, v9, v3
; GFX11-DL-FAKE16-NEXT: v_mad_u16 v3, v4, v5, v3
; GFX11-DL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-DL-FAKE16-NEXT: v_mad_u16 v0, v1, v0, v3
+; GFX11-DL-FAKE16-NEXT: v_mad_u16 v0, v0, v1, v3
; GFX11-DL-FAKE16-NEXT: global_store_b16 v2, v0, s[4:5]
; GFX11-DL-FAKE16-NEXT: s_endpgm
ptr addrspace(1) %src2,
@@ -2103,17 +2082,15 @@ define amdgpu_kernel void @udot4_acc32_vecMul(ptr addrspace(1) %src1,
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_dword v0, v[0:1]
; GFX8-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX8-NEXT: s_waitcnt vmcnt(1)
-; GFX8-NEXT: v_lshrrev_b16_e32 v1, 8, v3
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mul_u32_u24_sdwa v4, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT: v_lshrrev_b16_e32 v2, 8, v0
-; GFX8-NEXT: v_mul_u32_u24_sdwa v5, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
-; GFX8-NEXT: v_mul_u32_u24_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
+; GFX8-NEXT: v_mul_u32_u24_sdwa v1, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-NEXT: v_mul_u32_u24_sdwa v2, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v3, vcc, s0, v4
-; GFX8-NEXT: v_mad_u32_u24 v1, v1, v2, v3
-; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v5
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, s0, v1
+; GFX8-NEXT: v_mul_u32_u24_sdwa v4, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v2
+; GFX8-NEXT: v_mul_u32_u24_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v4
; GFX8-NEXT: v_add_u32_e32 v2, vcc, v1, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s4
; GFX8-NEXT: v_mov_b32_e32 v1, s5
@@ -2130,17 +2107,14 @@ define amdgpu_kernel void @udot4_acc32_vecMul(ptr addrspace(1) %src1,
; GFX9-NODL-NEXT: global_load_dword v2, v0, s[2:3]
; GFX9-NODL-NEXT: s_load_dword s0, s[6:7], 0x0
; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v3, 8, v1
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v4, 8, v2
-; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v6, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
+; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v3, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v4, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
+; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: v_add_u32_e32 v2, s0, v5
-; GFX9-NODL-NEXT: v_mad_u32_u24 v2, v3, v4, v2
-; GFX9-NODL-NEXT: v_add3_u32 v1, v2, v6, v1
+; GFX9-NODL-NEXT: v_add3_u32 v2, v3, s0, v4
+; GFX9-NODL-NEXT: v_add3_u32 v1, v2, v5, v1
; GFX9-NODL-NEXT: global_store_dword v0, v1, s[6:7]
; GFX9-NODL-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/lshr.v2i16.ll b/llvm/test/CodeGen/AMDGPU/lshr.v2i16.ll
index 8252a4422febb..214486cd7ab12 100644
--- a/llvm/test/CodeGen/AMDGPU/lshr.v2i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/lshr.v2i16.ll
@@ -453,13 +453,14 @@ define amdgpu_kernel void @lshr_v_imm_v2i16(ptr addrspace(1) %out, ptr addrspace
; VI-NEXT: v_add_u32_e32 v0, vcc, s2, v2
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-NEXT: flat_load_dword v3, v[0:1]
-; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v2
; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v2
+; VI-NEXT: s_mov_b32 s0, 0x5040c00
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_lshrrev_b32_e32 v2, 24, v3
-; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; VI-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; VI-NEXT: v_lshrrev_b32_e32 v2, 8, v3
+; VI-NEXT: v_lshrrev_b32_e32 v3, 24, v3
+; VI-NEXT: v_perm_b32 v2, v3, v2, s0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/min.ll b/llvm/test/CodeGen/AMDGPU/min.ll
index b6ca816550cc7..414165dbc3108 100644
--- a/llvm/test/CodeGen/AMDGPU/min.ll
+++ b/llvm/test/CodeGen/AMDGPU/min.ll
@@ -706,28 +706,26 @@ define amdgpu_kernel void @s_test_imin_sle_v4i8(ptr addrspace(1) %out, [8 x i32]
;
; VI-LABEL: s_test_imin_sle_v4i8:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[8:9], 0x28
-; VI-NEXT: s_load_dword s3, s[8:9], 0x4c
+; VI-NEXT: s_load_dword s2, s[8:9], 0x4c
+; VI-NEXT: s_load_dword s3, s[8:9], 0x28
; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
; VI-NEXT: s_add_i32 s12, s12, s17
; VI-NEXT: v_mov_b32_e32 v1, 0xc0c0400
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_sext_i32_i16 s5, s2
-; VI-NEXT: s_sext_i32_i16 s7, s3
-; VI-NEXT: s_ashr_i32 s7, s7, 8
-; VI-NEXT: s_ashr_i32 s5, s5, 8
-; VI-NEXT: s_min_i32 s5, s5, s7
-; VI-NEXT: s_sext_i32_i8 s7, s3
-; VI-NEXT: s_sext_i32_i8 s8, s2
+; VI-NEXT: s_sext_i32_i8 s4, s2
+; VI-NEXT: s_sext_i32_i8 s5, s3
+; VI-NEXT: s_bfe_i32 s6, s2, 0x80008
+; VI-NEXT: s_min_i32 s4, s5, s4
+; VI-NEXT: s_bfe_i32 s5, s3, 0x80008
+; VI-NEXT: s_min_i32 s5, s5, s6
+; VI-NEXT: v_mov_b32_e32 v0, s4
+; VI-NEXT: v_perm_b32 v0, s5, v0, v1
; VI-NEXT: s_ashr_i32 s4, s2, 24
-; VI-NEXT: s_ashr_i32 s6, s3, 24
-; VI-NEXT: s_min_i32 s7, s8, s7
-; VI-NEXT: s_bfe_i32 s3, s3, 0x80010
+; VI-NEXT: s_ashr_i32 s5, s3, 24
; VI-NEXT: s_bfe_i32 s2, s2, 0x80010
-; VI-NEXT: v_mov_b32_e32 v0, s7
-; VI-NEXT: s_min_i32 s2, s2, s3
-; VI-NEXT: v_perm_b32 v0, s5, v0, v1
-; VI-NEXT: s_min_i32 s4, s4, s6
+; VI-NEXT: s_bfe_i32 s3, s3, 0x80010
+; VI-NEXT: s_min_i32 s2, s3, s2
+; VI-NEXT: s_min_i32 s4, s5, s4
; VI-NEXT: v_mov_b32_e32 v1, s2
; VI-NEXT: v_mov_b32_e32 v2, 0x6050400
; VI-NEXT: v_perm_b32 v1, s4, v1, v2
@@ -742,28 +740,26 @@ define amdgpu_kernel void @s_test_imin_sle_v4i8(ptr addrspace(1) %out, [8 x i32]
;
; GFX9-LABEL: s_test_imin_sle_v4i8:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x28
-; GFX9-NEXT: s_load_dword s3, s[8:9], 0x4c
+; GFX9-NEXT: s_load_dword s2, s[8:9], 0x4c
+; GFX9-NEXT: s_load_dword s3, s[8:9], 0x28
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v2, 0xc0c0400
; GFX9-NEXT: v_mov_b32_e32 v3, 0x6050400
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_sext_i32_i16 s5, s2
-; GFX9-NEXT: s_sext_i32_i16 s7, s3
-; GFX9-NEXT: s_ashr_i32 s7, s7, 8
-; GFX9-NEXT: s_ashr_i32 s5, s5, 8
-; GFX9-NEXT: s_min_i32 s5, s5, s7
-; GFX9-NEXT: s_sext_i32_i8 s7, s3
-; GFX9-NEXT: s_sext_i32_i8 s8, s2
+; GFX9-NEXT: s_sext_i32_i8 s4, s2
+; GFX9-NEXT: s_sext_i32_i8 s5, s3
+; GFX9-NEXT: s_min_i32 s4, s5, s4
+; GFX9-NEXT: s_bfe_i32 s5, s2, 0x80008
+; GFX9-NEXT: s_bfe_i32 s6, s3, 0x80008
+; GFX9-NEXT: s_min_i32 s5, s6, s5
+; GFX9-NEXT: v_mov_b32_e32 v1, s4
+; GFX9-NEXT: v_perm_b32 v1, s5, v1, v2
; GFX9-NEXT: s_ashr_i32 s4, s2, 24
-; GFX9-NEXT: s_ashr_i32 s6, s3, 24
-; GFX9-NEXT: s_min_i32 s7, s8, s7
-; GFX9-NEXT: s_bfe_i32 s3, s3, 0x80010
+; GFX9-NEXT: s_ashr_i32 s5, s3, 24
; GFX9-NEXT: s_bfe_i32 s2, s2, 0x80010
-; GFX9-NEXT: v_mov_b32_e32 v1, s7
-; GFX9-NEXT: s_min_i32 s2, s2, s3
-; GFX9-NEXT: v_perm_b32 v1, s5, v1, v2
-; GFX9-NEXT: s_min_i32 s4, s4, s6
+; GFX9-NEXT: s_bfe_i32 s3, s3, 0x80010
+; GFX9-NEXT: s_min_i32 s2, s3, s2
+; GFX9-NEXT: s_min_i32 s4, s5, s4
; GFX9-NEXT: v_mov_b32_e32 v2, s2
; GFX9-NEXT: v_perm_b32 v2, s4, v2, v3
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
@@ -775,29 +771,27 @@ define amdgpu_kernel void @s_test_imin_sle_v4i8(ptr addrspace(1) %out, [8 x i32]
; GFX10-LABEL: s_test_imin_sle_v4i8:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_clause 0x2
-; GFX10-NEXT: s_load_dword s2, s[8:9], 0x28
-; GFX10-NEXT: s_load_dword s3, s[8:9], 0x4c
+; GFX10-NEXT: s_load_dword s2, s[8:9], 0x4c
+; GFX10-NEXT: s_load_dword s3, s[8:9], 0x28
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX10-NEXT: v_mov_b32_e32 v0, 0x6050400
; GFX10-NEXT: v_mov_b32_e32 v1, 0xc0c0400
; GFX10-NEXT: v_mov_b32_e32 v2, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_ashr_i32 s4, s2, 24
-; GFX10-NEXT: s_sext_i32_i16 s5, s2
-; GFX10-NEXT: s_ashr_i32 s6, s3, 24
-; GFX10-NEXT: s_sext_i32_i16 s7, s3
-; GFX10-NEXT: s_sext_i32_i8 s8, s3
-; GFX10-NEXT: s_sext_i32_i8 s9, s2
-; GFX10-NEXT: s_bfe_i32 s3, s3, 0x80010
+; GFX10-NEXT: s_sext_i32_i8 s4, s2
+; GFX10-NEXT: s_sext_i32_i8 s5, s3
+; GFX10-NEXT: s_bfe_i32 s6, s2, 0x80008
+; GFX10-NEXT: s_bfe_i32 s7, s3, 0x80008
+; GFX10-NEXT: s_ashr_i32 s8, s2, 24
+; GFX10-NEXT: s_ashr_i32 s9, s3, 24
; GFX10-NEXT: s_bfe_i32 s2, s2, 0x80010
-; GFX10-NEXT: s_min_i32 s4, s4, s6
-; GFX10-NEXT: s_min_i32 s2, s2, s3
-; GFX10-NEXT: s_ashr_i32 s7, s7, 8
-; GFX10-NEXT: v_perm_b32 v0, s4, s2, v0
-; GFX10-NEXT: s_ashr_i32 s5, s5, 8
-; GFX10-NEXT: s_min_i32 s3, s9, s8
-; GFX10-NEXT: s_min_i32 s2, s5, s7
-; GFX10-NEXT: v_perm_b32 v1, s2, s3, v1
+; GFX10-NEXT: s_bfe_i32 s3, s3, 0x80010
+; GFX10-NEXT: s_min_i32 s8, s9, s8
+; GFX10-NEXT: s_min_i32 s2, s3, s2
+; GFX10-NEXT: s_min_i32 s3, s7, s6
+; GFX10-NEXT: v_perm_b32 v0, s8, s2, v0
+; GFX10-NEXT: s_min_i32 s2, s5, s4
+; GFX10-NEXT: v_perm_b32 v1, s3, s2, v1
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX10-NEXT: v_or_b32_e32 v0, v1, v0
; GFX10-NEXT: global_store_dword v2, v0, s[0:1]
@@ -806,31 +800,30 @@ define amdgpu_kernel void @s_test_imin_sle_v4i8(ptr addrspace(1) %out, [8 x i32]
; GFX11-LABEL: s_test_imin_sle_v4i8:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_clause 0x2
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x28
-; GFX11-NEXT: s_load_b32 s3, s[4:5], 0x4c
+; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x4c
+; GFX11-NEXT: s_load_b32 s3, s[4:5], 0x28
; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
; GFX11-NEXT: v_mov_b32_e32 v0, 0x6050400
; GFX11-NEXT: v_mov_b32_e32 v1, 0xc0c0400
; GFX11-NEXT: v_mov_b32_e32 v2, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_ashr_i32 s4, s2, 24
-; GFX11-NEXT: s_sext_i32_i16 s5, s2
-; GFX11-NEXT: s_ashr_i32 s6, s3, 24
-; GFX11-NEXT: s_sext_i32_i16 s7, s3
-; GFX11-NEXT: s_sext_i32_i8 s8, s3
-; GFX11-NEXT: s_sext_i32_i8 s9, s2
-; GFX11-NEXT: s_bfe_i32 s3, s3, 0x80010
+; GFX11-NEXT: s_sext_i32_i8 s4, s2
+; GFX11-NEXT: s_sext_i32_i8 s5, s3
+; GFX11-NEXT: s_bfe_i32 s6, s2, 0x80008
+; GFX11-NEXT: s_bfe_i32 s7, s3, 0x80008
+; GFX11-NEXT: s_ashr_i32 s8, s2, 24
+; GFX11-NEXT: s_ashr_i32 s9, s3, 24
; GFX11-NEXT: s_bfe_i32 s2, s2, 0x80010
-; GFX11-NEXT: s_min_i32 s4, s4, s6
-; GFX11-NEXT: s_min_i32 s2, s2, s3
-; GFX11-NEXT: s_ashr_i32 s7, s7, 8
-; GFX11-NEXT: v_perm_b32 v0, s4, s2, v0
-; GFX11-NEXT: s_ashr_i32 s5, s5, 8
-; GFX11-NEXT: s_min_i32 s3, s9, s8
-; GFX11-NEXT: s_min_i32 s2, s5, s7
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_perm_b32 v1, s2, s3, v1
+; GFX11-NEXT: s_bfe_i32 s3, s3, 0x80010
+; GFX11-NEXT: s_min_i32 s8, s9, s8
+; GFX11-NEXT: s_min_i32 s2, s3, s2
+; GFX11-NEXT: s_min_i32 s3, s7, s6
+; GFX11-NEXT: v_perm_b32 v0, s8, s2, v0
+; GFX11-NEXT: s_min_i32 s2, s5, s4
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_perm_b32 v1, s3, s2, v1
; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_or_b32_e32 v0, v1, v0
; GFX11-NEXT: global_store_b32 v2, v0, s[0:1]
; GFX11-NEXT: s_endpgm
@@ -842,31 +835,30 @@ define amdgpu_kernel void @s_test_imin_sle_v4i8(ptr addrspace(1) %out, [8 x i32]
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NEXT: s_clause 0x2
-; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x28 nv
-; GFX1250-NEXT: s_load_b32 s3, s[4:5], 0x4c nv
+; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x4c nv
+; GFX1250-NEXT: s_load_b32 s3, s[4:5], 0x28 nv
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0x6050400
; GFX1250-NEXT: v_mov_b32_e32 v1, 0xc0c0400
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: s_ashr_i32 s4, s2, 24
-; GFX1250-NEXT: s_sext_i32_i16 s5, s2
-; GFX1250-NEXT: s_ashr_i32 s6, s3, 24
-; GFX1250-NEXT: s_sext_i32_i16 s7, s3
-; GFX1250-NEXT: s_sext_i32_i8 s8, s3
-; GFX1250-NEXT: s_sext_i32_i8 s9, s2
-; GFX1250-NEXT: s_bfe_i32 s3, s3, 0x80010
+; GFX1250-NEXT: s_sext_i32_i8 s4, s2
+; GFX1250-NEXT: s_sext_i32_i8 s5, s3
+; GFX1250-NEXT: s_bfe_i32 s6, s2, 0x80008
+; GFX1250-NEXT: s_bfe_i32 s7, s3, 0x80008
+; GFX1250-NEXT: s_ashr_i32 s8, s2, 24
+; GFX1250-NEXT: s_ashr_i32 s9, s3, 24
; GFX1250-NEXT: s_bfe_i32 s2, s2, 0x80010
-; GFX1250-NEXT: s_min_i32 s4, s4, s6
-; GFX1250-NEXT: s_min_i32 s2, s2, s3
-; GFX1250-NEXT: s_ashr_i32 s7, s7, 8
-; GFX1250-NEXT: v_perm_b32 v0, s4, s2, v0
-; GFX1250-NEXT: s_ashr_i32 s5, s5, 8
-; GFX1250-NEXT: s_min_i32 s3, s9, s8
-; GFX1250-NEXT: s_min_i32 s2, s5, s7
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_perm_b32 v1, s2, s3, v1
+; GFX1250-NEXT: s_bfe_i32 s3, s3, 0x80010
+; GFX1250-NEXT: s_min_i32 s8, s9, s8
+; GFX1250-NEXT: s_min_i32 s2, s3, s2
+; GFX1250-NEXT: s_min_i32 s3, s7, s6
+; GFX1250-NEXT: v_perm_b32 v0, s8, s2, v0
+; GFX1250-NEXT: s_min_i32 s2, s5, s4
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-NEXT: v_perm_b32 v1, s3, s2, v1
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-NEXT: v_or_b32_e32 v0, v1, v0
; GFX1250-NEXT: global_store_b32 v2, v0, s[0:1]
; GFX1250-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/permute_i8.ll b/llvm/test/CodeGen/AMDGPU/permute_i8.ll
index 7229d9545c95a..ab4221846fa33 100644
--- a/llvm/test/CodeGen/AMDGPU/permute_i8.ll
+++ b/llvm/test/CodeGen/AMDGPU/permute_i8.ll
@@ -1725,21 +1725,19 @@ define hidden void @sitofp_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_and_b32_e32 v4, 0x3ff, v31
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 2, v4
-; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
-; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, v2, v4
; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
-; GFX10-NEXT: global_load_dword v4, v[0:1], off
-; GFX10-NEXT: global_load_dword v9, v[2:3], off
+; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
+; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT: global_load_dword v4, v[2:3], off
+; GFX10-NEXT: global_load_dword v9, v[0:1], off
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_ashrrev_i16 v0, 8, v4
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v3, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_ashrrev_i16 v10, 8, v9
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v3, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v1, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
-; GFX10-NEXT: v_perm_b32 v4, v9, v4, 0x6010205
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v2, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v0, sext(v10) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v2, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v1, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v0, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1
+; GFX10-NEXT: v_perm_b32 v4, v4, v9, 0x6010205
; GFX10-NEXT: global_store_dwordx4 v[7:8], v[0:3], off
; GFX10-NEXT: global_store_dword v[5:6], v4, off
; GFX10-NEXT: s_setpc_b64 s[30:31]
@@ -1753,18 +1751,16 @@ define hidden void @sitofp_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in
; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v2, v4
; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX9-NEXT: global_load_dword v4, v[0:1], off
-; GFX9-NEXT: global_load_dword v9, v[2:3], off
+; GFX9-NEXT: global_load_dword v4, v[2:3], off
+; GFX9-NEXT: global_load_dword v9, v[0:1], off
; GFX9-NEXT: s_mov_b32 s4, 0x6010205
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_ashrrev_i16_e32 v0, 8, v4
+; GFX9-NEXT: v_cvt_f32_i32_sdwa v3, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_ashrrev_i16_e32 v10, 8, v9
-; GFX9-NEXT: v_cvt_f32_i32_sdwa v3, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
-; GFX9-NEXT: v_cvt_f32_i32_sdwa v1, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
-; GFX9-NEXT: v_cvt_f32_i32_sdwa v2, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
-; GFX9-NEXT: v_cvt_f32_i32_sdwa v0, sext(v10) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
-; GFX9-NEXT: v_perm_b32 v4, v9, v4, s4
+; GFX9-NEXT: v_cvt_f32_i32_sdwa v2, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1
+; GFX9-NEXT: v_cvt_f32_i32_sdwa v1, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
+; GFX9-NEXT: v_cvt_f32_i32_sdwa v0, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1
+; GFX9-NEXT: v_perm_b32 v4, v4, v9, s4
; GFX9-NEXT: global_store_dwordx4 v[7:8], v[0:3], off
; GFX9-NEXT: global_store_dword v[5:6], v4, off
; GFX9-NEXT: s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/sad.ll b/llvm/test/CodeGen/AMDGPU/sad.ll
index c73e60022b623..2233008b446b1 100644
--- a/llvm/test/CodeGen/AMDGPU/sad.ll
+++ b/llvm/test/CodeGen/AMDGPU/sad.ll
@@ -1425,12 +1425,11 @@ define amdgpu_kernel void @v_sad_u32_i8_pat1(ptr addrspace(1) %out, i8 %a, i8 %b
; GFX12-5-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX12-5-SDAG-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
; GFX12-5-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-5-SDAG-NEXT: s_lshr_b32 s3, s2, 8
-; GFX12-5-SDAG-NEXT: s_and_b32 s4, s2, 0xff
-; GFX12-5-SDAG-NEXT: s_and_b32 s3, s3, 0xff
+; GFX12-5-SDAG-NEXT: s_and_b32 s3, s2, 0xff
+; GFX12-5-SDAG-NEXT: s_bfe_u32 s4, s2, 0x80008
; GFX12-5-SDAG-NEXT: s_lshr_b32 s2, s2, 16
-; GFX12-5-SDAG-NEXT: s_min_u32 s5, s4, s3
-; GFX12-5-SDAG-NEXT: s_max_u32 s3, s4, s3
+; GFX12-5-SDAG-NEXT: s_min_u32 s5, s3, s4
+; GFX12-5-SDAG-NEXT: s_max_u32 s3, s3, s4
; GFX12-5-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-5-SDAG-NEXT: s_sub_co_i32 s3, s3, s5
; GFX12-5-SDAG-NEXT: s_add_co_i32 s2, s3, s2
@@ -1635,12 +1634,11 @@ define amdgpu_kernel void @s_sad_u32_i8_pat2(ptr addrspace(1) %out, i8 zeroext %
; GFX12-5-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX12-5-SDAG-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
; GFX12-5-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-5-SDAG-NEXT: s_lshr_b32 s3, s2, 8
-; GFX12-5-SDAG-NEXT: s_and_b32 s4, s2, 0xff
-; GFX12-5-SDAG-NEXT: s_and_b32 s3, s3, 0xff
+; GFX12-5-SDAG-NEXT: s_and_b32 s3, s2, 0xff
+; GFX12-5-SDAG-NEXT: s_bfe_u32 s4, s2, 0x80008
; GFX12-5-SDAG-NEXT: s_lshr_b32 s2, s2, 16
-; GFX12-5-SDAG-NEXT: s_min_u32 s5, s4, s3
-; GFX12-5-SDAG-NEXT: s_max_u32 s3, s4, s3
+; GFX12-5-SDAG-NEXT: s_min_u32 s5, s3, s4
+; GFX12-5-SDAG-NEXT: s_max_u32 s3, s3, s4
; GFX12-5-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-5-SDAG-NEXT: s_sub_co_i32 s3, s3, s5
; GFX12-5-SDAG-NEXT: s_add_co_i32 s2, s3, s2
diff --git a/llvm/test/CodeGen/AMDGPU/v4i8-byte-lane-ext.ll b/llvm/test/CodeGen/AMDGPU/v4i8-byte-lane-ext.ll
index ec76be49050d7..089dd79b78c36 100644
--- a/llvm/test/CodeGen/AMDGPU/v4i8-byte-lane-ext.ll
+++ b/llvm/test/CodeGen/AMDGPU/v4i8-byte-lane-ext.ll
@@ -12,13 +12,13 @@ define void @uitofp_v4i8_to_v4f64(ptr addrspace(1) %p, ptr addrspace(1) %o) {
; GFX9-NEXT: global_load_dword v0, v[0:1], off
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 24, v0
-; GFX9-NEXT: v_and_b32_e32 v4, 0xff, v0
-; GFX9-NEXT: v_lshrrev_b16_e32 v6, 8, v0
+; GFX9-NEXT: v_bfe_u32 v4, v0, 8, 8
+; GFX9-NEXT: v_and_b32_e32 v5, 0xff, v0
; GFX9-NEXT: v_bfe_u32 v0, v0, 16, 8
; GFX9-NEXT: v_cvt_f64_u32_e32 v[10:11], v1
; GFX9-NEXT: v_cvt_f64_u32_e32 v[8:9], v0
-; GFX9-NEXT: v_cvt_f64_u32_e32 v[4:5], v4
-; GFX9-NEXT: v_cvt_f64_u32_e32 v[6:7], v6
+; GFX9-NEXT: v_cvt_f64_u32_e32 v[6:7], v4
+; GFX9-NEXT: v_cvt_f64_u32_e32 v[4:5], v5
; GFX9-NEXT: global_store_dwordx4 v[2:3], v[8:11], off offset:16
; GFX9-NEXT: global_store_dwordx4 v[2:3], v[4:7], off
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -28,16 +28,15 @@ define void @uitofp_v4i8_to_v4f64(ptr addrspace(1) %p, ptr addrspace(1) %o) {
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v0, v[0:1], off
-; GFX10-NEXT: v_mov_b32_e32 v1, 0xffff
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_e32 v6, 24, v0
-; GFX10-NEXT: v_bfe_u32 v7, v0, 16, 8
-; GFX10-NEXT: v_and_b32_e32 v4, 0xff, v0
-; GFX10-NEXT: v_and_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_1
-; GFX10-NEXT: v_cvt_f64_u32_e32 v[10:11], v6
-; GFX10-NEXT: v_cvt_f64_u32_e32 v[8:9], v7
-; GFX10-NEXT: v_cvt_f64_u32_e32 v[4:5], v4
-; GFX10-NEXT: v_cvt_f64_u32_e32 v[6:7], v0
+; GFX10-NEXT: v_lshrrev_b32_e32 v4, 24, v0
+; GFX10-NEXT: v_bfe_u32 v5, v0, 16, 8
+; GFX10-NEXT: v_bfe_u32 v1, v0, 8, 8
+; GFX10-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX10-NEXT: v_cvt_f64_u32_e32 v[10:11], v4
+; GFX10-NEXT: v_cvt_f64_u32_e32 v[8:9], v5
+; GFX10-NEXT: v_cvt_f64_u32_e32 v[6:7], v1
+; GFX10-NEXT: v_cvt_f64_u32_e32 v[4:5], v0
; GFX10-NEXT: global_store_dwordx4 v[2:3], v[8:11], off offset:16
; GFX10-NEXT: global_store_dwordx4 v[2:3], v[4:7], off
; GFX10-NEXT: s_setpc_b64 s[30:31]
@@ -45,20 +44,18 @@ define void @uitofp_v4i8_to_v4f64(ptr addrspace(1) %p, ptr addrspace(1) %o) {
; GFX11-TRUE16-LABEL: uitofp_v4i8_to_v4f64:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b32 v1, v[0:1], off
+; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.l, 8, v1.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xff, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 24, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v1, 16, 8
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 24, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v0, 16, 8
+; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v0, 8, 8
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
-; GFX11-TRUE16-NEXT: v_cvt_f64_u32_e32 v[4:5], v4
+; GFX11-TRUE16-NEXT: v_cvt_f64_u32_e32 v[10:11], v4
+; GFX11-TRUE16-NEXT: v_cvt_f64_u32_e32 v[8:9], v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cvt_f64_u32_e32 v[10:11], v6
-; GFX11-TRUE16-NEXT: v_cvt_f64_u32_e32 v[8:9], v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cvt_f64_u32_e32 v[6:7], v0
+; GFX11-TRUE16-NEXT: v_cvt_f64_u32_e32 v[6:7], v1
+; GFX11-TRUE16-NEXT: v_cvt_f64_u32_e32 v[4:5], v0
; GFX11-TRUE16-NEXT: s_clause 0x1
; GFX11-TRUE16-NEXT: global_store_b128 v[2:3], v[8:11], off offset:16
; GFX11-TRUE16-NEXT: global_store_b128 v[2:3], v[4:7], off
@@ -69,18 +66,16 @@ define void @uitofp_v4i8_to_v4f64(ptr addrspace(1) %p, ptr addrspace(1) %o) {
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: global_load_b32 v0, v[0:1], off
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b16 v1, 8, v0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 0xff, v0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 24, v0
-; GFX11-FAKE16-NEXT: v_bfe_u32 v0, v0, 16, 8
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 24, v0
+; GFX11-FAKE16-NEXT: v_bfe_u32 v5, v0, 16, 8
+; GFX11-FAKE16-NEXT: v_bfe_u32 v1, v0, 8, 8
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-FAKE16-NEXT: v_cvt_f64_u32_e32 v[4:5], v4
+; GFX11-FAKE16-NEXT: v_cvt_f64_u32_e32 v[10:11], v4
+; GFX11-FAKE16-NEXT: v_cvt_f64_u32_e32 v[8:9], v5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_cvt_f64_u32_e32 v[10:11], v6
-; GFX11-FAKE16-NEXT: v_cvt_f64_u32_e32 v[8:9], v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-FAKE16-NEXT: v_cvt_f64_u32_e32 v[6:7], v1
+; GFX11-FAKE16-NEXT: v_cvt_f64_u32_e32 v[4:5], v0
; GFX11-FAKE16-NEXT: s_clause 0x1
; GFX11-FAKE16-NEXT: global_store_b128 v[2:3], v[8:11], off offset:16
; GFX11-FAKE16-NEXT: global_store_b128 v[2:3], v[4:7], off
@@ -90,21 +85,18 @@ define void @uitofp_v4i8_to_v4f64(ptr addrspace(1) %p, ptr addrspace(1) %o) {
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_load_b32 v1, v[0:1], off
+; GFX1250-NEXT: global_load_b32 v0, v[0:1], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_lshrrev_b32_e32 v4, 24, v0
+; GFX1250-NEXT: v_bfe_u32 v5, v0, 16, 8
; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_lshrrev_b16 v0.l, 8, v1.l
-; GFX1250-NEXT: s_mov_b32 s0, 0xff
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1250-NEXT: v_dual_lshrrev_b32 v6, 24, v1 :: v_dual_bitop2_b32 v4, s0, v1 bitop3:0x40
-; GFX1250-NEXT: v_bfe_u32 v1, v1, 16, 8
-; GFX1250-NEXT: v_cvt_u32_u16_e32 v0, v0.l
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1250-NEXT: v_cvt_f64_u32_e32 v[4:5], v4
-; GFX1250-NEXT: v_cvt_f64_u32_e32 v[10:11], v6
+; GFX1250-NEXT: v_bfe_u32 v1, v0, 8, 8
+; GFX1250-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX1250-NEXT: v_cvt_f64_u32_e32 v[10:11], v4
+; GFX1250-NEXT: v_cvt_f64_u32_e32 v[8:9], v5
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1250-NEXT: v_cvt_f64_u32_e32 v[8:9], v1
-; GFX1250-NEXT: v_cvt_f64_u32_e32 v[6:7], v0
+; GFX1250-NEXT: v_cvt_f64_u32_e32 v[6:7], v1
+; GFX1250-NEXT: v_cvt_f64_u32_e32 v[4:5], v0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: global_store_b128 v[2:3], v[8:11], off offset:16
; GFX1250-NEXT: global_store_b128 v[2:3], v[4:7], off
@@ -121,11 +113,10 @@ define void @sitofp_v4i8_to_v4f32(ptr addrspace(1) %p, ptr addrspace(1) %o) {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_load_dword v0, v[0:1], off
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_ashrrev_i16_e32 v1, 8, v0
; GFX9-NEXT: v_cvt_f32_i32_sdwa v7, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
; GFX9-NEXT: v_cvt_f32_i32_sdwa v6, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
+; GFX9-NEXT: v_cvt_f32_i32_sdwa v5, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1
; GFX9-NEXT: v_cvt_f32_i32_sdwa v4, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0
-; GFX9-NEXT: v_cvt_f32_i32_sdwa v5, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
; GFX9-NEXT: global_store_dwordx4 v[2:3], v[4:7], off
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -135,11 +126,10 @@ define void @sitofp_v4i8_to_v4f32(ptr addrspace(1) %p, ptr addrspace(1) %o) {
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v0, v[0:1], off
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_ashrrev_i16 v1, 8, v0
; GFX10-NEXT: v_cvt_f32_i32_sdwa v7, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
; GFX10-NEXT: v_cvt_f32_i32_sdwa v6, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v5, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1
; GFX10-NEXT: v_cvt_f32_i32_sdwa v4, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v5, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
; GFX10-NEXT: global_store_dwordx4 v[2:3], v[4:7], off
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -148,18 +138,16 @@ define void @sitofp_v4i8_to_v4f32(ptr addrspace(1) %p, ptr addrspace(1) %o) {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_ashrrev_i16 v1.l, 8, v0.l
-; GFX11-TRUE16-NEXT: v_ashrrev_i32_e32 v4, 24, v0
-; GFX11-TRUE16-NEXT: v_bfe_i32 v5, v0, 16, 8
+; GFX11-TRUE16-NEXT: v_ashrrev_i32_e32 v1, 24, v0
+; GFX11-TRUE16-NEXT: v_bfe_i32 v4, v0, 16, 8
+; GFX11-TRUE16-NEXT: v_bfe_i32 v5, v0, 8, 8
; GFX11-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 8
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 16
-; GFX11-TRUE16-NEXT: v_cvt_f32_i32_e32 v7, v4
+; GFX11-TRUE16-NEXT: v_cvt_f32_i32_e32 v7, v1
+; GFX11-TRUE16-NEXT: v_cvt_f32_i32_e32 v6, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cvt_f32_i32_e32 v6, v5
+; GFX11-TRUE16-NEXT: v_cvt_f32_i32_e32 v5, v5
; GFX11-TRUE16-NEXT: v_cvt_f32_i32_e32 v4, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cvt_f32_i32_e32 v5, v1
; GFX11-TRUE16-NEXT: global_store_b128 v[2:3], v[4:7], off
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -168,18 +156,16 @@ define void @sitofp_v4i8_to_v4f32(ptr addrspace(1) %p, ptr addrspace(1) %o) {
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: global_load_b32 v0, v[0:1], off
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_ashrrev_i16 v1, 8, v0
-; GFX11-FAKE16-NEXT: v_ashrrev_i32_e32 v4, 24, v0
-; GFX11-FAKE16-NEXT: v_bfe_i32 v5, v0, 16, 8
+; GFX11-FAKE16-NEXT: v_ashrrev_i32_e32 v1, 24, v0
+; GFX11-FAKE16-NEXT: v_bfe_i32 v4, v0, 16, 8
+; GFX11-FAKE16-NEXT: v_bfe_i32 v5, v0, 8, 8
; GFX11-FAKE16-NEXT: v_bfe_i32 v0, v0, 0, 8
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_bfe_i32 v1, v1, 0, 16
-; GFX11-FAKE16-NEXT: v_cvt_f32_i32_e32 v7, v4
+; GFX11-FAKE16-NEXT: v_cvt_f32_i32_e32 v7, v1
+; GFX11-FAKE16-NEXT: v_cvt_f32_i32_e32 v6, v4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_cvt_f32_i32_e32 v6, v5
+; GFX11-FAKE16-NEXT: v_cvt_f32_i32_e32 v5, v5
; GFX11-FAKE16-NEXT: v_cvt_f32_i32_e32 v4, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_cvt_f32_i32_e32 v5, v1
; GFX11-FAKE16-NEXT: global_store_b128 v[2:3], v[4:7], off
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -190,18 +176,16 @@ define void @sitofp_v4i8_to_v4f32(ptr addrspace(1) %p, ptr addrspace(1) %o) {
; GFX1250-NEXT: global_load_b32 v0, v[0:1], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_ashrrev_i16 v1.l, 8, v0.l
-; GFX1250-NEXT: v_ashrrev_i32_e32 v4, 24, v0
-; GFX1250-NEXT: v_bfe_i32 v5, v0, 16, 8
+; GFX1250-NEXT: v_ashrrev_i32_e32 v1, 24, v0
+; GFX1250-NEXT: v_bfe_i32 v4, v0, 16, 8
+; GFX1250-NEXT: v_bfe_i32 v5, v0, 8, 8
; GFX1250-NEXT: v_bfe_i32 v0, v0, 0, 8
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1250-NEXT: v_bfe_i32 v1, v1, 0, 16
-; GFX1250-NEXT: v_cvt_f32_i32_e32 v7, v4
+; GFX1250-NEXT: v_cvt_f32_i32_e32 v7, v1
+; GFX1250-NEXT: v_cvt_f32_i32_e32 v6, v4
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1250-NEXT: v_cvt_f32_i32_e32 v6, v5
+; GFX1250-NEXT: v_cvt_f32_i32_e32 v5, v5
; GFX1250-NEXT: v_cvt_f32_i32_e32 v4, v0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX1250-NEXT: v_cvt_f32_i32_e32 v5, v1
; GFX1250-NEXT: global_store_b128 v[2:3], v[4:7], off
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%v = load <4 x i8>, ptr addrspace(1) %p, align 4
@@ -215,20 +199,15 @@ define void @sitofp_2x_v4i8_to_v4f32(ptr addrspace(1) noalias %p, ptr addrspace(
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
-; GFX9-NEXT: v_mov_b32_e32 v4, 8
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_ashrrev_i16_sdwa v7, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_ashrrev_i16_e32 v5, 8, v0
-; GFX9-NEXT: v_ashrrev_i16_sdwa v11, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_ashrrev_i16_e32 v9, 8, v1
+; GFX9-NEXT: v_cvt_f32_i32_sdwa v7, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
; GFX9-NEXT: v_cvt_f32_i32_sdwa v6, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
+; GFX9-NEXT: v_cvt_f32_i32_sdwa v5, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1
; GFX9-NEXT: v_cvt_f32_i32_sdwa v4, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0
-; GFX9-NEXT: v_cvt_f32_i32_sdwa v5, sext(v5) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
-; GFX9-NEXT: v_cvt_f32_i32_sdwa v7, sext(v7) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX9-NEXT: v_cvt_f32_i32_sdwa v11, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
; GFX9-NEXT: v_cvt_f32_i32_sdwa v10, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
+; GFX9-NEXT: v_cvt_f32_i32_sdwa v9, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1
; GFX9-NEXT: v_cvt_f32_i32_sdwa v8, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0
-; GFX9-NEXT: v_cvt_f32_i32_sdwa v9, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
-; GFX9-NEXT: v_cvt_f32_i32_sdwa v11, sext(v11) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
; GFX9-NEXT: global_store_dwordx4 v[2:3], v[4:7], off
; GFX9-NEXT: global_store_dwordx4 v[2:3], v[8:11], off offset:16
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -239,20 +218,14 @@ define void @sitofp_2x_v4i8_to_v4f32(ptr addrspace(1) noalias %p, ptr addrspace(
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_e32 v5, 16, v0
-; GFX10-NEXT: v_lshrrev_b32_e32 v8, 16, v1
-; GFX10-NEXT: v_ashrrev_i16 v7, 8, v0
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v7, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
; GFX10-NEXT: v_cvt_f32_i32_sdwa v6, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v5, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1
; GFX10-NEXT: v_cvt_f32_i32_sdwa v4, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0
-; GFX10-NEXT: v_ashrrev_i16 v0, 8, v5
-; GFX10-NEXT: v_ashrrev_i16 v9, 8, v1
-; GFX10-NEXT: v_ashrrev_i16 v11, 8, v8
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v5, sext(v7) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v11, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
; GFX10-NEXT: v_cvt_f32_i32_sdwa v10, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v7, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v9, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1
; GFX10-NEXT: v_cvt_f32_i32_sdwa v8, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v9, sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v11, sext(v11) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
; GFX10-NEXT: global_store_dwordx4 v[2:3], v[4:7], off
; GFX10-NEXT: global_store_dwordx4 v[2:3], v[8:11], off offset:16
; GFX10-NEXT: s_setpc_b64 s[30:31]
@@ -262,28 +235,22 @@ define void @sitofp_2x_v4i8_to_v4f32(ptr addrspace(1) noalias %p, ptr addrspace(
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b64 v[0:1], v[0:1], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v1
-; GFX11-TRUE16-NEXT: v_ashrrev_i16 v5.l, 8, v0.l
-; GFX11-TRUE16-NEXT: v_ashrrev_i16 v7.l, 8, v1.l
-; GFX11-TRUE16-NEXT: v_bfe_i32 v8, v0, 16, 8
+; GFX11-TRUE16-NEXT: v_ashrrev_i32_e32 v4, 24, v0
+; GFX11-TRUE16-NEXT: v_bfe_i32 v5, v0, 16, 8
+; GFX11-TRUE16-NEXT: v_bfe_i32 v8, v0, 8, 8
; GFX11-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX11-TRUE16-NEXT: v_ashrrev_i16 v11.l, 8, v4.l
-; GFX11-TRUE16-NEXT: v_ashrrev_i16 v12.l, 8, v6.l
-; GFX11-TRUE16-NEXT: v_bfe_i32 v9, v1, 16, 8
-; GFX11-TRUE16-NEXT: v_bfe_i32 v7, v7, 0, 16
-; GFX11-TRUE16-NEXT: v_cvt_f32_i32_e32 v4, v0
-; GFX11-TRUE16-NEXT: v_bfe_i32 v0, v5, 0, 16
-; GFX11-TRUE16-NEXT: v_bfe_i32 v11, v11, 0, 16
+; GFX11-TRUE16-NEXT: v_ashrrev_i32_e32 v9, 24, v1
+; GFX11-TRUE16-NEXT: v_bfe_i32 v10, v1, 16, 8
+; GFX11-TRUE16-NEXT: v_bfe_i32 v12, v1, 8, 8
; GFX11-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX11-TRUE16-NEXT: v_bfe_i32 v12, v12, 0, 16
-; GFX11-TRUE16-NEXT: v_cvt_f32_i32_e32 v6, v8
-; GFX11-TRUE16-NEXT: v_cvt_f32_i32_e32 v10, v9
-; GFX11-TRUE16-NEXT: v_cvt_f32_i32_e32 v5, v0
-; GFX11-TRUE16-NEXT: v_cvt_f32_i32_e32 v9, v7
-; GFX11-TRUE16-NEXT: v_cvt_f32_i32_e32 v7, v11
+; GFX11-TRUE16-NEXT: v_cvt_f32_i32_e32 v7, v4
+; GFX11-TRUE16-NEXT: v_cvt_f32_i32_e32 v6, v5
+; GFX11-TRUE16-NEXT: v_cvt_f32_i32_e32 v5, v8
+; GFX11-TRUE16-NEXT: v_cvt_f32_i32_e32 v4, v0
+; GFX11-TRUE16-NEXT: v_cvt_f32_i32_e32 v11, v9
+; GFX11-TRUE16-NEXT: v_cvt_f32_i32_e32 v10, v10
+; GFX11-TRUE16-NEXT: v_cvt_f32_i32_e32 v9, v12
; GFX11-TRUE16-NEXT: v_cvt_f32_i32_e32 v8, v1
-; GFX11-TRUE16-NEXT: v_cvt_f32_i32_e32 v11, v12
; GFX11-TRUE16-NEXT: s_clause 0x1
; GFX11-TRUE16-NEXT: global_store_b128 v[2:3], v[4:7], off
; GFX11-TRUE16-NEXT: global_store_b128 v[2:3], v[8:11], off offset:16
@@ -294,28 +261,22 @@ define void @sitofp_2x_v4i8_to_v4f32(ptr addrspace(1) noalias %p, ptr addrspace(
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: global_load_b64 v[0:1], v[0:1], off
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 16, v0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 16, v1
-; GFX11-FAKE16-NEXT: v_ashrrev_i16 v5, 8, v0
-; GFX11-FAKE16-NEXT: v_ashrrev_i16 v7, 8, v1
-; GFX11-FAKE16-NEXT: v_bfe_i32 v8, v0, 16, 8
+; GFX11-FAKE16-NEXT: v_ashrrev_i32_e32 v4, 24, v0
+; GFX11-FAKE16-NEXT: v_bfe_i32 v5, v0, 16, 8
+; GFX11-FAKE16-NEXT: v_bfe_i32 v8, v0, 8, 8
; GFX11-FAKE16-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX11-FAKE16-NEXT: v_ashrrev_i16 v11, 8, v4
-; GFX11-FAKE16-NEXT: v_ashrrev_i16 v12, 8, v6
-; GFX11-FAKE16-NEXT: v_bfe_i32 v9, v1, 16, 8
-; GFX11-FAKE16-NEXT: v_bfe_i32 v7, v7, 0, 16
-; GFX11-FAKE16-NEXT: v_cvt_f32_i32_e32 v4, v0
-; GFX11-FAKE16-NEXT: v_bfe_i32 v0, v5, 0, 16
-; GFX11-FAKE16-NEXT: v_bfe_i32 v11, v11, 0, 16
+; GFX11-FAKE16-NEXT: v_ashrrev_i32_e32 v9, 24, v1
+; GFX11-FAKE16-NEXT: v_bfe_i32 v10, v1, 16, 8
+; GFX11-FAKE16-NEXT: v_bfe_i32 v12, v1, 8, 8
; GFX11-FAKE16-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX11-FAKE16-NEXT: v_bfe_i32 v12, v12, 0, 16
-; GFX11-FAKE16-NEXT: v_cvt_f32_i32_e32 v6, v8
-; GFX11-FAKE16-NEXT: v_cvt_f32_i32_e32 v10, v9
-; GFX11-FAKE16-NEXT: v_cvt_f32_i32_e32 v5, v0
-; GFX11-FAKE16-NEXT: v_cvt_f32_i32_e32 v9, v7
-; GFX11-FAKE16-NEXT: v_cvt_f32_i32_e32 v7, v11
+; GFX11-FAKE16-NEXT: v_cvt_f32_i32_e32 v7, v4
+; GFX11-FAKE16-NEXT: v_cvt_f32_i32_e32 v6, v5
+; GFX11-FAKE16-NEXT: v_cvt_f32_i32_e32 v5, v8
+; GFX11-FAKE16-NEXT: v_cvt_f32_i32_e32 v4, v0
+; GFX11-FAKE16-NEXT: v_cvt_f32_i32_e32 v11, v9
+; GFX11-FAKE16-NEXT: v_cvt_f32_i32_e32 v10, v10
+; GFX11-FAKE16-NEXT: v_cvt_f32_i32_e32 v9, v12
; GFX11-FAKE16-NEXT: v_cvt_f32_i32_e32 v8, v1
-; GFX11-FAKE16-NEXT: v_cvt_f32_i32_e32 v11, v12
; GFX11-FAKE16-NEXT: s_clause 0x1
; GFX11-FAKE16-NEXT: global_store_b128 v[2:3], v[4:7], off
; GFX11-FAKE16-NEXT: global_store_b128 v[2:3], v[8:11], off offset:16
@@ -327,28 +288,22 @@ define void @sitofp_2x_v4i8_to_v4f32(ptr addrspace(1) noalias %p, ptr addrspace(
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: global_load_b64 v[0:1], v[0:1], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_dual_lshrrev_b32 v4, 16, v0 :: v_dual_lshrrev_b32 v6, 16, v1
-; GFX1250-NEXT: v_ashrrev_i16 v5.l, 8, v0.l
-; GFX1250-NEXT: v_ashrrev_i16 v7.l, 8, v1.l
-; GFX1250-NEXT: v_bfe_i32 v8, v0, 16, 8
+; GFX1250-NEXT: v_dual_ashrrev_i32 v4, 24, v0 :: v_dual_ashrrev_i32 v9, 24, v1
+; GFX1250-NEXT: v_bfe_i32 v5, v0, 16, 8
+; GFX1250-NEXT: v_bfe_i32 v8, v0, 8, 8
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX1250-NEXT: v_ashrrev_i16 v11.l, 8, v4.l
-; GFX1250-NEXT: v_ashrrev_i16 v12.l, 8, v6.l
-; GFX1250-NEXT: v_bfe_i32 v9, v1, 16, 8
-; GFX1250-NEXT: v_bfe_i32 v7, v7, 0, 16
-; GFX1250-NEXT: v_cvt_f32_i32_e32 v4, v0
-; GFX1250-NEXT: v_bfe_i32 v0, v5, 0, 16
-; GFX1250-NEXT: v_bfe_i32 v11, v11, 0, 16
+; GFX1250-NEXT: v_bfe_i32 v10, v1, 16, 8
+; GFX1250-NEXT: v_bfe_i32 v12, v1, 8, 8
; GFX1250-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX1250-NEXT: v_bfe_i32 v12, v12, 0, 16
-; GFX1250-NEXT: v_cvt_f32_i32_e32 v6, v8
-; GFX1250-NEXT: v_cvt_f32_i32_e32 v10, v9
-; GFX1250-NEXT: v_cvt_f32_i32_e32 v5, v0
-; GFX1250-NEXT: v_cvt_f32_i32_e32 v9, v7
-; GFX1250-NEXT: v_cvt_f32_i32_e32 v7, v11
+; GFX1250-NEXT: v_cvt_f32_i32_e32 v7, v4
+; GFX1250-NEXT: v_cvt_f32_i32_e32 v6, v5
+; GFX1250-NEXT: v_cvt_f32_i32_e32 v5, v8
+; GFX1250-NEXT: v_cvt_f32_i32_e32 v4, v0
+; GFX1250-NEXT: v_cvt_f32_i32_e32 v11, v9
+; GFX1250-NEXT: v_cvt_f32_i32_e32 v10, v10
+; GFX1250-NEXT: v_cvt_f32_i32_e32 v9, v12
; GFX1250-NEXT: v_cvt_f32_i32_e32 v8, v1
-; GFX1250-NEXT: v_cvt_f32_i32_e32 v11, v12
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: global_store_b128 v[2:3], v[4:7], off
; GFX1250-NEXT: global_store_b128 v[2:3], v[8:11], off offset:16
@@ -371,14 +326,13 @@ define void @uitofp_2x_v2i8_to_v2f16(ptr addrspace(1) noalias %p, ptr addrspace(
; GFX9-NEXT: global_load_dword v0, v[0:1], off
; GFX9-NEXT: s_movk_i32 s4, 0xff
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 8, v0
-; GFX9-NEXT: v_and_b32_sdwa v4, v0, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT: v_and_b32_sdwa v1, v0, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT: v_cvt_f16_u16_sdwa v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1
; GFX9-NEXT: v_cvt_f16_u16_sdwa v5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0
-; GFX9-NEXT: v_cvt_f16_u16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
-; GFX9-NEXT: v_cvt_f16_u16_sdwa v6, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0
-; GFX9-NEXT: v_cvt_f16_u16_e32 v1, v4
-; GFX9-NEXT: v_pack_b32_f16 v1, v1, v0
-; GFX9-NEXT: v_pack_b32_f16 v0, v5, v6
+; GFX9-NEXT: v_cvt_f16_u16_sdwa v6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
+; GFX9-NEXT: v_cvt_f16_u16_e32 v1, v1
+; GFX9-NEXT: v_pack_b32_f16 v0, v5, v4
+; GFX9-NEXT: v_pack_b32_f16 v1, v1, v6
; GFX9-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -390,13 +344,12 @@ define void @uitofp_2x_v2i8_to_v2f16(ptr addrspace(1) noalias %p, ptr addrspace(
; GFX10-NEXT: v_mov_b32_e32 v1, 0xff
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_and_b32_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-NEXT: v_lshrrev_b32_e32 v4, 8, v0
+; GFX10-NEXT: v_cvt_f16_u16_sdwa v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1
; GFX10-NEXT: v_cvt_f16_u16_sdwa v5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0
-; GFX10-NEXT: v_cvt_f16_u16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
+; GFX10-NEXT: v_cvt_f16_u16_sdwa v6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
; GFX10-NEXT: v_cvt_f16_u16_e32 v1, v1
-; GFX10-NEXT: v_cvt_f16_u16_sdwa v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0
-; GFX10-NEXT: v_pack_b32_f16 v1, v1, v0
; GFX10-NEXT: v_pack_b32_f16 v0, v5, v4
+; GFX10-NEXT: v_pack_b32_f16 v1, v1, v6
; GFX10-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -405,19 +358,17 @@ define void @uitofp_2x_v2i8_to_v2f16(ptr addrspace(1) noalias %p, ptr addrspace(
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v1, v[0:1], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 8, v1
-; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v1.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.l, 8, v1.l
+; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.l
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 24, v1
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v4.l
-; GFX11-TRUE16-NEXT: v_cvt_f16_u16_e32 v0.l, v0.l
+; GFX11-TRUE16-NEXT: v_cvt_f16_u16_e32 v4.h, v0.l
+; GFX11-TRUE16-NEXT: v_cvt_f16_u16_e32 v4.l, v0.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cvt_f16_u16_e32 v1.h, v5.l
-; GFX11-TRUE16-NEXT: v_cvt_f16_u16_e32 v1.l, v1.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cvt_f16_u16_e32 v0.h, v0.h
-; GFX11-TRUE16-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GFX11-TRUE16-NEXT: v_cvt_f16_u16_e32 v5.h, v5.l
+; GFX11-TRUE16-NEXT: v_cvt_f16_u16_e32 v5.l, v1.l
+; GFX11-TRUE16-NEXT: global_store_b64 v[2:3], v[4:5], off
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: uitofp_2x_v2i8_to_v2f16:
@@ -426,21 +377,20 @@ define void @uitofp_2x_v2i8_to_v2f16(ptr addrspace(1) noalias %p, ptr addrspace(
; GFX11-FAKE16-NEXT: global_load_b32 v0, v[0:1], off
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 8, v0
+; GFX11-FAKE16-NEXT: v_lshrrev_b16 v4, 8, v0
; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xff, v0
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 24, v0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 0xff, v4
+; GFX11-FAKE16-NEXT: v_cvt_f16_u16_e32 v4, v4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-FAKE16-NEXT: v_cvt_f16_u16_e32 v5, v5
-; GFX11-FAKE16-NEXT: v_cvt_f16_u16_e32 v0, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_cvt_f16_u16_e32 v6, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-FAKE16-NEXT: v_cvt_f16_u16_e32 v1, v1
-; GFX11-FAKE16-NEXT: v_cvt_f16_u16_e32 v4, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_pack_b32_f16 v1, v1, v0
; GFX11-FAKE16-NEXT: v_pack_b32_f16 v0, v5, v4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_pack_b32_f16 v1, v1, v6
; GFX11-FAKE16-NEXT: global_store_b64 v[2:3], v[0:1], off
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -450,18 +400,18 @@ define void @uitofp_2x_v2i8_to_v2f16(ptr addrspace(1) noalias %p, ptr addrspace(
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: global_load_b32 v1, v[0:1], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_dual_lshrrev_b32 v4, 8, v1 :: v_dual_lshrrev_b32 v5, 24, v1
; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_and_b16 v0.l, 0xff, v1.l
+; GFX1250-NEXT: v_lshrrev_b16 v0.l, 8, v1.l
+; GFX1250-NEXT: v_and_b16 v0.h, 0xff, v1.l
+; GFX1250-NEXT: v_lshrrev_b32_e32 v5, 24, v1
; GFX1250-NEXT: v_and_b16 v1.l, 0xff, v1.h
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX1250-NEXT: v_and_b16 v0.h, 0xff, v4.l
-; GFX1250-NEXT: v_cvt_f16_u16_e32 v1.h, v5.l
-; GFX1250-NEXT: v_cvt_f16_u16_e32 v0.l, v0.l
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1250-NEXT: v_cvt_f16_u16_e32 v1.l, v1.l
-; GFX1250-NEXT: v_cvt_f16_u16_e32 v0.h, v0.h
-; GFX1250-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GFX1250-NEXT: v_cvt_f16_u16_e32 v4.h, v0.l
+; GFX1250-NEXT: v_cvt_f16_u16_e32 v4.l, v0.h
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1250-NEXT: v_cvt_f16_u16_e32 v5.h, v5.l
+; GFX1250-NEXT: v_cvt_f16_u16_e32 v5.l, v1.l
+; GFX1250-NEXT: global_store_b64 v[2:3], v[4:5], off
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%v0 = load <2 x i8>, ptr addrspace(1) %p, align 4
%c0 = uitofp <2 x i8> %v0 to <2 x half>
@@ -478,39 +428,32 @@ define i32 @zext_lshr_trunc_i16(i32 %x) {
; GFX9-LABEL: zext_lshr_trunc_i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_lshrrev_b16_e32 v0, 4, v0
+; GFX9-NEXT: v_bfe_u32 v0, v0, 4, 12
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: zext_lshr_trunc_i16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshrrev_b16 v0, 4, v0
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX10-NEXT: v_bfe_u32 v0, v0, 4, 12
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: zext_lshr_trunc_i16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.l, 4, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
+; GFX11-TRUE16-NEXT: v_bfe_u32 v0, v0, 4, 12
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: zext_lshr_trunc_i16:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b16 v0, 4, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-FAKE16-NEXT: v_bfe_u32 v0, v0, 4, 12
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: zext_lshr_trunc_i16:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_lshrrev_b16 v0.l, 4, v0.l
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cvt_u32_u16_e32 v0, v0.l
+; GFX1250-NEXT: v_bfe_u32 v0, v0, 4, 12
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%t = trunc i32 %x to i16
%s = lshr i16 %t, 4
@@ -522,40 +465,32 @@ define i32 @sext_ashr_trunc_i16(i32 %x) {
; GFX9-LABEL: sext_ashr_trunc_i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_ashrrev_i16_e32 v0, 8, v0
-; GFX9-NEXT: v_bfe_i32 v0, v0, 0, 16
+; GFX9-NEXT: v_bfe_i32 v0, v0, 8, 8
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: sext_ashr_trunc_i16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_ashrrev_i16 v0, 8, v0
-; GFX10-NEXT: v_bfe_i32 v0, v0, 0, 16
+; GFX10-NEXT: v_bfe_i32 v0, v0, 8, 8
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: sext_ashr_trunc_i16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_ashrrev_i16 v0.l, 8, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 16
+; GFX11-TRUE16-NEXT: v_bfe_i32 v0, v0, 8, 8
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: sext_ashr_trunc_i16:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_ashrrev_i16 v0, 8, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_bfe_i32 v0, v0, 0, 16
+; GFX11-FAKE16-NEXT: v_bfe_i32 v0, v0, 8, 8
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: sext_ashr_trunc_i16:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_ashrrev_i16 v0.l, 8, v0.l
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_bfe_i32 v0, v0, 0, 16
+; GFX1250-NEXT: v_bfe_i32 v0, v0, 8, 8
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%t = trunc i32 %x to i16
%s = ashr i16 %t, 8
@@ -566,28 +501,22 @@ define i32 @sext_ashr_trunc_i16(i32 %x) {
define amdgpu_ps i32 @s_sext_ashr_trunc_i16(i32 inreg %x) {
; GFX9-LABEL: s_sext_ashr_trunc_i16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_sext_i32_i16 s0, s0
-; GFX9-NEXT: s_ashr_i32 s0, s0, 8
+; GFX9-NEXT: s_bfe_i32 s0, s0, 0x80008
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: s_sext_ashr_trunc_i16:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_sext_i32_i16 s0, s0
-; GFX10-NEXT: s_ashr_i32 s0, s0, 8
+; GFX10-NEXT: s_bfe_i32 s0, s0, 0x80008
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-TRUE16-LABEL: s_sext_ashr_trunc_i16:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_sext_i32_i16 s0, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_ashr_i32 s0, s0, 8
+; GFX11-TRUE16-NEXT: s_bfe_i32 s0, s0, 0x80008
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
; GFX11-FAKE16-LABEL: s_sext_ashr_trunc_i16:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_sext_i32_i16 s0, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_ashr_i32 s0, s0, 8
+; GFX11-FAKE16-NEXT: s_bfe_i32 s0, s0, 0x80008
; GFX11-FAKE16-NEXT: ; return to shader part epilog
;
; GFX1250-LABEL: s_sext_ashr_trunc_i16:
@@ -596,9 +525,7 @@ define amdgpu_ps i32 @s_sext_ashr_trunc_i16(i32 inreg %x) {
; GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_sext_i32_i16 s0, s0
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: s_ashr_i32 s0, s0, 8
+; GFX1250-NEXT: s_bfe_i32 s0, s0, 0x80008
; GFX1250-NEXT: ; return to shader part epilog
%t = trunc i32 %x to i16
%s = ashr i16 %t, 8
@@ -661,25 +588,25 @@ define i32 @zext_lshr_trunc_i16_multi_use(i32 %x, ptr addrspace(1) %o) {
define amdgpu_ps void @anyext_lshr_trunc_i16(<4 x i32> inreg %rsrc, i32 %x) {
; GFX9-LABEL: anyext_lshr_trunc_i16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: v_lshrrev_b16_e32 v0, 8, v0
+; GFX9-NEXT: v_bfe_u32 v0, v0, 8, 8
; GFX9-NEXT: buffer_store_short v0, off, s[0:3], 0
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: anyext_lshr_trunc_i16:
; GFX10: ; %bb.0:
-; GFX10-NEXT: v_lshrrev_b16 v0, 8, v0
+; GFX10-NEXT: v_bfe_u32 v0, v0, 8, 8
; GFX10-NEXT: buffer_store_short v0, off, s[0:3], 0
; GFX10-NEXT: s_endpgm
;
; GFX11-TRUE16-LABEL: anyext_lshr_trunc_i16:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.l, 8, v0.l
+; GFX11-TRUE16-NEXT: v_bfe_u32 v0, v0, 8, 8
; GFX11-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0
; GFX11-TRUE16-NEXT: s_endpgm
;
; GFX11-FAKE16-LABEL: anyext_lshr_trunc_i16:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: v_lshrrev_b16 v0, 8, v0
+; GFX11-FAKE16-NEXT: v_bfe_u32 v0, v0, 8, 8
; GFX11-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0
; GFX11-FAKE16-NEXT: s_endpgm
;
@@ -689,7 +616,7 @@ define amdgpu_ps void @anyext_lshr_trunc_i16(<4 x i32> inreg %rsrc, i32 %x) {
; GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: v_lshrrev_b16 v0.l, 8, v0.l
+; GFX1250-NEXT: v_bfe_u32 v0, v0, 8, 8
; GFX1250-NEXT: buffer_store_b16 v0, off, s[0:3], null
; GFX1250-NEXT: s_endpgm
%t = trunc i32 %x to i16
More information about the llvm-branch-commits
mailing list