[llvm-branch-commits] [llvm] [AMDGPU] Narrow provably-I32 address offsets (PR #224737)
Krzysztof Drewniak via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Fri Sep 18 18:55:22 PDT 2026
https://github.com/krzysz00 updated https://github.com/llvm/llvm-project/pull/224737
>From 61ce430a79fd4a75626a233a8f2b9963d1c441f4 Mon Sep 17 00:00:00 2001
From: Krzysztof Drewniak <Krzysztof.Drewniak at amd.com>
Date: Fri, 18 Sep 2026 18:19:12 +0000
Subject: [PATCH] [AMDGPU] Narrow provably-I32 address offsets
Add a `tryNarrowToI32()` function that uses KnowsBits to detect 64-bit
offests that could be in a 32-bit register, allowing us to match
base + offset cases that don't involve literal base + (ext offset)
nodes.
Apply this to global_* instructions and the scalar memory instructions
that take a scalar offset.
This does leave a few dead copies when we have to bail out of the SMEM
handling for creatining a negative offset, but those don't have
end-to-end effect and it looks like you could already get dead MOVs
from that codepath.
AI disclosure: Claude wrote the code, I looked at it.
---
llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp | 48 +-
.../AMDGPU/AMDGPUInstructionSelector.cpp | 36 +-
.../Target/AMDGPU/AMDGPUInstructionSelector.h | 5 +
.../AMDGPU/GlobalISel/extractelement.i8.ll | 150 ++---
llvm/test/CodeGen/AMDGPU/ctlz.ll | 14 +-
llvm/test/CodeGen/AMDGPU/ctlz_zero_poison.ll | 18 +-
llvm/test/CodeGen/AMDGPU/cttz.ll | 14 +-
.../AMDGPU/global-saddr-narrow-offset.ll | 93 ++-
.../AMDGPU/memory-legalizer-global-lastuse.ll | 66 +-
.../memory-legalizer-global-nontemporal.ll | 568 +++++++-----------
.../memory-legalizer-global-volatile.ll | 350 ++++-------
.../AMDGPU/promote-constOffset-to-imm.ll | 19 +-
.../AMDGPU/rewrite-vgpr-mfma-to-agpr.ll | 5 +-
.../AMDGPU/smfmac_alloc_failure_no_agpr_O0.ll | 39 +-
.../CodeGen/AMDGPU/smrd-narrow-soffset.ll | 45 +-
llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll | 15 +-
16 files changed, 551 insertions(+), 934 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
index 0d212f9dd39b2..69f3332ca0abb 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
@@ -2023,8 +2023,26 @@ static SDValue matchExtFromI32orI32(SDValue Op, bool IsSigned,
return (ExtSrc.getValueType() == MVT::i32) ? ExtSrc : SDValue();
}
+// Return an i32 equal to \p Op if it already trivially exists. This includes
+// both i32 values themselves, the inputs to extensions from i32, and i64 values
+// with irrelevant high-halves (they can just be the low half).
+static SDValue tryNarrowToI32(SDValue Op, bool IsSigned, SelectionDAG *DAG) {
+ if (SDValue Ext = matchExtFromI32orI32(Op, IsSigned, DAG))
+ return Ext;
+
+ if (Op.getValueType() != MVT::i64)
+ return SDValue();
+
+ if (IsSigned ? DAG->ComputeNumSignBits(Op) > 32
+ : DAG->computeKnownBits(Op).countMinLeadingZeros() >= 32)
+ return DAG->getTargetExtractSubreg(AMDGPU::sub0, SDLoc(Op), MVT::i32, Op);
+
+ return SDValue();
+}
+
// Match (64-bit SGPR base) + (zext vgpr offset) + sext(imm offset)
-// or (64-bit SGPR base) + (sext vgpr offset) + sext(imm offset)
+// or (64-bit SGPR base) + (sext vgpr offset) + sext(imm offset) or cases where
+// the zext/sext has been folded into the offset but it's still a 32-bit value.
bool AMDGPUDAGToDAGISel::SelectGlobalSAddr(SDNode *N, SDValue Addr,
SDValue &SAddr, SDValue &VOffset,
SDValue &Offset, bool &ScaleOffset,
@@ -2083,16 +2101,16 @@ bool AMDGPUDAGToDAGISel::SelectGlobalSAddr(SDNode *N, SDValue Addr,
}
}
- // Match the variable offset.
- if (Addr->isAnyAdd()) {
+ // Match the variable offset and don't push constants back into VGPRs.
+ if (Addr->isAnyAdd() && !isa<ConstantSDNode>(Addr.getOperand(1))) {
LHS = Addr.getOperand(0);
if (!LHS->isDivergent()) {
// add (i64 sgpr), (*_extend (i32 vgpr))
RHS = Addr.getOperand(1);
ScaleOffset = SelectScaleOffset(N, RHS, Subtarget->hasSignedGVSOffset());
- if (SDValue ExtRHS = matchExtFromI32orI32(
- RHS, Subtarget->hasSignedGVSOffset(), CurDAG)) {
+ if (SDValue ExtRHS =
+ tryNarrowToI32(RHS, Subtarget->hasSignedGVSOffset(), CurDAG)) {
SAddr = LHS;
VOffset = ExtRHS;
}
@@ -2100,10 +2118,11 @@ bool AMDGPUDAGToDAGISel::SelectGlobalSAddr(SDNode *N, SDValue Addr,
RHS = Addr.getOperand(1);
if (!SAddr && !RHS->isDivergent()) {
- // add (*_extend (i32 vgpr)), (i64 sgpr)
+ // add (*_extend (i32 vgpr)), (i64 sgpr), where the extend may have been
+ // folded.
ScaleOffset = SelectScaleOffset(N, LHS, Subtarget->hasSignedGVSOffset());
- if (SDValue ExtLHS = matchExtFromI32orI32(
- LHS, Subtarget->hasSignedGVSOffset(), CurDAG)) {
+ if (SDValue ExtLHS =
+ tryNarrowToI32(LHS, Subtarget->hasSignedGVSOffset(), CurDAG)) {
SAddr = RHS;
VOffset = ExtLHS;
}
@@ -2479,19 +2498,12 @@ bool AMDGPUDAGToDAGISel::SelectSMRDOffset(SDNode *N, SDValue ByteOffsetNode,
if (!SOffset)
return false;
- if (ByteOffsetNode.getValueType().isScalarInteger() &&
- ByteOffsetNode.getValueType().getSizeInBits() == 32) {
- *SOffset = ByteOffsetNode;
+ if (SDValue S32 =
+ tryNarrowToI32(ByteOffsetNode, /*IsSigned=*/false, CurDAG)) {
+ *SOffset = S32;
return isSOffsetLegalWithImmOffset(SOffset, Imm32Only, IsBuffer,
ImmOffset);
}
- if (ByteOffsetNode.getOpcode() == ISD::ZERO_EXTEND) {
- if (ByteOffsetNode.getOperand(0).getValueType().getSizeInBits() == 32) {
- *SOffset = ByteOffsetNode.getOperand(0);
- return isSOffsetLegalWithImmOffset(SOffset, Imm32Only, IsBuffer,
- ImmOffset);
- }
- }
return false;
}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
index 140cf58e8fdd3..2c647ee1130cc 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
@@ -3728,6 +3728,27 @@ AMDGPUInstructionSelector::matchExtendFromS32OrS32(Register Reg,
return matchZeroExtendFromS32OrS32(Reg);
}
+Register
+AMDGPUInstructionSelector::tryNarrowToI32(Register Reg, bool IsSigned,
+ MachineInstr &InsertPt) const {
+ if (Register Ext = matchExtendFromS32OrS32(Reg, IsSigned))
+ return Ext;
+
+ if (MRI->getType(Reg) != LLT::scalar(64))
+ return Register();
+
+ if (IsSigned ? VT->computeNumSignBits(Reg) <= 32
+ : VT->getKnownBits(Reg).countMinLeadingZeros() < 32)
+ return Register();
+
+ Register Low = MRI->createVirtualRegister(
+ isSGPR(Reg) ? &AMDGPU::SReg_32RegClass : &AMDGPU::VGPR_32RegClass);
+ BuildMI(*InsertPt.getParent(), InsertPt, InsertPt.getDebugLoc(),
+ TII.get(AMDGPU::COPY), Low)
+ .addReg(Reg, {}, AMDGPU::sub0);
+ return Low;
+}
+
Register AMDGPUInstructionSelector::matchAnyExtendFromS32(Register Reg) const {
Register AnyExtSrc;
if (mi_match(Reg, *MRI, m_GAnyExt(m_Reg(AnyExtSrc))))
@@ -5908,7 +5929,7 @@ bool AMDGPUInstructionSelector::selectSmrdOffset(MachineOperand &Root,
if (ScaleOffset)
*ScaleOffset =
selectScaleOffset(Root, OffsetReg, false /* IsSigned */);
- OffsetReg = matchZeroExtendFromS32OrS32(OffsetReg);
+ OffsetReg = tryNarrowToI32(OffsetReg, /*IsSigned=*/false, *MI);
if (OffsetReg) {
Base = GEPI2.SgprParts[0];
*SOffset = OffsetReg;
@@ -5957,7 +5978,7 @@ bool AMDGPUInstructionSelector::selectSmrdOffset(MachineOperand &Root,
Register OffsetReg = GEPI.SgprParts[1];
if (ScaleOffset)
*ScaleOffset = selectScaleOffset(Root, OffsetReg, false /* IsSigned */);
- OffsetReg = matchZeroExtendFromS32OrS32(OffsetReg);
+ OffsetReg = tryNarrowToI32(OffsetReg, /*IsSigned=*/false, *MI);
if (OffsetReg) {
Base = GEPI.SgprParts[0];
*SOffset = OffsetReg;
@@ -6183,16 +6204,17 @@ AMDGPUInstructionSelector::selectGlobalSAddr(MachineOperand &Root,
// Look through the SGPR->VGPR copy.
Register SAddr =
getSrcRegIgnoringCopies(AddrDef->MI->getOperand(1).getReg(), *MRI);
+ Register PtrBaseOffset = AddrDef->MI->getOperand(2).getReg();
- if (isSGPR(SAddr)) {
- Register PtrBaseOffset = AddrDef->MI->getOperand(2).getReg();
-
+ // Don't push constants back out.
+ if (isSGPR(SAddr) && !getIConstantVRegVal(PtrBaseOffset, *MRI)) {
// It's possible voffset is an SGPR here, but the copy to VGPR will be
// inserted later.
bool ScaleOffset = selectScaleOffset(Root, PtrBaseOffset,
Subtarget->hasSignedGVSOffset());
- if (Register VOffset = matchExtendFromS32OrS32(
- PtrBaseOffset, Subtarget->hasSignedGVSOffset())) {
+ if (Register VOffset =
+ tryNarrowToI32(PtrBaseOffset, Subtarget->hasSignedGVSOffset(),
+ *Root.getParent())) {
if (NeedIOffset)
return {{[=](MachineInstrBuilder &MIB) { // saddr
MIB.addReg(SAddr);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
index 71e815bf122d8..6828f809e9caf 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
@@ -459,6 +459,11 @@ class AMDGPUInstructionSelector final : public InstructionSelector {
/// Match either sign or zero extend depending on the \p IsSigned from a
/// 32-bit value to 64-bits, or \p Reg itself if it is 32-bit.
Register matchExtendFromS32OrS32(Register Reg, bool IsSigned) const;
+ /// Return an i32 equal to \p Op if it already trivially exists. This includes
+ /// both i32 values themselves, the inputs to extensions from i32, and i64
+ /// values with irrelevant high-halves (they can just be the low half).
+ Register tryNarrowToI32(Register Reg, bool IsSigned,
+ MachineInstr &InsertPt) const;
/// Match an any extend from a 32-bit value to 64-bit.
Register matchAnyExtendFromS32(Register Reg) const;
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.i8.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.i8.ll
index faf2898562d57..2e4ce5602d4ca 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.i8.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.i8.ll
@@ -9,11 +9,8 @@ define amdgpu_ps i8 @extractelement_sgpr_v4i8_sgpr_idx(ptr addrspace(4) inreg %p
; GFX9-LABEL: extractelement_sgpr_v4i8_sgpr_idx:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_and_b32 s0, s4, 3
-; GFX9-NEXT: s_ashr_i32 s1, s0, 31
-; GFX9-NEXT: s_add_u32 s0, s2, s0
-; GFX9-NEXT: s_addc_u32 s1, s3, s1
-; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: global_load_ubyte v0, v0, s[0:1]
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: global_load_ubyte v0, v0, s[2:3]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s0, v0
; GFX9-NEXT: ; return to shader part epilog
@@ -49,11 +46,8 @@ define amdgpu_ps i8 @extractelement_sgpr_v4i8_sgpr_idx(ptr addrspace(4) inreg %p
; GFX10-LABEL: extractelement_sgpr_v4i8_sgpr_idx:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_and_b32 s0, s4, 3
-; GFX10-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-NEXT: s_ashr_i32 s1, s0, 31
-; GFX10-NEXT: s_add_u32 s0, s2, s0
-; GFX10-NEXT: s_addc_u32 s1, s3, s1
-; GFX10-NEXT: global_load_ubyte v0, v0, s[0:1]
+; GFX10-NEXT: v_mov_b32_e32 v0, s0
+; GFX10-NEXT: global_load_ubyte v0, v0, s[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_readfirstlane_b32 s0, v0
; GFX10-NEXT: ; return to shader part epilog
@@ -61,11 +55,9 @@ define amdgpu_ps i8 @extractelement_sgpr_v4i8_sgpr_idx(ptr addrspace(4) inreg %p
; GFX11-LABEL: extractelement_sgpr_v4i8_sgpr_idx:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_and_b32 s0, s4, 3
-; GFX11-NEXT: v_mov_b32_e32 v0, 0
-; GFX11-NEXT: s_ashr_i32 s1, s0, 31
-; GFX11-NEXT: s_add_u32 s0, s2, s0
-; GFX11-NEXT: s_addc_u32 s1, s3, s1
-; GFX11-NEXT: global_load_u8 v0, v0, s[0:1]
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-NEXT: global_load_u8 v0, v0, s[2:3]
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-NEXT: ; return to shader part epilog
@@ -210,13 +202,8 @@ define i8 @extractelement_vgpr_v4i8_vgpr_idx(ptr addrspace(1) %ptr, i32 %idx) {
define amdgpu_ps i8 @extractelement_sgpr_v4i8_vgpr_idx(ptr addrspace(4) inreg %ptr, i32 %idx) {
; GFX9-LABEL: extractelement_sgpr_v4i8_vgpr_idx:
; GFX9: ; %bb.0:
-; GFX9-NEXT: v_and_b32_e32 v2, 3, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_ashrrev_i32_e32 v3, 31, v2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v2
-; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc
-; GFX9-NEXT: global_load_ubyte v0, v[0:1], off
+; GFX9-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX9-NEXT: global_load_ubyte v0, v0, s[2:3]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s0, v0
; GFX9-NEXT: ; return to shader part epilog
@@ -249,27 +236,16 @@ define amdgpu_ps i8 @extractelement_sgpr_v4i8_vgpr_idx(ptr addrspace(4) inreg %p
;
; GFX10-LABEL: extractelement_sgpr_v4i8_vgpr_idx:
; GFX10: ; %bb.0:
-; GFX10-NEXT: v_and_b32_e32 v2, 3, v0
-; GFX10-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-NEXT: v_ashrrev_i32_e32 v3, 31, v2
-; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v3, vcc_lo
-; GFX10-NEXT: global_load_ubyte v0, v[0:1], off
+; GFX10-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX10-NEXT: global_load_ubyte v0, v0, s[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_readfirstlane_b32 s0, v0
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: extractelement_sgpr_v4i8_vgpr_idx:
; GFX11: ; %bb.0:
-; GFX11-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_and_b32 v2, 3, v0
-; GFX11-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_ashrrev_i32_e32 v3, 31, v2
-; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
-; GFX11-NEXT: global_load_u8 v0, v[0:1], off
+; GFX11-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX11-NEXT: global_load_u8 v0, v0, s[2:3]
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-NEXT: ; return to shader part epilog
@@ -662,11 +638,8 @@ define amdgpu_ps i8 @extractelement_sgpr_v8i8_sgpr_idx(ptr addrspace(4) inreg %p
; GFX9-LABEL: extractelement_sgpr_v8i8_sgpr_idx:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_and_b32 s0, s4, 7
-; GFX9-NEXT: s_ashr_i32 s1, s0, 31
-; GFX9-NEXT: s_add_u32 s0, s2, s0
-; GFX9-NEXT: s_addc_u32 s1, s3, s1
-; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: global_load_ubyte v0, v0, s[0:1]
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: global_load_ubyte v0, v0, s[2:3]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s0, v0
; GFX9-NEXT: ; return to shader part epilog
@@ -702,11 +675,8 @@ define amdgpu_ps i8 @extractelement_sgpr_v8i8_sgpr_idx(ptr addrspace(4) inreg %p
; GFX10-LABEL: extractelement_sgpr_v8i8_sgpr_idx:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_and_b32 s0, s4, 7
-; GFX10-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-NEXT: s_ashr_i32 s1, s0, 31
-; GFX10-NEXT: s_add_u32 s0, s2, s0
-; GFX10-NEXT: s_addc_u32 s1, s3, s1
-; GFX10-NEXT: global_load_ubyte v0, v0, s[0:1]
+; GFX10-NEXT: v_mov_b32_e32 v0, s0
+; GFX10-NEXT: global_load_ubyte v0, v0, s[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_readfirstlane_b32 s0, v0
; GFX10-NEXT: ; return to shader part epilog
@@ -714,11 +684,9 @@ define amdgpu_ps i8 @extractelement_sgpr_v8i8_sgpr_idx(ptr addrspace(4) inreg %p
; GFX11-LABEL: extractelement_sgpr_v8i8_sgpr_idx:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_and_b32 s0, s4, 7
-; GFX11-NEXT: v_mov_b32_e32 v0, 0
-; GFX11-NEXT: s_ashr_i32 s1, s0, 31
-; GFX11-NEXT: s_add_u32 s0, s2, s0
-; GFX11-NEXT: s_addc_u32 s1, s3, s1
-; GFX11-NEXT: global_load_u8 v0, v0, s[0:1]
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-NEXT: global_load_u8 v0, v0, s[2:3]
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-NEXT: ; return to shader part epilog
@@ -863,13 +831,8 @@ define i8 @extractelement_vgpr_v8i8_vgpr_idx(ptr addrspace(1) %ptr, i32 %idx) {
define amdgpu_ps i8 @extractelement_sgpr_v8i8_vgpr_idx(ptr addrspace(4) inreg %ptr, i32 %idx) {
; GFX9-LABEL: extractelement_sgpr_v8i8_vgpr_idx:
; GFX9: ; %bb.0:
-; GFX9-NEXT: v_and_b32_e32 v2, 7, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_ashrrev_i32_e32 v3, 31, v2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v2
-; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc
-; GFX9-NEXT: global_load_ubyte v0, v[0:1], off
+; GFX9-NEXT: v_and_b32_e32 v0, 7, v0
+; GFX9-NEXT: global_load_ubyte v0, v0, s[2:3]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s0, v0
; GFX9-NEXT: ; return to shader part epilog
@@ -902,27 +865,16 @@ define amdgpu_ps i8 @extractelement_sgpr_v8i8_vgpr_idx(ptr addrspace(4) inreg %p
;
; GFX10-LABEL: extractelement_sgpr_v8i8_vgpr_idx:
; GFX10: ; %bb.0:
-; GFX10-NEXT: v_and_b32_e32 v2, 7, v0
-; GFX10-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-NEXT: v_ashrrev_i32_e32 v3, 31, v2
-; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v3, vcc_lo
-; GFX10-NEXT: global_load_ubyte v0, v[0:1], off
+; GFX10-NEXT: v_and_b32_e32 v0, 7, v0
+; GFX10-NEXT: global_load_ubyte v0, v0, s[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_readfirstlane_b32 s0, v0
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: extractelement_sgpr_v8i8_vgpr_idx:
; GFX11: ; %bb.0:
-; GFX11-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_and_b32 v2, 7, v0
-; GFX11-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_ashrrev_i32_e32 v3, 31, v2
-; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
-; GFX11-NEXT: global_load_u8 v0, v[0:1], off
+; GFX11-NEXT: v_and_b32_e32 v0, 7, v0
+; GFX11-NEXT: global_load_u8 v0, v0, s[2:3]
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-NEXT: ; return to shader part epilog
@@ -1699,11 +1651,8 @@ define amdgpu_ps i8 @extractelement_sgpr_v16i8_sgpr_idx(ptr addrspace(4) inreg %
; GFX9-LABEL: extractelement_sgpr_v16i8_sgpr_idx:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_and_b32 s0, s4, 15
-; GFX9-NEXT: s_ashr_i32 s1, s0, 31
-; GFX9-NEXT: s_add_u32 s0, s2, s0
-; GFX9-NEXT: s_addc_u32 s1, s3, s1
-; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: global_load_ubyte v0, v0, s[0:1]
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: global_load_ubyte v0, v0, s[2:3]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s0, v0
; GFX9-NEXT: ; return to shader part epilog
@@ -1739,11 +1688,8 @@ define amdgpu_ps i8 @extractelement_sgpr_v16i8_sgpr_idx(ptr addrspace(4) inreg %
; GFX10-LABEL: extractelement_sgpr_v16i8_sgpr_idx:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_and_b32 s0, s4, 15
-; GFX10-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-NEXT: s_ashr_i32 s1, s0, 31
-; GFX10-NEXT: s_add_u32 s0, s2, s0
-; GFX10-NEXT: s_addc_u32 s1, s3, s1
-; GFX10-NEXT: global_load_ubyte v0, v0, s[0:1]
+; GFX10-NEXT: v_mov_b32_e32 v0, s0
+; GFX10-NEXT: global_load_ubyte v0, v0, s[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_readfirstlane_b32 s0, v0
; GFX10-NEXT: ; return to shader part epilog
@@ -1751,11 +1697,9 @@ define amdgpu_ps i8 @extractelement_sgpr_v16i8_sgpr_idx(ptr addrspace(4) inreg %
; GFX11-LABEL: extractelement_sgpr_v16i8_sgpr_idx:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_and_b32 s0, s4, 15
-; GFX11-NEXT: v_mov_b32_e32 v0, 0
-; GFX11-NEXT: s_ashr_i32 s1, s0, 31
-; GFX11-NEXT: s_add_u32 s0, s2, s0
-; GFX11-NEXT: s_addc_u32 s1, s3, s1
-; GFX11-NEXT: global_load_u8 v0, v0, s[0:1]
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-NEXT: global_load_u8 v0, v0, s[2:3]
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-NEXT: ; return to shader part epilog
@@ -1900,13 +1844,8 @@ define i8 @extractelement_vgpr_v16i8_vgpr_idx(ptr addrspace(1) %ptr, i32 %idx) {
define amdgpu_ps i8 @extractelement_sgpr_v16i8_vgpr_idx(ptr addrspace(4) inreg %ptr, i32 %idx) {
; GFX9-LABEL: extractelement_sgpr_v16i8_vgpr_idx:
; GFX9: ; %bb.0:
-; GFX9-NEXT: v_and_b32_e32 v2, 15, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_ashrrev_i32_e32 v3, 31, v2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v2
-; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc
-; GFX9-NEXT: global_load_ubyte v0, v[0:1], off
+; GFX9-NEXT: v_and_b32_e32 v0, 15, v0
+; GFX9-NEXT: global_load_ubyte v0, v0, s[2:3]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s0, v0
; GFX9-NEXT: ; return to shader part epilog
@@ -1939,27 +1878,16 @@ define amdgpu_ps i8 @extractelement_sgpr_v16i8_vgpr_idx(ptr addrspace(4) inreg %
;
; GFX10-LABEL: extractelement_sgpr_v16i8_vgpr_idx:
; GFX10: ; %bb.0:
-; GFX10-NEXT: v_and_b32_e32 v2, 15, v0
-; GFX10-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-NEXT: v_ashrrev_i32_e32 v3, 31, v2
-; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v3, vcc_lo
-; GFX10-NEXT: global_load_ubyte v0, v[0:1], off
+; GFX10-NEXT: v_and_b32_e32 v0, 15, v0
+; GFX10-NEXT: global_load_ubyte v0, v0, s[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_readfirstlane_b32 s0, v0
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: extractelement_sgpr_v16i8_vgpr_idx:
; GFX11: ; %bb.0:
-; GFX11-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_and_b32 v2, 15, v0
-; GFX11-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_ashrrev_i32_e32 v3, 31, v2
-; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
-; GFX11-NEXT: global_load_u8 v0, v[0:1], off
+; GFX11-NEXT: v_and_b32_e32 v0, 15, v0
+; GFX11-NEXT: global_load_u8 v0, v0, s[2:3]
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-NEXT: ; return to shader part epilog
diff --git a/llvm/test/CodeGen/AMDGPU/ctlz.ll b/llvm/test/CodeGen/AMDGPU/ctlz.ll
index d9b6edf00b82e..2a5fab7c35881 100644
--- a/llvm/test/CodeGen/AMDGPU/ctlz.ll
+++ b/llvm/test/CodeGen/AMDGPU/ctlz.ll
@@ -2048,13 +2048,8 @@ define amdgpu_kernel void @v_ctlz_i8_sel_eq_neg1(ptr addrspace(1) noalias %out,
; GFX10-GISEL-LABEL: v_ctlz_i8_sel_eq_neg1:
; GFX10-GISEL: ; %bb.0:
; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: v_ashrrev_i32_e32 v3, 31, v0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s3
-; GFX10-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v1, v0
-; GFX10-GISEL-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v2, v3, vcc_lo
-; GFX10-GISEL-NEXT: global_load_ubyte v0, v[0:1], off
+; GFX10-GISEL-NEXT: global_load_ubyte v0, v0, s[2:3]
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10-GISEL-NEXT: v_ffbh_u32_e32 v1, v0
; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
@@ -2275,13 +2270,8 @@ define amdgpu_kernel void @v_ctlz_i7_sel_eq_neg1(ptr addrspace(1) noalias %out,
; GFX10-GISEL-LABEL: v_ctlz_i7_sel_eq_neg1:
; GFX10-GISEL: ; %bb.0:
; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: v_ashrrev_i32_e32 v3, 31, v0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s3
-; GFX10-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v1, v0
-; GFX10-GISEL-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v2, v3, vcc_lo
-; GFX10-GISEL-NEXT: global_load_ubyte v0, v[0:1], off
+; GFX10-GISEL-NEXT: global_load_ubyte v0, v0, s[2:3]
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10-GISEL-NEXT: v_and_b32_e32 v0, 0x7f, v0
; GFX10-GISEL-NEXT: v_ffbh_u32_e32 v1, v0
diff --git a/llvm/test/CodeGen/AMDGPU/ctlz_zero_poison.ll b/llvm/test/CodeGen/AMDGPU/ctlz_zero_poison.ll
index d38a473a9a31f..09056a306883e 100644
--- a/llvm/test/CodeGen/AMDGPU/ctlz_zero_poison.ll
+++ b/llvm/test/CodeGen/AMDGPU/ctlz_zero_poison.ll
@@ -1191,14 +1191,9 @@ define amdgpu_kernel void @v_ctlz_zero_poison_i8(ptr addrspace(1) noalias %out,
; GFX9-GISEL-LABEL: v_ctlz_zero_poison_i8:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX9-GISEL-NEXT: v_ashrrev_i32_e32 v3, 31, v0
-; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, s3
-; GFX9-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v1, v0
-; GFX9-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, v2, v3, vcc
-; GFX9-GISEL-NEXT: global_load_ubyte v0, v[0:1], off
; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: global_load_ubyte v0, v0, s[2:3]
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v0, 24, v0
; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v0, v0
@@ -1730,14 +1725,9 @@ define amdgpu_kernel void @v_ctlz_zero_poison_i8_sel_eq_neg1(ptr addrspace(1) no
; GFX9-GISEL-LABEL: v_ctlz_zero_poison_i8_sel_eq_neg1:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX9-GISEL-NEXT: v_ashrrev_i32_e32 v3, 31, v0
-; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, s3
-; GFX9-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v1, v0
-; GFX9-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, v2, v3, vcc
-; GFX9-GISEL-NEXT: global_load_ubyte v0, v[0:1], off
; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0xffff
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: global_load_ubyte v0, v0, s[2:3]
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v2, 24, v0
; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v2, v2
diff --git a/llvm/test/CodeGen/AMDGPU/cttz.ll b/llvm/test/CodeGen/AMDGPU/cttz.ll
index 81227d358ad29..88f7f65488c64 100644
--- a/llvm/test/CodeGen/AMDGPU/cttz.ll
+++ b/llvm/test/CodeGen/AMDGPU/cttz.ll
@@ -1347,13 +1347,8 @@ define amdgpu_kernel void @v_cttz_i32_sel_ne_bitwidth(ptr addrspace(1) noalias %
; GFX10-GISEL-LABEL: v_cttz_i8_sel_eq_neg1:
; GFX10-GISEL: ; %bb.0:
; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: v_ashrrev_i32_e32 v3, 31, v0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s3
-; GFX10-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v1, v0
-; GFX10-GISEL-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v2, v3, vcc_lo
-; GFX10-GISEL-NEXT: global_load_ubyte v0, v[0:1], off
+; GFX10-GISEL-NEXT: global_load_ubyte v0, v0, s[2:3]
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10-GISEL-NEXT: v_or_b32_e32 v1, 0x100, v0
; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
@@ -1550,13 +1545,8 @@ define amdgpu_kernel void @v_cttz_i7_sel_eq_neg1(ptr addrspace(1) noalias %out,
; GFX10-GISEL-LABEL: v_cttz_i7_sel_eq_neg1:
; GFX10-GISEL: ; %bb.0:
; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-GISEL-NEXT: v_ashrrev_i32_e32 v3, 31, v0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s3
-; GFX10-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v1, v0
-; GFX10-GISEL-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v2, v3, vcc_lo
-; GFX10-GISEL-NEXT: global_load_ubyte v0, v[0:1], off
+; GFX10-GISEL-NEXT: global_load_ubyte v0, v0, s[2:3]
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10-GISEL-NEXT: v_or_b32_e32 v1, 0x80, v0
; GFX10-GISEL-NEXT: v_and_b32_e32 v0, 0x7f, v0
diff --git a/llvm/test/CodeGen/AMDGPU/global-saddr-narrow-offset.ll b/llvm/test/CodeGen/AMDGPU/global-saddr-narrow-offset.ll
index c33ad672e3948..1f2074ba13b4d 100644
--- a/llvm/test/CodeGen/AMDGPU/global-saddr-narrow-offset.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-saddr-narrow-offset.ll
@@ -11,16 +11,13 @@ define amdgpu_kernel void @variable_shift(ptr addrspace(1) %p, i32 %n) {
; GFX9-SDAG: ; %bb.0:
; GFX9-SDAG-NEXT: s_load_dword s2, s[8:9], 0x8
; GFX9-SDAG-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX9-SDAG-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-SDAG-NEXT: s_and_b32 s3, s2, 3
; GFX9-SDAG-NEXT: v_lshlrev_b64 v[0:1], s3, v[0:1]
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, s1
-; GFX9-SDAG-NEXT: v_add_co_u32_e32 v0, vcc, s0, v0
-; GFX9-SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, v2, v1, vcc
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, s2
-; GFX9-SDAG-NEXT: global_store_dword v[0:1], v2, off
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s2
+; GFX9-SDAG-NEXT: global_store_dword v0, v1, s[0:1]
; GFX9-SDAG-NEXT: s_endpgm
;
; GFX12-SDAG-LABEL: variable_shift:
@@ -29,13 +26,10 @@ define amdgpu_kernel void @variable_shift(ptr addrspace(1) %p, i32 %n) {
; GFX12-SDAG-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0xff, v0
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-NEXT: s_and_b32 s3, s2, 3
-; GFX12-SDAG-NEXT: v_mov_b32_e32 v2, s2
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
; GFX12-SDAG-NEXT: v_lshlrev_b64_e32 v[0:1], s3, v[0:1]
-; GFX12-SDAG-NEXT: v_add_co_u32 v0, vcc_lo, s0, v0
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, s1, v1, vcc_lo
-; GFX12-SDAG-NEXT: global_store_b32 v[0:1], v2, off
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v1, s2
+; GFX12-SDAG-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX12-SDAG-NEXT: s_endpgm
;
; GFX1250-SDAG-LABEL: variable_shift:
@@ -46,30 +40,27 @@ define amdgpu_kernel void @variable_shift(ptr addrspace(1) %p, i32 %n) {
; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-NEXT: s_load_b96 s[0:2], s[4:5], 0x0 nv
; GFX1250-SDAG-NEXT: s_mov_b32 s3, 0xff
-; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s3, v0 bitop3:0x40
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1250-SDAG-NEXT: s_and_b32 s3, s2, 3
-; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, s2
-; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], s3, s[0:1]
-; GFX1250-SDAG-NEXT: global_store_b32 v[0:1], v2, off
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1250-SDAG-NEXT: v_lshlrev_b64_e32 v[0:1], s3, v[0:1]
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, s2
+; GFX1250-SDAG-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX1250-SDAG-NEXT: s_endpgm
;
; GFX9-GISEL-LABEL: variable_shift:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_load_dword s2, s[8:9], 0x8
; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-GISEL-NEXT: s_and_b32 s3, s2, 3
; GFX9-GISEL-NEXT: v_lshlrev_b64 v[0:1], s3, v[0:1]
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, s0
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, s1
-; GFX9-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
-; GFX9-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, v3, v1, vcc
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, s2
-; GFX9-GISEL-NEXT: global_store_dword v[0:1], v2, off
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s2
+; GFX9-GISEL-NEXT: global_store_dword v0, v1, s[0:1]
; GFX9-GISEL-NEXT: s_endpgm
;
; GFX12-GISEL-LABEL: variable_shift:
@@ -78,14 +69,10 @@ define amdgpu_kernel void @variable_shift(ptr addrspace(1) %p, i32 %n) {
; GFX12-GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0xff, v0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: s_and_b32 s3, s2, 3
-; GFX12-GISEL-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
; GFX12-GISEL-NEXT: v_lshlrev_b64_e32 v[0:1], s3, v[0:1]
-; GFX12-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, v3, v1, vcc_lo
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v2, s2
-; GFX12-GISEL-NEXT: global_store_b32 v[0:1], v2, off
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v1, s2
+; GFX12-GISEL-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX12-GISEL-NEXT: s_endpgm
;
; GFX1250-GISEL-LABEL: variable_shift:
@@ -96,13 +83,14 @@ define amdgpu_kernel void @variable_shift(ptr addrspace(1) %p, i32 %n) {
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_load_b96 s[0:2], s[4:5], 0x0 nv
; GFX1250-GISEL-NEXT: s_mov_b32 s3, 0xff
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s3, v0 bitop3:0x40
; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-NEXT: s_and_b32 s3, s2, 3
-; GFX1250-GISEL-NEXT: v_mov_b32_e32 v2, s2
-; GFX1250-GISEL-NEXT: v_lshl_add_u64 v[0:1], v[0:1], s3, s[0:1]
-; GFX1250-GISEL-NEXT: global_store_b32 v[0:1], v2, off
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1250-GISEL-NEXT: v_lshlrev_b64_e32 v[0:1], s3, v[0:1]
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v1, s2
+; GFX1250-GISEL-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX1250-GISEL-NEXT: s_endpgm
%id = call i32 @llvm.amdgcn.workitem.id.x()
%m = and i32 %id, 255
@@ -182,9 +170,8 @@ define amdgpu_kernel void @sext_across_block(ptr addrspace(1) %p, ptr addrspace(
; GFX1250-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
; GFX1250-SDAG-NEXT: s_cbranch_scc1 .LBB1_2
; GFX1250-SDAG-NEXT: ; %bb.1: ; %then
-; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 2, s[0:1]
-; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, s6
-; GFX1250-SDAG-NEXT: global_store_b32 v[0:1], v2, off
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, s6
+; GFX1250-SDAG-NEXT: global_store_b32 v0, v1, s[0:1] scale_offset
; GFX1250-SDAG-NEXT: .LBB1_2: ; %exit
; GFX1250-SDAG-NEXT: s_endpgm
;
@@ -457,13 +444,11 @@ define amdgpu_kernel void @uniform_narrow_offset(ptr addrspace(1) %p, i32 %n, i3
; GFX9-SDAG: ; %bb.0:
; GFX9-SDAG-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-SDAG-NEXT: s_mov_b32 s5, 0
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-SDAG-NEXT: s_and_b32 s4, s2, 0xff
; GFX9-SDAG-NEXT: s_and_b32 s2, s3, 3
; GFX9-SDAG-NEXT: s_lshl_b64 s[2:3], s[4:5], s2
-; GFX9-SDAG-NEXT: s_add_u32 s0, s0, s2
-; GFX9-SDAG-NEXT: s_addc_u32 s1, s1, s3
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s2
; GFX9-SDAG-NEXT: global_store_dword v1, v0, s[0:1]
; GFX9-SDAG-NEXT: s_endpgm
;
@@ -471,13 +456,12 @@ define amdgpu_kernel void @uniform_narrow_offset(ptr addrspace(1) %p, i32 %n, i3
; GFX12-SDAG: ; %bb.0:
; GFX12-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX12-SDAG-NEXT: s_mov_b32 s5, 0
-; GFX12-SDAG-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-NEXT: s_and_b32 s4, s2, 0xff
; GFX12-SDAG-NEXT: s_and_b32 s2, s3, 3
; GFX12-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-SDAG-NEXT: s_lshl_b64 s[2:3], s[4:5], s2
-; GFX12-SDAG-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[2:3]
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v1, s2 :: v_dual_and_b32 v0, 0x3ff, v0
; GFX12-SDAG-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX12-SDAG-NEXT: s_endpgm
;
@@ -490,15 +474,13 @@ define amdgpu_kernel void @uniform_narrow_offset(ptr addrspace(1) %p, i32 %n, i3
; GFX1250-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
; GFX1250-SDAG-NEXT: s_mov_b32 s5, 0
-; GFX1250-SDAG-NEXT: s_mov_b32 s4, 0x3ff
-; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
-; GFX1250-SDAG-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1250-SDAG-NEXT: s_and_b32 s4, s2, 0xff
; GFX1250-SDAG-NEXT: s_and_b32 s2, s3, 3
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1250-SDAG-NEXT: s_lshl_b64 s[2:3], s[4:5], s2
-; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-SDAG-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[2:3]
+; GFX1250-SDAG-NEXT: s_mov_b32 s3, 0x3ff
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v1, s2 :: v_dual_bitop2_b32 v0, s3, v0 bitop3:0x40
; GFX1250-SDAG-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX1250-SDAG-NEXT: s_endpgm
;
@@ -506,13 +488,11 @@ define amdgpu_kernel void @uniform_narrow_offset(ptr addrspace(1) %p, i32 %n, i3
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-GISEL-NEXT: s_mov_b32 s5, 0
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-GISEL-NEXT: s_and_b32 s4, s2, 0xff
; GFX9-GISEL-NEXT: s_and_b32 s2, s3, 3
; GFX9-GISEL-NEXT: s_lshl_b64 s[2:3], s[4:5], s2
-; GFX9-GISEL-NEXT: s_add_u32 s0, s0, s2
-; GFX9-GISEL-NEXT: s_addc_u32 s1, s1, s3
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s2
; GFX9-GISEL-NEXT: global_store_dword v1, v0, s[0:1]
; GFX9-GISEL-NEXT: s_endpgm
;
@@ -520,14 +500,12 @@ define amdgpu_kernel void @uniform_narrow_offset(ptr addrspace(1) %p, i32 %n, i3
; GFX12-GISEL: ; %bb.0:
; GFX12-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX12-GISEL-NEXT: s_mov_b32 s5, 0
-; GFX12-GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: s_and_b32 s4, s2, 0xff
; GFX12-GISEL-NEXT: s_and_b32 s2, s3, 3
; GFX12-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-GISEL-NEXT: s_lshl_b64 s[2:3], s[4:5], s2
-; GFX12-GISEL-NEXT: s_add_co_u32 s0, s0, s2
-; GFX12-GISEL-NEXT: s_add_co_ci_u32 s1, s1, s3
+; GFX12-GISEL-NEXT: v_dual_mov_b32 v1, s2 :: v_dual_and_b32 v0, 0x3ff, v0
; GFX12-GISEL-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX12-GISEL-NEXT: s_endpgm
;
@@ -540,16 +518,13 @@ define amdgpu_kernel void @uniform_narrow_offset(ptr addrspace(1) %p, i32 %n, i3
; GFX1250-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
; GFX1250-GISEL-NEXT: s_mov_b32 s5, 0
-; GFX1250-GISEL-NEXT: s_mov_b32 s4, 0x3ff
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, s4, v0 bitop3:0x40
; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-NEXT: s_and_b32 s4, s2, 0xff
; GFX1250-GISEL-NEXT: s_and_b32 s2, s3, 3
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1250-GISEL-NEXT: s_lshl_b64 s[2:3], s[4:5], s2
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-GISEL-NEXT: s_add_co_u32 s0, s0, s2
-; GFX1250-GISEL-NEXT: s_add_co_ci_u32 s1, s1, s3
+; GFX1250-GISEL-NEXT: s_mov_b32 s3, 0x3ff
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v1, s2 :: v_dual_bitop2_b32 v0, s3, v0 bitop3:0x40
; GFX1250-GISEL-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX1250-GISEL-NEXT: s_endpgm
%a = and i32 %n, 255
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-lastuse.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-lastuse.ll
index 0df3bb2beaa66..bf722fcac405f 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-lastuse.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-lastuse.ll
@@ -63,35 +63,26 @@ entry:
define amdgpu_kernel void @global_last_use_load_1(ptr addrspace(1) %in, ptr addrspace(1) %out) {
; GFX12-LABEL: global_last_use_load_1:
; GFX12: ; %bb.0: ; %entry
-; GFX12-NEXT: s_mov_b64 s[0:1], s[4:5]
; GFX12-NEXT: v_mov_b32_e32 v1, v0
-; GFX12-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
+; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
-; GFX12-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
-; GFX12-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
-; GFX12-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
+; GFX12-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_mov_b32 s2, 0x3ff
-; GFX12-NEXT: v_and_b32_e64 v1, v1, s2
+; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
+; GFX12-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-NEXT: s_mov_b32 s4, 0x3ff
+; GFX12-NEXT: v_and_b32_e64 v1, v1, s4
; GFX12-NEXT: v_ashrrev_i32_e64 v3, 31, v1
; GFX12-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
; GFX12-NEXT: v_mov_b32_e32 v2, v3
-; GFX12-NEXT: s_mov_b32 s2, 2
+; GFX12-NEXT: s_mov_b32 s4, 2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_lshlrev_b64_e64 v[2:3], s2, v[1:2]
-; GFX12-NEXT: s_mov_b32 s3, s4
-; GFX12-NEXT: v_mov_b32_e32 v1, v2
-; GFX12-NEXT: s_mov_b32 s2, s5
-; GFX12-NEXT: v_mov_b32_e32 v2, v3
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_add_co_u32 v1, s3, s3, v1
-; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v3, s2, s2, v2, s3
-; GFX12-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
-; GFX12-NEXT: v_mov_b32_e32 v2, v3
-; GFX12-NEXT: global_load_b32 v1, v[1:2], off th:TH_LOAD_LU
+; GFX12-NEXT: v_lshlrev_b64_e64 v[1:2], s4, v[1:2]
+; GFX12-NEXT: ; kill: def $vgpr1 killed $vgpr1 killed $vgpr1_vgpr2 killed $exec
+; GFX12-NEXT: global_load_b32 v1, v1, s[2:3] th:TH_LOAD_LU
; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX12-NEXT: s_endpgm
;
@@ -207,35 +198,26 @@ entry:
define amdgpu_kernel void @global_last_use_and_nontemporal_load(ptr addrspace(1) %in, ptr addrspace(1) %out) {
; GFX12-LABEL: global_last_use_and_nontemporal_load:
; GFX12: ; %bb.0: ; %entry
-; GFX12-NEXT: s_mov_b64 s[0:1], s[4:5]
; GFX12-NEXT: v_mov_b32_e32 v1, v0
-; GFX12-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
+; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
-; GFX12-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
-; GFX12-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
-; GFX12-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
+; GFX12-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_mov_b32 s2, 0x3ff
-; GFX12-NEXT: v_and_b32_e64 v1, v1, s2
+; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
+; GFX12-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-NEXT: s_mov_b32 s4, 0x3ff
+; GFX12-NEXT: v_and_b32_e64 v1, v1, s4
; GFX12-NEXT: v_ashrrev_i32_e64 v3, 31, v1
; GFX12-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
; GFX12-NEXT: v_mov_b32_e32 v2, v3
-; GFX12-NEXT: s_mov_b32 s2, 2
+; GFX12-NEXT: s_mov_b32 s4, 2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_lshlrev_b64_e64 v[2:3], s2, v[1:2]
-; GFX12-NEXT: s_mov_b32 s3, s4
-; GFX12-NEXT: v_mov_b32_e32 v1, v2
-; GFX12-NEXT: s_mov_b32 s2, s5
-; GFX12-NEXT: v_mov_b32_e32 v2, v3
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_add_co_u32 v1, s3, s3, v1
-; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v3, s2, s2, v2, s3
-; GFX12-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
-; GFX12-NEXT: v_mov_b32_e32 v2, v3
-; GFX12-NEXT: global_load_b32 v1, v[1:2], off th:TH_LOAD_LU
+; GFX12-NEXT: v_lshlrev_b64_e64 v[1:2], s4, v[1:2]
+; GFX12-NEXT: ; kill: def $vgpr1 killed $vgpr1 killed $vgpr1_vgpr2 killed $exec
+; GFX12-NEXT: global_load_b32 v1, v1, s[2:3] th:TH_LOAD_LU
; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX12-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-nontemporal.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-nontemporal.ll
index ac0145c1256f3..0fd2e2746e29b 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-nontemporal.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-nontemporal.ll
@@ -324,61 +324,43 @@ define amdgpu_kernel void @global_nontemporal_load_1(
;
; GFX10-WGP-LABEL: global_nontemporal_load_1:
; GFX10-WGP: ; %bb.0: ; %entry
-; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[8:9]
; GFX10-WGP-NEXT: v_mov_b32_e32 v1, v0
-; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
+; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x8
-; GFX10-WGP-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
-; GFX10-WGP-NEXT: s_nop 0
-; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
+; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
+; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
+; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
; GFX10-WGP-NEXT: v_ashrrev_i32_e64 v3, 31, v1
; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
-; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-WGP-NEXT: s_mov_b32 s6, 2
-; GFX10-WGP-NEXT: v_lshlrev_b64 v[2:3], s6, v[1:2]
-; GFX10-WGP-NEXT: s_mov_b32 s7, s8
-; GFX10-WGP-NEXT: v_mov_b32_e32 v1, v2
-; GFX10-WGP-NEXT: s_mov_b32 s6, s9
-; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
-; GFX10-WGP-NEXT: v_add_co_u32 v1, s7, s7, v1
-; GFX10-WGP-NEXT: v_add_co_ci_u32_e64 v3, s6, s6, v2, s7
-; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
-; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
-; GFX10-WGP-NEXT: global_load_dword v1, v[1:2], off slc
-; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT: s_mov_b32 s8, 2
+; GFX10-WGP-NEXT: v_lshlrev_b64 v[1:2], s8, v[1:2]
+; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 killed $vgpr1_vgpr2 killed $exec
+; GFX10-WGP-NEXT: global_load_dword v1, v1, s[6:7] slc
+; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-WGP-NEXT: global_store_dword v0, v1, s[4:5]
; GFX10-WGP-NEXT: s_endpgm
;
; GFX10-CU-LABEL: global_nontemporal_load_1:
; GFX10-CU: ; %bb.0: ; %entry
-; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[8:9]
; GFX10-CU-NEXT: v_mov_b32_e32 v1, v0
-; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
+; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x8
-; GFX10-CU-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
-; GFX10-CU-NEXT: s_nop 0
-; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
+; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
+; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
+; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
; GFX10-CU-NEXT: v_ashrrev_i32_e64 v3, 31, v1
; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
-; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-CU-NEXT: s_mov_b32 s6, 2
-; GFX10-CU-NEXT: v_lshlrev_b64 v[2:3], s6, v[1:2]
-; GFX10-CU-NEXT: s_mov_b32 s7, s8
-; GFX10-CU-NEXT: v_mov_b32_e32 v1, v2
-; GFX10-CU-NEXT: s_mov_b32 s6, s9
-; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
-; GFX10-CU-NEXT: v_add_co_u32 v1, s7, s7, v1
-; GFX10-CU-NEXT: v_add_co_ci_u32_e64 v3, s6, s6, v2, s7
-; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
-; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
-; GFX10-CU-NEXT: global_load_dword v1, v[1:2], off slc
-; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT: s_mov_b32 s8, 2
+; GFX10-CU-NEXT: v_lshlrev_b64 v[1:2], s8, v[1:2]
+; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 killed $vgpr1_vgpr2 killed $exec
+; GFX10-CU-NEXT: global_load_dword v1, v1, s[6:7] slc
+; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-CU-NEXT: global_store_dword v0, v1, s[4:5]
; GFX10-CU-NEXT: s_endpgm
;
@@ -416,247 +398,189 @@ define amdgpu_kernel void @global_nontemporal_load_1(
;
; GFX90A-NOTTGSPLIT-LABEL: global_nontemporal_load_1:
; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
-; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[8:9]
; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
+; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x8
-; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
-; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
-; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
-; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
+; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
+; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NOTTGSPLIT-NEXT: s_mov_b32 s6, 0x3ff
-; GFX90A-NOTTGSPLIT-NEXT: v_and_b32_e64 v2, v1, s6
+; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
+; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
+; GFX90A-NOTTGSPLIT-NEXT: s_mov_b32 s8, 0x3ff
+; GFX90A-NOTTGSPLIT-NEXT: v_and_b32_e64 v2, v1, s8
; GFX90A-NOTTGSPLIT-NEXT: v_ashrrev_i32_e64 v1, 31, v2
; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
-; GFX90A-NOTTGSPLIT-NEXT: s_mov_b32 s6, 2
-; GFX90A-NOTTGSPLIT-NEXT: v_lshlrev_b64 v[2:3], s6, v[2:3]
-; GFX90A-NOTTGSPLIT-NEXT: s_mov_b32 s6, s8
+; GFX90A-NOTTGSPLIT-NEXT: s_mov_b32 s8, 2
+; GFX90A-NOTTGSPLIT-NEXT: v_lshlrev_b64 v[2:3], s8, v[2:3]
; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, v2
-; GFX90A-NOTTGSPLIT-NEXT: s_mov_b32 s8, s9
-; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr3 killed $vgpr3 killed $vgpr2_vgpr3 killed $exec
-; GFX90A-NOTTGSPLIT-NEXT: v_add_co_u32_e64 v2, s[6:7], s6, v1
-; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s8
-; GFX90A-NOTTGSPLIT-NEXT: v_addc_co_u32_e64 v1, s[6:7], v1, v3, s[6:7]
-; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
-; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
-; GFX90A-NOTTGSPLIT-NEXT: global_load_dword v1, v[2:3], off glc slc
-; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NOTTGSPLIT-NEXT: global_load_dword v1, v1, s[6:7] glc slc
+; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
;
; GFX90A-TGSPLIT-LABEL: global_nontemporal_load_1:
; GFX90A-TGSPLIT: ; %bb.0: ; %entry
-; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[8:9]
; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
+; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x8
-; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
-; GFX90A-TGSPLIT-NEXT: s_nop 0
-; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
-; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
+; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
+; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-TGSPLIT-NEXT: s_mov_b32 s6, 0x3ff
-; GFX90A-TGSPLIT-NEXT: v_and_b32_e64 v2, v1, s6
+; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
+; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
+; GFX90A-TGSPLIT-NEXT: s_mov_b32 s8, 0x3ff
+; GFX90A-TGSPLIT-NEXT: v_and_b32_e64 v2, v1, s8
; GFX90A-TGSPLIT-NEXT: v_ashrrev_i32_e64 v1, 31, v2
; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
-; GFX90A-TGSPLIT-NEXT: s_mov_b32 s6, 2
-; GFX90A-TGSPLIT-NEXT: v_lshlrev_b64 v[2:3], s6, v[2:3]
-; GFX90A-TGSPLIT-NEXT: s_mov_b32 s6, s8
+; GFX90A-TGSPLIT-NEXT: s_mov_b32 s8, 2
+; GFX90A-TGSPLIT-NEXT: v_lshlrev_b64 v[2:3], s8, v[2:3]
; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, v2
-; GFX90A-TGSPLIT-NEXT: s_mov_b32 s8, s9
-; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr3 killed $vgpr3 killed $vgpr2_vgpr3 killed $exec
-; GFX90A-TGSPLIT-NEXT: v_add_co_u32_e64 v2, s[6:7], s6, v1
-; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s8
-; GFX90A-TGSPLIT-NEXT: v_addc_co_u32_e64 v1, s[6:7], v1, v3, s[6:7]
-; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
-; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
-; GFX90A-TGSPLIT-NEXT: global_load_dword v1, v[2:3], off glc slc
-; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-TGSPLIT-NEXT: global_load_dword v1, v1, s[6:7] glc slc
+; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-TGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
; GFX90A-TGSPLIT-NEXT: s_endpgm
;
; GFX942-NOTTGSPLIT-LABEL: global_nontemporal_load_1:
; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry
-; GFX942-NOTTGSPLIT-NEXT: s_mov_b64 s[0:1], s[4:5]
; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, v0
-; GFX942-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
+; GFX942-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x8
-; GFX942-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX942-NOTTGSPLIT-NEXT: s_nop 0
-; GFX942-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8
-; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
+; GFX942-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX942-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NOTTGSPLIT-NEXT: s_mov_b32 s2, 0x3ff
-; GFX942-NOTTGSPLIT-NEXT: v_and_b32_e64 v2, v1, s2
+; GFX942-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
+; GFX942-NOTTGSPLIT-NEXT: s_mov_b32 s4, 0x3ff
+; GFX942-NOTTGSPLIT-NEXT: v_and_b32_e64 v2, v1, s4
; GFX942-NOTTGSPLIT-NEXT: v_ashrrev_i32_e64 v1, 31, v2
; GFX942-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
-; GFX942-NOTTGSPLIT-NEXT: s_mov_b32 s2, 2
-; GFX942-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
-; GFX942-NOTTGSPLIT-NEXT: v_lshl_add_u64 v[2:3], v[2:3], s2, v[4:5]
-; GFX942-NOTTGSPLIT-NEXT: global_load_dword v1, v[2:3], off nt
-; GFX942-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NOTTGSPLIT-NEXT: s_mov_b32 s4, 2
+; GFX942-NOTTGSPLIT-NEXT: v_lshlrev_b64 v[2:3], s4, v[2:3]
+; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, v2
+; GFX942-NOTTGSPLIT-NEXT: global_load_dword v1, v1, s[2:3] nt
+; GFX942-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[0:1]
; GFX942-NOTTGSPLIT-NEXT: s_endpgm
;
; GFX942-TGSPLIT-LABEL: global_nontemporal_load_1:
; GFX942-TGSPLIT: ; %bb.0: ; %entry
-; GFX942-TGSPLIT-NEXT: s_mov_b64 s[0:1], s[4:5]
; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, v0
-; GFX942-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
+; GFX942-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x8
-; GFX942-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX942-TGSPLIT-NEXT: s_nop 0
-; GFX942-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8
-; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
+; GFX942-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX942-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-TGSPLIT-NEXT: s_mov_b32 s2, 0x3ff
-; GFX942-TGSPLIT-NEXT: v_and_b32_e64 v2, v1, s2
+; GFX942-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
+; GFX942-TGSPLIT-NEXT: s_mov_b32 s4, 0x3ff
+; GFX942-TGSPLIT-NEXT: v_and_b32_e64 v2, v1, s4
; GFX942-TGSPLIT-NEXT: v_ashrrev_i32_e64 v1, 31, v2
; GFX942-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
-; GFX942-TGSPLIT-NEXT: s_mov_b32 s2, 2
-; GFX942-TGSPLIT-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
-; GFX942-TGSPLIT-NEXT: v_lshl_add_u64 v[2:3], v[2:3], s2, v[4:5]
-; GFX942-TGSPLIT-NEXT: global_load_dword v1, v[2:3], off nt
-; GFX942-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
+; GFX942-TGSPLIT-NEXT: s_mov_b32 s4, 2
+; GFX942-TGSPLIT-NEXT: v_lshlrev_b64 v[2:3], s4, v[2:3]
+; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, v2
+; GFX942-TGSPLIT-NEXT: global_load_dword v1, v1, s[2:3] nt
+; GFX942-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-TGSPLIT-NEXT: global_store_dword v0, v1, s[0:1]
; GFX942-TGSPLIT-NEXT: s_endpgm
;
; GFX11-WGP-LABEL: global_nontemporal_load_1:
; GFX11-WGP: ; %bb.0: ; %entry
-; GFX11-WGP-NEXT: s_mov_b64 s[0:1], s[4:5]
; GFX11-WGP-NEXT: v_mov_b32_e32 v1, v0
-; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
+; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
-; GFX11-WGP-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
-; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
-; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
+; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
+; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-WGP-NEXT: s_mov_b32 s2, 0x3ff
-; GFX11-WGP-NEXT: v_and_b32_e64 v1, v1, s2
+; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
+; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
+; GFX11-WGP-NEXT: s_mov_b32 s4, 0x3ff
+; GFX11-WGP-NEXT: v_and_b32_e64 v1, v1, s4
; GFX11-WGP-NEXT: v_ashrrev_i32_e64 v3, 31, v1
; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
-; GFX11-WGP-NEXT: s_mov_b32 s2, 2
-; GFX11-WGP-NEXT: v_lshlrev_b64 v[2:3], s2, v[1:2]
-; GFX11-WGP-NEXT: s_mov_b32 s3, s4
-; GFX11-WGP-NEXT: v_mov_b32_e32 v1, v2
-; GFX11-WGP-NEXT: s_mov_b32 s2, s5
-; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
-; GFX11-WGP-NEXT: v_add_co_u32 v1, s3, s3, v1
-; GFX11-WGP-NEXT: v_add_co_ci_u32_e64 v3, s2, s2, v2, s3
-; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
-; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
-; GFX11-WGP-NEXT: global_load_b32 v1, v[1:2], off slc dlc
-; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
+; GFX11-WGP-NEXT: s_mov_b32 s4, 2
+; GFX11-WGP-NEXT: v_lshlrev_b64 v[1:2], s4, v[1:2]
+; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 killed $vgpr1_vgpr2 killed $exec
+; GFX11-WGP-NEXT: global_load_b32 v1, v1, s[2:3] slc dlc
+; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX11-WGP-NEXT: s_endpgm
;
; GFX11-CU-LABEL: global_nontemporal_load_1:
; GFX11-CU: ; %bb.0: ; %entry
-; GFX11-CU-NEXT: s_mov_b64 s[0:1], s[4:5]
; GFX11-CU-NEXT: v_mov_b32_e32 v1, v0
-; GFX11-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
+; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
-; GFX11-CU-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
-; GFX11-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
-; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
+; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
+; GFX11-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-CU-NEXT: s_mov_b32 s2, 0x3ff
-; GFX11-CU-NEXT: v_and_b32_e64 v1, v1, s2
+; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
+; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
+; GFX11-CU-NEXT: s_mov_b32 s4, 0x3ff
+; GFX11-CU-NEXT: v_and_b32_e64 v1, v1, s4
; GFX11-CU-NEXT: v_ashrrev_i32_e64 v3, 31, v1
; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
-; GFX11-CU-NEXT: s_mov_b32 s2, 2
-; GFX11-CU-NEXT: v_lshlrev_b64 v[2:3], s2, v[1:2]
-; GFX11-CU-NEXT: s_mov_b32 s3, s4
-; GFX11-CU-NEXT: v_mov_b32_e32 v1, v2
-; GFX11-CU-NEXT: s_mov_b32 s2, s5
-; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
-; GFX11-CU-NEXT: v_add_co_u32 v1, s3, s3, v1
-; GFX11-CU-NEXT: v_add_co_ci_u32_e64 v3, s2, s2, v2, s3
-; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
-; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
-; GFX11-CU-NEXT: global_load_b32 v1, v[1:2], off slc dlc
-; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
+; GFX11-CU-NEXT: s_mov_b32 s4, 2
+; GFX11-CU-NEXT: v_lshlrev_b64 v[1:2], s4, v[1:2]
+; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 killed $vgpr1_vgpr2 killed $exec
+; GFX11-CU-NEXT: global_load_b32 v1, v1, s[2:3] slc dlc
+; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX11-CU-NEXT: s_endpgm
;
; GFX12-WGP-LABEL: global_nontemporal_load_1:
; GFX12-WGP: ; %bb.0: ; %entry
-; GFX12-WGP-NEXT: s_mov_b64 s[0:1], s[4:5]
; GFX12-WGP-NEXT: v_mov_b32_e32 v1, v0
-; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
-; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
-; GFX12-WGP-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
-; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
-; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
+; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
-; GFX12-WGP-NEXT: s_mov_b32 s2, 0x3ff
-; GFX12-WGP-NEXT: v_and_b32_e64 v1, v1, s2
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
+; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-WGP-NEXT: s_mov_b32 s4, 0x3ff
+; GFX12-WGP-NEXT: v_and_b32_e64 v1, v1, s4
; GFX12-WGP-NEXT: v_ashrrev_i32_e64 v3, 31, v1
; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
-; GFX12-WGP-NEXT: s_mov_b32 s2, 2
-; GFX12-WGP-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-WGP-NEXT: v_lshlrev_b64_e64 v[2:3], s2, v[1:2]
-; GFX12-WGP-NEXT: s_mov_b32 s3, s4
-; GFX12-WGP-NEXT: v_mov_b32_e32 v1, v2
-; GFX12-WGP-NEXT: s_mov_b32 s2, s5
-; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
+; GFX12-WGP-NEXT: s_mov_b32 s4, 2
; GFX12-WGP-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-WGP-NEXT: v_add_co_u32 v1, s3, s3, v1
-; GFX12-WGP-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-WGP-NEXT: v_add_co_ci_u32_e64 v3, s2, s2, v2, s3
-; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
-; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
-; GFX12-WGP-NEXT: global_load_b32 v1, v[1:2], off th:TH_LOAD_NT
+; GFX12-WGP-NEXT: v_lshlrev_b64_e64 v[1:2], s4, v[1:2]
+; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 killed $vgpr1_vgpr2 killed $exec
+; GFX12-WGP-NEXT: global_load_b32 v1, v1, s[2:3] th:TH_LOAD_NT
; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX12-WGP-NEXT: s_endpgm
;
; GFX12-CU-LABEL: global_nontemporal_load_1:
; GFX12-CU: ; %bb.0: ; %entry
-; GFX12-CU-NEXT: s_mov_b64 s[0:1], s[4:5]
; GFX12-CU-NEXT: v_mov_b32_e32 v1, v0
-; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
; GFX12-CU-NEXT: s_wait_kmcnt 0x0
-; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
-; GFX12-CU-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
-; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
-; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
+; GFX12-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
; GFX12-CU-NEXT: s_wait_kmcnt 0x0
-; GFX12-CU-NEXT: s_mov_b32 s2, 0x3ff
-; GFX12-CU-NEXT: v_and_b32_e64 v1, v1, s2
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
+; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-CU-NEXT: s_mov_b32 s4, 0x3ff
+; GFX12-CU-NEXT: v_and_b32_e64 v1, v1, s4
; GFX12-CU-NEXT: v_ashrrev_i32_e64 v3, 31, v1
; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
-; GFX12-CU-NEXT: s_mov_b32 s2, 2
-; GFX12-CU-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-CU-NEXT: v_lshlrev_b64_e64 v[2:3], s2, v[1:2]
-; GFX12-CU-NEXT: s_mov_b32 s3, s4
-; GFX12-CU-NEXT: v_mov_b32_e32 v1, v2
-; GFX12-CU-NEXT: s_mov_b32 s2, s5
-; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
+; GFX12-CU-NEXT: s_mov_b32 s4, 2
; GFX12-CU-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-CU-NEXT: v_add_co_u32 v1, s3, s3, v1
-; GFX12-CU-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-CU-NEXT: v_add_co_ci_u32_e64 v3, s2, s2, v2, s3
-; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
-; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
-; GFX12-CU-NEXT: global_load_b32 v1, v[1:2], off th:TH_LOAD_NT
+; GFX12-CU-NEXT: v_lshlrev_b64_e64 v[1:2], s4, v[1:2]
+; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 killed $vgpr1_vgpr2 killed $exec
+; GFX12-CU-NEXT: global_load_b32 v1, v1, s[2:3] th:TH_LOAD_NT
; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX12-CU-NEXT: s_endpgm
;
@@ -996,28 +920,20 @@ define amdgpu_kernel void @global_nontemporal_store_1(
; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
+; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
; GFX10-WGP-NEXT: s_nop 0
-; GFX10-WGP-NEXT: s_load_dwordx2 s[8:9], s[8:9], 0x8
-; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-WGP-NEXT: s_load_dword s4, s[4:5], 0x0
+; GFX10-WGP-NEXT: s_load_dword s6, s[6:7], 0x0
; GFX10-WGP-NEXT: v_ashrrev_i32_e64 v2, 31, v0
; GFX10-WGP-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
; GFX10-WGP-NEXT: v_mov_b32_e32 v1, v2
-; GFX10-WGP-NEXT: s_mov_b32 s5, 2
-; GFX10-WGP-NEXT: v_lshlrev_b64 v[1:2], s5, v[0:1]
-; GFX10-WGP-NEXT: s_mov_b32 s6, s8
-; GFX10-WGP-NEXT: v_mov_b32_e32 v0, v1
-; GFX10-WGP-NEXT: s_mov_b32 s5, s9
-; GFX10-WGP-NEXT: v_mov_b32_e32 v1, v2
-; GFX10-WGP-NEXT: v_add_co_u32 v0, s6, s6, v0
-; GFX10-WGP-NEXT: v_add_co_ci_u32_e64 v2, s5, s5, v1, s6
-; GFX10-WGP-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
-; GFX10-WGP-NEXT: v_mov_b32_e32 v1, v2
+; GFX10-WGP-NEXT: s_mov_b32 s7, 2
+; GFX10-WGP-NEXT: v_lshlrev_b64 v[0:1], s7, v[0:1]
+; GFX10-WGP-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s4
-; GFX10-WGP-NEXT: global_store_dword v[0:1], v2, off glc slc
+; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s6
+; GFX10-WGP-NEXT: global_store_dword v0, v1, s[4:5] glc slc
; GFX10-WGP-NEXT: s_endpgm
;
; GFX10-CU-LABEL: global_nontemporal_store_1:
@@ -1025,28 +941,20 @@ define amdgpu_kernel void @global_nontemporal_store_1(
; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
+; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
; GFX10-CU-NEXT: s_nop 0
-; GFX10-CU-NEXT: s_load_dwordx2 s[8:9], s[8:9], 0x8
-; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-CU-NEXT: s_load_dword s4, s[4:5], 0x0
+; GFX10-CU-NEXT: s_load_dword s6, s[6:7], 0x0
; GFX10-CU-NEXT: v_ashrrev_i32_e64 v2, 31, v0
; GFX10-CU-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
; GFX10-CU-NEXT: v_mov_b32_e32 v1, v2
-; GFX10-CU-NEXT: s_mov_b32 s5, 2
-; GFX10-CU-NEXT: v_lshlrev_b64 v[1:2], s5, v[0:1]
-; GFX10-CU-NEXT: s_mov_b32 s6, s8
-; GFX10-CU-NEXT: v_mov_b32_e32 v0, v1
-; GFX10-CU-NEXT: s_mov_b32 s5, s9
-; GFX10-CU-NEXT: v_mov_b32_e32 v1, v2
-; GFX10-CU-NEXT: v_add_co_u32 v0, s6, s6, v0
-; GFX10-CU-NEXT: v_add_co_ci_u32_e64 v2, s5, s5, v1, s6
-; GFX10-CU-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
-; GFX10-CU-NEXT: v_mov_b32_e32 v1, v2
+; GFX10-CU-NEXT: s_mov_b32 s7, 2
+; GFX10-CU-NEXT: v_lshlrev_b64 v[0:1], s7, v[0:1]
+; GFX10-CU-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-CU-NEXT: v_mov_b32_e32 v2, s4
-; GFX10-CU-NEXT: global_store_dword v[0:1], v2, off glc slc
+; GFX10-CU-NEXT: v_mov_b32_e32 v1, s6
+; GFX10-CU-NEXT: global_store_dword v0, v1, s[4:5] glc slc
; GFX10-CU-NEXT: s_endpgm
;
; SKIP-CACHE-INV-LABEL: global_nontemporal_store_1:
@@ -1079,31 +987,22 @@ define amdgpu_kernel void @global_nontemporal_store_1(
; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
+; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
-; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[8:9], s[8:9], 0x8
-; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX90A-NOTTGSPLIT-NEXT: s_mov_b32 s5, 0x3ff
-; GFX90A-NOTTGSPLIT-NEXT: v_and_b32_e64 v0, v0, s5
+; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[6:7], 0x0
+; GFX90A-NOTTGSPLIT-NEXT: s_mov_b32 s7, 0x3ff
+; GFX90A-NOTTGSPLIT-NEXT: v_and_b32_e64 v0, v0, s7
; GFX90A-NOTTGSPLIT-NEXT: v_ashrrev_i32_e64 v2, 31, v0
; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, v2
-; GFX90A-NOTTGSPLIT-NEXT: s_mov_b32 s5, 2
-; GFX90A-NOTTGSPLIT-NEXT: v_lshlrev_b64 v[2:3], s5, v[0:1]
-; GFX90A-NOTTGSPLIT-NEXT: s_mov_b32 s6, s8
-; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, v2
-; GFX90A-NOTTGSPLIT-NEXT: s_mov_b32 s5, s9
-; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, v3
-; GFX90A-NOTTGSPLIT-NEXT: v_add_co_u32_e64 v0, s[6:7], s6, v0
-; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s5
-; GFX90A-NOTTGSPLIT-NEXT: v_addc_co_u32_e64 v2, s[6:7], v1, v2, s[6:7]
-; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
-; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, v2
+; GFX90A-NOTTGSPLIT-NEXT: s_mov_b32 s7, 2
+; GFX90A-NOTTGSPLIT-NEXT: v_lshlrev_b64 v[0:1], s7, v[0:1]
+; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4
-; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v[0:1], v2, off glc slc
+; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
+; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[4:5] glc slc
; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
;
; GFX90A-TGSPLIT-LABEL: global_nontemporal_store_1:
@@ -1111,31 +1010,22 @@ define amdgpu_kernel void @global_nontemporal_store_1(
; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
+; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
; GFX90A-TGSPLIT-NEXT: s_nop 0
-; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[8:9], s[8:9], 0x8
-; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX90A-TGSPLIT-NEXT: s_mov_b32 s5, 0x3ff
-; GFX90A-TGSPLIT-NEXT: v_and_b32_e64 v0, v0, s5
+; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[6:7], 0x0
+; GFX90A-TGSPLIT-NEXT: s_mov_b32 s7, 0x3ff
+; GFX90A-TGSPLIT-NEXT: v_and_b32_e64 v0, v0, s7
; GFX90A-TGSPLIT-NEXT: v_ashrrev_i32_e64 v2, 31, v0
; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, v2
-; GFX90A-TGSPLIT-NEXT: s_mov_b32 s5, 2
-; GFX90A-TGSPLIT-NEXT: v_lshlrev_b64 v[2:3], s5, v[0:1]
-; GFX90A-TGSPLIT-NEXT: s_mov_b32 s6, s8
-; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, v2
-; GFX90A-TGSPLIT-NEXT: s_mov_b32 s5, s9
-; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, v3
-; GFX90A-TGSPLIT-NEXT: v_add_co_u32_e64 v0, s[6:7], s6, v0
-; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s5
-; GFX90A-TGSPLIT-NEXT: v_addc_co_u32_e64 v2, s[6:7], v1, v2, s[6:7]
-; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
-; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, v2
+; GFX90A-TGSPLIT-NEXT: s_mov_b32 s7, 2
+; GFX90A-TGSPLIT-NEXT: v_lshlrev_b64 v[0:1], s7, v[0:1]
+; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4
-; GFX90A-TGSPLIT-NEXT: global_store_dword v[0:1], v2, off glc slc
+; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
+; GFX90A-TGSPLIT-NEXT: global_store_dword v0, v1, s[4:5] glc slc
; GFX90A-TGSPLIT-NEXT: s_endpgm
;
; GFX942-NOTTGSPLIT-LABEL: global_nontemporal_store_1:
@@ -1143,22 +1033,22 @@ define amdgpu_kernel void @global_nontemporal_store_1(
; GFX942-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX942-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX942-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8
-; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[0:1], 0x0
-; GFX942-NOTTGSPLIT-NEXT: s_mov_b32 s1, 0x3ff
-; GFX942-NOTTGSPLIT-NEXT: v_and_b32_e64 v0, v0, s1
+; GFX942-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX942-NOTTGSPLIT-NEXT: s_nop 0
+; GFX942-NOTTGSPLIT-NEXT: s_load_dword s2, s[2:3], 0x0
+; GFX942-NOTTGSPLIT-NEXT: s_mov_b32 s3, 0x3ff
+; GFX942-NOTTGSPLIT-NEXT: v_and_b32_e64 v0, v0, s3
; GFX942-NOTTGSPLIT-NEXT: v_ashrrev_i32_e64 v2, 31, v0
; GFX942-NOTTGSPLIT-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, v2
-; GFX942-NOTTGSPLIT-NEXT: s_mov_b32 s1, 2
-; GFX942-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX942-NOTTGSPLIT-NEXT: v_lshl_add_u64 v[0:1], v[0:1], s1, v[2:3]
+; GFX942-NOTTGSPLIT-NEXT: s_mov_b32 s3, 2
+; GFX942-NOTTGSPLIT-NEXT: v_lshlrev_b64 v[0:1], s3, v[0:1]
+; GFX942-NOTTGSPLIT-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s0
-; GFX942-NOTTGSPLIT-NEXT: global_store_dword v[0:1], v2, off nt
+; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
+; GFX942-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] nt
; GFX942-NOTTGSPLIT-NEXT: s_endpgm
;
; GFX942-TGSPLIT-LABEL: global_nontemporal_store_1:
@@ -1166,22 +1056,22 @@ define amdgpu_kernel void @global_nontemporal_store_1(
; GFX942-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX942-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX942-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8
-; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[0:1], 0x0
-; GFX942-TGSPLIT-NEXT: s_mov_b32 s1, 0x3ff
-; GFX942-TGSPLIT-NEXT: v_and_b32_e64 v0, v0, s1
+; GFX942-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX942-TGSPLIT-NEXT: s_nop 0
+; GFX942-TGSPLIT-NEXT: s_load_dword s2, s[2:3], 0x0
+; GFX942-TGSPLIT-NEXT: s_mov_b32 s3, 0x3ff
+; GFX942-TGSPLIT-NEXT: v_and_b32_e64 v0, v0, s3
; GFX942-TGSPLIT-NEXT: v_ashrrev_i32_e64 v2, 31, v0
; GFX942-TGSPLIT-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, v2
-; GFX942-TGSPLIT-NEXT: s_mov_b32 s1, 2
-; GFX942-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX942-TGSPLIT-NEXT: v_lshl_add_u64 v[0:1], v[0:1], s1, v[2:3]
+; GFX942-TGSPLIT-NEXT: s_mov_b32 s3, 2
+; GFX942-TGSPLIT-NEXT: v_lshlrev_b64 v[0:1], s3, v[0:1]
+; GFX942-TGSPLIT-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v2, s0
-; GFX942-TGSPLIT-NEXT: global_store_dword v[0:1], v2, off nt
+; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
+; GFX942-TGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] nt
; GFX942-TGSPLIT-NEXT: s_endpgm
;
; GFX11-WGP-LABEL: global_nontemporal_store_1:
@@ -1189,29 +1079,21 @@ define amdgpu_kernel void @global_nontemporal_store_1(
; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
+; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
-; GFX11-WGP-NEXT: s_load_b64 s[4:5], s[4:5], 0x8
-; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
-; GFX11-WGP-NEXT: s_mov_b32 s1, 0x3ff
-; GFX11-WGP-NEXT: v_and_b32_e64 v0, v0, s1
+; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
+; GFX11-WGP-NEXT: s_load_b32 s2, s[2:3], 0x0
+; GFX11-WGP-NEXT: s_mov_b32 s3, 0x3ff
+; GFX11-WGP-NEXT: v_and_b32_e64 v0, v0, s3
; GFX11-WGP-NEXT: v_ashrrev_i32_e64 v2, 31, v0
; GFX11-WGP-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
; GFX11-WGP-NEXT: v_mov_b32_e32 v1, v2
-; GFX11-WGP-NEXT: s_mov_b32 s1, 2
-; GFX11-WGP-NEXT: v_lshlrev_b64 v[1:2], s1, v[0:1]
-; GFX11-WGP-NEXT: s_mov_b32 s2, s4
-; GFX11-WGP-NEXT: v_mov_b32_e32 v0, v1
-; GFX11-WGP-NEXT: s_mov_b32 s1, s5
-; GFX11-WGP-NEXT: v_mov_b32_e32 v1, v2
-; GFX11-WGP-NEXT: v_add_co_u32 v0, s2, s2, v0
-; GFX11-WGP-NEXT: v_add_co_ci_u32_e64 v2, s1, s1, v1, s2
-; GFX11-WGP-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
-; GFX11-WGP-NEXT: v_mov_b32_e32 v1, v2
+; GFX11-WGP-NEXT: s_mov_b32 s3, 2
+; GFX11-WGP-NEXT: v_lshlrev_b64 v[0:1], s3, v[0:1]
+; GFX11-WGP-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0
-; GFX11-WGP-NEXT: global_store_b32 v[0:1], v2, off glc slc dlc
+; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s2
+; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1] glc slc dlc
; GFX11-WGP-NEXT: s_endpgm
;
; GFX11-CU-LABEL: global_nontemporal_store_1:
@@ -1219,29 +1101,21 @@ define amdgpu_kernel void @global_nontemporal_store_1(
; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
+; GFX11-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
-; GFX11-CU-NEXT: s_load_b64 s[4:5], s[4:5], 0x8
-; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
-; GFX11-CU-NEXT: s_mov_b32 s1, 0x3ff
-; GFX11-CU-NEXT: v_and_b32_e64 v0, v0, s1
+; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
+; GFX11-CU-NEXT: s_load_b32 s2, s[2:3], 0x0
+; GFX11-CU-NEXT: s_mov_b32 s3, 0x3ff
+; GFX11-CU-NEXT: v_and_b32_e64 v0, v0, s3
; GFX11-CU-NEXT: v_ashrrev_i32_e64 v2, 31, v0
; GFX11-CU-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
; GFX11-CU-NEXT: v_mov_b32_e32 v1, v2
-; GFX11-CU-NEXT: s_mov_b32 s1, 2
-; GFX11-CU-NEXT: v_lshlrev_b64 v[1:2], s1, v[0:1]
-; GFX11-CU-NEXT: s_mov_b32 s2, s4
-; GFX11-CU-NEXT: v_mov_b32_e32 v0, v1
-; GFX11-CU-NEXT: s_mov_b32 s1, s5
-; GFX11-CU-NEXT: v_mov_b32_e32 v1, v2
-; GFX11-CU-NEXT: v_add_co_u32 v0, s2, s2, v0
-; GFX11-CU-NEXT: v_add_co_ci_u32_e64 v2, s1, s1, v1, s2
-; GFX11-CU-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
-; GFX11-CU-NEXT: v_mov_b32_e32 v1, v2
+; GFX11-CU-NEXT: s_mov_b32 s3, 2
+; GFX11-CU-NEXT: v_lshlrev_b64 v[0:1], s3, v[0:1]
+; GFX11-CU-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
-; GFX11-CU-NEXT: global_store_b32 v[0:1], v2, off glc slc dlc
+; GFX11-CU-NEXT: v_mov_b32_e32 v1, s2
+; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1] glc slc dlc
; GFX11-CU-NEXT: s_endpgm
;
; GFX12-WGP-LABEL: global_nontemporal_store_1:
@@ -1249,31 +1123,22 @@ define amdgpu_kernel void @global_nontemporal_store_1(
; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
+; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
-; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
-; GFX12-WGP-NEXT: s_load_b64 s[4:5], s[4:5], 0x8
-; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
-; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
-; GFX12-WGP-NEXT: s_mov_b32 s1, 0x3ff
-; GFX12-WGP-NEXT: v_and_b32_e64 v0, v0, s1
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
+; GFX12-WGP-NEXT: s_load_b32 s2, s[2:3], 0x0
+; GFX12-WGP-NEXT: s_mov_b32 s3, 0x3ff
+; GFX12-WGP-NEXT: v_and_b32_e64 v0, v0, s3
; GFX12-WGP-NEXT: v_ashrrev_i32_e64 v2, 31, v0
; GFX12-WGP-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
; GFX12-WGP-NEXT: v_mov_b32_e32 v1, v2
-; GFX12-WGP-NEXT: s_mov_b32 s1, 2
+; GFX12-WGP-NEXT: s_mov_b32 s3, 2
; GFX12-WGP-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-WGP-NEXT: v_lshlrev_b64_e64 v[1:2], s1, v[0:1]
-; GFX12-WGP-NEXT: s_mov_b32 s2, s4
-; GFX12-WGP-NEXT: v_mov_b32_e32 v0, v1
-; GFX12-WGP-NEXT: s_mov_b32 s1, s5
-; GFX12-WGP-NEXT: v_mov_b32_e32 v1, v2
-; GFX12-WGP-NEXT: v_add_co_u32 v0, s2, s2, v0
-; GFX12-WGP-NEXT: s_wait_alu depctr_sa_sdst(0) depctr_va_sdst(0)
-; GFX12-WGP-NEXT: v_add_co_ci_u32_e64 v2, s1, s1, v1, s2
-; GFX12-WGP-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
-; GFX12-WGP-NEXT: v_mov_b32_e32 v1, v2
+; GFX12-WGP-NEXT: v_lshlrev_b64_e64 v[0:1], s3, v[0:1]
+; GFX12-WGP-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
-; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
-; GFX12-WGP-NEXT: global_store_b32 v[0:1], v2, off th:TH_STORE_NT
+; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s2
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1] th:TH_STORE_NT
; GFX12-WGP-NEXT: s_endpgm
;
; GFX12-CU-LABEL: global_nontemporal_store_1:
@@ -1281,31 +1146,22 @@ define amdgpu_kernel void @global_nontemporal_store_1(
; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
; GFX12-CU-NEXT: s_wait_kmcnt 0x0
; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
+; GFX12-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
; GFX12-CU-NEXT: s_wait_kmcnt 0x0
-; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
-; GFX12-CU-NEXT: s_load_b64 s[4:5], s[4:5], 0x8
-; GFX12-CU-NEXT: s_wait_kmcnt 0x0
-; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
-; GFX12-CU-NEXT: s_mov_b32 s1, 0x3ff
-; GFX12-CU-NEXT: v_and_b32_e64 v0, v0, s1
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
+; GFX12-CU-NEXT: s_load_b32 s2, s[2:3], 0x0
+; GFX12-CU-NEXT: s_mov_b32 s3, 0x3ff
+; GFX12-CU-NEXT: v_and_b32_e64 v0, v0, s3
; GFX12-CU-NEXT: v_ashrrev_i32_e64 v2, 31, v0
; GFX12-CU-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
; GFX12-CU-NEXT: v_mov_b32_e32 v1, v2
-; GFX12-CU-NEXT: s_mov_b32 s1, 2
+; GFX12-CU-NEXT: s_mov_b32 s3, 2
; GFX12-CU-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-CU-NEXT: v_lshlrev_b64_e64 v[1:2], s1, v[0:1]
-; GFX12-CU-NEXT: s_mov_b32 s2, s4
-; GFX12-CU-NEXT: v_mov_b32_e32 v0, v1
-; GFX12-CU-NEXT: s_mov_b32 s1, s5
-; GFX12-CU-NEXT: v_mov_b32_e32 v1, v2
-; GFX12-CU-NEXT: v_add_co_u32 v0, s2, s2, v0
-; GFX12-CU-NEXT: s_wait_alu depctr_sa_sdst(0) depctr_va_sdst(0)
-; GFX12-CU-NEXT: v_add_co_ci_u32_e64 v2, s1, s1, v1, s2
-; GFX12-CU-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
-; GFX12-CU-NEXT: v_mov_b32_e32 v1, v2
+; GFX12-CU-NEXT: v_lshlrev_b64_e64 v[0:1], s3, v[0:1]
+; GFX12-CU-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
; GFX12-CU-NEXT: s_wait_kmcnt 0x0
-; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
-; GFX12-CU-NEXT: global_store_b32 v[0:1], v2, off th:TH_STORE_NT
+; GFX12-CU-NEXT: v_mov_b32_e32 v1, s2
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1] th:TH_STORE_NT
; GFX12-CU-NEXT: s_endpgm
;
; GFX1250-LABEL: global_nontemporal_store_1:
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-volatile.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-volatile.ll
index 49056d184f927..bf805926867ea 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-volatile.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-volatile.ll
@@ -271,61 +271,43 @@ define amdgpu_kernel void @global_volatile_load_1(
;
; GFX10-WGP-LABEL: global_volatile_load_1:
; GFX10-WGP: ; %bb.0: ; %entry
-; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[8:9]
; GFX10-WGP-NEXT: v_mov_b32_e32 v1, v0
-; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
+; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x8
-; GFX10-WGP-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
-; GFX10-WGP-NEXT: s_nop 0
-; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
+; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
+; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
+; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
; GFX10-WGP-NEXT: v_ashrrev_i32_e64 v3, 31, v1
; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
-; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-WGP-NEXT: s_mov_b32 s6, 2
-; GFX10-WGP-NEXT: v_lshlrev_b64 v[2:3], s6, v[1:2]
-; GFX10-WGP-NEXT: s_mov_b32 s7, s8
-; GFX10-WGP-NEXT: v_mov_b32_e32 v1, v2
-; GFX10-WGP-NEXT: s_mov_b32 s6, s9
-; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
-; GFX10-WGP-NEXT: v_add_co_u32 v1, s7, s7, v1
-; GFX10-WGP-NEXT: v_add_co_ci_u32_e64 v3, s6, s6, v2, s7
-; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
-; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
-; GFX10-WGP-NEXT: global_load_dword v1, v[1:2], off glc dlc
-; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT: s_mov_b32 s8, 2
+; GFX10-WGP-NEXT: v_lshlrev_b64 v[1:2], s8, v[1:2]
+; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 killed $vgpr1_vgpr2 killed $exec
+; GFX10-WGP-NEXT: global_load_dword v1, v1, s[6:7] glc dlc
+; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-WGP-NEXT: global_store_dword v0, v1, s[4:5]
; GFX10-WGP-NEXT: s_endpgm
;
; GFX10-CU-LABEL: global_volatile_load_1:
; GFX10-CU: ; %bb.0: ; %entry
-; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[8:9]
; GFX10-CU-NEXT: v_mov_b32_e32 v1, v0
-; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
+; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x8
-; GFX10-CU-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
-; GFX10-CU-NEXT: s_nop 0
-; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
+; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
+; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
+; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
; GFX10-CU-NEXT: v_ashrrev_i32_e64 v3, 31, v1
; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
-; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-CU-NEXT: s_mov_b32 s6, 2
-; GFX10-CU-NEXT: v_lshlrev_b64 v[2:3], s6, v[1:2]
-; GFX10-CU-NEXT: s_mov_b32 s7, s8
-; GFX10-CU-NEXT: v_mov_b32_e32 v1, v2
-; GFX10-CU-NEXT: s_mov_b32 s6, s9
-; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
-; GFX10-CU-NEXT: v_add_co_u32 v1, s7, s7, v1
-; GFX10-CU-NEXT: v_add_co_ci_u32_e64 v3, s6, s6, v2, s7
-; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
-; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
-; GFX10-CU-NEXT: global_load_dword v1, v[1:2], off glc dlc
-; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT: s_mov_b32 s8, 2
+; GFX10-CU-NEXT: v_lshlrev_b64 v[1:2], s8, v[1:2]
+; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 killed $vgpr1_vgpr2 killed $exec
+; GFX10-CU-NEXT: global_load_dword v1, v1, s[6:7] glc dlc
+; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-CU-NEXT: global_store_dword v0, v1, s[4:5]
; GFX10-CU-NEXT: s_endpgm
;
@@ -363,135 +345,101 @@ define amdgpu_kernel void @global_volatile_load_1(
;
; GFX11-WGP-LABEL: global_volatile_load_1:
; GFX11-WGP: ; %bb.0: ; %entry
-; GFX11-WGP-NEXT: s_mov_b64 s[0:1], s[4:5]
; GFX11-WGP-NEXT: v_mov_b32_e32 v1, v0
-; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
+; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
-; GFX11-WGP-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
-; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
-; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
+; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
+; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-WGP-NEXT: s_mov_b32 s2, 0x3ff
-; GFX11-WGP-NEXT: v_and_b32_e64 v1, v1, s2
+; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
+; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
+; GFX11-WGP-NEXT: s_mov_b32 s4, 0x3ff
+; GFX11-WGP-NEXT: v_and_b32_e64 v1, v1, s4
; GFX11-WGP-NEXT: v_ashrrev_i32_e64 v3, 31, v1
; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
-; GFX11-WGP-NEXT: s_mov_b32 s2, 2
-; GFX11-WGP-NEXT: v_lshlrev_b64 v[2:3], s2, v[1:2]
-; GFX11-WGP-NEXT: s_mov_b32 s3, s4
-; GFX11-WGP-NEXT: v_mov_b32_e32 v1, v2
-; GFX11-WGP-NEXT: s_mov_b32 s2, s5
-; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
-; GFX11-WGP-NEXT: v_add_co_u32 v1, s3, s3, v1
-; GFX11-WGP-NEXT: v_add_co_ci_u32_e64 v3, s2, s2, v2, s3
-; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
-; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
-; GFX11-WGP-NEXT: global_load_b32 v1, v[1:2], off glc dlc
-; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
+; GFX11-WGP-NEXT: s_mov_b32 s4, 2
+; GFX11-WGP-NEXT: v_lshlrev_b64 v[1:2], s4, v[1:2]
+; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 killed $vgpr1_vgpr2 killed $exec
+; GFX11-WGP-NEXT: global_load_b32 v1, v1, s[2:3] glc dlc
+; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX11-WGP-NEXT: s_endpgm
;
; GFX11-CU-LABEL: global_volatile_load_1:
; GFX11-CU: ; %bb.0: ; %entry
-; GFX11-CU-NEXT: s_mov_b64 s[0:1], s[4:5]
; GFX11-CU-NEXT: v_mov_b32_e32 v1, v0
-; GFX11-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
+; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
-; GFX11-CU-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
-; GFX11-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
-; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
+; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
+; GFX11-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-CU-NEXT: s_mov_b32 s2, 0x3ff
-; GFX11-CU-NEXT: v_and_b32_e64 v1, v1, s2
+; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
+; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
+; GFX11-CU-NEXT: s_mov_b32 s4, 0x3ff
+; GFX11-CU-NEXT: v_and_b32_e64 v1, v1, s4
; GFX11-CU-NEXT: v_ashrrev_i32_e64 v3, 31, v1
; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
-; GFX11-CU-NEXT: s_mov_b32 s2, 2
-; GFX11-CU-NEXT: v_lshlrev_b64 v[2:3], s2, v[1:2]
-; GFX11-CU-NEXT: s_mov_b32 s3, s4
-; GFX11-CU-NEXT: v_mov_b32_e32 v1, v2
-; GFX11-CU-NEXT: s_mov_b32 s2, s5
-; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
-; GFX11-CU-NEXT: v_add_co_u32 v1, s3, s3, v1
-; GFX11-CU-NEXT: v_add_co_ci_u32_e64 v3, s2, s2, v2, s3
-; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
-; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
-; GFX11-CU-NEXT: global_load_b32 v1, v[1:2], off glc dlc
-; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
+; GFX11-CU-NEXT: s_mov_b32 s4, 2
+; GFX11-CU-NEXT: v_lshlrev_b64 v[1:2], s4, v[1:2]
+; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 killed $vgpr1_vgpr2 killed $exec
+; GFX11-CU-NEXT: global_load_b32 v1, v1, s[2:3] glc dlc
+; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX11-CU-NEXT: s_endpgm
;
; GFX12-WGP-LABEL: global_volatile_load_1:
; GFX12-WGP: ; %bb.0: ; %entry
-; GFX12-WGP-NEXT: s_mov_b64 s[0:1], s[4:5]
; GFX12-WGP-NEXT: v_mov_b32_e32 v1, v0
-; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
-; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
-; GFX12-WGP-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
-; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
-; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
+; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
-; GFX12-WGP-NEXT: s_mov_b32 s2, 0x3ff
-; GFX12-WGP-NEXT: v_and_b32_e64 v1, v1, s2
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
+; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-WGP-NEXT: s_mov_b32 s4, 0x3ff
+; GFX12-WGP-NEXT: v_and_b32_e64 v1, v1, s4
; GFX12-WGP-NEXT: v_ashrrev_i32_e64 v3, 31, v1
; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
-; GFX12-WGP-NEXT: s_mov_b32 s2, 2
+; GFX12-WGP-NEXT: s_mov_b32 s4, 2
; GFX12-WGP-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-WGP-NEXT: v_lshlrev_b64_e64 v[2:3], s2, v[1:2]
-; GFX12-WGP-NEXT: s_mov_b32 s3, s4
-; GFX12-WGP-NEXT: v_mov_b32_e32 v1, v2
-; GFX12-WGP-NEXT: s_mov_b32 s2, s5
-; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
-; GFX12-WGP-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-WGP-NEXT: v_add_co_u32 v1, s3, s3, v1
-; GFX12-WGP-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-WGP-NEXT: v_add_co_ci_u32_e64 v3, s2, s2, v2, s3
-; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
-; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
-; GFX12-WGP-NEXT: global_load_b32 v1, v[1:2], off scope:SCOPE_SYS
+; GFX12-WGP-NEXT: v_lshlrev_b64_e64 v[1:2], s4, v[1:2]
+; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 killed $vgpr1_vgpr2 killed $exec
+; GFX12-WGP-NEXT: global_load_b32 v1, v1, s[2:3] scope:SCOPE_SYS
; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX12-WGP-NEXT: s_endpgm
;
; GFX12-CU-LABEL: global_volatile_load_1:
; GFX12-CU: ; %bb.0: ; %entry
-; GFX12-CU-NEXT: s_mov_b64 s[0:1], s[4:5]
; GFX12-CU-NEXT: v_mov_b32_e32 v1, v0
-; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
; GFX12-CU-NEXT: s_wait_kmcnt 0x0
-; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
-; GFX12-CU-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
-; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
-; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
+; GFX12-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
; GFX12-CU-NEXT: s_wait_kmcnt 0x0
-; GFX12-CU-NEXT: s_mov_b32 s2, 0x3ff
-; GFX12-CU-NEXT: v_and_b32_e64 v1, v1, s2
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
+; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-CU-NEXT: s_mov_b32 s4, 0x3ff
+; GFX12-CU-NEXT: v_and_b32_e64 v1, v1, s4
; GFX12-CU-NEXT: v_ashrrev_i32_e64 v3, 31, v1
; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
-; GFX12-CU-NEXT: s_mov_b32 s2, 2
+; GFX12-CU-NEXT: s_mov_b32 s4, 2
; GFX12-CU-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-CU-NEXT: v_lshlrev_b64_e64 v[2:3], s2, v[1:2]
-; GFX12-CU-NEXT: s_mov_b32 s3, s4
-; GFX12-CU-NEXT: v_mov_b32_e32 v1, v2
-; GFX12-CU-NEXT: s_mov_b32 s2, s5
-; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
-; GFX12-CU-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-CU-NEXT: v_add_co_u32 v1, s3, s3, v1
-; GFX12-CU-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-CU-NEXT: v_add_co_ci_u32_e64 v3, s2, s2, v2, s3
-; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
-; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
-; GFX12-CU-NEXT: global_load_b32 v1, v[1:2], off scope:SCOPE_SYS
+; GFX12-CU-NEXT: v_lshlrev_b64_e64 v[1:2], s4, v[1:2]
+; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 killed $vgpr1_vgpr2 killed $exec
+; GFX12-CU-NEXT: global_load_b32 v1, v1, s[2:3] scope:SCOPE_SYS
; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
; GFX12-CU-NEXT: s_wait_samplecnt 0x0
; GFX12-CU-NEXT: s_wait_loadcnt 0x0
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX12-CU-NEXT: s_endpgm
;
@@ -795,28 +743,20 @@ define amdgpu_kernel void @global_volatile_store_1(
; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
+; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
; GFX10-WGP-NEXT: s_nop 0
-; GFX10-WGP-NEXT: s_load_dwordx2 s[8:9], s[8:9], 0x8
-; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-WGP-NEXT: s_load_dword s4, s[4:5], 0x0
+; GFX10-WGP-NEXT: s_load_dword s6, s[6:7], 0x0
; GFX10-WGP-NEXT: v_ashrrev_i32_e64 v2, 31, v0
; GFX10-WGP-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
; GFX10-WGP-NEXT: v_mov_b32_e32 v1, v2
-; GFX10-WGP-NEXT: s_mov_b32 s5, 2
-; GFX10-WGP-NEXT: v_lshlrev_b64 v[1:2], s5, v[0:1]
-; GFX10-WGP-NEXT: s_mov_b32 s6, s8
-; GFX10-WGP-NEXT: v_mov_b32_e32 v0, v1
-; GFX10-WGP-NEXT: s_mov_b32 s5, s9
-; GFX10-WGP-NEXT: v_mov_b32_e32 v1, v2
-; GFX10-WGP-NEXT: v_add_co_u32 v0, s6, s6, v0
-; GFX10-WGP-NEXT: v_add_co_ci_u32_e64 v2, s5, s5, v1, s6
-; GFX10-WGP-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
-; GFX10-WGP-NEXT: v_mov_b32_e32 v1, v2
+; GFX10-WGP-NEXT: s_mov_b32 s7, 2
+; GFX10-WGP-NEXT: v_lshlrev_b64 v[0:1], s7, v[0:1]
+; GFX10-WGP-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s4
-; GFX10-WGP-NEXT: global_store_dword v[0:1], v2, off
+; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s6
+; GFX10-WGP-NEXT: global_store_dword v0, v1, s[4:5]
; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-WGP-NEXT: s_endpgm
;
@@ -825,28 +765,20 @@ define amdgpu_kernel void @global_volatile_store_1(
; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
+; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
; GFX10-CU-NEXT: s_nop 0
-; GFX10-CU-NEXT: s_load_dwordx2 s[8:9], s[8:9], 0x8
-; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-CU-NEXT: s_load_dword s4, s[4:5], 0x0
+; GFX10-CU-NEXT: s_load_dword s6, s[6:7], 0x0
; GFX10-CU-NEXT: v_ashrrev_i32_e64 v2, 31, v0
; GFX10-CU-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
; GFX10-CU-NEXT: v_mov_b32_e32 v1, v2
-; GFX10-CU-NEXT: s_mov_b32 s5, 2
-; GFX10-CU-NEXT: v_lshlrev_b64 v[1:2], s5, v[0:1]
-; GFX10-CU-NEXT: s_mov_b32 s6, s8
-; GFX10-CU-NEXT: v_mov_b32_e32 v0, v1
-; GFX10-CU-NEXT: s_mov_b32 s5, s9
-; GFX10-CU-NEXT: v_mov_b32_e32 v1, v2
-; GFX10-CU-NEXT: v_add_co_u32 v0, s6, s6, v0
-; GFX10-CU-NEXT: v_add_co_ci_u32_e64 v2, s5, s5, v1, s6
-; GFX10-CU-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
-; GFX10-CU-NEXT: v_mov_b32_e32 v1, v2
+; GFX10-CU-NEXT: s_mov_b32 s7, 2
+; GFX10-CU-NEXT: v_lshlrev_b64 v[0:1], s7, v[0:1]
+; GFX10-CU-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-CU-NEXT: v_mov_b32_e32 v2, s4
-; GFX10-CU-NEXT: global_store_dword v[0:1], v2, off
+; GFX10-CU-NEXT: v_mov_b32_e32 v1, s6
+; GFX10-CU-NEXT: global_store_dword v0, v1, s[4:5]
; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-CU-NEXT: s_endpgm
;
@@ -881,29 +813,21 @@ define amdgpu_kernel void @global_volatile_store_1(
; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
+; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
-; GFX11-WGP-NEXT: s_load_b64 s[4:5], s[4:5], 0x8
-; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
-; GFX11-WGP-NEXT: s_mov_b32 s1, 0x3ff
-; GFX11-WGP-NEXT: v_and_b32_e64 v0, v0, s1
+; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
+; GFX11-WGP-NEXT: s_load_b32 s2, s[2:3], 0x0
+; GFX11-WGP-NEXT: s_mov_b32 s3, 0x3ff
+; GFX11-WGP-NEXT: v_and_b32_e64 v0, v0, s3
; GFX11-WGP-NEXT: v_ashrrev_i32_e64 v2, 31, v0
; GFX11-WGP-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
; GFX11-WGP-NEXT: v_mov_b32_e32 v1, v2
-; GFX11-WGP-NEXT: s_mov_b32 s1, 2
-; GFX11-WGP-NEXT: v_lshlrev_b64 v[1:2], s1, v[0:1]
-; GFX11-WGP-NEXT: s_mov_b32 s2, s4
-; GFX11-WGP-NEXT: v_mov_b32_e32 v0, v1
-; GFX11-WGP-NEXT: s_mov_b32 s1, s5
-; GFX11-WGP-NEXT: v_mov_b32_e32 v1, v2
-; GFX11-WGP-NEXT: v_add_co_u32 v0, s2, s2, v0
-; GFX11-WGP-NEXT: v_add_co_ci_u32_e64 v2, s1, s1, v1, s2
-; GFX11-WGP-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
-; GFX11-WGP-NEXT: v_mov_b32_e32 v1, v2
+; GFX11-WGP-NEXT: s_mov_b32 s3, 2
+; GFX11-WGP-NEXT: v_lshlrev_b64 v[0:1], s3, v[0:1]
+; GFX11-WGP-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0
-; GFX11-WGP-NEXT: global_store_b32 v[0:1], v2, off dlc
+; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s2
+; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1] dlc
; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-WGP-NEXT: s_endpgm
;
@@ -912,29 +836,21 @@ define amdgpu_kernel void @global_volatile_store_1(
; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
+; GFX11-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
-; GFX11-CU-NEXT: s_load_b64 s[4:5], s[4:5], 0x8
-; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
-; GFX11-CU-NEXT: s_mov_b32 s1, 0x3ff
-; GFX11-CU-NEXT: v_and_b32_e64 v0, v0, s1
+; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
+; GFX11-CU-NEXT: s_load_b32 s2, s[2:3], 0x0
+; GFX11-CU-NEXT: s_mov_b32 s3, 0x3ff
+; GFX11-CU-NEXT: v_and_b32_e64 v0, v0, s3
; GFX11-CU-NEXT: v_ashrrev_i32_e64 v2, 31, v0
; GFX11-CU-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
; GFX11-CU-NEXT: v_mov_b32_e32 v1, v2
-; GFX11-CU-NEXT: s_mov_b32 s1, 2
-; GFX11-CU-NEXT: v_lshlrev_b64 v[1:2], s1, v[0:1]
-; GFX11-CU-NEXT: s_mov_b32 s2, s4
-; GFX11-CU-NEXT: v_mov_b32_e32 v0, v1
-; GFX11-CU-NEXT: s_mov_b32 s1, s5
-; GFX11-CU-NEXT: v_mov_b32_e32 v1, v2
-; GFX11-CU-NEXT: v_add_co_u32 v0, s2, s2, v0
-; GFX11-CU-NEXT: v_add_co_ci_u32_e64 v2, s1, s1, v1, s2
-; GFX11-CU-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
-; GFX11-CU-NEXT: v_mov_b32_e32 v1, v2
+; GFX11-CU-NEXT: s_mov_b32 s3, 2
+; GFX11-CU-NEXT: v_lshlrev_b64 v[0:1], s3, v[0:1]
+; GFX11-CU-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
-; GFX11-CU-NEXT: global_store_b32 v[0:1], v2, off dlc
+; GFX11-CU-NEXT: v_mov_b32_e32 v1, s2
+; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1] dlc
; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-CU-NEXT: s_endpgm
;
@@ -943,36 +859,27 @@ define amdgpu_kernel void @global_volatile_store_1(
; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
+; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
-; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
-; GFX12-WGP-NEXT: s_load_b64 s[4:5], s[4:5], 0x8
-; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
-; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
-; GFX12-WGP-NEXT: s_mov_b32 s1, 0x3ff
-; GFX12-WGP-NEXT: v_and_b32_e64 v0, v0, s1
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
+; GFX12-WGP-NEXT: s_load_b32 s2, s[2:3], 0x0
+; GFX12-WGP-NEXT: s_mov_b32 s3, 0x3ff
+; GFX12-WGP-NEXT: v_and_b32_e64 v0, v0, s3
; GFX12-WGP-NEXT: v_ashrrev_i32_e64 v2, 31, v0
; GFX12-WGP-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
; GFX12-WGP-NEXT: v_mov_b32_e32 v1, v2
-; GFX12-WGP-NEXT: s_mov_b32 s1, 2
+; GFX12-WGP-NEXT: s_mov_b32 s3, 2
; GFX12-WGP-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-WGP-NEXT: v_lshlrev_b64_e64 v[1:2], s1, v[0:1]
-; GFX12-WGP-NEXT: s_mov_b32 s2, s4
-; GFX12-WGP-NEXT: v_mov_b32_e32 v0, v1
-; GFX12-WGP-NEXT: s_mov_b32 s1, s5
-; GFX12-WGP-NEXT: v_mov_b32_e32 v1, v2
-; GFX12-WGP-NEXT: v_add_co_u32 v0, s2, s2, v0
-; GFX12-WGP-NEXT: s_wait_alu depctr_sa_sdst(0) depctr_va_sdst(0)
-; GFX12-WGP-NEXT: v_add_co_ci_u32_e64 v2, s1, s1, v1, s2
-; GFX12-WGP-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
-; GFX12-WGP-NEXT: v_mov_b32_e32 v1, v2
+; GFX12-WGP-NEXT: v_lshlrev_b64_e64 v[0:1], s3, v[0:1]
+; GFX12-WGP-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
-; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s2
; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
; GFX12-WGP-NEXT: s_wait_storecnt 0x0
-; GFX12-WGP-NEXT: global_store_b32 v[0:1], v2, off scope:SCOPE_SYS
+; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1] scope:SCOPE_SYS
; GFX12-WGP-NEXT: s_wait_storecnt 0x0
; GFX12-WGP-NEXT: s_endpgm
;
@@ -981,36 +888,27 @@ define amdgpu_kernel void @global_volatile_store_1(
; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
; GFX12-CU-NEXT: s_wait_kmcnt 0x0
; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
+; GFX12-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
; GFX12-CU-NEXT: s_wait_kmcnt 0x0
-; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
-; GFX12-CU-NEXT: s_load_b64 s[4:5], s[4:5], 0x8
-; GFX12-CU-NEXT: s_wait_kmcnt 0x0
-; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
-; GFX12-CU-NEXT: s_mov_b32 s1, 0x3ff
-; GFX12-CU-NEXT: v_and_b32_e64 v0, v0, s1
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
+; GFX12-CU-NEXT: s_load_b32 s2, s[2:3], 0x0
+; GFX12-CU-NEXT: s_mov_b32 s3, 0x3ff
+; GFX12-CU-NEXT: v_and_b32_e64 v0, v0, s3
; GFX12-CU-NEXT: v_ashrrev_i32_e64 v2, 31, v0
; GFX12-CU-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
; GFX12-CU-NEXT: v_mov_b32_e32 v1, v2
-; GFX12-CU-NEXT: s_mov_b32 s1, 2
+; GFX12-CU-NEXT: s_mov_b32 s3, 2
; GFX12-CU-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-CU-NEXT: v_lshlrev_b64_e64 v[1:2], s1, v[0:1]
-; GFX12-CU-NEXT: s_mov_b32 s2, s4
-; GFX12-CU-NEXT: v_mov_b32_e32 v0, v1
-; GFX12-CU-NEXT: s_mov_b32 s1, s5
-; GFX12-CU-NEXT: v_mov_b32_e32 v1, v2
-; GFX12-CU-NEXT: v_add_co_u32 v0, s2, s2, v0
-; GFX12-CU-NEXT: s_wait_alu depctr_sa_sdst(0) depctr_va_sdst(0)
-; GFX12-CU-NEXT: v_add_co_ci_u32_e64 v2, s1, s1, v1, s2
-; GFX12-CU-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
-; GFX12-CU-NEXT: v_mov_b32_e32 v1, v2
+; GFX12-CU-NEXT: v_lshlrev_b64_e64 v[0:1], s3, v[0:1]
+; GFX12-CU-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
; GFX12-CU-NEXT: s_wait_kmcnt 0x0
-; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-CU-NEXT: v_mov_b32_e32 v1, s2
; GFX12-CU-NEXT: s_wait_loadcnt 0x0
; GFX12-CU-NEXT: s_wait_samplecnt 0x0
; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
; GFX12-CU-NEXT: s_wait_kmcnt 0x0
; GFX12-CU-NEXT: s_wait_storecnt 0x0
-; GFX12-CU-NEXT: global_store_b32 v[0:1], v2, off scope:SCOPE_SYS
+; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1] scope:SCOPE_SYS
; GFX12-CU-NEXT: s_wait_storecnt 0x0
; GFX12-CU-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm.ll b/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm.ll
index b113380874cd3..018ab475ac718 100644
--- a/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm.ll
+++ b/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm.ll
@@ -581,10 +581,7 @@ define hidden amdgpu_kernel void @clmem_read(ptr addrspace(1) %buffer) {
; GFX900-NEXT: s_and_b64 vcc, exec, vcc
; GFX900-NEXT: s_cbranch_vccz .LBB1_1
; GFX900-NEXT: ; %bb.4: ; %while.end
-; GFX900-NEXT: v_mov_b32_e32 v1, s35
-; GFX900-NEXT: v_add_co_u32_e32 v0, vcc, s34, v6
-; GFX900-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
-; GFX900-NEXT: global_store_dwordx2 v[0:1], v[4:5], off
+; GFX900-NEXT: global_store_dwordx2 v6, v[4:5], s[34:35]
; GFX900-NEXT: s_endpgm
;
; GFX10-LABEL: clmem_read:
@@ -695,9 +692,7 @@ define hidden amdgpu_kernel void @clmem_read(ptr addrspace(1) %buffer) {
; GFX10-NEXT: s_and_b32 vcc_lo, exec_lo, s0
; GFX10-NEXT: s_cbranch_vccz .LBB1_1
; GFX10-NEXT: ; %bb.4: ; %while.end
-; GFX10-NEXT: v_add_co_u32 v0, s0, s34, v6
-; GFX10-NEXT: v_add_co_ci_u32_e64 v1, s0, s35, 0, s0
-; GFX10-NEXT: global_store_dwordx2 v[0:1], v[2:3], off
+; GFX10-NEXT: global_store_dwordx2 v6, v[2:3], s[34:35]
; GFX10-NEXT: s_endpgm
;
; GFX90A-LABEL: clmem_read:
@@ -805,10 +800,7 @@ define hidden amdgpu_kernel void @clmem_read(ptr addrspace(1) %buffer) {
; GFX90A-NEXT: s_and_b64 vcc, exec, vcc
; GFX90A-NEXT: s_cbranch_vccz .LBB1_1
; GFX90A-NEXT: ; %bb.4: ; %while.end
-; GFX90A-NEXT: v_mov_b32_e32 v1, s35
-; GFX90A-NEXT: v_add_co_u32_e32 v0, vcc, s34, v6
-; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
-; GFX90A-NEXT: global_store_dwordx2 v[0:1], v[2:3], off
+; GFX90A-NEXT: global_store_dwordx2 v6, v[2:3], s[34:35]
; GFX90A-NEXT: s_endpgm
;
; GFX11-LABEL: clmem_read:
@@ -924,10 +916,7 @@ define hidden amdgpu_kernel void @clmem_read(ptr addrspace(1) %buffer) {
; GFX11-NEXT: s_and_b32 vcc_lo, exec_lo, s0
; GFX11-NEXT: s_cbranch_vccz .LBB1_1
; GFX11-NEXT: ; %bb.4: ; %while.end
-; GFX11-NEXT: v_add_co_u32 v0, s0, s34, v6
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, s35, 0, s0
-; GFX11-NEXT: global_store_b64 v[0:1], v[2:3], off
+; GFX11-NEXT: global_store_b64 v6, v[2:3], s[34:35]
; GFX11-NEXT: s_endpgm
entry:
%call = tail call i64 @_Z13get_global_idj(i32 0)
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr.ll b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr.ll
index 0e260f1262499..aa5eea560428c 100644
--- a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr.ll
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr.ll
@@ -19,10 +19,9 @@ define amdgpu_kernel void @respect_optnone(double %arg0, double %arg1, ptr addrs
; CHECK-NEXT: v_and_b32_e64 v0, v0, s6
; CHECK-NEXT: v_ashrrev_i32_e64 v1, 31, v0
; CHECK-NEXT: s_mov_b32 s6, 3
+; CHECK-NEXT: v_lshlrev_b64 v[0:1], s6, v[0:1]
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: v_mov_b64_e32 v[2:3], s[4:5]
-; CHECK-NEXT: v_lshl_add_u64 v[0:1], v[0:1], s6, v[2:3]
-; CHECK-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
+; CHECK-NEXT: global_load_dwordx2 v[0:1], v0, s[4:5]
; CHECK-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; CHECK-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
; CHECK-NEXT: s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/smfmac_alloc_failure_no_agpr_O0.ll b/llvm/test/CodeGen/AMDGPU/smfmac_alloc_failure_no_agpr_O0.ll
index 626f547c31952..4afee369b8585 100644
--- a/llvm/test/CodeGen/AMDGPU/smfmac_alloc_failure_no_agpr_O0.ll
+++ b/llvm/test/CodeGen/AMDGPU/smfmac_alloc_failure_no_agpr_O0.ll
@@ -26,29 +26,30 @@ define amdgpu_kernel void @test_smfmac_f32_32x32x32_f16__vgpr(ptr addrspace(1) %
; CHECK-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
; CHECK-NEXT: v_mov_b32_e32 v3, v1
; CHECK-NEXT: s_mov_b32 s3, 6
-; CHECK-NEXT: v_lshlrev_b64 v[2:3], s3, v[2:3]
+; CHECK-NEXT: v_lshlrev_b64 v[4:5], s3, v[2:3]
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: v_lshl_add_u64 v[2:3], s[0:1], 0, v[2:3]
-; CHECK-NEXT: global_load_dwordx4 v[4:7], v[2:3], off offset:16
+; CHECK-NEXT: v_lshl_add_u64 v[2:3], s[0:1], 0, v[4:5]
+; CHECK-NEXT: v_mov_b32_e32 v1, v4
+; CHECK-NEXT: global_load_dwordx4 v[4:7], v1, s[0:1] offset:16
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: v_mov_b32_e32 v1, v7
-; CHECK-NEXT: v_mov_b32_e32 v12, v6
-; CHECK-NEXT: v_mov_b32_e32 v13, v5
-; CHECK-NEXT: v_mov_b32_e32 v14, v4
-; CHECK-NEXT: global_load_dwordx4 v[4:7], v[2:3], off
+; CHECK-NEXT: v_mov_b32_e32 v12, v7
+; CHECK-NEXT: v_mov_b32_e32 v13, v6
+; CHECK-NEXT: v_mov_b32_e32 v14, v5
+; CHECK-NEXT: v_mov_b32_e32 v15, v4
+; CHECK-NEXT: global_load_dwordx4 v[4:7], v1, s[0:1]
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: v_mov_b32_e32 v15, v7
-; CHECK-NEXT: v_mov_b32_e32 v16, v6
-; CHECK-NEXT: v_mov_b32_e32 v17, v5
+; CHECK-NEXT: v_mov_b32_e32 v16, v7
+; CHECK-NEXT: v_mov_b32_e32 v17, v6
+; CHECK-NEXT: v_mov_b32_e32 v18, v5
; CHECK-NEXT: ; kill: def $vgpr4 killed $vgpr4 killed $vgpr4_vgpr5_vgpr6_vgpr7 killed $exec
; CHECK-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11 killed $exec
-; CHECK-NEXT: v_mov_b32_e32 v5, v17
-; CHECK-NEXT: v_mov_b32_e32 v6, v16
-; CHECK-NEXT: v_mov_b32_e32 v7, v15
-; CHECK-NEXT: v_mov_b32_e32 v8, v14
-; CHECK-NEXT: v_mov_b32_e32 v9, v13
-; CHECK-NEXT: v_mov_b32_e32 v10, v12
-; CHECK-NEXT: v_mov_b32_e32 v11, v1
+; CHECK-NEXT: v_mov_b32_e32 v5, v18
+; CHECK-NEXT: v_mov_b32_e32 v6, v17
+; CHECK-NEXT: v_mov_b32_e32 v7, v16
+; CHECK-NEXT: v_mov_b32_e32 v8, v15
+; CHECK-NEXT: v_mov_b32_e32 v9, v14
+; CHECK-NEXT: v_mov_b32_e32 v10, v13
+; CHECK-NEXT: v_mov_b32_e32 v11, v12
; CHECK-NEXT: v_mov_b32_e32 v25, v11
; CHECK-NEXT: v_mov_b32_e32 v26, v10
; CHECK-NEXT: v_mov_b32_e32 v27, v9
@@ -57,7 +58,7 @@ define amdgpu_kernel void @test_smfmac_f32_32x32x32_f16__vgpr(ptr addrspace(1) %
; CHECK-NEXT: v_mov_b32_e32 v30, v6
; CHECK-NEXT: v_mov_b32_e32 v31, v5
; CHECK-NEXT: v_mov_b32_e32 v8, v4
-; CHECK-NEXT: global_load_dwordx4 v[10:13], v[2:3], off offset:32
+; CHECK-NEXT: global_load_dwordx4 v[10:13], v1, s[0:1] offset:32
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: v_mov_b32_e32 v5, v13
; CHECK-NEXT: v_mov_b32_e32 v6, v12
diff --git a/llvm/test/CodeGen/AMDGPU/smrd-narrow-soffset.ll b/llvm/test/CodeGen/AMDGPU/smrd-narrow-soffset.ll
index cea7e984980e9..dc2c769eac762 100644
--- a/llvm/test/CodeGen/AMDGPU/smrd-narrow-soffset.ll
+++ b/llvm/test/CodeGen/AMDGPU/smrd-narrow-soffset.ll
@@ -15,9 +15,7 @@ define amdgpu_kernel void @variable_shift(ptr addrspace(4) %p, i32 %n, i32 %amt,
; GFX9-SDAG-NEXT: s_and_b32 s4, s2, 0xff
; GFX9-SDAG-NEXT: s_and_b32 s2, s3, 3
; GFX9-SDAG-NEXT: s_lshl_b64 s[2:3], s[4:5], s2
-; GFX9-SDAG-NEXT: s_add_u32 s0, s0, s2
-; GFX9-SDAG-NEXT: s_addc_u32 s1, s1, s3
-; GFX9-SDAG-NEXT: s_load_dword s0, s[0:1], 0x0
+; GFX9-SDAG-NEXT: s_load_dword s0, s[0:1], s2 offset:0x0
; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s0
; GFX9-SDAG-NEXT: global_store_dword v0, v1, s[6:7]
@@ -36,10 +34,9 @@ define amdgpu_kernel void @variable_shift(ptr addrspace(4) %p, i32 %n, i32 %amt,
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1250-SDAG-NEXT: s_and_b32 s6, s2, 0xff
; GFX1250-SDAG-NEXT: s_and_b32 s2, s3, 3
-; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1250-SDAG-NEXT: s_lshl_b64 s[2:3], s[6:7], s2
-; GFX1250-SDAG-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[2:3]
-; GFX1250-SDAG-NEXT: s_load_b32 s0, s[0:1], 0x0 nv
+; GFX1250-SDAG-NEXT: s_load_b32 s0, s[0:1], s2 offset:0x0 nv
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
; GFX1250-SDAG-NEXT: global_store_b32 v0, v1, s[4:5]
@@ -55,9 +52,7 @@ define amdgpu_kernel void @variable_shift(ptr addrspace(4) %p, i32 %n, i32 %amt,
; GFX9-GISEL-NEXT: s_and_b32 s4, s2, 0xff
; GFX9-GISEL-NEXT: s_and_b32 s2, s3, 3
; GFX9-GISEL-NEXT: s_lshl_b64 s[2:3], s[4:5], s2
-; GFX9-GISEL-NEXT: s_add_u32 s0, s0, s2
-; GFX9-GISEL-NEXT: s_addc_u32 s1, s1, s3
-; GFX9-GISEL-NEXT: s_load_dword s0, s[0:1], 0x0
+; GFX9-GISEL-NEXT: s_load_dword s0, s[0:1], s2 offset:0x0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s0
; GFX9-GISEL-NEXT: global_store_dword v1, v0, s[6:7]
@@ -77,11 +72,9 @@ define amdgpu_kernel void @variable_shift(ptr addrspace(4) %p, i32 %n, i32 %amt,
; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-NEXT: s_and_b32 s6, s2, 0xff
; GFX1250-GISEL-NEXT: s_and_b32 s2, s3, 3
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1250-GISEL-NEXT: s_lshl_b64 s[2:3], s[6:7], s2
-; GFX1250-GISEL-NEXT: s_add_co_u32 s0, s0, s2
-; GFX1250-GISEL-NEXT: s_add_co_ci_u32 s1, s1, s3
-; GFX1250-GISEL-NEXT: s_load_b32 s0, s[0:1], 0x0 nv
+; GFX1250-GISEL-NEXT: s_load_b32 s0, s[0:1], s2 offset:0x0 nv
; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-NEXT: v_mov_b32_e32 v0, s0
; GFX1250-GISEL-NEXT: global_store_b32 v1, v0, s[4:5]
@@ -108,9 +101,7 @@ define amdgpu_kernel void @variable_shift_plus_imm(ptr addrspace(4) %p, i32 %n,
; GFX9-SDAG-NEXT: s_and_b32 s4, s2, 0xff
; GFX9-SDAG-NEXT: s_and_b32 s2, s3, 3
; GFX9-SDAG-NEXT: s_lshl_b64 s[2:3], s[4:5], s2
-; GFX9-SDAG-NEXT: s_add_u32 s0, s0, s2
-; GFX9-SDAG-NEXT: s_addc_u32 s1, s1, s3
-; GFX9-SDAG-NEXT: s_load_dword s0, s[0:1], 0x10
+; GFX9-SDAG-NEXT: s_load_dword s0, s[0:1], s2 offset:0x10
; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s0
; GFX9-SDAG-NEXT: global_store_dword v0, v1, s[6:7]
@@ -129,10 +120,9 @@ define amdgpu_kernel void @variable_shift_plus_imm(ptr addrspace(4) %p, i32 %n,
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1250-SDAG-NEXT: s_and_b32 s6, s2, 0xff
; GFX1250-SDAG-NEXT: s_and_b32 s2, s3, 3
-; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1250-SDAG-NEXT: s_lshl_b64 s[2:3], s[6:7], s2
-; GFX1250-SDAG-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[2:3]
-; GFX1250-SDAG-NEXT: s_load_b32 s0, s[0:1], 0x10 nv
+; GFX1250-SDAG-NEXT: s_load_b32 s0, s[0:1], s2 offset:0x10 nv
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
; GFX1250-SDAG-NEXT: global_store_b32 v0, v1, s[4:5]
@@ -148,9 +138,7 @@ define amdgpu_kernel void @variable_shift_plus_imm(ptr addrspace(4) %p, i32 %n,
; GFX9-GISEL-NEXT: s_and_b32 s4, s2, 0xff
; GFX9-GISEL-NEXT: s_and_b32 s2, s3, 3
; GFX9-GISEL-NEXT: s_lshl_b64 s[2:3], s[4:5], s2
-; GFX9-GISEL-NEXT: s_add_u32 s0, s0, s2
-; GFX9-GISEL-NEXT: s_addc_u32 s1, s1, s3
-; GFX9-GISEL-NEXT: s_load_dword s0, s[0:1], 0x10
+; GFX9-GISEL-NEXT: s_load_dword s0, s[0:1], s2 offset:0x10
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s0
; GFX9-GISEL-NEXT: global_store_dword v1, v0, s[6:7]
@@ -170,11 +158,9 @@ define amdgpu_kernel void @variable_shift_plus_imm(ptr addrspace(4) %p, i32 %n,
; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-NEXT: s_and_b32 s6, s2, 0xff
; GFX1250-GISEL-NEXT: s_and_b32 s2, s3, 3
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1250-GISEL-NEXT: s_lshl_b64 s[2:3], s[6:7], s2
-; GFX1250-GISEL-NEXT: s_add_co_u32 s0, s0, s2
-; GFX1250-GISEL-NEXT: s_add_co_ci_u32 s1, s1, s3
-; GFX1250-GISEL-NEXT: s_load_b32 s0, s[0:1], 0x10 nv
+; GFX1250-GISEL-NEXT: s_load_b32 s0, s[0:1], s2 offset:0x10 nv
; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-NEXT: v_mov_b32_e32 v0, s0
; GFX1250-GISEL-NEXT: global_store_b32 v1, v0, s[4:5]
@@ -207,9 +193,7 @@ define amdgpu_kernel void @zext_across_block(ptr addrspace(4) %p, i32 %n, ptr ad
; GFX9-SDAG-NEXT: ; %bb.1: ; %load
; GFX9-SDAG-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-SDAG-NEXT: s_add_u32 s0, s0, s4
-; GFX9-SDAG-NEXT: s_addc_u32 s1, s1, 0
-; GFX9-SDAG-NEXT: s_load_dword s0, s[0:1], 0x0
+; GFX9-SDAG-NEXT: s_load_dword s0, s[0:1], s4 offset:0x0
; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s0
; GFX9-SDAG-NEXT: global_store_dword v0, v1, s[2:3]
@@ -235,8 +219,7 @@ define amdgpu_kernel void @zext_across_block(ptr addrspace(4) %p, i32 %n, ptr ad
; GFX1250-SDAG-NEXT: ; %bb.1: ; %load
; GFX1250-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1250-SDAG-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[6:7]
-; GFX1250-SDAG-NEXT: s_load_b32 s0, s[0:1], 0x0 nv
+; GFX1250-SDAG-NEXT: s_load_b32 s0, s[0:1], s6 offset:0x0 nv
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, s0
; GFX1250-SDAG-NEXT: global_store_b32 v0, v1, s[2:3]
diff --git a/llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll b/llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll
index f4702bc2f8311..9d4604c46c642 100644
--- a/llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll
+++ b/llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll
@@ -476,13 +476,10 @@ define amdgpu_kernel void @livevariables_update_missed_block(ptr addrspace(1) %s
; SI-NEXT: bb.1.if.then:
; SI-NEXT: successors: %bb.7(0x80000000)
; SI-NEXT: {{ $}}
- ; SI-NEXT: early-clobber %34:sreg_64_xexec = S_LOAD_DWORDX2_IMM_ec killed [[COPY]](p4), 36, 0 :: (dereferenceable invariant load (s64) from %ir.src1.kernarg.offset, align 4, addrspace 4)
- ; SI-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_ADD_CO_U32_e64 %34.sub0, killed %52, 0, implicit $exec
- ; SI-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_ADDC_U32_e64 0, killed %34.sub1, killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
- ; SI-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE killed [[V_ADD_CO_U32_e64_]], %subreg.sub0, killed [[V_ADDC_U32_e64_]], %subreg.sub1
- ; SI-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE killed [[REG_SEQUENCE]], 0, 0, implicit $exec :: (load (s8) from %ir.i10, addrspace 1)
+ ; SI-NEXT: early-clobber %34:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM_ec killed [[COPY]](p4), 36, 0 :: (dereferenceable invariant load (s64) from %ir.src1.kernarg.offset, align 4, addrspace 4)
+ ; SI-NEXT: [[GLOBAL_LOAD_UBYTE_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE_SADDR killed %34, killed %44, 0, 0, implicit $exec :: (load (s8) from %ir.i10, addrspace 1)
; SI-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO 0, implicit $exec
- ; SI-NEXT: GLOBAL_STORE_BYTE killed [[V_MOV_B]], killed [[GLOBAL_LOAD_UBYTE]], 0, 0, implicit $exec :: (store (s8) into `ptr addrspace(1) null`, addrspace 1)
+ ; SI-NEXT: GLOBAL_STORE_BYTE killed [[V_MOV_B]], killed [[GLOBAL_LOAD_UBYTE_SADDR]], 0, 0, implicit $exec :: (store (s8) into `ptr addrspace(1) null`, addrspace 1)
; SI-NEXT: S_BRANCH %bb.7
; SI-NEXT: {{ $}}
; SI-NEXT: bb.2.if.then9:
@@ -499,20 +496,20 @@ define amdgpu_kernel void @livevariables_update_missed_block(ptr addrspace(1) %s
; SI-NEXT: successors: %bb.6(0x80000000)
; SI-NEXT: {{ $}}
; SI-NEXT: [[V_MOV_B1:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO 0, implicit $exec
- ; SI-NEXT: [[GLOBAL_LOAD_UBYTE1:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE killed [[V_MOV_B1]], 0, 0, implicit $exec :: ("amdgpu-noclobber" load (s8) from `ptr addrspace(1) null`, addrspace 1)
+ ; SI-NEXT: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE killed [[V_MOV_B1]], 0, 0, implicit $exec :: ("amdgpu-noclobber" load (s8) from `ptr addrspace(1) null`, addrspace 1)
; SI-NEXT: S_BRANCH %bb.6
; SI-NEXT: {{ $}}
; SI-NEXT: bb.5.Flow:
; SI-NEXT: successors: %bb.1(0x40000000), %bb.7(0x40000000)
; SI-NEXT: {{ $}}
- ; SI-NEXT: [[PHI:%[0-9]+]]:vgpr_32 = PHI [[COPY1]](i32), %bb.0, undef %53:vgpr_32, %bb.6
+ ; SI-NEXT: [[PHI:%[0-9]+]]:vgpr_32 = PHI [[COPY1]](i32), %bb.0, undef %45:vgpr_32, %bb.6
; SI-NEXT: [[SI_ELSE:%[0-9]+]]:sreg_32 = SI_ELSE killed [[SI_IF]], %bb.7, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
; SI-NEXT: S_BRANCH %bb.1
; SI-NEXT: {{ $}}
; SI-NEXT: bb.6.sw.bb18:
; SI-NEXT: successors: %bb.5(0x80000000)
; SI-NEXT: {{ $}}
- ; SI-NEXT: [[PHI1:%[0-9]+]]:vgpr_32 = PHI undef %41:vgpr_32, %bb.3, [[GLOBAL_LOAD_UBYTE1]], %bb.4
+ ; SI-NEXT: [[PHI1:%[0-9]+]]:vgpr_32 = PHI undef %41:vgpr_32, %bb.3, [[GLOBAL_LOAD_UBYTE]], %bb.4
; SI-NEXT: [[V_MOV_B2:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO 0, implicit $exec
; SI-NEXT: GLOBAL_STORE_BYTE killed [[V_MOV_B2]], killed [[PHI1]], 0, 0, implicit $exec :: (store (s8) into `ptr addrspace(1) null`, addrspace 1)
; SI-NEXT: S_BRANCH %bb.5
More information about the llvm-branch-commits
mailing list