[llvm] [AMDGPU][GlobalISel] Eliminate READANYLANE round trips in mixed build_vector/merge (PR #210449)
Anshil Gandhi via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 20 19:15:45 PDT 2026
https://github.com/gandhi56 updated https://github.com/llvm/llvm-project/pull/210449
>From db1490a415dd12d394fbce1dfd9700721cd0969f Mon Sep 17 00:00:00 2001
From: Anshil Gandhi <Anshil.Gandhi at amd.com>
Date: Wed, 5 Aug 2026 17:34:19 -0500
Subject: [PATCH 1/2] [AMDGPU][GlobalISel] Pre-commit tests for readanylane
merge regbank combine (NFC)
Add regbank-combiner tests covering a copy to vgpr whose source is a
merge or build_vector of G_AMDGPU_READANYLANE results and uniform
values. These currently keep the round trip through sgprs, and also
cover the cases where the merge has another user and where all merge
sources are uniform.
Co-authored-by: Cursor <cursoragent at cursor.com>
---
.../AMDGPU/GlobalISel/regbankcombiner-merge-readanylane.mir | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankcombiner-merge-readanylane.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankcombiner-merge-readanylane.mir
index e39604cb12a87..cffbcff999574 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankcombiner-merge-readanylane.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankcombiner-merge-readanylane.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
-# RUN: llc -mtriple=amdgpu12.00-amd-amdhsa -run-pass=amdgpu-regbank-combiner %s -verify-machineinstrs -o - | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -run-pass=amdgpu-regbank-combiner %s -verify-machineinstrs -o - | FileCheck %s
---
name: readanylane_and_uniform_merge_to_virtual_vgpr
>From df9f345337ea19accfb3dfb1dc6c719f268ae361 Mon Sep 17 00:00:00 2001
From: Anshil Gandhi <Anshil.Gandhi at amd.com>
Date: Fri, 17 Jul 2026 16:46:12 -0500
Subject: [PATCH 2/2] [AMDGPU][GlobalISel] Fold redundant uniform
G_AMDGPU_READANYLANE in RegBankCombiner
---
.../llvm/CodeGen/GlobalISel/MIPatternMatch.h | 4 +
llvm/lib/Target/AMDGPU/AMDGPUCombine.td | 11 +-
.../lib/Target/AMDGPU/AMDGPUGlobalISelUtils.h | 10 +
.../Target/AMDGPU/AMDGPURegBankCombiner.cpp | 78 ++++
.../Target/AMDGPU/AMDGPURegBankLegalize.cpp | 7 -
.../bug_shuffle_vector_to_scalar.ll | 13 +-
.../AMDGPU/GlobalISel/insertelement.i8.ll | 177 ++++----
.../llvm.amdgcn.raw.buffer.load.tfe.ll | 144 ++-----
.../llvm.amdgcn.struct.buffer.load.format.ll | 34 +-
.../llvm.amdgcn.struct.buffer.load.tfe.ll | 144 ++-----
...vm.amdgcn.struct.ptr.buffer.load.format.ll | 17 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll | 16 +-
.../regbankcombiner-merge-readanylane.mir | 43 +-
.../AMDGPU/GlobalISel/strict_fma.f32.ll | 15 +-
.../AMDGPU/llvm.amdgcn.intersect_ray.ll | 213 ++++------
.../llvm.amdgcn.struct.buffer.load.format.ll | 115 ++----
llvm/test/CodeGen/AMDGPU/packed-fp32.ll | 386 ++++++------------
llvm/test/CodeGen/AMDGPU/packed-fp64.ll | 36 +-
18 files changed, 555 insertions(+), 908 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/MIPatternMatch.h b/llvm/include/llvm/CodeGen/GlobalISel/MIPatternMatch.h
index 47b3b6fcc0aca..f99c258ce35d9 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/MIPatternMatch.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/MIPatternMatch.h
@@ -645,6 +645,10 @@ inline GInstrBind<GVScale> m_GVScale(GVScale *&Inst) { return Inst; }
inline GInstrBind<GBuildVector> m_GBuildVector(GBuildVector *&Inst) {
return Inst;
}
+/// Matches any of G_MERGE_VALUES, G_BUILD_VECTOR or G_CONCAT_VECTORS.
+inline GInstrBind<GMergeLikeInstr> m_GMergeLikeInstr(GMergeLikeInstr *&Inst) {
+ return Inst;
+}
inline GInstrBind<GConcatVectors> m_GConcatVectors(GConcatVectors *&Inst) {
return Inst;
}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCombine.td b/llvm/lib/Target/AMDGPU/AMDGPUCombine.td
index 0c348bb647c0c..934a2804edc95 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCombine.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCombine.td
@@ -184,6 +184,14 @@ def fminimum_to_minmax3 : minmax_to_minmax3_opcodes<G_FMINIMUM>;
def fmax_ieee_to_minmax3 : minmax_to_minmax3_opcodes<G_FMAXNUM_IEEE>;
def fmin_ieee_to_minmax3 : minmax_to_minmax3_opcodes<G_FMINNUM_IEEE>;
+// Copy to vgpr of a merge of readanylane results and uniform values can use the
+// values before they were moved to sgprs.
+def merge_readanylane_to_vgpr : GICombineRule<
+ (defs root:$copy, register_vector_matchinfo:$matchinfo),
+ (match (COPY $dst, $src):$copy,
+ [{ return matchMergeReadAnyLaneToVgpr(*${copy}, ${matchinfo}); }]),
+ (apply [{ applyMergeReadAnyLaneToVgpr(*${copy}, ${matchinfo}); }])>;
+
// (and/or i64:x, i64:y) -> i64:(merge (and/or lo_32(x), lo_32(y)), (and/or hi_32(x), hi_32(y)))
// when either x or y is all ones in low or high parts
class combine_binop_s64_with_s32_mask<Instruction opcode> : GICombineRule<
@@ -270,5 +278,6 @@ def AMDGPURegBankCombiner : GICombiner<
cast_of_cast_combines, sext_trunc, zext_of_shift_amount_combines,
d16_load, smax_to_minmax3, smin_to_minmax3, umax_to_minmax3,
umin_to_minmax3, fmax_to_minmax3, fmin_to_minmax3, fmaximum_to_minmax3,
- fminimum_to_minmax3, fmax_ieee_to_minmax3, fmin_ieee_to_minmax3]> {
+ fminimum_to_minmax3, fmax_ieee_to_minmax3, fmin_ieee_to_minmax3,
+ merge_readanylane_to_vgpr]> {
}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.h b/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.h
index 5e1000ee0ab26..7af71fd53ba28 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.h
@@ -9,7 +9,9 @@
#ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUGLOBALISELUTILS_H
#define LLVM_LIB_TARGET_AMDGPU_AMDGPUGLOBALISELUTILS_H
+#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
#include "llvm/ADT/DenseSet.h"
+#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
#include "llvm/CodeGen/Register.h"
#include <utility>
@@ -49,6 +51,14 @@ class IntrinsicLaneMaskAnalyzer {
void initLaneMaskIntrinsics(MachineFunction &MF);
};
+// AMDGPU-specific pattern matchers
+template <typename SrcTy>
+inline MIPatternMatch::UnaryOp_match<SrcTy, AMDGPU::G_AMDGPU_READANYLANE>
+m_GAMDGPUReadAnyLane(const SrcTy &Src) {
+ return MIPatternMatch::UnaryOp_match<SrcTy, AMDGPU::G_AMDGPU_READANYLANE>(
+ Src);
+}
+
void buildReadAnyLane(MachineIRBuilder &B, Register SgprDst, Register VgprSrc,
const RegisterBankInfo &RBI);
void buildReadFirstLane(MachineIRBuilder &B, Register SgprDst, Register VgprSrc,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankCombiner.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankCombiner.cpp
index e218bdd826bab..3a49712dfd565 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankCombiner.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankCombiner.cpp
@@ -12,6 +12,7 @@
//===----------------------------------------------------------------------===//
#include "AMDGPU.h"
+#include "AMDGPUGlobalISelUtils.h"
#include "AMDGPULegalizerInfo.h"
#include "AMDGPURegisterBankInfo.h"
#include "GCNSubtarget.h"
@@ -63,6 +64,7 @@ class AMDGPURegBankCombinerImpl : public Combiner {
bool tryCombineAll(MachineInstr &I) const override;
bool isVgprRegBank(Register Reg) const;
+ bool isSgprRegBank(Register Reg) const;
Register getAsVgpr(Register Reg) const;
struct MinMaxMedOpc {
@@ -103,6 +105,11 @@ class AMDGPURegBankCombinerImpl : public Combiner {
void applyMinMaxToMinMax3(MachineInstr &MI,
MinMaxToMinMax3MatchInfo &MatchInfo) const;
+ bool matchMergeReadAnyLaneToVgpr(MachineInstr &Copy,
+ SmallVector<Register> &Srcs) const;
+ void applyMergeReadAnyLaneToVgpr(MachineInstr &Copy,
+ SmallVectorImpl<Register> &Srcs) const;
+
private:
SIModeRegisterDefaults getMode() const;
bool getIEEE() const;
@@ -143,6 +150,10 @@ bool AMDGPURegBankCombinerImpl::isVgprRegBank(Register Reg) const {
return RBI.getRegBank(Reg, MRI, TRI)->getID() == AMDGPU::VGPRRegBankID;
}
+bool AMDGPURegBankCombinerImpl::isSgprRegBank(Register Reg) const {
+ return RBI.getRegBank(Reg, MRI, TRI)->getID() == AMDGPU::SGPRRegBankID;
+}
+
Register AMDGPURegBankCombinerImpl::getAsVgpr(Register Reg) const {
if (isVgprRegBank(Reg))
return Reg;
@@ -575,6 +586,73 @@ bool AMDGPURegBankCombinerImpl::matchMinMaxToMinMax3(
return true;
}
+// Sgpr0 = G_AMDGPU_READANYLANE Vgpr0
+// Src = G_MERGE_LIKE Sgpr0, Sgpr1, ...
+// Dst = COPY Src
+// ->
+// Vgpr1 = COPY Sgpr1
+// Dst = G_MERGE_LIKE Vgpr0, Vgpr1, ...
+//
+// Merge sources that are not readanylanes have to be uniform, copying them to a
+// vgpr is a broadcast. Requires at least one readanylane source, otherwise this
+// would only move the copy from the merge result to its sources.
+bool AMDGPURegBankCombinerImpl::matchMergeReadAnyLaneToVgpr(
+ MachineInstr &Copy, SmallVector<Register> &Srcs) const {
+ Register Dst = Copy.getOperand(0).getReg();
+ Register Src = Copy.getOperand(1).getReg();
+
+ if (!Dst.isVirtual() || !MRI.getRegBankOrNull(Dst) || !isVgprRegBank(Dst))
+ return false;
+
+ if (!Src.isVirtual() || MRI.getRegClassOrNull(Src))
+ return false;
+
+ GMergeLikeInstr *Merge;
+ if (!mi_match(Src, MRI, m_GMergeLikeInstr(Merge)))
+ return false;
+
+ bool HasReadAnyLaneSrc = false;
+ for (unsigned I = 0; I < Merge->getNumSources(); ++I) {
+ Register RALSrc;
+ Register MergeSrc = Merge->getSourceReg(I);
+ if (mi_match(MergeSrc, MRI, AMDGPU::m_GAMDGPUReadAnyLane(m_Reg(RALSrc)))) {
+ Srcs.push_back(RALSrc);
+ HasReadAnyLaneSrc = true;
+ continue;
+ }
+
+ if (isSgprRegBank(MergeSrc)) {
+ Srcs.push_back(MergeSrc);
+ continue;
+ }
+
+ return false;
+ }
+
+ if (!HasReadAnyLaneSrc)
+ return false;
+ return true;
+}
+
+void AMDGPURegBankCombinerImpl::applyMergeReadAnyLaneToVgpr(
+ MachineInstr &Copy, SmallVectorImpl<Register> &Srcs) const {
+ const RegisterBank &VgprRB = RBI.getRegBank(AMDGPU::VGPRRegBankID);
+ SmallVector<Register, 4> VgprSrcs;
+ for (Register Src : Srcs) {
+ if (isVgprRegBank(Src)) {
+ VgprSrcs.push_back(Src);
+ continue;
+ }
+
+ auto VgprCopy = B.buildCopy({&VgprRB, MRI.getType(Src)}, Src);
+ VgprSrcs.push_back(VgprCopy.getReg(0));
+ }
+
+ Register Dst = Copy.getOperand(0).getReg();
+ B.buildMergeLikeInstr(Dst, VgprSrcs);
+ Copy.eraseFromParent();
+}
+
bool AMDGPURegBankCombinerImpl::applyD16Load(
unsigned D16Opc, MachineInstr &DstMI, MachineInstr *SmallLoad,
Register SrcReg32ToOverwriteD16) const {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalize.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalize.cpp
index 19ba2f5ca6fb5..89d08432a90ad 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalize.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalize.cpp
@@ -40,13 +40,6 @@ using namespace llvm::MIPatternMatch;
namespace {
-// AMDGPU-specific pattern matchers
-template <typename SrcTy>
-inline UnaryOp_match<SrcTy, AMDGPU::G_AMDGPU_READANYLANE>
-m_GAMDGPUReadAnyLane(const SrcTy &Src) {
- return UnaryOp_match<SrcTy, AMDGPU::G_AMDGPU_READANYLANE>(Src);
-}
-
class AMDGPURegBankLegalize : public MachineFunctionPass {
public:
static char ID;
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/bug_shuffle_vector_to_scalar.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/bug_shuffle_vector_to_scalar.ll
index 9d40f03d99670..f08dd1012dd9a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/bug_shuffle_vector_to_scalar.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/bug_shuffle_vector_to_scalar.ll
@@ -8,21 +8,16 @@ define amdgpu_gs <4 x float> @_amdgpu_gs_main() {
; CHECK-LABEL: _amdgpu_gs_main:
; CHECK: ; %bb.0: ; %bb
; CHECK-NEXT: v_mov_b32_e32 v0, 16
-; CHECK-NEXT: ds_read2_b32 v[3:4], v0 offset1:1
+; CHECK-NEXT: ds_read2_b32 v[0:1], v0 offset1:1
; CHECK-NEXT: s_mov_b32 s0, 0
; CHECK-NEXT: s_mov_b32 s1, s0
; CHECK-NEXT: s_mov_b32 s2, s0
; CHECK-NEXT: s_mov_b32 s3, s0
+; CHECK-NEXT: v_mov_b32_e32 v2, 0
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: v_readfirstlane_b32 s4, v3
-; CHECK-NEXT: v_mov_b32_e32 v0, s4
-; CHECK-NEXT: v_mov_b32_e32 v1, s5
-; CHECK-NEXT: v_mov_b32_e32 v2, s6
-; CHECK-NEXT: v_mov_b32_e32 v3, s7
-; CHECK-NEXT: v_mov_b32_e32 v5, 0
-; CHECK-NEXT: buffer_store_dwordx4 v[0:3], v5, s[0:3], 0 idxen
+; CHECK-NEXT: buffer_store_dwordx4 v[0:3], v2, s[0:3], 0 idxen
; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: v_mov_b32_e32 v0, v4
+; CHECK-NEXT: v_mov_b32_e32 v0, v1
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: ; return to shader part epilog
bb:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.i8.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.i8.ll
index 620cba152f5a2..2b0a81b6d278b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.i8.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.i8.ll
@@ -248,9 +248,8 @@ define amdgpu_ps void @insertelement_s_v2i8_v_s(ptr addrspace(4) inreg %ptr, i8
; GFX9-NEXT: v_mov_b32_e32 v3, 0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s0, v1
-; GFX9-NEXT: s_lshr_b32 s1, s0, 8
-; GFX9-NEXT: v_mov_b32_e32 v2, s1
-; GFX9-NEXT: v_mov_b32_e32 v1, s0
+; GFX9-NEXT: s_lshr_b32 s0, s0, 8
+; GFX9-NEXT: v_mov_b32_e32 v2, s0
; GFX9-NEXT: s_set_gpr_idx_on s4, gpr_idx(DST)
; GFX9-NEXT: v_mov_b32_e32 v1, v0
; GFX9-NEXT: s_set_gpr_idx_off
@@ -270,9 +269,8 @@ define amdgpu_ps void @insertelement_s_v2i8_v_s(ptr addrspace(4) inreg %ptr, i8
; GFX8-NEXT: v_mov_b32_e32 v3, 0
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_readfirstlane_b32 s0, v1
-; GFX8-NEXT: s_lshr_b32 s1, s0, 8
-; GFX8-NEXT: v_mov_b32_e32 v2, s1
-; GFX8-NEXT: v_mov_b32_e32 v1, s0
+; GFX8-NEXT: s_lshr_b32 s0, s0, 8
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
; GFX8-NEXT: v_movreld_b32_e32 v1, v0
; GFX8-NEXT: v_and_b32_e32 v0, 0xff, v2
; GFX8-NEXT: v_lshlrev_b16_e32 v0, 8, v0
@@ -292,9 +290,8 @@ define amdgpu_ps void @insertelement_s_v2i8_v_s(ptr addrspace(4) inreg %ptr, i8
; GFX7-NEXT: s_mov_b64 s[0:1], 0
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_readfirstlane_b32 s4, v1
-; GFX7-NEXT: s_lshr_b32 s5, s4, 8
-; GFX7-NEXT: v_mov_b32_e32 v1, s4
-; GFX7-NEXT: v_mov_b32_e32 v2, s5
+; GFX7-NEXT: s_lshr_b32 s4, s4, 8
+; GFX7-NEXT: v_mov_b32_e32 v2, s4
; GFX7-NEXT: v_movreld_b32_e32 v1, v0
; GFX7-NEXT: v_and_b32_e32 v0, 0xff, v1
; GFX7-NEXT: v_and_b32_e32 v1, 0xff, v2
@@ -311,9 +308,8 @@ define amdgpu_ps void @insertelement_s_v2i8_v_s(ptr addrspace(4) inreg %ptr, i8
; GFX10-NEXT: global_load_ushort v1, v1, s[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_readfirstlane_b32 s0, v1
-; GFX10-NEXT: s_lshr_b32 s1, s0, 8
-; GFX10-NEXT: v_mov_b32_e32 v1, s0
-; GFX10-NEXT: v_mov_b32_e32 v2, s1
+; GFX10-NEXT: s_lshr_b32 s0, s0, 8
+; GFX10-NEXT: v_mov_b32_e32 v2, s0
; GFX10-NEXT: v_movreld_b32_e32 v1, v0
; GFX10-NEXT: v_mov_b32_e32 v0, 0xff
; GFX10-NEXT: v_and_b32_sdwa v0, v2, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -329,17 +325,17 @@ define amdgpu_ps void @insertelement_s_v2i8_v_s(ptr addrspace(4) inreg %ptr, i8
; GFX11-TRUE16-NEXT: global_load_u16 v1, v1, s[2:3]
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v1
-; GFX11-TRUE16-NEXT: s_and_b32 s1, 0xffff, s0
+; GFX11-TRUE16-NEXT: s_and_b32 s0, 0xffff, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s1, 8
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s0 :: v_dual_mov_b32 v2, s1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_lshr_b32 s0, s0, 8
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, s0
; GFX11-TRUE16-NEXT: v_movreld_b32_e32 v1, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.l
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, 8, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.h, v0.l
; GFX11-TRUE16-NEXT: global_store_b16 v[1:2], v0, off
; GFX11-TRUE16-NEXT: s_endpgm
@@ -351,16 +347,15 @@ define amdgpu_ps void @insertelement_s_v2i8_v_s(ptr addrspace(4) inreg %ptr, i8
; GFX11-FAKE16-NEXT: global_load_u16 v1, v1, s[2:3]
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v1
-; GFX11-FAKE16-NEXT: s_lshr_b32 s1, s0, 8
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, s0 :: v_dual_mov_b32 v2, s1
+; GFX11-FAKE16-NEXT: s_lshr_b32 s0, s0, 8
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, s0
; GFX11-FAKE16-NEXT: v_movreld_b32_e32 v1, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v2
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v2, 0xff, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v3, 8, v0
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v2, v3
; GFX11-FAKE16-NEXT: global_store_b16 v[0:1], v2, off
; GFX11-FAKE16-NEXT: s_endpgm
@@ -379,17 +374,16 @@ define amdgpu_ps void @insertelement_s_v2i8_s_v(ptr addrspace(4) inreg %ptr, i8
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s0, v1
-; GFX9-NEXT: s_lshr_b32 s1, s0, 8
+; GFX9-NEXT: s_lshr_b32 s0, s0, 8
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v1, v2, vcc
; GFX9-NEXT: v_mov_b32_e32 v1, s0
-; GFX9-NEXT: v_mov_b32_e32 v3, s1
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v1, v2, vcc
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v2, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX9-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: v_mov_b32_e32 v1, 0
-; GFX9-NEXT: v_or_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: global_store_short v[0:1], v2, off
; GFX9-NEXT: s_endpgm
;
@@ -402,17 +396,16 @@ define amdgpu_ps void @insertelement_s_v2i8_s_v(ptr addrspace(4) inreg %ptr, i8
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_readfirstlane_b32 s0, v1
-; GFX8-NEXT: s_lshr_b32 s1, s0, 8
+; GFX8-NEXT: s_lshr_b32 s0, s0, 8
+; GFX8-NEXT: v_cndmask_b32_e32 v3, v1, v2, vcc
; GFX8-NEXT: v_mov_b32_e32 v1, s0
-; GFX8-NEXT: v_mov_b32_e32 v3, s1
-; GFX8-NEXT: v_cndmask_b32_e32 v4, v1, v2, vcc
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v3, v2, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX8-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX8-NEXT: v_lshlrev_b16_e32 v2, 8, v0
; GFX8-NEXT: v_mov_b32_e32 v0, 0
; GFX8-NEXT: v_mov_b32_e32 v1, 0
-; GFX8-NEXT: v_or_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX8-NEXT: flat_store_short v[0:1], v2
; GFX8-NEXT: s_endpgm
;
@@ -428,10 +421,9 @@ define amdgpu_ps void @insertelement_s_v2i8_s_v(ptr addrspace(4) inreg %ptr, i8
; GFX7-NEXT: s_mov_b64 s[0:1], 0
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_readfirstlane_b32 s4, v1
-; GFX7-NEXT: s_lshr_b32 s5, s4, 8
-; GFX7-NEXT: v_mov_b32_e32 v1, s4
-; GFX7-NEXT: v_mov_b32_e32 v3, s5
+; GFX7-NEXT: s_lshr_b32 s4, s4, 8
; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
+; GFX7-NEXT: v_mov_b32_e32 v3, s4
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
; GFX7-NEXT: v_cndmask_b32_e32 v0, v3, v2, vcc
; GFX7-NEXT: v_and_b32_e32 v0, 0xff, v0
@@ -444,69 +436,65 @@ define amdgpu_ps void @insertelement_s_v2i8_s_v(ptr addrspace(4) inreg %ptr, i8
; GFX10-LABEL: insertelement_s_v2i8_s_v:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-NEXT: v_mov_b32_e32 v2, s4
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0
; GFX10-NEXT: v_mov_b32_e32 v3, 0xff
; GFX10-NEXT: global_load_ushort v1, v1, s[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_readfirstlane_b32 s0, v1
-; GFX10-NEXT: s_lshr_b32 s1, s0, 8
-; GFX10-NEXT: v_mov_b32_e32 v2, s0
-; GFX10-NEXT: v_mov_b32_e32 v1, s1
-; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, s4, vcc_lo
+; GFX10-NEXT: s_lshr_b32 s0, s0, 8
+; GFX10-NEXT: v_cndmask_b32_e32 v2, s0, v2, vcc_lo
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX10-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-NEXT: v_and_b32_sdwa v3, v1, v3 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, s4, vcc_lo
+; GFX10-NEXT: v_and_b32_sdwa v2, v2, v3 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_cndmask_b32_e64 v4, v1, s4, vcc_lo
; GFX10-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-NEXT: v_or_b32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX10-NEXT: global_store_short v[0:1], v2, off
; GFX10-NEXT: s_endpgm
;
; GFX11-TRUE16-LABEL: insertelement_s_v2i8_s_v:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s4
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0
; GFX11-TRUE16-NEXT: global_load_u16 v1, v1, s[2:3]
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v1
-; GFX11-TRUE16-NEXT: s_and_b32 s1, 0xffff, s0
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, s0
-; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s1, 8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, s1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v1, v1, s4, vcc_lo
+; GFX11-TRUE16-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_lshr_b32 s0, s0, 8
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, s0, v2, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v2, v2, s4, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v1, v1, s4, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.l
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, 8, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v2.l
+; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, 8, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.h, v0.l
; GFX11-TRUE16-NEXT: global_store_b16 v[1:2], v0, off
; GFX11-TRUE16-NEXT: s_endpgm
;
; GFX11-FAKE16-LABEL: insertelement_s_v2i8_s_v:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s4
; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0
; GFX11-FAKE16-NEXT: global_load_u16 v1, v1, s[2:3]
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v1
-; GFX11-FAKE16-NEXT: s_lshr_b32 s1, s0, 8
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v1, s1
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v1, v1, s4, vcc_lo
+; GFX11-FAKE16-NEXT: s_lshr_b32 s0, s0, 8
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v2, s0, v2, vcc_lo
; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v0, v2, s4, vcc_lo
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v3, 8, v1
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v2, 0xff, v0
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v0, v1, s4, vcc_lo
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xff, v0
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v3, 8, v1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, 0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v2, v3
; GFX11-FAKE16-NEXT: global_store_b16 v[0:1], v2, off
@@ -525,10 +513,9 @@ define amdgpu_ps void @insertelement_s_v2i8_v_v(ptr addrspace(4) inreg %ptr, i8
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s0, v2
-; GFX9-NEXT: s_lshr_b32 s1, s0, 8
-; GFX9-NEXT: v_mov_b32_e32 v2, s0
-; GFX9-NEXT: v_mov_b32_e32 v3, s1
+; GFX9-NEXT: s_lshr_b32 s0, s0, 8
; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
+; GFX9-NEXT: v_mov_b32_e32 v3, s0
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v1
; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc
; GFX9-NEXT: v_and_b32_e32 v0, 0xff, v0
@@ -547,10 +534,9 @@ define amdgpu_ps void @insertelement_s_v2i8_v_v(ptr addrspace(4) inreg %ptr, i8
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_readfirstlane_b32 s0, v2
-; GFX8-NEXT: s_lshr_b32 s1, s0, 8
-; GFX8-NEXT: v_mov_b32_e32 v2, s0
-; GFX8-NEXT: v_mov_b32_e32 v3, s1
+; GFX8-NEXT: s_lshr_b32 s0, s0, 8
; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
+; GFX8-NEXT: v_mov_b32_e32 v3, s0
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v1
; GFX8-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc
; GFX8-NEXT: v_and_b32_e32 v0, 0xff, v0
@@ -572,10 +558,9 @@ define amdgpu_ps void @insertelement_s_v2i8_v_v(ptr addrspace(4) inreg %ptr, i8
; GFX7-NEXT: s_mov_b64 s[0:1], 0
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_readfirstlane_b32 s4, v2
-; GFX7-NEXT: s_lshr_b32 s5, s4, 8
-; GFX7-NEXT: v_mov_b32_e32 v2, s4
-; GFX7-NEXT: v_mov_b32_e32 v3, s5
+; GFX7-NEXT: s_lshr_b32 s4, s4, 8
; GFX7-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
+; GFX7-NEXT: v_mov_b32_e32 v3, s4
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v1
; GFX7-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc
; GFX7-NEXT: v_and_b32_e32 v0, 0xff, v0
@@ -589,18 +574,18 @@ define amdgpu_ps void @insertelement_s_v2i8_v_v(ptr addrspace(4) inreg %ptr, i8
; GFX10: ; %bb.0:
; GFX10-NEXT: v_mov_b32_e32 v2, 0
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v1
-; GFX10-NEXT: v_mov_b32_e32 v3, 0xff
+; GFX10-NEXT: v_mov_b32_e32 v4, 0xff
; GFX10-NEXT: global_load_ushort v2, v2, s[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_readfirstlane_b32 s0, v2
-; GFX10-NEXT: s_lshr_b32 s1, s0, 8
-; GFX10-NEXT: v_cndmask_b32_e32 v2, s1, v0, vcc_lo
+; GFX10-NEXT: s_lshr_b32 s0, s0, 8
+; GFX10-NEXT: v_cndmask_b32_e32 v3, s0, v0, vcc_lo
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
; GFX10-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-NEXT: v_and_b32_sdwa v2, v2, v3 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT: v_cndmask_b32_e32 v4, s0, v0, vcc_lo
+; GFX10-NEXT: v_and_b32_sdwa v3, v3, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc_lo
; GFX10-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-NEXT: v_or_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT: v_or_b32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX10-NEXT: global_store_short v[0:1], v2, off
; GFX10-NEXT: s_endpgm
;
@@ -611,19 +596,18 @@ define amdgpu_ps void @insertelement_s_v2i8_v_v(ptr addrspace(4) inreg %ptr, i8
; GFX11-TRUE16-NEXT: global_load_u16 v2, v2, s[2:3]
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v2
-; GFX11-TRUE16-NEXT: s_and_b32 s1, 0xffff, s0
+; GFX11-TRUE16-NEXT: s_and_b32 s0, 0xffff, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s1, 8
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, s1, v0, vcc_lo
+; GFX11-TRUE16-NEXT: s_lshr_b32 s0, s0, 8
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, s0, v0, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, s0, v0, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v1, v2, v0 :: v_dual_mov_b32 v2, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v3.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, 8, v0.l
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, 8, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.h, v0.l
; GFX11-TRUE16-NEXT: global_store_b16 v[1:2], v0, off
; GFX11-TRUE16-NEXT: s_endpgm
@@ -635,15 +619,14 @@ define amdgpu_ps void @insertelement_s_v2i8_v_v(ptr addrspace(4) inreg %ptr, i8
; GFX11-FAKE16-NEXT: global_load_u16 v2, v2, s[2:3]
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v2
-; GFX11-FAKE16-NEXT: s_lshr_b32 s1, s0, 8
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v2, s1, v0, vcc_lo
+; GFX11-FAKE16-NEXT: s_lshr_b32 s0, s0, 8
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v3, s0, v0, vcc_lo
; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v2
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, s0, v0, vcc_lo
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc_lo
; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xff, v0
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_and_b32 v1, 0xff, v3
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v3, 8, v1
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, 0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.load.tfe.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.load.tfe.ll
index 0d0ab1388c0d6..0d153d6bc0664 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.load.tfe.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.load.tfe.ll
@@ -749,16 +749,13 @@ define amdgpu_ps void @raw_buffer_load_v2i32_tfe(<4 x i32> inreg %rsrc, ptr addr
; GFX67-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_OFFSET]].sub0
; GFX67-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_OFFSET]].sub1
; GFX67-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_OFFSET]].sub2
- ; GFX67-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
- ; GFX67-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
- ; GFX67-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; GFX67-NEXT: [[COPY11:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE3]]
+ ; GFX67-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY9]], %subreg.sub1
; GFX67-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 0
; GFX67-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
; GFX67-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_1]], %subreg.sub0, [[S_MOV_B32_2]], %subreg.sub1
; GFX67-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
; GFX67-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE4]], %subreg.sub2_sub3
- ; GFX67-NEXT: BUFFER_STORE_DWORDX2_ADDR64 [[COPY11]], [[REG_SEQUENCE1]], [[REG_SEQUENCE5]], 0, 0, 0, 0, implicit $exec :: (store (<2 x i32>) into %ir.data_addr, addrspace 1)
+ ; GFX67-NEXT: BUFFER_STORE_DWORDX2_ADDR64 [[REG_SEQUENCE3]], [[REG_SEQUENCE1]], [[REG_SEQUENCE5]], 0, 0, 0, 0, implicit $exec :: (store (<2 x i32>) into %ir.data_addr, addrspace 1)
; GFX67-NEXT: [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 0
; GFX67-NEXT: [[S_MOV_B32_4:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
; GFX67-NEXT: [[REG_SEQUENCE6:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_3]], %subreg.sub0, [[S_MOV_B32_4]], %subreg.sub1
@@ -787,11 +784,8 @@ define amdgpu_ps void @raw_buffer_load_v2i32_tfe(<4 x i32> inreg %rsrc, ptr addr
; GFX8-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_OFFSET]].sub0
; GFX8-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_OFFSET]].sub1
; GFX8-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_OFFSET]].sub2
- ; GFX8-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
- ; GFX8-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
- ; GFX8-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; GFX8-NEXT: [[COPY11:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE3]]
- ; GFX8-NEXT: FLAT_STORE_DWORDX2 [[REG_SEQUENCE1]], [[COPY11]], 0, 0, implicit $exec, implicit $flat_scr :: (store (<2 x i32>) into %ir.data_addr, addrspace 1)
+ ; GFX8-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY9]], %subreg.sub1
+ ; GFX8-NEXT: FLAT_STORE_DWORDX2 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec, implicit $flat_scr :: (store (<2 x i32>) into %ir.data_addr, addrspace 1)
; GFX8-NEXT: FLAT_STORE_DWORD [[REG_SEQUENCE2]], [[COPY10]], 0, 0, implicit $exec, implicit $flat_scr :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX8-NEXT: S_ENDPGM 0
;
@@ -815,11 +809,8 @@ define amdgpu_ps void @raw_buffer_load_v2i32_tfe(<4 x i32> inreg %rsrc, ptr addr
; GFX910-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_OFFSET]].sub0
; GFX910-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_OFFSET]].sub1
; GFX910-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_OFFSET]].sub2
- ; GFX910-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
- ; GFX910-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
- ; GFX910-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; GFX910-NEXT: [[COPY11:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE3]]
- ; GFX910-NEXT: GLOBAL_STORE_DWORDX2 [[REG_SEQUENCE1]], [[COPY11]], 0, 0, implicit $exec :: (store (<2 x i32>) into %ir.data_addr, addrspace 1)
+ ; GFX910-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY9]], %subreg.sub1
+ ; GFX910-NEXT: GLOBAL_STORE_DWORDX2 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (<2 x i32>) into %ir.data_addr, addrspace 1)
; GFX910-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY10]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX910-NEXT: S_ENDPGM 0
;
@@ -843,11 +834,8 @@ define amdgpu_ps void @raw_buffer_load_v2i32_tfe(<4 x i32> inreg %rsrc, ptr addr
; GFX11-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_OFFSET]].sub0
; GFX11-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_OFFSET]].sub1
; GFX11-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_OFFSET]].sub2
- ; GFX11-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
- ; GFX11-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
- ; GFX11-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; GFX11-NEXT: [[COPY11:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE3]]
- ; GFX11-NEXT: GLOBAL_STORE_DWORDX2 [[REG_SEQUENCE1]], [[COPY11]], 0, 0, implicit $exec :: (store (<2 x i32>) into %ir.data_addr, addrspace 1)
+ ; GFX11-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY9]], %subreg.sub1
+ ; GFX11-NEXT: GLOBAL_STORE_DWORDX2 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (<2 x i32>) into %ir.data_addr, addrspace 1)
; GFX11-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY10]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX11-NEXT: S_ENDPGM 0
;
@@ -870,11 +858,8 @@ define amdgpu_ps void @raw_buffer_load_v2i32_tfe(<4 x i32> inreg %rsrc, ptr addr
; GFX12-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_OFFSET]].sub0
; GFX12-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_OFFSET]].sub1
; GFX12-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_OFFSET]].sub2
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
- ; GFX12-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; GFX12-NEXT: [[COPY11:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE3]]
- ; GFX12-NEXT: GLOBAL_STORE_DWORDX2 [[REG_SEQUENCE1]], [[COPY11]], 0, 0, implicit $exec :: (store (<2 x i32>) into %ir.data_addr, addrspace 1)
+ ; GFX12-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY9]], %subreg.sub1
+ ; GFX12-NEXT: GLOBAL_STORE_DWORDX2 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (<2 x i32>) into %ir.data_addr, addrspace 1)
; GFX12-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY10]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX12-NEXT: S_ENDPGM 0
;
@@ -897,11 +882,8 @@ define amdgpu_ps void @raw_buffer_load_v2i32_tfe(<4 x i32> inreg %rsrc, ptr addr
; GFX1250-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_OFFSET]].sub0
; GFX1250-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_OFFSET]].sub1
; GFX1250-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_OFFSET]].sub2
- ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
- ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
- ; GFX1250-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; GFX1250-NEXT: [[COPY11:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE3]]
- ; GFX1250-NEXT: GLOBAL_STORE_DWORDX2 [[REG_SEQUENCE1]], [[COPY11]], 0, 0, implicit $exec :: (store (<2 x i32>) into %ir.data_addr, addrspace 1)
+ ; GFX1250-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY9]], %subreg.sub1
+ ; GFX1250-NEXT: GLOBAL_STORE_DWORDX2 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (<2 x i32>) into %ir.data_addr, addrspace 1)
; GFX1250-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY10]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX1250-NEXT: S_ENDPGM 0
%res = call { <2 x i32>, i32 } @llvm.amdgcn.raw.buffer.load.sl_v2i32i32s(<4 x i32> %rsrc, i32 0, i32 0, i32 0)
@@ -1167,17 +1149,13 @@ define amdgpu_ps void @raw_buffer_load_v3i32_tfe(<4 x i32> inreg %rsrc, ptr addr
; GFX7-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_OFFSET]].sub1
; GFX7-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_OFFSET]].sub2
; GFX7-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_OFFSET]].sub3
- ; GFX7-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
- ; GFX7-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
- ; GFX7-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
- ; GFX7-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_96 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2
- ; GFX7-NEXT: [[COPY12:%[0-9]+]]:vreg_96 = COPY [[REG_SEQUENCE3]]
+ ; GFX7-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_96 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY9]], %subreg.sub1, [[COPY10]], %subreg.sub2
; GFX7-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 0
; GFX7-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
; GFX7-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_1]], %subreg.sub0, [[S_MOV_B32_2]], %subreg.sub1
; GFX7-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
; GFX7-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE4]], %subreg.sub2_sub3
- ; GFX7-NEXT: BUFFER_STORE_DWORDX3_ADDR64 [[COPY12]], [[REG_SEQUENCE1]], [[REG_SEQUENCE5]], 0, 0, 0, 0, implicit $exec :: (store (<3 x i32>) into %ir.data_addr, align 16, addrspace 1)
+ ; GFX7-NEXT: BUFFER_STORE_DWORDX3_ADDR64 [[REG_SEQUENCE3]], [[REG_SEQUENCE1]], [[REG_SEQUENCE5]], 0, 0, 0, 0, implicit $exec :: (store (<3 x i32>) into %ir.data_addr, align 16, addrspace 1)
; GFX7-NEXT: [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 0
; GFX7-NEXT: [[S_MOV_B32_4:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
; GFX7-NEXT: [[REG_SEQUENCE6:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_3]], %subreg.sub0, [[S_MOV_B32_4]], %subreg.sub1
@@ -1207,12 +1185,8 @@ define amdgpu_ps void @raw_buffer_load_v3i32_tfe(<4 x i32> inreg %rsrc, ptr addr
; GFX8-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_OFFSET]].sub1
; GFX8-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_OFFSET]].sub2
; GFX8-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_OFFSET]].sub3
- ; GFX8-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
- ; GFX8-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
- ; GFX8-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
- ; GFX8-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_96 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2
- ; GFX8-NEXT: [[COPY12:%[0-9]+]]:vreg_96 = COPY [[REG_SEQUENCE3]]
- ; GFX8-NEXT: FLAT_STORE_DWORDX3 [[REG_SEQUENCE1]], [[COPY12]], 0, 0, implicit $exec, implicit $flat_scr :: (store (<3 x i32>) into %ir.data_addr, align 16, addrspace 1)
+ ; GFX8-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_96 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY9]], %subreg.sub1, [[COPY10]], %subreg.sub2
+ ; GFX8-NEXT: FLAT_STORE_DWORDX3 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec, implicit $flat_scr :: (store (<3 x i32>) into %ir.data_addr, align 16, addrspace 1)
; GFX8-NEXT: FLAT_STORE_DWORD [[REG_SEQUENCE2]], [[COPY11]], 0, 0, implicit $exec, implicit $flat_scr :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX8-NEXT: S_ENDPGM 0
;
@@ -1237,12 +1211,8 @@ define amdgpu_ps void @raw_buffer_load_v3i32_tfe(<4 x i32> inreg %rsrc, ptr addr
; GFX910-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_OFFSET]].sub1
; GFX910-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_OFFSET]].sub2
; GFX910-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_OFFSET]].sub3
- ; GFX910-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
- ; GFX910-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
- ; GFX910-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
- ; GFX910-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_96 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2
- ; GFX910-NEXT: [[COPY12:%[0-9]+]]:vreg_96 = COPY [[REG_SEQUENCE3]]
- ; GFX910-NEXT: GLOBAL_STORE_DWORDX3 [[REG_SEQUENCE1]], [[COPY12]], 0, 0, implicit $exec :: (store (<3 x i32>) into %ir.data_addr, align 16, addrspace 1)
+ ; GFX910-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_96 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY9]], %subreg.sub1, [[COPY10]], %subreg.sub2
+ ; GFX910-NEXT: GLOBAL_STORE_DWORDX3 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (<3 x i32>) into %ir.data_addr, align 16, addrspace 1)
; GFX910-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY11]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX910-NEXT: S_ENDPGM 0
;
@@ -1267,12 +1237,8 @@ define amdgpu_ps void @raw_buffer_load_v3i32_tfe(<4 x i32> inreg %rsrc, ptr addr
; GFX11-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_OFFSET]].sub1
; GFX11-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_OFFSET]].sub2
; GFX11-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_OFFSET]].sub3
- ; GFX11-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
- ; GFX11-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
- ; GFX11-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
- ; GFX11-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_96 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:vreg_96 = COPY [[REG_SEQUENCE3]]
- ; GFX11-NEXT: GLOBAL_STORE_DWORDX3 [[REG_SEQUENCE1]], [[COPY12]], 0, 0, implicit $exec :: (store (<3 x i32>) into %ir.data_addr, align 16, addrspace 1)
+ ; GFX11-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_96 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY9]], %subreg.sub1, [[COPY10]], %subreg.sub2
+ ; GFX11-NEXT: GLOBAL_STORE_DWORDX3 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (<3 x i32>) into %ir.data_addr, align 16, addrspace 1)
; GFX11-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY11]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX11-NEXT: S_ENDPGM 0
;
@@ -1296,12 +1262,8 @@ define amdgpu_ps void @raw_buffer_load_v3i32_tfe(<4 x i32> inreg %rsrc, ptr addr
; GFX12-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET]].sub1
; GFX12-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET]].sub2
; GFX12-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET]].sub3
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
- ; GFX12-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_96 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:vreg_96 = COPY [[REG_SEQUENCE3]]
- ; GFX12-NEXT: GLOBAL_STORE_DWORDX3 [[REG_SEQUENCE1]], [[COPY12]], 0, 0, implicit $exec :: (store (<3 x i32>) into %ir.data_addr, align 16, addrspace 1)
+ ; GFX12-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_96 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY9]], %subreg.sub1, [[COPY10]], %subreg.sub2
+ ; GFX12-NEXT: GLOBAL_STORE_DWORDX3 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (<3 x i32>) into %ir.data_addr, align 16, addrspace 1)
; GFX12-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY11]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX12-NEXT: S_ENDPGM 0
;
@@ -1325,12 +1287,8 @@ define amdgpu_ps void @raw_buffer_load_v3i32_tfe(<4 x i32> inreg %rsrc, ptr addr
; GFX1250-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET]].sub1
; GFX1250-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET]].sub2
; GFX1250-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET]].sub3
- ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
- ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
- ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
- ; GFX1250-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_96 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2
- ; GFX1250-NEXT: [[COPY12:%[0-9]+]]:vreg_96_align2 = COPY [[REG_SEQUENCE3]]
- ; GFX1250-NEXT: GLOBAL_STORE_DWORDX3 [[REG_SEQUENCE1]], [[COPY12]], 0, 0, implicit $exec :: (store (<3 x i32>) into %ir.data_addr, align 16, addrspace 1)
+ ; GFX1250-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_96_align2 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY9]], %subreg.sub1, [[COPY10]], %subreg.sub2
+ ; GFX1250-NEXT: GLOBAL_STORE_DWORDX3 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (<3 x i32>) into %ir.data_addr, align 16, addrspace 1)
; GFX1250-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY11]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX1250-NEXT: S_ENDPGM 0
%res = call { <3 x i32>, i32 } @llvm.amdgcn.raw.buffer.load.sl_v3i32i32s(<4 x i32> %rsrc, i32 0, i32 0, i32 0)
@@ -1607,18 +1565,13 @@ define amdgpu_ps void @raw_buffer_load_v4i32_tfe(<4 x i32> inreg %rsrc, ptr addr
; GFX67-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_OFFSET]].sub2
; GFX67-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_OFFSET]].sub3
; GFX67-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_OFFSET]].sub4
- ; GFX67-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
- ; GFX67-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
- ; GFX67-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
- ; GFX67-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY11]], implicit $exec
- ; GFX67-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
- ; GFX67-NEXT: [[COPY13:%[0-9]+]]:vreg_128 = COPY [[REG_SEQUENCE3]]
+ ; GFX67-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY9]], %subreg.sub1, [[COPY10]], %subreg.sub2, [[COPY11]], %subreg.sub3
; GFX67-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 0
; GFX67-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
; GFX67-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_1]], %subreg.sub0, [[S_MOV_B32_2]], %subreg.sub1
; GFX67-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
; GFX67-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE4]], %subreg.sub2_sub3
- ; GFX67-NEXT: BUFFER_STORE_DWORDX4_ADDR64 [[COPY13]], [[REG_SEQUENCE1]], [[REG_SEQUENCE5]], 0, 0, 0, 0, implicit $exec :: (store (<4 x i32>) into %ir.data_addr, addrspace 1)
+ ; GFX67-NEXT: BUFFER_STORE_DWORDX4_ADDR64 [[REG_SEQUENCE3]], [[REG_SEQUENCE1]], [[REG_SEQUENCE5]], 0, 0, 0, 0, implicit $exec :: (store (<4 x i32>) into %ir.data_addr, addrspace 1)
; GFX67-NEXT: [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 0
; GFX67-NEXT: [[S_MOV_B32_4:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
; GFX67-NEXT: [[REG_SEQUENCE6:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_3]], %subreg.sub0, [[S_MOV_B32_4]], %subreg.sub1
@@ -1649,13 +1602,8 @@ define amdgpu_ps void @raw_buffer_load_v4i32_tfe(<4 x i32> inreg %rsrc, ptr addr
; GFX8-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_OFFSET]].sub2
; GFX8-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_OFFSET]].sub3
; GFX8-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_OFFSET]].sub4
- ; GFX8-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
- ; GFX8-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
- ; GFX8-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
- ; GFX8-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY11]], implicit $exec
- ; GFX8-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
- ; GFX8-NEXT: [[COPY13:%[0-9]+]]:vreg_128 = COPY [[REG_SEQUENCE3]]
- ; GFX8-NEXT: FLAT_STORE_DWORDX4 [[REG_SEQUENCE1]], [[COPY13]], 0, 0, implicit $exec, implicit $flat_scr :: (store (<4 x i32>) into %ir.data_addr, addrspace 1)
+ ; GFX8-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY9]], %subreg.sub1, [[COPY10]], %subreg.sub2, [[COPY11]], %subreg.sub3
+ ; GFX8-NEXT: FLAT_STORE_DWORDX4 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec, implicit $flat_scr :: (store (<4 x i32>) into %ir.data_addr, addrspace 1)
; GFX8-NEXT: FLAT_STORE_DWORD [[REG_SEQUENCE2]], [[COPY12]], 0, 0, implicit $exec, implicit $flat_scr :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX8-NEXT: S_ENDPGM 0
;
@@ -1681,13 +1629,8 @@ define amdgpu_ps void @raw_buffer_load_v4i32_tfe(<4 x i32> inreg %rsrc, ptr addr
; GFX910-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_OFFSET]].sub2
; GFX910-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_OFFSET]].sub3
; GFX910-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_OFFSET]].sub4
- ; GFX910-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
- ; GFX910-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
- ; GFX910-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
- ; GFX910-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY11]], implicit $exec
- ; GFX910-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
- ; GFX910-NEXT: [[COPY13:%[0-9]+]]:vreg_128 = COPY [[REG_SEQUENCE3]]
- ; GFX910-NEXT: GLOBAL_STORE_DWORDX4 [[REG_SEQUENCE1]], [[COPY13]], 0, 0, implicit $exec :: (store (<4 x i32>) into %ir.data_addr, addrspace 1)
+ ; GFX910-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY9]], %subreg.sub1, [[COPY10]], %subreg.sub2, [[COPY11]], %subreg.sub3
+ ; GFX910-NEXT: GLOBAL_STORE_DWORDX4 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (<4 x i32>) into %ir.data_addr, addrspace 1)
; GFX910-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY12]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX910-NEXT: S_ENDPGM 0
;
@@ -1713,13 +1656,8 @@ define amdgpu_ps void @raw_buffer_load_v4i32_tfe(<4 x i32> inreg %rsrc, ptr addr
; GFX11-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_OFFSET]].sub2
; GFX11-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_OFFSET]].sub3
; GFX11-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_OFFSET]].sub4
- ; GFX11-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
- ; GFX11-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
- ; GFX11-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
- ; GFX11-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY11]], implicit $exec
- ; GFX11-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:vreg_128 = COPY [[REG_SEQUENCE3]]
- ; GFX11-NEXT: GLOBAL_STORE_DWORDX4 [[REG_SEQUENCE1]], [[COPY13]], 0, 0, implicit $exec :: (store (<4 x i32>) into %ir.data_addr, addrspace 1)
+ ; GFX11-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY9]], %subreg.sub1, [[COPY10]], %subreg.sub2, [[COPY11]], %subreg.sub3
+ ; GFX11-NEXT: GLOBAL_STORE_DWORDX4 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (<4 x i32>) into %ir.data_addr, addrspace 1)
; GFX11-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY12]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX11-NEXT: S_ENDPGM 0
;
@@ -1744,13 +1682,8 @@ define amdgpu_ps void @raw_buffer_load_v4i32_tfe(<4 x i32> inreg %rsrc, ptr addr
; GFX12-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET]].sub2
; GFX12-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET]].sub3
; GFX12-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET]].sub4
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY11]], implicit $exec
- ; GFX12-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:vreg_128 = COPY [[REG_SEQUENCE3]]
- ; GFX12-NEXT: GLOBAL_STORE_DWORDX4 [[REG_SEQUENCE1]], [[COPY13]], 0, 0, implicit $exec :: (store (<4 x i32>) into %ir.data_addr, addrspace 1)
+ ; GFX12-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY9]], %subreg.sub1, [[COPY10]], %subreg.sub2, [[COPY11]], %subreg.sub3
+ ; GFX12-NEXT: GLOBAL_STORE_DWORDX4 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (<4 x i32>) into %ir.data_addr, addrspace 1)
; GFX12-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY12]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX12-NEXT: S_ENDPGM 0
;
@@ -1775,13 +1708,8 @@ define amdgpu_ps void @raw_buffer_load_v4i32_tfe(<4 x i32> inreg %rsrc, ptr addr
; GFX1250-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET]].sub2
; GFX1250-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET]].sub3
; GFX1250-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET]].sub4
- ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
- ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
- ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
- ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY11]], implicit $exec
- ; GFX1250-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
- ; GFX1250-NEXT: [[COPY13:%[0-9]+]]:vreg_128_align2 = COPY [[REG_SEQUENCE3]]
- ; GFX1250-NEXT: GLOBAL_STORE_DWORDX4 [[REG_SEQUENCE1]], [[COPY13]], 0, 0, implicit $exec :: (store (<4 x i32>) into %ir.data_addr, addrspace 1)
+ ; GFX1250-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY9]], %subreg.sub1, [[COPY10]], %subreg.sub2, [[COPY11]], %subreg.sub3
+ ; GFX1250-NEXT: GLOBAL_STORE_DWORDX4 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (<4 x i32>) into %ir.data_addr, addrspace 1)
; GFX1250-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY12]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX1250-NEXT: S_ENDPGM 0
%res = call { <4 x i32>, i32 } @llvm.amdgcn.raw.buffer.load.sl_v4i32i32s(<4 x i32> %rsrc, i32 0, i32 0, i32 0)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.format.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.format.ll
index ed4c1416c5f17..0fd166c22e7e5 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.format.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.format.ll
@@ -414,13 +414,8 @@ define amdgpu_cs void @struct_buffer_load_format_v4i32_tfe(<4 x i32> inreg %rsrc
; GFX8-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_XYZW_TFE_IDXEN]].sub2
; GFX8-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_XYZW_TFE_IDXEN]].sub3
; GFX8-NEXT: [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_XYZW_TFE_IDXEN]].sub4
- ; GFX8-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
- ; GFX8-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
- ; GFX8-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY11]], implicit $exec
- ; GFX8-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY12]], implicit $exec
- ; GFX8-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
- ; GFX8-NEXT: [[COPY14:%[0-9]+]]:vreg_128 = COPY [[REG_SEQUENCE3]]
- ; GFX8-NEXT: FLAT_STORE_DWORDX4 [[REG_SEQUENCE1]], [[COPY14]], 0, 0, implicit $exec, implicit $flat_scr :: (store (<4 x i32>) into %ir.value, addrspace 1)
+ ; GFX8-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1, [[COPY11]], %subreg.sub2, [[COPY12]], %subreg.sub3
+ ; GFX8-NEXT: FLAT_STORE_DWORDX4 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec, implicit $flat_scr :: (store (<4 x i32>) into %ir.value, addrspace 1)
; GFX8-NEXT: FLAT_STORE_DWORD [[REG_SEQUENCE2]], [[COPY13]], 0, 0, implicit $exec, implicit $flat_scr :: (store (i32) into %ir.status, addrspace 1)
; GFX8-NEXT: S_ENDPGM 0
;
@@ -447,13 +442,8 @@ define amdgpu_cs void @struct_buffer_load_format_v4i32_tfe(<4 x i32> inreg %rsrc
; GFX12-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_XYZW_TFE_VBUFFER_IDXEN]].sub2
; GFX12-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_XYZW_TFE_VBUFFER_IDXEN]].sub3
; GFX12-NEXT: [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_XYZW_TFE_VBUFFER_IDXEN]].sub4
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY11]], implicit $exec
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY12]], implicit $exec
- ; GFX12-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:vreg_128 = COPY [[REG_SEQUENCE3]]
- ; GFX12-NEXT: GLOBAL_STORE_DWORDX4 [[REG_SEQUENCE1]], [[COPY14]], 0, 0, implicit $exec :: (store (<4 x i32>) into %ir.value, addrspace 1)
+ ; GFX12-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1, [[COPY11]], %subreg.sub2, [[COPY12]], %subreg.sub3
+ ; GFX12-NEXT: GLOBAL_STORE_DWORDX4 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (<4 x i32>) into %ir.value, addrspace 1)
; GFX12-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY13]], 0, 0, implicit $exec :: (store (i32) into %ir.status, addrspace 1)
; GFX12-NEXT: S_ENDPGM 0
%load = call { <4 x i32>, i32 } @llvm.amdgcn.struct.buffer.load.format.sl_v4i32i32s(<4 x i32> %rsrc, i32 0, i32 0, i32 0, i32 0)
@@ -490,12 +480,8 @@ define amdgpu_cs void @struct_buffer_load_format_v3i32_tfe(<4 x i32> inreg %rsrc
; GFX8-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_XYZ_TFE_IDXEN]].sub1
; GFX8-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_XYZ_TFE_IDXEN]].sub2
; GFX8-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_XYZ_TFE_IDXEN]].sub3
- ; GFX8-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
- ; GFX8-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
- ; GFX8-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY11]], implicit $exec
- ; GFX8-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_96 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2
- ; GFX8-NEXT: [[COPY13:%[0-9]+]]:vreg_96 = COPY [[REG_SEQUENCE3]]
- ; GFX8-NEXT: FLAT_STORE_DWORDX3 [[REG_SEQUENCE1]], [[COPY13]], 0, 0, implicit $exec, implicit $flat_scr :: (store (<3 x i32>) into %ir.value, align 16, addrspace 1)
+ ; GFX8-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_96 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1, [[COPY11]], %subreg.sub2
+ ; GFX8-NEXT: FLAT_STORE_DWORDX3 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec, implicit $flat_scr :: (store (<3 x i32>) into %ir.value, align 16, addrspace 1)
; GFX8-NEXT: FLAT_STORE_DWORD [[REG_SEQUENCE2]], [[COPY12]], 0, 0, implicit $exec, implicit $flat_scr :: (store (i32) into %ir.status, addrspace 1)
; GFX8-NEXT: S_ENDPGM 0
;
@@ -521,12 +507,8 @@ define amdgpu_cs void @struct_buffer_load_format_v3i32_tfe(<4 x i32> inreg %rsrc
; GFX12-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_XYZ_TFE_VBUFFER_IDXEN]].sub1
; GFX12-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_XYZ_TFE_VBUFFER_IDXEN]].sub2
; GFX12-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_XYZ_TFE_VBUFFER_IDXEN]].sub3
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY11]], implicit $exec
- ; GFX12-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_96 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:vreg_96 = COPY [[REG_SEQUENCE3]]
- ; GFX12-NEXT: GLOBAL_STORE_DWORDX3 [[REG_SEQUENCE1]], [[COPY13]], 0, 0, implicit $exec :: (store (<3 x i32>) into %ir.value, align 16, addrspace 1)
+ ; GFX12-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_96 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1, [[COPY11]], %subreg.sub2
+ ; GFX12-NEXT: GLOBAL_STORE_DWORDX3 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (<3 x i32>) into %ir.value, align 16, addrspace 1)
; GFX12-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY12]], 0, 0, implicit $exec :: (store (i32) into %ir.status, addrspace 1)
; GFX12-NEXT: S_ENDPGM 0
%load = call { <3 x i32>, i32 } @llvm.amdgcn.struct.buffer.load.format.sl_v3i32i32s(<4 x i32> %rsrc, i32 0, i32 0, i32 0, i32 0)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.tfe.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.tfe.ll
index 81a44835c2c1e..20011be7866f3 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.tfe.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.tfe.ll
@@ -788,16 +788,13 @@ define amdgpu_ps void @raw_buffer_load_v2i32_tfe(<4 x i32> inreg %rsrc, ptr addr
; GFX67-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_IDXEN]].sub0
; GFX67-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_IDXEN]].sub1
; GFX67-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_IDXEN]].sub2
- ; GFX67-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
- ; GFX67-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
- ; GFX67-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; GFX67-NEXT: [[COPY12:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE3]]
+ ; GFX67-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1
; GFX67-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 0
; GFX67-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
; GFX67-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_1]], %subreg.sub0, [[S_MOV_B32_2]], %subreg.sub1
; GFX67-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
; GFX67-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE4]], %subreg.sub2_sub3
- ; GFX67-NEXT: BUFFER_STORE_DWORDX2_ADDR64 [[COPY12]], [[REG_SEQUENCE1]], [[REG_SEQUENCE5]], 0, 0, 0, 0, implicit $exec :: (store (<2 x i32>) into %ir.data_addr, addrspace 1)
+ ; GFX67-NEXT: BUFFER_STORE_DWORDX2_ADDR64 [[REG_SEQUENCE3]], [[REG_SEQUENCE1]], [[REG_SEQUENCE5]], 0, 0, 0, 0, implicit $exec :: (store (<2 x i32>) into %ir.data_addr, addrspace 1)
; GFX67-NEXT: [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 0
; GFX67-NEXT: [[S_MOV_B32_4:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
; GFX67-NEXT: [[REG_SEQUENCE6:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_3]], %subreg.sub0, [[S_MOV_B32_4]], %subreg.sub1
@@ -827,11 +824,8 @@ define amdgpu_ps void @raw_buffer_load_v2i32_tfe(<4 x i32> inreg %rsrc, ptr addr
; GFX8-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_IDXEN]].sub0
; GFX8-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_IDXEN]].sub1
; GFX8-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_IDXEN]].sub2
- ; GFX8-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
- ; GFX8-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
- ; GFX8-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; GFX8-NEXT: [[COPY12:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE3]]
- ; GFX8-NEXT: FLAT_STORE_DWORDX2 [[REG_SEQUENCE1]], [[COPY12]], 0, 0, implicit $exec, implicit $flat_scr :: (store (<2 x i32>) into %ir.data_addr, addrspace 1)
+ ; GFX8-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1
+ ; GFX8-NEXT: FLAT_STORE_DWORDX2 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec, implicit $flat_scr :: (store (<2 x i32>) into %ir.data_addr, addrspace 1)
; GFX8-NEXT: FLAT_STORE_DWORD [[REG_SEQUENCE2]], [[COPY11]], 0, 0, implicit $exec, implicit $flat_scr :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX8-NEXT: S_ENDPGM 0
;
@@ -856,11 +850,8 @@ define amdgpu_ps void @raw_buffer_load_v2i32_tfe(<4 x i32> inreg %rsrc, ptr addr
; GFX910-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_IDXEN]].sub0
; GFX910-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_IDXEN]].sub1
; GFX910-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_IDXEN]].sub2
- ; GFX910-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
- ; GFX910-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
- ; GFX910-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; GFX910-NEXT: [[COPY12:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE3]]
- ; GFX910-NEXT: GLOBAL_STORE_DWORDX2 [[REG_SEQUENCE1]], [[COPY12]], 0, 0, implicit $exec :: (store (<2 x i32>) into %ir.data_addr, addrspace 1)
+ ; GFX910-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1
+ ; GFX910-NEXT: GLOBAL_STORE_DWORDX2 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (<2 x i32>) into %ir.data_addr, addrspace 1)
; GFX910-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY11]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX910-NEXT: S_ENDPGM 0
;
@@ -885,11 +876,8 @@ define amdgpu_ps void @raw_buffer_load_v2i32_tfe(<4 x i32> inreg %rsrc, ptr addr
; GFX11-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_IDXEN]].sub0
; GFX11-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_IDXEN]].sub1
; GFX11-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_IDXEN]].sub2
- ; GFX11-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
- ; GFX11-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
- ; GFX11-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE3]]
- ; GFX11-NEXT: GLOBAL_STORE_DWORDX2 [[REG_SEQUENCE1]], [[COPY12]], 0, 0, implicit $exec :: (store (<2 x i32>) into %ir.data_addr, addrspace 1)
+ ; GFX11-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1
+ ; GFX11-NEXT: GLOBAL_STORE_DWORDX2 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (<2 x i32>) into %ir.data_addr, addrspace 1)
; GFX11-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY11]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX11-NEXT: S_ENDPGM 0
;
@@ -914,11 +902,8 @@ define amdgpu_ps void @raw_buffer_load_v2i32_tfe(<4 x i32> inreg %rsrc, ptr addr
; GFX12-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_IDXEN]].sub0
; GFX12-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_IDXEN]].sub1
; GFX12-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_IDXEN]].sub2
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
- ; GFX12-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE3]]
- ; GFX12-NEXT: GLOBAL_STORE_DWORDX2 [[REG_SEQUENCE1]], [[COPY12]], 0, 0, implicit $exec :: (store (<2 x i32>) into %ir.data_addr, addrspace 1)
+ ; GFX12-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1
+ ; GFX12-NEXT: GLOBAL_STORE_DWORDX2 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (<2 x i32>) into %ir.data_addr, addrspace 1)
; GFX12-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY11]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX12-NEXT: S_ENDPGM 0
;
@@ -943,11 +928,8 @@ define amdgpu_ps void @raw_buffer_load_v2i32_tfe(<4 x i32> inreg %rsrc, ptr addr
; GFX1250-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_IDXEN]].sub0
; GFX1250-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_IDXEN]].sub1
; GFX1250-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_IDXEN]].sub2
- ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
- ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
- ; GFX1250-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; GFX1250-NEXT: [[COPY12:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE3]]
- ; GFX1250-NEXT: GLOBAL_STORE_DWORDX2 [[REG_SEQUENCE1]], [[COPY12]], 0, 0, implicit $exec :: (store (<2 x i32>) into %ir.data_addr, addrspace 1)
+ ; GFX1250-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1
+ ; GFX1250-NEXT: GLOBAL_STORE_DWORDX2 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (<2 x i32>) into %ir.data_addr, addrspace 1)
; GFX1250-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY11]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX1250-NEXT: S_ENDPGM 0
%res = call { <2 x i32>, i32 } @llvm.amdgcn.struct.buffer.load.sl_v2i32i32s(<4 x i32> %rsrc, i32 0, i32 0, i32 0, i32 0)
@@ -1223,17 +1205,13 @@ define amdgpu_ps void @raw_buffer_load_v3i32_tfe(<4 x i32> inreg %rsrc, ptr addr
; GFX7-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_IDXEN]].sub1
; GFX7-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_IDXEN]].sub2
; GFX7-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_IDXEN]].sub3
- ; GFX7-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
- ; GFX7-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
- ; GFX7-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY11]], implicit $exec
- ; GFX7-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_96 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2
- ; GFX7-NEXT: [[COPY13:%[0-9]+]]:vreg_96 = COPY [[REG_SEQUENCE3]]
+ ; GFX7-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_96 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1, [[COPY11]], %subreg.sub2
; GFX7-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 0
; GFX7-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
; GFX7-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_1]], %subreg.sub0, [[S_MOV_B32_2]], %subreg.sub1
; GFX7-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
; GFX7-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE4]], %subreg.sub2_sub3
- ; GFX7-NEXT: BUFFER_STORE_DWORDX3_ADDR64 [[COPY13]], [[REG_SEQUENCE1]], [[REG_SEQUENCE5]], 0, 0, 0, 0, implicit $exec :: (store (<3 x i32>) into %ir.data_addr, align 16, addrspace 1)
+ ; GFX7-NEXT: BUFFER_STORE_DWORDX3_ADDR64 [[REG_SEQUENCE3]], [[REG_SEQUENCE1]], [[REG_SEQUENCE5]], 0, 0, 0, 0, implicit $exec :: (store (<3 x i32>) into %ir.data_addr, align 16, addrspace 1)
; GFX7-NEXT: [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 0
; GFX7-NEXT: [[S_MOV_B32_4:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
; GFX7-NEXT: [[REG_SEQUENCE6:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_3]], %subreg.sub0, [[S_MOV_B32_4]], %subreg.sub1
@@ -1264,12 +1242,8 @@ define amdgpu_ps void @raw_buffer_load_v3i32_tfe(<4 x i32> inreg %rsrc, ptr addr
; GFX8-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_IDXEN]].sub1
; GFX8-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_IDXEN]].sub2
; GFX8-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_IDXEN]].sub3
- ; GFX8-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
- ; GFX8-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
- ; GFX8-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY11]], implicit $exec
- ; GFX8-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_96 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2
- ; GFX8-NEXT: [[COPY13:%[0-9]+]]:vreg_96 = COPY [[REG_SEQUENCE3]]
- ; GFX8-NEXT: FLAT_STORE_DWORDX3 [[REG_SEQUENCE1]], [[COPY13]], 0, 0, implicit $exec, implicit $flat_scr :: (store (<3 x i32>) into %ir.data_addr, align 16, addrspace 1)
+ ; GFX8-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_96 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1, [[COPY11]], %subreg.sub2
+ ; GFX8-NEXT: FLAT_STORE_DWORDX3 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec, implicit $flat_scr :: (store (<3 x i32>) into %ir.data_addr, align 16, addrspace 1)
; GFX8-NEXT: FLAT_STORE_DWORD [[REG_SEQUENCE2]], [[COPY12]], 0, 0, implicit $exec, implicit $flat_scr :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX8-NEXT: S_ENDPGM 0
;
@@ -1295,12 +1269,8 @@ define amdgpu_ps void @raw_buffer_load_v3i32_tfe(<4 x i32> inreg %rsrc, ptr addr
; GFX910-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_IDXEN]].sub1
; GFX910-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_IDXEN]].sub2
; GFX910-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_IDXEN]].sub3
- ; GFX910-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
- ; GFX910-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
- ; GFX910-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY11]], implicit $exec
- ; GFX910-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_96 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2
- ; GFX910-NEXT: [[COPY13:%[0-9]+]]:vreg_96 = COPY [[REG_SEQUENCE3]]
- ; GFX910-NEXT: GLOBAL_STORE_DWORDX3 [[REG_SEQUENCE1]], [[COPY13]], 0, 0, implicit $exec :: (store (<3 x i32>) into %ir.data_addr, align 16, addrspace 1)
+ ; GFX910-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_96 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1, [[COPY11]], %subreg.sub2
+ ; GFX910-NEXT: GLOBAL_STORE_DWORDX3 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (<3 x i32>) into %ir.data_addr, align 16, addrspace 1)
; GFX910-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY12]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX910-NEXT: S_ENDPGM 0
;
@@ -1326,12 +1296,8 @@ define amdgpu_ps void @raw_buffer_load_v3i32_tfe(<4 x i32> inreg %rsrc, ptr addr
; GFX11-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_IDXEN]].sub1
; GFX11-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_IDXEN]].sub2
; GFX11-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_IDXEN]].sub3
- ; GFX11-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
- ; GFX11-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
- ; GFX11-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY11]], implicit $exec
- ; GFX11-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_96 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:vreg_96 = COPY [[REG_SEQUENCE3]]
- ; GFX11-NEXT: GLOBAL_STORE_DWORDX3 [[REG_SEQUENCE1]], [[COPY13]], 0, 0, implicit $exec :: (store (<3 x i32>) into %ir.data_addr, align 16, addrspace 1)
+ ; GFX11-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_96 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1, [[COPY11]], %subreg.sub2
+ ; GFX11-NEXT: GLOBAL_STORE_DWORDX3 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (<3 x i32>) into %ir.data_addr, align 16, addrspace 1)
; GFX11-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY12]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX11-NEXT: S_ENDPGM 0
;
@@ -1357,12 +1323,8 @@ define amdgpu_ps void @raw_buffer_load_v3i32_tfe(<4 x i32> inreg %rsrc, ptr addr
; GFX12-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN]].sub1
; GFX12-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN]].sub2
; GFX12-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN]].sub3
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY11]], implicit $exec
- ; GFX12-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_96 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:vreg_96 = COPY [[REG_SEQUENCE3]]
- ; GFX12-NEXT: GLOBAL_STORE_DWORDX3 [[REG_SEQUENCE1]], [[COPY13]], 0, 0, implicit $exec :: (store (<3 x i32>) into %ir.data_addr, align 16, addrspace 1)
+ ; GFX12-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_96 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1, [[COPY11]], %subreg.sub2
+ ; GFX12-NEXT: GLOBAL_STORE_DWORDX3 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (<3 x i32>) into %ir.data_addr, align 16, addrspace 1)
; GFX12-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY12]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX12-NEXT: S_ENDPGM 0
;
@@ -1388,12 +1350,8 @@ define amdgpu_ps void @raw_buffer_load_v3i32_tfe(<4 x i32> inreg %rsrc, ptr addr
; GFX1250-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN]].sub1
; GFX1250-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN]].sub2
; GFX1250-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN]].sub3
- ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
- ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
- ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY11]], implicit $exec
- ; GFX1250-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_96 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2
- ; GFX1250-NEXT: [[COPY13:%[0-9]+]]:vreg_96_align2 = COPY [[REG_SEQUENCE3]]
- ; GFX1250-NEXT: GLOBAL_STORE_DWORDX3 [[REG_SEQUENCE1]], [[COPY13]], 0, 0, implicit $exec :: (store (<3 x i32>) into %ir.data_addr, align 16, addrspace 1)
+ ; GFX1250-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_96_align2 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1, [[COPY11]], %subreg.sub2
+ ; GFX1250-NEXT: GLOBAL_STORE_DWORDX3 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (<3 x i32>) into %ir.data_addr, align 16, addrspace 1)
; GFX1250-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY12]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX1250-NEXT: S_ENDPGM 0
%res = call { <3 x i32>, i32 } @llvm.amdgcn.struct.buffer.load.sl_v3i32i32s(<4 x i32> %rsrc, i32 0, i32 0, i32 0, i32 0)
@@ -1680,18 +1638,13 @@ define amdgpu_ps void @raw_buffer_load_v4i32_tfe(<4 x i32> inreg %rsrc, ptr addr
; GFX67-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_IDXEN]].sub2
; GFX67-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_IDXEN]].sub3
; GFX67-NEXT: [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_IDXEN]].sub4
- ; GFX67-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
- ; GFX67-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
- ; GFX67-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY11]], implicit $exec
- ; GFX67-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY12]], implicit $exec
- ; GFX67-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
- ; GFX67-NEXT: [[COPY14:%[0-9]+]]:vreg_128 = COPY [[REG_SEQUENCE3]]
+ ; GFX67-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1, [[COPY11]], %subreg.sub2, [[COPY12]], %subreg.sub3
; GFX67-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 0
; GFX67-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
; GFX67-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_1]], %subreg.sub0, [[S_MOV_B32_2]], %subreg.sub1
; GFX67-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
; GFX67-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE4]], %subreg.sub2_sub3
- ; GFX67-NEXT: BUFFER_STORE_DWORDX4_ADDR64 [[COPY14]], [[REG_SEQUENCE1]], [[REG_SEQUENCE5]], 0, 0, 0, 0, implicit $exec :: (store (<4 x i32>) into %ir.data_addr, addrspace 1)
+ ; GFX67-NEXT: BUFFER_STORE_DWORDX4_ADDR64 [[REG_SEQUENCE3]], [[REG_SEQUENCE1]], [[REG_SEQUENCE5]], 0, 0, 0, 0, implicit $exec :: (store (<4 x i32>) into %ir.data_addr, addrspace 1)
; GFX67-NEXT: [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 0
; GFX67-NEXT: [[S_MOV_B32_4:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
; GFX67-NEXT: [[REG_SEQUENCE6:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_3]], %subreg.sub0, [[S_MOV_B32_4]], %subreg.sub1
@@ -1723,13 +1676,8 @@ define amdgpu_ps void @raw_buffer_load_v4i32_tfe(<4 x i32> inreg %rsrc, ptr addr
; GFX8-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_IDXEN]].sub2
; GFX8-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_IDXEN]].sub3
; GFX8-NEXT: [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_IDXEN]].sub4
- ; GFX8-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
- ; GFX8-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
- ; GFX8-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY11]], implicit $exec
- ; GFX8-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY12]], implicit $exec
- ; GFX8-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
- ; GFX8-NEXT: [[COPY14:%[0-9]+]]:vreg_128 = COPY [[REG_SEQUENCE3]]
- ; GFX8-NEXT: FLAT_STORE_DWORDX4 [[REG_SEQUENCE1]], [[COPY14]], 0, 0, implicit $exec, implicit $flat_scr :: (store (<4 x i32>) into %ir.data_addr, addrspace 1)
+ ; GFX8-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1, [[COPY11]], %subreg.sub2, [[COPY12]], %subreg.sub3
+ ; GFX8-NEXT: FLAT_STORE_DWORDX4 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec, implicit $flat_scr :: (store (<4 x i32>) into %ir.data_addr, addrspace 1)
; GFX8-NEXT: FLAT_STORE_DWORD [[REG_SEQUENCE2]], [[COPY13]], 0, 0, implicit $exec, implicit $flat_scr :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX8-NEXT: S_ENDPGM 0
;
@@ -1756,13 +1704,8 @@ define amdgpu_ps void @raw_buffer_load_v4i32_tfe(<4 x i32> inreg %rsrc, ptr addr
; GFX910-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_IDXEN]].sub2
; GFX910-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_IDXEN]].sub3
; GFX910-NEXT: [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_IDXEN]].sub4
- ; GFX910-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
- ; GFX910-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
- ; GFX910-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY11]], implicit $exec
- ; GFX910-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY12]], implicit $exec
- ; GFX910-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
- ; GFX910-NEXT: [[COPY14:%[0-9]+]]:vreg_128 = COPY [[REG_SEQUENCE3]]
- ; GFX910-NEXT: GLOBAL_STORE_DWORDX4 [[REG_SEQUENCE1]], [[COPY14]], 0, 0, implicit $exec :: (store (<4 x i32>) into %ir.data_addr, addrspace 1)
+ ; GFX910-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1, [[COPY11]], %subreg.sub2, [[COPY12]], %subreg.sub3
+ ; GFX910-NEXT: GLOBAL_STORE_DWORDX4 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (<4 x i32>) into %ir.data_addr, addrspace 1)
; GFX910-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY13]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX910-NEXT: S_ENDPGM 0
;
@@ -1789,13 +1732,8 @@ define amdgpu_ps void @raw_buffer_load_v4i32_tfe(<4 x i32> inreg %rsrc, ptr addr
; GFX11-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_IDXEN]].sub2
; GFX11-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_IDXEN]].sub3
; GFX11-NEXT: [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_IDXEN]].sub4
- ; GFX11-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
- ; GFX11-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
- ; GFX11-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY11]], implicit $exec
- ; GFX11-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY12]], implicit $exec
- ; GFX11-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:vreg_128 = COPY [[REG_SEQUENCE3]]
- ; GFX11-NEXT: GLOBAL_STORE_DWORDX4 [[REG_SEQUENCE1]], [[COPY14]], 0, 0, implicit $exec :: (store (<4 x i32>) into %ir.data_addr, addrspace 1)
+ ; GFX11-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1, [[COPY11]], %subreg.sub2, [[COPY12]], %subreg.sub3
+ ; GFX11-NEXT: GLOBAL_STORE_DWORDX4 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (<4 x i32>) into %ir.data_addr, addrspace 1)
; GFX11-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY13]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX11-NEXT: S_ENDPGM 0
;
@@ -1822,13 +1760,8 @@ define amdgpu_ps void @raw_buffer_load_v4i32_tfe(<4 x i32> inreg %rsrc, ptr addr
; GFX12-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN]].sub2
; GFX12-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN]].sub3
; GFX12-NEXT: [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN]].sub4
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY11]], implicit $exec
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY12]], implicit $exec
- ; GFX12-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:vreg_128 = COPY [[REG_SEQUENCE3]]
- ; GFX12-NEXT: GLOBAL_STORE_DWORDX4 [[REG_SEQUENCE1]], [[COPY14]], 0, 0, implicit $exec :: (store (<4 x i32>) into %ir.data_addr, addrspace 1)
+ ; GFX12-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1, [[COPY11]], %subreg.sub2, [[COPY12]], %subreg.sub3
+ ; GFX12-NEXT: GLOBAL_STORE_DWORDX4 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (<4 x i32>) into %ir.data_addr, addrspace 1)
; GFX12-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY13]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX12-NEXT: S_ENDPGM 0
;
@@ -1855,13 +1788,8 @@ define amdgpu_ps void @raw_buffer_load_v4i32_tfe(<4 x i32> inreg %rsrc, ptr addr
; GFX1250-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN]].sub2
; GFX1250-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN]].sub3
; GFX1250-NEXT: [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN]].sub4
- ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
- ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
- ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY11]], implicit $exec
- ; GFX1250-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY12]], implicit $exec
- ; GFX1250-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
- ; GFX1250-NEXT: [[COPY14:%[0-9]+]]:vreg_128_align2 = COPY [[REG_SEQUENCE3]]
- ; GFX1250-NEXT: GLOBAL_STORE_DWORDX4 [[REG_SEQUENCE1]], [[COPY14]], 0, 0, implicit $exec :: (store (<4 x i32>) into %ir.data_addr, addrspace 1)
+ ; GFX1250-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1, [[COPY11]], %subreg.sub2, [[COPY12]], %subreg.sub3
+ ; GFX1250-NEXT: GLOBAL_STORE_DWORDX4 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (<4 x i32>) into %ir.data_addr, addrspace 1)
; GFX1250-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY13]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX1250-NEXT: S_ENDPGM 0
%res = call { <4 x i32>, i32 } @llvm.amdgcn.struct.buffer.load.sl_v4i32i32s(<4 x i32> %rsrc, i32 0, i32 0, i32 0, i32 0)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.load.format.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.load.format.ll
index 5f57d603a7c89..c78b29a5fcc0f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.load.format.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.load.format.ll
@@ -235,13 +235,8 @@ define amdgpu_cs void @struct_ptr_buffer_load_format_v4i32_tfe(ptr addrspace(8)
; CHECK-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_XYZW_TFE_IDXEN]].sub2
; CHECK-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_XYZW_TFE_IDXEN]].sub3
; CHECK-NEXT: [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_XYZW_TFE_IDXEN]].sub4
- ; CHECK-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
- ; CHECK-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
- ; CHECK-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY11]], implicit $exec
- ; CHECK-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY12]], implicit $exec
- ; CHECK-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
- ; CHECK-NEXT: [[COPY14:%[0-9]+]]:vreg_128 = COPY [[REG_SEQUENCE3]]
- ; CHECK-NEXT: FLAT_STORE_DWORDX4 [[REG_SEQUENCE]], [[COPY14]], 0, 0, implicit $exec, implicit $flat_scr :: (store (<4 x i32>) into %ir.value, addrspace 1)
+ ; CHECK-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1, [[COPY11]], %subreg.sub2, [[COPY12]], %subreg.sub3
+ ; CHECK-NEXT: FLAT_STORE_DWORDX4 [[REG_SEQUENCE]], [[REG_SEQUENCE3]], 0, 0, implicit $exec, implicit $flat_scr :: (store (<4 x i32>) into %ir.value, addrspace 1)
; CHECK-NEXT: FLAT_STORE_DWORD [[REG_SEQUENCE1]], [[COPY13]], 0, 0, implicit $exec, implicit $flat_scr :: (store (i32) into %ir.status, addrspace 1)
; CHECK-NEXT: S_ENDPGM 0
%load = call { <4 x i32>, i32 } @llvm.amdgcn.struct.ptr.buffer.load.format.sl_v4i32i32s(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0)
@@ -278,12 +273,8 @@ define amdgpu_cs void @struct_ptr_buffer_load_format_v3i32_tfe(ptr addrspace(8)
; CHECK-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_XYZ_TFE_IDXEN]].sub1
; CHECK-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_XYZ_TFE_IDXEN]].sub2
; CHECK-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_XYZ_TFE_IDXEN]].sub3
- ; CHECK-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
- ; CHECK-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
- ; CHECK-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY11]], implicit $exec
- ; CHECK-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_96 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2
- ; CHECK-NEXT: [[COPY13:%[0-9]+]]:vreg_96 = COPY [[REG_SEQUENCE3]]
- ; CHECK-NEXT: FLAT_STORE_DWORDX3 [[REG_SEQUENCE]], [[COPY13]], 0, 0, implicit $exec, implicit $flat_scr :: (store (<3 x i32>) into %ir.value, align 16, addrspace 1)
+ ; CHECK-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_96 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1, [[COPY11]], %subreg.sub2
+ ; CHECK-NEXT: FLAT_STORE_DWORDX3 [[REG_SEQUENCE]], [[REG_SEQUENCE3]], 0, 0, implicit $exec, implicit $flat_scr :: (store (<3 x i32>) into %ir.value, align 16, addrspace 1)
; CHECK-NEXT: FLAT_STORE_DWORD [[REG_SEQUENCE1]], [[COPY12]], 0, 0, implicit $exec, implicit $flat_scr :: (store (i32) into %ir.status, addrspace 1)
; CHECK-NEXT: S_ENDPGM 0
%load = call { <3 x i32>, i32 } @llvm.amdgcn.struct.ptr.buffer.load.format.sl_v3i32i32s(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll
index 46fb53085ebf0..d121f4dd8dc42 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll
@@ -3863,15 +3863,13 @@ define amdgpu_kernel void @s_mul_u64_zext_with_sregs(ptr addrspace(1) %out, ptr
; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GFX7-NEXT: v_mov_b32_e32 v0, 0x50
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_load_dword s3, s[2:3], 0x0
-; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_load_dword s2, s[2:3], 0x0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mul_hi_u32 v0, s3, v0
-; GFX7-NEXT: s_mul_i32 s4, s3, 0x50
+; GFX7-NEXT: v_mul_hi_u32 v1, s2, v0
+; GFX7-NEXT: s_mul_i32 s3, s2, 0x50
+; GFX7-NEXT: v_mov_b32_e32 v0, s3
+; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: v_readfirstlane_b32 s5, v0
-; GFX7-NEXT: v_mov_b32_e32 v0, s4
-; GFX7-NEXT: v_mov_b32_e32 v1, s5
; GFX7-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
@@ -3884,11 +3882,9 @@ define amdgpu_kernel void @s_mul_u64_zext_with_sregs(ptr addrspace(1) %out, ptr
; GFX8-NEXT: v_mov_b32_e32 v3, s1
; GFX8-NEXT: v_mov_b32_e32 v2, s0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mul_hi_u32 v0, s2, v0
+; GFX8-NEXT: v_mul_hi_u32 v1, s2, v0
; GFX8-NEXT: s_mulk_i32 s2, 0x50
-; GFX8-NEXT: v_readfirstlane_b32 s3, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s2
-; GFX8-NEXT: v_mov_b32_e32 v1, s3
; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; GFX8-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankcombiner-merge-readanylane.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankcombiner-merge-readanylane.mir
index cffbcff999574..fe6f6605f94d0 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankcombiner-merge-readanylane.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankcombiner-merge-readanylane.mir
@@ -20,10 +20,9 @@ body: |
; CHECK-NEXT: [[COPY3:%[0-9]+]]:sgpr(s32) = COPY $sgpr3
; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[COPY3]](s32)
; CHECK-NEXT: [[FCANONICALIZE:%[0-9]+]]:vgpr(s32) = G_FCANONICALIZE [[COPY4]]
- ; CHECK-NEXT: [[AMDGPU_READANYLANE:%[0-9]+]]:sgpr(s32) = G_AMDGPU_READANYLANE [[FCANONICALIZE]]
- ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<2 x s32>) = G_BUILD_VECTOR [[COPY2]](s32), [[AMDGPU_READANYLANE]](s32)
- ; CHECK-NEXT: [[COPY5:%[0-9]+]]:vgpr(<2 x s32>) = COPY [[BUILD_VECTOR]](<2 x s32>)
- ; CHECK-NEXT: G_STORE [[COPY5]](<2 x s32>), [[MV]](p1) :: (store (<2 x s32>), addrspace 1)
+ ; CHECK-NEXT: [[COPY5:%[0-9]+]]:vgpr(s32) = COPY [[COPY2]](s32)
+ ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:vgpr(<2 x s32>) = G_BUILD_VECTOR [[COPY5]](s32), [[FCANONICALIZE]](s32)
+ ; CHECK-NEXT: G_STORE [[BUILD_VECTOR]](<2 x s32>), [[MV]](p1) :: (store (<2 x s32>), addrspace 1)
; CHECK-NEXT: S_ENDPGM 0
%0:sgpr(s32) = COPY $sgpr0
%1:sgpr(s32) = COPY $sgpr1
@@ -58,9 +57,10 @@ body: |
; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
; CHECK-NEXT: [[AMDGPU_READANYLANE:%[0-9]+]]:sgpr(s32) = G_AMDGPU_READANYLANE [[COPY2]]
; CHECK-NEXT: [[MV1:%[0-9]+]]:sgpr(s64) = G_MERGE_VALUES [[COPY]](s32), [[AMDGPU_READANYLANE]](s32)
- ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(s64) = COPY [[MV1]](s64)
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32)
+ ; CHECK-NEXT: [[MV2:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[COPY3]](s32), [[COPY2]](s32)
; CHECK-NEXT: G_STORE [[MV1]](s64), [[MV]](p1) :: (store (s64), addrspace 1)
- ; CHECK-NEXT: G_STORE [[COPY3]](s64), [[MV]](p1) :: (store (s64), addrspace 1)
+ ; CHECK-NEXT: G_STORE [[MV2]](s64), [[MV]](p1) :: (store (s64), addrspace 1)
; CHECK-NEXT: S_ENDPGM 0
%0:sgpr(s32) = COPY $sgpr0
%1:sgpr(s32) = COPY $sgpr1
@@ -74,6 +74,37 @@ body: |
S_ENDPGM 0
...
+# The merge feeds a valu instruction that can take an sgpr operand, so
+# SIFoldOperands would fold the uniform value back in and undo the rewrite.
+---
+name: readanylane_and_uniform_merge_valu_use
+legalized: true
+regBankSelected: true
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr0, $vgpr0
+
+ ; CHECK-LABEL: name: readanylane_and_uniform_merge_valu_use
+ ; CHECK: liveins: $sgpr0, $vgpr0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32)
+ ; CHECK-NEXT: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[COPY2]](s32), [[COPY1]](s32)
+ ; CHECK-NEXT: [[ADD:%[0-9]+]]:vgpr(s64) = G_ADD [[MV]], [[MV]]
+ ; CHECK-NEXT: $vgpr0_vgpr1 = COPY [[ADD]](s64)
+ ; CHECK-NEXT: S_ENDPGM 0
+ %0:sgpr(s32) = COPY $sgpr0
+ %1:vgpr(s32) = COPY $vgpr0
+ %2:sgpr(s32) = G_AMDGPU_READANYLANE %1
+ %3:sgpr(s64) = G_MERGE_VALUES %0(s32), %2(s32)
+ %4:vgpr(s64) = COPY %3(s64)
+ %5:vgpr(s64) = G_ADD %4, %4
+ $vgpr0_vgpr1 = COPY %5(s64)
+ S_ENDPGM 0
+...
+
# All merge sources are uniform, moving the copy to the sources would not
# remove any readanylane.
---
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fma.f32.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fma.f32.ll
index cd2f12548aec6..b409bd5f3f673 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fma.f32.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fma.f32.ll
@@ -267,20 +267,13 @@ define void @v_constained_fma_v3f32_fpexcept_strict_uni(<3 x float> inreg %x, <3
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: s_mov_b32 s4, s3
; GFX942-NEXT: s_mov_b32 s5, s16
+; GFX942-NEXT: s_mov_b32 s6, s17
; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[4:5]
; GFX942-NEXT: v_mov_b64_e32 v[4:5], s[18:19]
-; GFX942-NEXT: s_mov_b32 s6, s17
; GFX942-NEXT: v_pk_fma_f32 v[2:3], s[0:1], v[2:3], v[4:5]
-; GFX942-NEXT: v_mov_b64_e32 v[4:5], s[20:21]
-; GFX942-NEXT: v_readfirstlane_b32 s0, v2
-; GFX942-NEXT: v_readfirstlane_b32 s1, v3
-; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7]
-; GFX942-NEXT: v_pk_fma_f32 v[2:3], s[2:3], v[2:3], v[4:5]
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_readfirstlane_b32 s2, v2
-; GFX942-NEXT: v_mov_b32_e32 v3, s1
-; GFX942-NEXT: v_mov_b32_e32 v2, s0
-; GFX942-NEXT: v_mov_b32_e32 v4, s2
+; GFX942-NEXT: v_mov_b64_e32 v[4:5], s[6:7]
+; GFX942-NEXT: v_mov_b64_e32 v[6:7], s[20:21]
+; GFX942-NEXT: v_pk_fma_f32 v[4:5], s[2:3], v[4:5], v[6:7]
; GFX942-NEXT: global_store_dwordx3 v[0:1], v[2:4], off
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.intersect_ray.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.intersect_ray.ll
index 173739fa8f35f..968bfd2a45ab2 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.intersect_ray.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.intersect_ray.ll
@@ -228,77 +228,40 @@ define amdgpu_ps <4 x float> @image_bvh_intersect_ray_a16(i32 inreg %node_ptr, f
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX10-SDAG-NEXT: ; return to shader part epilog
;
-; GFX1013-GISEL-LABEL: image_bvh_intersect_ray_a16:
-; GFX1013-GISEL: ; %bb.0: ; %main_body
-; GFX1013-GISEL-NEXT: s_and_b32 s8, s8, 0xffff
-; GFX1013-GISEL-NEXT: s_mov_b32 s16, s9
-; GFX1013-GISEL-NEXT: v_alignbit_b32 v0, s8, s7, 16
-; GFX1013-GISEL-NEXT: s_lshr_b32 s9, s5, 16
-; GFX1013-GISEL-NEXT: s_and_b32 s5, s5, 0xffff
-; GFX1013-GISEL-NEXT: s_lshl_b32 s8, s9, 16
-; GFX1013-GISEL-NEXT: s_and_b32 s9, s7, 0xffff
-; GFX1013-GISEL-NEXT: v_readfirstlane_b32 s7, v0
-; GFX1013-GISEL-NEXT: s_and_b32 s6, s6, 0xffff
-; GFX1013-GISEL-NEXT: s_lshl_b32 s9, s9, 16
-; GFX1013-GISEL-NEXT: s_or_b32 s5, s5, s8
-; GFX1013-GISEL-NEXT: s_or_b32 s6, s6, s9
-; GFX1013-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; GFX1013-GISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX1013-GISEL-NEXT: v_mov_b32_e32 v2, s2
-; GFX1013-GISEL-NEXT: v_mov_b32_e32 v3, s3
-; GFX1013-GISEL-NEXT: v_mov_b32_e32 v4, s4
-; GFX1013-GISEL-NEXT: v_mov_b32_e32 v5, s5
-; GFX1013-GISEL-NEXT: v_mov_b32_e32 v6, s6
-; GFX1013-GISEL-NEXT: v_mov_b32_e32 v7, s7
-; GFX1013-GISEL-NEXT: s_mov_b32 s17, s10
-; GFX1013-GISEL-NEXT: s_mov_b32 s18, s11
-; GFX1013-GISEL-NEXT: s_mov_b32 s19, s12
-; GFX1013-GISEL-NEXT: image_bvh_intersect_ray v[0:3], v[0:7], s[16:19] a16
-; GFX1013-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX1013-GISEL-NEXT: v_readfirstlane_b32 s0, v0
-; GFX1013-GISEL-NEXT: v_readfirstlane_b32 s1, v1
-; GFX1013-GISEL-NEXT: v_readfirstlane_b32 s2, v2
-; GFX1013-GISEL-NEXT: v_readfirstlane_b32 s3, v3
-; GFX1013-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; GFX1013-GISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX1013-GISEL-NEXT: v_mov_b32_e32 v2, s2
-; GFX1013-GISEL-NEXT: v_mov_b32_e32 v3, s3
-; GFX1013-GISEL-NEXT: ; return to shader part epilog
-;
-; GFX1030-GISEL-LABEL: image_bvh_intersect_ray_a16:
-; GFX1030-GISEL: ; %bb.0: ; %main_body
-; GFX1030-GISEL-NEXT: s_mov_b32 s16, s9
-; GFX1030-GISEL-NEXT: s_lshr_b32 s9, s5, 16
-; GFX1030-GISEL-NEXT: s_and_b32 s5, s5, 0xffff
-; GFX1030-GISEL-NEXT: s_lshl_b32 s9, s9, 16
-; GFX1030-GISEL-NEXT: s_and_b32 s6, s6, 0xffff
-; GFX1030-GISEL-NEXT: s_or_b32 s5, s5, s9
-; GFX1030-GISEL-NEXT: s_and_b32 s9, s7, 0xffff
-; GFX1030-GISEL-NEXT: s_and_b32 s8, s8, 0xffff
-; GFX1030-GISEL-NEXT: s_lshl_b32 s9, s9, 16
-; GFX1030-GISEL-NEXT: v_alignbit_b32 v7, s8, s7, 16
-; GFX1030-GISEL-NEXT: s_or_b32 s6, s6, s9
-; GFX1030-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; GFX1030-GISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX1030-GISEL-NEXT: v_mov_b32_e32 v2, s2
-; GFX1030-GISEL-NEXT: v_mov_b32_e32 v3, s3
-; GFX1030-GISEL-NEXT: v_mov_b32_e32 v4, s4
-; GFX1030-GISEL-NEXT: v_mov_b32_e32 v5, s5
-; GFX1030-GISEL-NEXT: v_mov_b32_e32 v6, s6
-; GFX1030-GISEL-NEXT: s_mov_b32 s17, s10
-; GFX1030-GISEL-NEXT: s_mov_b32 s18, s11
-; GFX1030-GISEL-NEXT: s_mov_b32 s19, s12
-; GFX1030-GISEL-NEXT: image_bvh_intersect_ray v[0:3], v[0:7], s[16:19] a16
-; GFX1030-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX1030-GISEL-NEXT: v_readfirstlane_b32 s0, v0
-; GFX1030-GISEL-NEXT: v_readfirstlane_b32 s1, v1
-; GFX1030-GISEL-NEXT: v_readfirstlane_b32 s2, v2
-; GFX1030-GISEL-NEXT: v_readfirstlane_b32 s3, v3
-; GFX1030-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; GFX1030-GISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX1030-GISEL-NEXT: v_mov_b32_e32 v2, s2
-; GFX1030-GISEL-NEXT: v_mov_b32_e32 v3, s3
-; GFX1030-GISEL-NEXT: ; return to shader part epilog
+; GFX10-GISEL-LABEL: image_bvh_intersect_ray_a16:
+; GFX10-GISEL: ; %bb.0: ; %main_body
+; GFX10-GISEL-NEXT: s_mov_b32 s16, s9
+; GFX10-GISEL-NEXT: s_lshr_b32 s9, s5, 16
+; GFX10-GISEL-NEXT: s_and_b32 s5, s5, 0xffff
+; GFX10-GISEL-NEXT: s_lshl_b32 s9, s9, 16
+; GFX10-GISEL-NEXT: s_and_b32 s6, s6, 0xffff
+; GFX10-GISEL-NEXT: s_or_b32 s5, s5, s9
+; GFX10-GISEL-NEXT: s_and_b32 s9, s7, 0xffff
+; GFX10-GISEL-NEXT: s_and_b32 s8, s8, 0xffff
+; GFX10-GISEL-NEXT: s_lshl_b32 s9, s9, 16
+; GFX10-GISEL-NEXT: v_alignbit_b32 v7, s8, s7, 16
+; GFX10-GISEL-NEXT: s_or_b32 s6, s6, s9
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, s3
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v4, s4
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v5, s5
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v6, s6
+; GFX10-GISEL-NEXT: s_mov_b32 s17, s10
+; GFX10-GISEL-NEXT: s_mov_b32 s18, s11
+; GFX10-GISEL-NEXT: s_mov_b32 s19, s12
+; GFX10-GISEL-NEXT: image_bvh_intersect_ray v[0:3], v[0:7], s[16:19] a16
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s1, v1
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s2, v2
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s3, v3
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, s3
+; GFX10-GISEL-NEXT: ; return to shader part epilog
;
; GFX11-SDAG-LABEL: image_bvh_intersect_ray_a16:
; GFX11-SDAG: ; %bb.0: ; %main_body
@@ -530,79 +493,41 @@ define amdgpu_ps <4 x float> @image_bvh64_intersect_ray_a16(i64 inreg %node_ptr,
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX10-SDAG-NEXT: ; return to shader part epilog
;
-; GFX1013-GISEL-LABEL: image_bvh64_intersect_ray_a16:
-; GFX1013-GISEL: ; %bb.0: ; %main_body
-; GFX1013-GISEL-NEXT: s_and_b32 s9, s9, 0xffff
-; GFX1013-GISEL-NEXT: s_mov_b32 s16, s10
-; GFX1013-GISEL-NEXT: v_alignbit_b32 v0, s9, s8, 16
-; GFX1013-GISEL-NEXT: s_lshr_b32 s10, s6, 16
-; GFX1013-GISEL-NEXT: s_and_b32 s6, s6, 0xffff
-; GFX1013-GISEL-NEXT: s_lshl_b32 s9, s10, 16
-; GFX1013-GISEL-NEXT: s_and_b32 s10, s8, 0xffff
-; GFX1013-GISEL-NEXT: v_readfirstlane_b32 s8, v0
-; GFX1013-GISEL-NEXT: s_and_b32 s7, s7, 0xffff
-; GFX1013-GISEL-NEXT: s_lshl_b32 s10, s10, 16
-; GFX1013-GISEL-NEXT: s_or_b32 s6, s6, s9
-; GFX1013-GISEL-NEXT: s_or_b32 s7, s7, s10
-; GFX1013-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; GFX1013-GISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX1013-GISEL-NEXT: v_mov_b32_e32 v2, s2
-; GFX1013-GISEL-NEXT: v_mov_b32_e32 v3, s3
-; GFX1013-GISEL-NEXT: v_mov_b32_e32 v4, s4
-; GFX1013-GISEL-NEXT: v_mov_b32_e32 v5, s5
-; GFX1013-GISEL-NEXT: v_mov_b32_e32 v6, s6
-; GFX1013-GISEL-NEXT: v_mov_b32_e32 v7, s7
-; GFX1013-GISEL-NEXT: v_mov_b32_e32 v8, s8
-; GFX1013-GISEL-NEXT: s_mov_b32 s17, s11
-; GFX1013-GISEL-NEXT: s_mov_b32 s18, s12
-; GFX1013-GISEL-NEXT: s_mov_b32 s19, s13
-; GFX1013-GISEL-NEXT: image_bvh64_intersect_ray v[0:3], v[0:8], s[16:19] a16
-; GFX1013-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX1013-GISEL-NEXT: v_readfirstlane_b32 s0, v0
-; GFX1013-GISEL-NEXT: v_readfirstlane_b32 s1, v1
-; GFX1013-GISEL-NEXT: v_readfirstlane_b32 s2, v2
-; GFX1013-GISEL-NEXT: v_readfirstlane_b32 s3, v3
-; GFX1013-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; GFX1013-GISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX1013-GISEL-NEXT: v_mov_b32_e32 v2, s2
-; GFX1013-GISEL-NEXT: v_mov_b32_e32 v3, s3
-; GFX1013-GISEL-NEXT: ; return to shader part epilog
-;
-; GFX1030-GISEL-LABEL: image_bvh64_intersect_ray_a16:
-; GFX1030-GISEL: ; %bb.0: ; %main_body
-; GFX1030-GISEL-NEXT: s_mov_b32 s16, s10
-; GFX1030-GISEL-NEXT: s_lshr_b32 s10, s6, 16
-; GFX1030-GISEL-NEXT: s_and_b32 s6, s6, 0xffff
-; GFX1030-GISEL-NEXT: s_lshl_b32 s10, s10, 16
-; GFX1030-GISEL-NEXT: s_and_b32 s7, s7, 0xffff
-; GFX1030-GISEL-NEXT: s_or_b32 s6, s6, s10
-; GFX1030-GISEL-NEXT: s_and_b32 s10, s8, 0xffff
-; GFX1030-GISEL-NEXT: s_and_b32 s9, s9, 0xffff
-; GFX1030-GISEL-NEXT: s_lshl_b32 s10, s10, 16
-; GFX1030-GISEL-NEXT: v_alignbit_b32 v8, s9, s8, 16
-; GFX1030-GISEL-NEXT: s_or_b32 s7, s7, s10
-; GFX1030-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; GFX1030-GISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX1030-GISEL-NEXT: v_mov_b32_e32 v2, s2
-; GFX1030-GISEL-NEXT: v_mov_b32_e32 v3, s3
-; GFX1030-GISEL-NEXT: v_mov_b32_e32 v4, s4
-; GFX1030-GISEL-NEXT: v_mov_b32_e32 v5, s5
-; GFX1030-GISEL-NEXT: v_mov_b32_e32 v6, s6
-; GFX1030-GISEL-NEXT: v_mov_b32_e32 v7, s7
-; GFX1030-GISEL-NEXT: s_mov_b32 s17, s11
-; GFX1030-GISEL-NEXT: s_mov_b32 s18, s12
-; GFX1030-GISEL-NEXT: s_mov_b32 s19, s13
-; GFX1030-GISEL-NEXT: image_bvh64_intersect_ray v[0:3], v[0:8], s[16:19] a16
-; GFX1030-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX1030-GISEL-NEXT: v_readfirstlane_b32 s0, v0
-; GFX1030-GISEL-NEXT: v_readfirstlane_b32 s1, v1
-; GFX1030-GISEL-NEXT: v_readfirstlane_b32 s2, v2
-; GFX1030-GISEL-NEXT: v_readfirstlane_b32 s3, v3
-; GFX1030-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; GFX1030-GISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX1030-GISEL-NEXT: v_mov_b32_e32 v2, s2
-; GFX1030-GISEL-NEXT: v_mov_b32_e32 v3, s3
-; GFX1030-GISEL-NEXT: ; return to shader part epilog
+; GFX10-GISEL-LABEL: image_bvh64_intersect_ray_a16:
+; GFX10-GISEL: ; %bb.0: ; %main_body
+; GFX10-GISEL-NEXT: s_mov_b32 s16, s10
+; GFX10-GISEL-NEXT: s_lshr_b32 s10, s6, 16
+; GFX10-GISEL-NEXT: s_and_b32 s6, s6, 0xffff
+; GFX10-GISEL-NEXT: s_lshl_b32 s10, s10, 16
+; GFX10-GISEL-NEXT: s_and_b32 s7, s7, 0xffff
+; GFX10-GISEL-NEXT: s_or_b32 s6, s6, s10
+; GFX10-GISEL-NEXT: s_and_b32 s10, s8, 0xffff
+; GFX10-GISEL-NEXT: s_and_b32 s9, s9, 0xffff
+; GFX10-GISEL-NEXT: s_lshl_b32 s10, s10, 16
+; GFX10-GISEL-NEXT: v_alignbit_b32 v8, s9, s8, 16
+; GFX10-GISEL-NEXT: s_or_b32 s7, s7, s10
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, s3
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v4, s4
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v5, s5
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v6, s6
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v7, s7
+; GFX10-GISEL-NEXT: s_mov_b32 s17, s11
+; GFX10-GISEL-NEXT: s_mov_b32 s18, s12
+; GFX10-GISEL-NEXT: s_mov_b32 s19, s13
+; GFX10-GISEL-NEXT: image_bvh64_intersect_ray v[0:3], v[0:8], s[16:19] a16
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s1, v1
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s2, v2
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s3, v3
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, s3
+; GFX10-GISEL-NEXT: ; return to shader part epilog
;
; GFX11-SDAG-LABEL: image_bvh64_intersect_ray_a16:
; GFX11-SDAG: ; %bb.0: ; %main_body
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.ll
index e0e7eea0d07f0..eb81882c550e5 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.ll
@@ -871,36 +871,17 @@ define amdgpu_cs float @buffer_load_v4i32_tfe(<4 x i32> inreg %rsrc, ptr addrspa
; NOPRT-NEXT: v_mov_b32_e32 v0, v6
; NOPRT-NEXT: ; return to shader part epilog
;
-; GFX12-SDAG-LABEL: buffer_load_v4i32_tfe:
-; GFX12-SDAG: ; %bb.0:
-; GFX12-SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v7, 2
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v4, v2
-; GFX12-SDAG-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v6, v2
-; GFX12-SDAG-NEXT: buffer_load_format_xyzw v[2:6], v7, s[0:3], null idxen tfe
-; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX12-SDAG-NEXT: global_store_b128 v[0:1], v[2:5], off
-; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, v6
-; GFX12-SDAG-NEXT: ; return to shader part epilog
-;
-; GFX12-GISEL-LABEL: buffer_load_v4i32_tfe:
-; GFX12-GISEL: ; %bb.0:
-; GFX12-GISEL-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v7, 2
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v4, v2
-; GFX12-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v6, v2
-; GFX12-GISEL-NEXT: buffer_load_format_xyzw v[2:6], v7, s[0:3], null idxen tfe
-; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX12-GISEL-NEXT: v_readfirstlane_b32 s0, v2
-; GFX12-GISEL-NEXT: v_readfirstlane_b32 s1, v3
-; GFX12-GISEL-NEXT: v_readfirstlane_b32 s2, v4
-; GFX12-GISEL-NEXT: v_readfirstlane_b32 s3, v5
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-GISEL-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
-; GFX12-GISEL-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
-; GFX12-GISEL-NEXT: global_store_b128 v[0:1], v[2:5], off
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, v6
-; GFX12-GISEL-NEXT: ; return to shader part epilog
+; GFX12-LABEL: buffer_load_v4i32_tfe:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v7, 2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v4, v2
+; GFX12-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v6, v2
+; GFX12-NEXT: buffer_load_format_xyzw v[2:6], v7, s[0:3], null idxen tfe
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX12-NEXT: v_mov_b32_e32 v0, v6
+; GFX12-NEXT: ; return to shader part epilog
%load = call { <4 x i32>, i32 } @llvm.amdgcn.struct.buffer.load.format.sl_v4i32i32s(<4 x i32> %rsrc, i32 2, i32 0, i32 0, i32 0)
%data = extractvalue { <4 x i32>, i32 } %load, 0
store <4 x i32> %data, ptr addrspace(1) %out
@@ -1057,35 +1038,17 @@ define amdgpu_cs float @buffer_load_v3i32_tfe(<4 x i32> inreg %rsrc, ptr addrspa
; NOPRT-NEXT: v_mov_b32_e32 v0, v5
; NOPRT-NEXT: ; return to shader part epilog
;
-; GFX12-SDAG-LABEL: buffer_load_v3i32_tfe:
-; GFX12-SDAG: ; %bb.0:
-; GFX12-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v4, v2
-; GFX12-SDAG-NEXT: v_mov_b32_e32 v5, v2
-; GFX12-SDAG-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], null idxen tfe
-; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX12-SDAG-NEXT: global_store_b96 v[0:1], v[2:4], off
-; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, v5
-; GFX12-SDAG-NEXT: ; return to shader part epilog
-;
-; GFX12-GISEL-LABEL: buffer_load_v3i32_tfe:
-; GFX12-GISEL: ; %bb.0:
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v4, v2
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v5, v2
-; GFX12-GISEL-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], null idxen tfe
-; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX12-GISEL-NEXT: v_readfirstlane_b32 s0, v2
-; GFX12-GISEL-NEXT: v_readfirstlane_b32 s1, v3
-; GFX12-GISEL-NEXT: v_readfirstlane_b32 s2, v4
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-GISEL-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v4, s2
-; GFX12-GISEL-NEXT: global_store_b96 v[0:1], v[2:4], off
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, v5
-; GFX12-GISEL-NEXT: ; return to shader part epilog
+; GFX12-LABEL: buffer_load_v3i32_tfe:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v4, v2
+; GFX12-NEXT: v_mov_b32_e32 v5, v2
+; GFX12-NEXT: buffer_load_format_xyz v[2:5], v2, s[0:3], null idxen tfe
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_store_b96 v[0:1], v[2:4], off
+; GFX12-NEXT: v_mov_b32_e32 v0, v5
+; GFX12-NEXT: ; return to shader part epilog
%load = call { <3 x i32>, i32 } @llvm.amdgcn.struct.buffer.load.format.sl_v3i32i32s(<4 x i32> %rsrc, i32 0, i32 0, i32 0, i32 0)
%data = extractvalue { <3 x i32>, i32 } %load, 0
store <3 x i32> %data, ptr addrspace(1) %out
@@ -1236,30 +1199,16 @@ define amdgpu_cs float @buffer_load_v2i32_tfe(<4 x i32> inreg %rsrc, ptr addrspa
; NOPRT-NEXT: v_mov_b32_e32 v0, v4
; NOPRT-NEXT: ; return to shader part epilog
;
-; GFX12-SDAG-LABEL: buffer_load_v2i32_tfe:
-; GFX12-SDAG: ; %bb.0:
-; GFX12-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v4, v2
-; GFX12-SDAG-NEXT: buffer_load_format_xy v[2:4], v2, s[0:3], null idxen tfe
-; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX12-SDAG-NEXT: global_store_b64 v[0:1], v[2:3], off
-; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, v4
-; GFX12-SDAG-NEXT: ; return to shader part epilog
-;
-; GFX12-GISEL-LABEL: buffer_load_v2i32_tfe:
-; GFX12-GISEL: ; %bb.0:
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v4, v2
-; GFX12-GISEL-NEXT: buffer_load_format_xy v[2:4], v2, s[0:3], null idxen tfe
-; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX12-GISEL-NEXT: v_readfirstlane_b32 s0, v2
-; GFX12-GISEL-NEXT: v_readfirstlane_b32 s1, v3
-; GFX12-GISEL-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
-; GFX12-GISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, v4
-; GFX12-GISEL-NEXT: ; return to shader part epilog
+; GFX12-LABEL: buffer_load_v2i32_tfe:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v4, v2
+; GFX12-NEXT: buffer_load_format_xy v[2:4], v2, s[0:3], null idxen tfe
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_store_b64 v[0:1], v[2:3], off
+; GFX12-NEXT: v_mov_b32_e32 v0, v4
+; GFX12-NEXT: ; return to shader part epilog
%load = call { <2 x i32>, i32 } @llvm.amdgcn.struct.buffer.load.format.sl_v2i32i32s(<4 x i32> %rsrc, i32 0, i32 0, i32 0, i32 0)
%data = extractvalue { <2 x i32>, i32 } %load, 0
store <2 x i32> %data, ptr addrspace(1) %out
diff --git a/llvm/test/CodeGen/AMDGPU/packed-fp32.ll b/llvm/test/CodeGen/AMDGPU/packed-fp32.ll
index 55f60193236a0..e85e4c7d4d350 100644
--- a/llvm/test/CodeGen/AMDGPU/packed-fp32.ll
+++ b/llvm/test/CodeGen/AMDGPU/packed-fp32.ll
@@ -1133,37 +1133,20 @@ define amdgpu_kernel void @fadd_v2_v_fneg_lo(ptr addrspace(1) %a, float %x) {
; PACKED-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; PACKED-SDAG-NEXT: s_endpgm
;
-; GFX90A-GISEL-LABEL: fadd_v2_v_fneg_lo:
-; GFX90A-GISEL: ; %bb.0:
-; GFX90A-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX90A-GISEL-NEXT: s_load_dword s3, s[4:5], 0x2c
-; GFX90A-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX90A-GISEL-NEXT: v_lshlrev_b32_e32 v2, 3, v0
-; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-GISEL-NEXT: global_load_dwordx2 v[0:1], v2, s[0:1]
-; GFX90A-GISEL-NEXT: v_max_f32_e64 v3, -s3, -s3
-; GFX90A-GISEL-NEXT: v_readfirstlane_b32 s2, v3
-; GFX90A-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-GISEL-NEXT: v_pk_add_f32 v[0:1], v[0:1], s[2:3]
-; GFX90A-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX90A-GISEL-NEXT: s_endpgm
-;
-; GFX942-GISEL-LABEL: fadd_v2_v_fneg_lo:
-; GFX942-GISEL: ; %bb.0:
-; GFX942-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX942-GISEL-NEXT: s_load_dword s3, s[4:5], 0x2c
-; GFX942-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX942-GISEL-NEXT: v_lshlrev_b32_e32 v2, 3, v0
-; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: global_load_dwordx2 v[0:1], v2, s[0:1]
-; GFX942-GISEL-NEXT: v_max_f32_e64 v3, -s3, -s3
-; GFX942-GISEL-NEXT: s_nop 0
-; GFX942-GISEL-NEXT: v_readfirstlane_b32 s2, v3
-; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX942-GISEL-NEXT: s_nop 0
-; GFX942-GISEL-NEXT: v_pk_add_f32 v[0:1], v[0:1], s[2:3]
-; GFX942-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX942-GISEL-NEXT: s_endpgm
+; PACKED-GISEL-LABEL: fadd_v2_v_fneg_lo:
+; PACKED-GISEL: ; %bb.0:
+; PACKED-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; PACKED-GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; PACKED-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; PACKED-GISEL-NEXT: v_lshlrev_b32_e32 v4, 3, v0
+; PACKED-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; PACKED-GISEL-NEXT: global_load_dwordx2 v[0:1], v4, s[0:1]
+; PACKED-GISEL-NEXT: v_mov_b32_e32 v3, s2
+; PACKED-GISEL-NEXT: v_max_f32_e64 v2, -s2, -s2
+; PACKED-GISEL-NEXT: s_waitcnt vmcnt(0)
+; PACKED-GISEL-NEXT: v_pk_add_f32 v[0:1], v[0:1], v[2:3]
+; PACKED-GISEL-NEXT: global_store_dwordx2 v4, v[0:1], s[0:1]
+; PACKED-GISEL-NEXT: s_endpgm
;
; GFX1250-SDAG-LABEL: fadd_v2_v_fneg_lo:
; GFX1250-SDAG: ; %bb.0:
@@ -1190,11 +1173,8 @@ define amdgpu_kernel void @fadd_v2_v_fneg_lo(ptr addrspace(1) %a, float %x) {
; GFX1250-GISEL-NEXT: v_and_b32_e32 v4, 0x3ff, v0
; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-NEXT: global_load_b64 v[0:1], v4, s[0:1] scale_offset
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v3, s2
; GFX1250-GISEL-NEXT: v_max_num_f32_e64 v2, -s2, -s2
-; GFX1250-GISEL-NEXT: s_mov_b32 s5, s2
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s4, v2
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[4:5]
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_pk_add_f32 v[0:1], v[0:1], v[2:3]
@@ -1238,37 +1218,20 @@ define amdgpu_kernel void @fadd_v2_v_fneg_hi(ptr addrspace(1) %a, float %x) {
; PACKED-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; PACKED-SDAG-NEXT: s_endpgm
;
-; GFX90A-GISEL-LABEL: fadd_v2_v_fneg_hi:
-; GFX90A-GISEL: ; %bb.0:
-; GFX90A-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX90A-GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX90A-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX90A-GISEL-NEXT: v_lshlrev_b32_e32 v2, 3, v0
-; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-GISEL-NEXT: global_load_dwordx2 v[0:1], v2, s[0:1]
-; GFX90A-GISEL-NEXT: v_max_f32_e64 v3, -s2, -s2
-; GFX90A-GISEL-NEXT: v_readfirstlane_b32 s3, v3
-; GFX90A-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-GISEL-NEXT: v_pk_add_f32 v[0:1], v[0:1], s[2:3]
-; GFX90A-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX90A-GISEL-NEXT: s_endpgm
-;
-; GFX942-GISEL-LABEL: fadd_v2_v_fneg_hi:
-; GFX942-GISEL: ; %bb.0:
-; GFX942-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX942-GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX942-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX942-GISEL-NEXT: v_lshlrev_b32_e32 v2, 3, v0
-; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: global_load_dwordx2 v[0:1], v2, s[0:1]
-; GFX942-GISEL-NEXT: v_max_f32_e64 v3, -s2, -s2
-; GFX942-GISEL-NEXT: s_nop 0
-; GFX942-GISEL-NEXT: v_readfirstlane_b32 s3, v3
-; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX942-GISEL-NEXT: s_nop 0
-; GFX942-GISEL-NEXT: v_pk_add_f32 v[0:1], v[0:1], s[2:3]
-; GFX942-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX942-GISEL-NEXT: s_endpgm
+; PACKED-GISEL-LABEL: fadd_v2_v_fneg_hi:
+; PACKED-GISEL: ; %bb.0:
+; PACKED-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; PACKED-GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; PACKED-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; PACKED-GISEL-NEXT: v_lshlrev_b32_e32 v4, 3, v0
+; PACKED-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; PACKED-GISEL-NEXT: global_load_dwordx2 v[0:1], v4, s[0:1]
+; PACKED-GISEL-NEXT: v_mov_b32_e32 v2, s2
+; PACKED-GISEL-NEXT: v_max_f32_e64 v3, -s2, -s2
+; PACKED-GISEL-NEXT: s_waitcnt vmcnt(0)
+; PACKED-GISEL-NEXT: v_pk_add_f32 v[0:1], v[0:1], v[2:3]
+; PACKED-GISEL-NEXT: global_store_dwordx2 v4, v[0:1], s[0:1]
+; PACKED-GISEL-NEXT: s_endpgm
;
; GFX1250-SDAG-LABEL: fadd_v2_v_fneg_hi:
; GFX1250-SDAG: ; %bb.0:
@@ -1295,10 +1258,8 @@ define amdgpu_kernel void @fadd_v2_v_fneg_hi(ptr addrspace(1) %a, float %x) {
; GFX1250-GISEL-NEXT: v_and_b32_e32 v4, 0x3ff, v0
; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-NEXT: global_load_b64 v[0:1], v4, s[0:1] scale_offset
-; GFX1250-GISEL-NEXT: v_max_num_f32_e64 v2, -s2, -s2
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s3, v2
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v2, s2
+; GFX1250-GISEL-NEXT: v_max_num_f32_e64 v3, -s2, -s2
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_pk_add_f32 v[0:1], v[0:1], v[2:3]
@@ -1340,35 +1301,19 @@ define amdgpu_kernel void @fadd_v2_v_fneg_lo2(ptr addrspace(1) %a, float %x, flo
; PACKED-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; PACKED-SDAG-NEXT: s_endpgm
;
-; GFX90A-GISEL-LABEL: fadd_v2_v_fneg_lo2:
-; GFX90A-GISEL: ; %bb.0:
-; GFX90A-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX90A-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX90A-GISEL-NEXT: v_lshlrev_b32_e32 v2, 3, v0
-; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-GISEL-NEXT: global_load_dwordx2 v[0:1], v2, s[0:1]
-; GFX90A-GISEL-NEXT: v_max_f32_e64 v3, -s2, -s2
-; GFX90A-GISEL-NEXT: v_readfirstlane_b32 s2, v3
-; GFX90A-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-GISEL-NEXT: v_pk_add_f32 v[0:1], v[0:1], s[2:3]
-; GFX90A-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX90A-GISEL-NEXT: s_endpgm
-;
-; GFX942-GISEL-LABEL: fadd_v2_v_fneg_lo2:
-; GFX942-GISEL: ; %bb.0:
-; GFX942-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX942-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX942-GISEL-NEXT: v_lshlrev_b32_e32 v2, 3, v0
-; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: global_load_dwordx2 v[0:1], v2, s[0:1]
-; GFX942-GISEL-NEXT: v_max_f32_e64 v3, -s2, -s2
-; GFX942-GISEL-NEXT: s_nop 0
-; GFX942-GISEL-NEXT: v_readfirstlane_b32 s2, v3
-; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX942-GISEL-NEXT: s_nop 0
-; GFX942-GISEL-NEXT: v_pk_add_f32 v[0:1], v[0:1], s[2:3]
-; GFX942-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX942-GISEL-NEXT: s_endpgm
+; PACKED-GISEL-LABEL: fadd_v2_v_fneg_lo2:
+; PACKED-GISEL: ; %bb.0:
+; PACKED-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; PACKED-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; PACKED-GISEL-NEXT: v_lshlrev_b32_e32 v4, 3, v0
+; PACKED-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; PACKED-GISEL-NEXT: global_load_dwordx2 v[0:1], v4, s[0:1]
+; PACKED-GISEL-NEXT: v_max_f32_e64 v2, -s2, -s2
+; PACKED-GISEL-NEXT: v_mov_b32_e32 v3, s3
+; PACKED-GISEL-NEXT: s_waitcnt vmcnt(0)
+; PACKED-GISEL-NEXT: v_pk_add_f32 v[0:1], v[0:1], v[2:3]
+; PACKED-GISEL-NEXT: global_store_dwordx2 v4, v[0:1], s[0:1]
+; PACKED-GISEL-NEXT: s_endpgm
;
; GFX1250-SDAG-LABEL: fadd_v2_v_fneg_lo2:
; GFX1250-SDAG: ; %bb.0:
@@ -1398,9 +1343,7 @@ define amdgpu_kernel void @fadd_v2_v_fneg_lo2(ptr addrspace(1) %a, float %x, flo
; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-NEXT: global_load_b64 v[0:1], v4, s[0:1] scale_offset
; GFX1250-GISEL-NEXT: v_max_num_f32_e64 v2, -s2, -s2
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s2, v2
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_pk_add_f32 v[0:1], v[0:1], v[2:3]
@@ -1442,36 +1385,19 @@ define amdgpu_kernel void @fadd_v2_v_fneg_hi2(ptr addrspace(1) %a, float %x, flo
; PACKED-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; PACKED-SDAG-NEXT: s_endpgm
;
-; GFX90A-GISEL-LABEL: fadd_v2_v_fneg_hi2:
-; GFX90A-GISEL: ; %bb.0:
-; GFX90A-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX90A-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX90A-GISEL-NEXT: v_lshlrev_b32_e32 v2, 3, v0
-; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-GISEL-NEXT: global_load_dwordx2 v[0:1], v2, s[0:1]
-; GFX90A-GISEL-NEXT: v_max_f32_e64 v3, -s2, -s2
-; GFX90A-GISEL-NEXT: v_readfirstlane_b32 s5, v3
-; GFX90A-GISEL-NEXT: s_mov_b32 s4, s3
-; GFX90A-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-GISEL-NEXT: v_pk_add_f32 v[0:1], v[0:1], s[4:5]
-; GFX90A-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX90A-GISEL-NEXT: s_endpgm
-;
-; GFX942-GISEL-LABEL: fadd_v2_v_fneg_hi2:
-; GFX942-GISEL: ; %bb.0:
-; GFX942-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX942-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX942-GISEL-NEXT: v_lshlrev_b32_e32 v2, 3, v0
-; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: global_load_dwordx2 v[0:1], v2, s[0:1]
-; GFX942-GISEL-NEXT: v_max_f32_e64 v3, -s2, -s2
-; GFX942-GISEL-NEXT: s_mov_b32 s4, s3
-; GFX942-GISEL-NEXT: v_readfirstlane_b32 s5, v3
-; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX942-GISEL-NEXT: s_nop 0
-; GFX942-GISEL-NEXT: v_pk_add_f32 v[0:1], v[0:1], s[4:5]
-; GFX942-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX942-GISEL-NEXT: s_endpgm
+; PACKED-GISEL-LABEL: fadd_v2_v_fneg_hi2:
+; PACKED-GISEL: ; %bb.0:
+; PACKED-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; PACKED-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; PACKED-GISEL-NEXT: v_lshlrev_b32_e32 v4, 3, v0
+; PACKED-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; PACKED-GISEL-NEXT: global_load_dwordx2 v[0:1], v4, s[0:1]
+; PACKED-GISEL-NEXT: v_max_f32_e64 v3, -s2, -s2
+; PACKED-GISEL-NEXT: v_mov_b32_e32 v2, s3
+; PACKED-GISEL-NEXT: s_waitcnt vmcnt(0)
+; PACKED-GISEL-NEXT: v_pk_add_f32 v[0:1], v[0:1], v[2:3]
+; PACKED-GISEL-NEXT: global_store_dwordx2 v4, v[0:1], s[0:1]
+; PACKED-GISEL-NEXT: s_endpgm
;
; GFX1250-SDAG-LABEL: fadd_v2_v_fneg_hi2:
; GFX1250-SDAG: ; %bb.0:
@@ -1500,11 +1426,8 @@ define amdgpu_kernel void @fadd_v2_v_fneg_hi2(ptr addrspace(1) %a, float %x, flo
; GFX1250-GISEL-NEXT: v_and_b32_e32 v4, 0x3ff, v0
; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-NEXT: global_load_b64 v[0:1], v4, s[0:1] scale_offset
-; GFX1250-GISEL-NEXT: v_max_num_f32_e64 v2, -s2, -s2
-; GFX1250-GISEL-NEXT: s_mov_b32 s4, s3
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s5, v2
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[4:5]
+; GFX1250-GISEL-NEXT: v_max_num_f32_e64 v3, -s2, -s2
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v2, s3
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_pk_add_f32 v[0:1], v[0:1], v[2:3]
@@ -3778,38 +3701,19 @@ define amdgpu_kernel void @shuffle_add_f32(ptr addrspace(1) %out, ptr addrspace(
; PACKED-SDAG-NEXT: global_store_dwordx2 v4, v[0:1], s[0:1]
; PACKED-SDAG-NEXT: s_endpgm
;
-; GFX90A-GISEL-LABEL: shuffle_add_f32:
-; GFX90A-GISEL: ; %bb.0: ; %bb
-; GFX90A-GISEL-NEXT: s_load_dword s0, s[4:5], 0x2c
-; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v2, s0
-; GFX90A-GISEL-NEXT: ds_read_b64 v[0:1], v2
-; GFX90A-GISEL-NEXT: ds_read_b64 v[2:3], v2 offset:8
-; GFX90A-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-GISEL-NEXT: v_readfirstlane_b32 s3, v2
-; GFX90A-GISEL-NEXT: v_readfirstlane_b32 s2, v3
-; GFX90A-GISEL-NEXT: v_pk_add_f32 v[0:1], v[0:1], s[2:3]
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX90A-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX90A-GISEL-NEXT: s_endpgm
-;
-; GFX942-GISEL-LABEL: shuffle_add_f32:
-; GFX942-GISEL: ; %bb.0: ; %bb
-; GFX942-GISEL-NEXT: s_load_dword s0, s[4:5], 0x2c
-; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v2, s0
-; GFX942-GISEL-NEXT: ds_read_b64 v[0:1], v2
-; GFX942-GISEL-NEXT: ds_read_b64 v[2:3], v2 offset:8
-; GFX942-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: v_readfirstlane_b32 s3, v2
-; GFX942-GISEL-NEXT: v_readfirstlane_b32 s2, v3
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX942-GISEL-NEXT: s_nop 0
-; GFX942-GISEL-NEXT: v_pk_add_f32 v[0:1], v[0:1], s[2:3]
-; GFX942-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX942-GISEL-NEXT: s_endpgm
+; PACKED-GISEL-LABEL: shuffle_add_f32:
+; PACKED-GISEL: ; %bb.0: ; %bb
+; PACKED-GISEL-NEXT: s_load_dword s0, s[4:5], 0x2c
+; PACKED-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; PACKED-GISEL-NEXT: v_mov_b32_e32 v2, s0
+; PACKED-GISEL-NEXT: ds_read_b64 v[0:1], v2
+; PACKED-GISEL-NEXT: ds_read_b64 v[2:3], v2 offset:8
+; PACKED-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; PACKED-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; PACKED-GISEL-NEXT: v_pk_add_f32 v[0:1], v[0:1], v[2:3] op_sel:[0,1] op_sel_hi:[1,0]
+; PACKED-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; PACKED-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; PACKED-GISEL-NEXT: s_endpgm
;
; GFX1250-SDAG-LABEL: shuffle_add_f32:
; GFX1250-SDAG: ; %bb.0: ; %bb
@@ -3839,11 +3743,7 @@ define amdgpu_kernel void @shuffle_add_f32(ptr addrspace(1) %out, ptr addrspace(
; GFX1250-GISEL-NEXT: ds_load_b64 v[0:1], v2
; GFX1250-GISEL-NEXT: ds_load_b64 v[2:3], v2 offset:8
; GFX1250-GISEL-NEXT: s_wait_dscnt 0x0
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s3, v2
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s2, v3
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX1250-GISEL-NEXT: v_pk_add_f32 v[0:1], v[0:1], v[2:3]
+; GFX1250-GISEL-NEXT: v_pk_add_f32 v[0:1], v[0:1], v[2:3] op_sel:[0,1] op_sel_hi:[1,0]
; GFX1250-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX1250-GISEL-NEXT: s_endpgm
@@ -3893,47 +3793,23 @@ define amdgpu_kernel void @shuffle_neg_add_f32(ptr addrspace(1) %out, ptr addrsp
; PACKED-SDAG-NEXT: global_store_dwordx2 v4, v[0:1], s[0:1]
; PACKED-SDAG-NEXT: s_endpgm
;
-; GFX90A-GISEL-LABEL: shuffle_neg_add_f32:
-; GFX90A-GISEL: ; %bb.0: ; %bb
-; GFX90A-GISEL-NEXT: s_load_dword s0, s[4:5], 0x2c
-; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v2, s0
-; GFX90A-GISEL-NEXT: ds_read_b64 v[0:1], v2
-; GFX90A-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-GISEL-NEXT: ds_read_b32 v3, v0
-; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-GISEL-NEXT: ds_read_b64 v[2:3], v2 offset:8
-; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-GISEL-NEXT: v_pk_mul_f32 v[2:3], -1.0, v[2:3] op_sel_hi:[0,1]
-; GFX90A-GISEL-NEXT: v_readfirstlane_b32 s3, v2
-; GFX90A-GISEL-NEXT: v_readfirstlane_b32 s2, v3
-; GFX90A-GISEL-NEXT: v_pk_add_f32 v[0:1], v[0:1], s[2:3]
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX90A-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX90A-GISEL-NEXT: s_endpgm
-;
-; GFX942-GISEL-LABEL: shuffle_neg_add_f32:
-; GFX942-GISEL: ; %bb.0: ; %bb
-; GFX942-GISEL-NEXT: s_load_dword s0, s[4:5], 0x2c
-; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v2, s0
-; GFX942-GISEL-NEXT: ds_read_b64 v[0:1], v2
-; GFX942-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: ds_read_b32 v3, v0
-; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: ds_read_b64 v[2:3], v2 offset:8
-; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: v_pk_mul_f32 v[2:3], -1.0, v[2:3] op_sel_hi:[0,1]
-; GFX942-GISEL-NEXT: s_nop 0
-; GFX942-GISEL-NEXT: v_readfirstlane_b32 s3, v2
-; GFX942-GISEL-NEXT: v_readfirstlane_b32 s2, v3
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX942-GISEL-NEXT: s_nop 0
-; GFX942-GISEL-NEXT: v_pk_add_f32 v[0:1], v[0:1], s[2:3]
-; GFX942-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX942-GISEL-NEXT: s_endpgm
+; PACKED-GISEL-LABEL: shuffle_neg_add_f32:
+; PACKED-GISEL: ; %bb.0: ; %bb
+; PACKED-GISEL-NEXT: s_load_dword s0, s[4:5], 0x2c
+; PACKED-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; PACKED-GISEL-NEXT: v_mov_b32_e32 v2, s0
+; PACKED-GISEL-NEXT: ds_read_b64 v[0:1], v2
+; PACKED-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; PACKED-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; PACKED-GISEL-NEXT: ds_read_b32 v3, v0
+; PACKED-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; PACKED-GISEL-NEXT: ds_read_b64 v[2:3], v2 offset:8
+; PACKED-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; PACKED-GISEL-NEXT: v_pk_mul_f32 v[2:3], -1.0, v[2:3] op_sel_hi:[0,1]
+; PACKED-GISEL-NEXT: v_pk_add_f32 v[0:1], v[0:1], v[2:3] op_sel:[0,1] op_sel_hi:[1,0]
+; PACKED-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; PACKED-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; PACKED-GISEL-NEXT: s_endpgm
;
; GFX1250-SDAG-LABEL: shuffle_neg_add_f32:
; GFX1250-SDAG: ; %bb.0: ; %bb
@@ -3970,12 +3846,8 @@ define amdgpu_kernel void @shuffle_neg_add_f32(ptr addrspace(1) %out, ptr addrsp
; GFX1250-GISEL-NEXT: ds_load_b64 v[2:3], v2 offset:8
; GFX1250-GISEL-NEXT: s_wait_dscnt 0x0
; GFX1250-GISEL-NEXT: v_pk_mul_f32 v[2:3], -1.0, v[2:3] op_sel_hi:[0,1]
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s3, v2
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s2, v3
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX1250-GISEL-NEXT: v_pk_add_f32 v[0:1], v[0:1], v[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_pk_add_f32 v[0:1], v[0:1], v[2:3] op_sel:[0,1] op_sel_hi:[1,0]
; GFX1250-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX1250-GISEL-NEXT: s_endpgm
@@ -4017,11 +3889,11 @@ define amdgpu_kernel void @fadd_fadd_fsub_0(<2 x float> %arg) {
; GFX90A-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-GISEL-NEXT: v_pk_add_f32 v[0:1], s[0:1], 0
-; GFX90A-GISEL-NEXT: v_readfirstlane_b32 s2, v1
-; GFX90A-GISEL-NEXT: v_pk_add_f32 v[0:1], s[2:3], 0
-; GFX90A-GISEL-NEXT: v_readfirstlane_b32 s1, v0
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: flat_store_dwordx2 v[0:1], v[0:1]
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v0, v1
+; GFX90A-GISEL-NEXT: v_pk_add_f32 v[0:1], v[0:1], 0
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v2, s0
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v3, v0
+; GFX90A-GISEL-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX90A-GISEL-NEXT: s_endpgm
;
; GFX942-GISEL-LABEL: fadd_fadd_fsub_0:
@@ -4029,15 +3901,12 @@ define amdgpu_kernel void @fadd_fadd_fsub_0(<2 x float> %arg) {
; GFX942-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-GISEL-NEXT: v_pk_add_f32 v[0:1], s[0:1], 0
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v2, s0
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, v1
+; GFX942-GISEL-NEXT: v_pk_add_f32 v[0:1], v[0:1], 0
; GFX942-GISEL-NEXT: s_nop 0
-; GFX942-GISEL-NEXT: v_readfirstlane_b32 s2, v1
-; GFX942-GISEL-NEXT: s_nop 1
-; GFX942-GISEL-NEXT: v_pk_add_f32 v[0:1], s[2:3], 0
-; GFX942-GISEL-NEXT: s_nop 0
-; GFX942-GISEL-NEXT: v_readfirstlane_b32 s1, v0
-; GFX942-GISEL-NEXT: s_nop 1
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX942-GISEL-NEXT: flat_store_dwordx2 v[0:1], v[0:1]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v3, v0
+; GFX942-GISEL-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX942-GISEL-NEXT: s_endpgm
;
; GFX1250-SDAG-LABEL: fadd_fadd_fsub_0:
@@ -4064,16 +3933,14 @@ define amdgpu_kernel void @fadd_fadd_fsub_0(<2 x float> %arg) {
; GFX1250-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v2, s0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_pk_add_f32 v[0:1], v[0:1], 0
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s2, v1
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v0, v1
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
; GFX1250-GISEL-NEXT: v_pk_add_f32 v[0:1], v[0:1], 0
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s1, v0
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX1250-GISEL-NEXT: flat_store_b64 v[0:1], v[0:1]
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v3, v0
+; GFX1250-GISEL-NEXT: flat_store_b64 v[0:1], v[2:3]
; GFX1250-GISEL-NEXT: s_endpgm
bb:
%i12 = fadd <2 x float> zeroinitializer, %arg
@@ -4120,16 +3987,15 @@ define amdgpu_kernel void @fadd_fadd_fsub(<2 x float> %arg, <2 x float> %arg1, p
; GFX90A-GISEL: ; %bb.0: ; %bb
; GFX90A-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX90A-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX90A-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX90A-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
; GFX90A-GISEL-NEXT: v_pk_add_f32 v[0:1], s[0:1], v[0:1]
-; GFX90A-GISEL-NEXT: v_readfirstlane_b32 s4, v1
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v0, v1
; GFX90A-GISEL-NEXT: v_pk_add_f32 v[0:1], s[2:3], v[0:1]
-; GFX90A-GISEL-NEXT: v_readfirstlane_b32 s1, v0
-; GFX90A-GISEL-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
-; GFX90A-GISEL-NEXT: v_pk_add_f32 v[0:1], v[0:1], s[2:3] neg_lo:[0,1] neg_hi:[0,1]
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v2, s0
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v3, v0
+; GFX90A-GISEL-NEXT: v_pk_add_f32 v[0:1], v[2:3], s[2:3] neg_lo:[0,1] neg_hi:[0,1]
+; GFX90A-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX90A-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7]
; GFX90A-GISEL-NEXT: s_endpgm
;
@@ -4137,20 +4003,16 @@ define amdgpu_kernel void @fadd_fadd_fsub(<2 x float> %arg, <2 x float> %arg1, p
; GFX942-GISEL: ; %bb.0: ; %bb
; GFX942-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
; GFX942-GISEL-NEXT: v_pk_add_f32 v[0:1], s[0:1], v[0:1]
-; GFX942-GISEL-NEXT: s_nop 0
-; GFX942-GISEL-NEXT: v_readfirstlane_b32 s4, v1
-; GFX942-GISEL-NEXT: s_nop 1
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[4:5]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v2, s0
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, v1
; GFX942-GISEL-NEXT: v_pk_add_f32 v[0:1], s[2:3], v[0:1]
; GFX942-GISEL-NEXT: s_nop 0
-; GFX942-GISEL-NEXT: v_readfirstlane_b32 s1, v0
-; GFX942-GISEL-NEXT: s_nop 1
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX942-GISEL-NEXT: v_pk_add_f32 v[0:1], v[0:1], s[2:3] neg_lo:[0,1] neg_hi:[0,1]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v3, v0
+; GFX942-GISEL-NEXT: v_pk_add_f32 v[0:1], v[2:3], s[2:3] neg_lo:[0,1] neg_hi:[0,1]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX942-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7]
; GFX942-GISEL-NEXT: s_endpgm
;
@@ -4187,17 +4049,15 @@ define amdgpu_kernel void @fadd_fadd_fsub(<2 x float> %arg, <2 x float> %arg1, p
; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v4, s0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_pk_add_f32 v[0:1], v[0:1], v[2:3]
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s2, v1
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v0, v1
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
; GFX1250-GISEL-NEXT: v_pk_add_f32 v[0:1], v[2:3], v[0:1]
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s1, v0
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v5, v0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_pk_add_f32 v[0:1], v[0:1], v[2:3] neg_lo:[0,1] neg_hi:[0,1]
+; GFX1250-GISEL-NEXT: v_pk_add_f32 v[0:1], v[4:5], v[2:3] neg_lo:[0,1] neg_hi:[0,1]
; GFX1250-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-GISEL-NEXT: global_store_b64 v2, v[0:1], s[6:7]
; GFX1250-GISEL-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/packed-fp64.ll b/llvm/test/CodeGen/AMDGPU/packed-fp64.ll
index d7c94df43b354..807cdb0307eb7 100644
--- a/llvm/test/CodeGen/AMDGPU/packed-fp64.ll
+++ b/llvm/test/CodeGen/AMDGPU/packed-fp64.ll
@@ -2147,58 +2147,50 @@ define amdgpu_kernel void @fma_vector_vector_neg_scalar_lo_scalar_hi(ptr addrspa
; GFX1251-GISEL-NEXT: ds_load_2addr_b32 v[10:11], v0 offset0:6 offset1:7
; GFX1251-GISEL-NEXT: ds_load_2addr_b32 v[0:1], v2 offset1:1
; GFX1251-GISEL-NEXT: ds_load_2addr_b32 v[2:3], v2 offset0:4 offset1:5
-; GFX1251-GISEL-NEXT: s_wait_dscnt 0x2
-; GFX1251-GISEL-NEXT: v_readfirstlane_b32 s10, v10
+; GFX1251-GISEL-NEXT: s_wait_dscnt 0x3
+; GFX1251-GISEL-NEXT: v_readfirstlane_b32 s8, v8
+; GFX1251-GISEL-NEXT: v_readfirstlane_b32 s9, v9
; GFX1251-GISEL-NEXT: s_wait_dscnt 0x1
; GFX1251-GISEL-NEXT: v_readfirstlane_b32 s4, v0
; GFX1251-GISEL-NEXT: v_readfirstlane_b32 s5, v1
; GFX1251-GISEL-NEXT: s_wait_dscnt 0x0
; GFX1251-GISEL-NEXT: v_readfirstlane_b32 s6, v2
; GFX1251-GISEL-NEXT: v_readfirstlane_b32 s7, v3
+; GFX1251-GISEL-NEXT: v_readfirstlane_b32 s10, v10
; GFX1251-GISEL-NEXT: v_readfirstlane_b32 s11, v11
; GFX1251-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[4:5]
-; GFX1251-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX1251-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX1251-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[6:7]
; GFX1251-GISEL-NEXT: v_readfirstlane_b32 s4, v4
; GFX1251-GISEL-NEXT: v_readfirstlane_b32 s5, v5
; GFX1251-GISEL-NEXT: v_readfirstlane_b32 s6, v6
; GFX1251-GISEL-NEXT: v_readfirstlane_b32 s7, v7
; GFX1251-GISEL-NEXT: v_pk_max_num_f64 v[0:3], v[0:3], v[0:3]
-; GFX1251-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1251-GISEL-NEXT: v_readfirstlane_b32 s2, v0
-; GFX1251-GISEL-NEXT: v_readfirstlane_b32 s3, v1
+; GFX1251-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[8:9]
+; GFX1251-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[10:11]
; GFX1251-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1251-GISEL-NEXT: v_readfirstlane_b32 s8, v2
-; GFX1251-GISEL-NEXT: v_readfirstlane_b32 s9, v3
-; GFX1251-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1251-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
-; GFX1251-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[8:9]
-; GFX1251-GISEL-NEXT: v_readfirstlane_b32 s8, v8
-; GFX1251-GISEL-NEXT: v_readfirstlane_b32 s9, v9
-; GFX1251-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX1251-GISEL-NEXT: v_xor_b32_e32 v1, 0x80000000, v1
-; GFX1251-GISEL-NEXT: v_readfirstlane_b32 s12, v0
; GFX1251-GISEL-NEXT: v_xor_b32_e32 v3, 0x80000000, v3
+; GFX1251-GISEL-NEXT: v_readfirstlane_b32 s12, v0
; GFX1251-GISEL-NEXT: v_readfirstlane_b32 s14, v2
-; GFX1251-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[8:9]
+; GFX1251-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX1251-GISEL-NEXT: v_readfirstlane_b32 s13, v1
-; GFX1251-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[4:5]
; GFX1251-GISEL-NEXT: v_readfirstlane_b32 s15, v3
+; GFX1251-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[4:5]
; GFX1251-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[6:7]
-; GFX1251-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[10:11]
+; GFX1251-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX1251-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[12:13]
-; GFX1251-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1251-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[14:15]
+; GFX1251-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1251-GISEL-NEXT: v_pk_fma_f64 v[0:3], v[0:3], v[4:7], v[8:11]
; GFX1251-GISEL-NEXT: v_mov_b32_e32 v4, 0
-; GFX1251-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX1251-GISEL-NEXT: v_readfirstlane_b32 s4, v0
+; GFX1251-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX1251-GISEL-NEXT: v_readfirstlane_b32 s5, v1
-; GFX1251-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX1251-GISEL-NEXT: v_readfirstlane_b32 s6, v2
; GFX1251-GISEL-NEXT: v_readfirstlane_b32 s7, v3
+; GFX1251-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1251-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[4:5]
-; GFX1251-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1251-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[6:7]
; GFX1251-GISEL-NEXT: global_store_b128 v4, v[0:3], s[0:1]
; GFX1251-GISEL-NEXT: s_endpgm
More information about the llvm-commits
mailing list