[llvm] Feat/sink gep constant offset (PR #140027)
via llvm-commits
llvm-commits at lists.llvm.org
Thu May 15 19:55:55 PDT 2025
https://github.com/StevenYangCC updated https://github.com/llvm/llvm-project/pull/140027
>From 788ed913819ef8ecf6b4ef492205f409aa9b4a38 Mon Sep 17 00:00:00 2001
From: "chengcang.yang" <yangchengcang at gmail.com>
Date: Thu, 15 May 2025 20:08:24 +0800
Subject: [PATCH] [SeparateConstOffsetFromGEP] Sink constant offset in GEP
chain to tail.
Summary:
Sink constant offsets down the GEP chain to the tail helps reduce
register usage. For example:
%gep0 = getelementptr half, ptr addrspace(3) %ptr, i32 512
%gep1 = getelementptr half, ptr addrspace(3) %gep0, i32 %ofst0
%gep2 = getelementptr half, ptr addrspace(3) %gep1, i32 %ofst1
%data = load half, ptr addrspace(3) %gep2, align 2
==>
%gep0 = getelementptr half, ptr addrspace(3) %ptr, i32 %ofst0
%gep1 = getelementptr half, ptr addrspace(3) %gep0, i32 %ofst1
%gep2 = getelementptr half, ptr addrspace(3) %gep1, i32 512
%data = load half, ptr addrspace(3) %gep2, align 2
---
.../Scalar/SeparateConstOffsetFromGEP.cpp | 145 +++
.../CodeGen/AArch64/aarch64-loop-gep-opt.ll | 1 -
.../CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.ll | 36 +-
...vm.amdgcn.sched.group.barrier.iterative.ll | 108 +--
.../AMDGPU/llvm.amdgcn.sched.group.barrier.ll | 850 +++++++++---------
...ne-sink-temporal-divergence-swdev407790.ll | 10 +-
llvm/test/CodeGen/PowerPC/split-index-tc.ll | 2 -
.../SeparateConstOffsetFromGEP/reorder-gep.ll | 27 +-
8 files changed, 666 insertions(+), 513 deletions(-)
diff --git a/llvm/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp b/llvm/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp
index 320b79203c0b3..a7bac1bb2d3fc 100644
--- a/llvm/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp
+++ b/llvm/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp
@@ -456,6 +456,22 @@ class SeparateConstOffsetFromGEP {
/// A helper that reunites sexts in an instruction.
bool reuniteExts(Instruction *I);
+ /// Sink constant offset in a GEP chain to tail. For example,
+ /// %gep0 = getelementptr half, ptr addrspace(3) %ptr, i32 512
+ /// %gep1 = getelementptr half, ptr addrspace(3) %gep0, i32 %ofst0
+ /// %gep2 = getelementptr half, ptr addrspace(3) %gep1, i32 %ofst1
+ /// %data = load half, ptr addrspace(3) %gep2, align 2
+ /// ==>
+ /// %gep0 = getelementptr half, ptr addrspace(3) %ptr, i32 %ofst0
+ /// %gep1 = getelementptr half, ptr addrspace(3) %gep0, i32 %ofst1
+ /// %gep2 = getelementptr half, ptr addrspace(3) %gep1, i32 512
+ /// %data = load half, ptr addrspace(3) %gep2, align 2
+ bool sinkGEPConstantOffset(Function &F);
+
+ /// A helper that does sink action for a root in a gep chain.
+ /// Return true if Ptr is a candidate for upper GEP in recursive calling.
+ bool sinkGEPConstantOffset(Value *Ptr, bool &Changed);
+
/// Find the closest dominator of <Dominatee> that is equivalent to <Key>.
Instruction *findClosestMatchingDominator(
ExprKey Key, Instruction *Dominatee,
@@ -1255,6 +1271,8 @@ bool SeparateConstOffsetFromGEP::run(Function &F) {
Changed |= reuniteExts(F);
+ Changed |= sinkGEPConstantOffset(F);
+
if (VerifyNoDeadCode)
verifyNoDeadCode(F);
@@ -1344,6 +1362,133 @@ bool SeparateConstOffsetFromGEP::reuniteExts(Function &F) {
return Changed;
}
+bool SeparateConstOffsetFromGEP::sinkGEPConstantOffset(Value *Ptr,
+ bool &Changed) {
+ // The purpose of this function is to sink the constant offsets in the GEP
+ // chain to the tail of the chain.
+ // This algorithm is implemented recursively, the algorithm starts from the
+ // tail of the chain through the DFS method and shifts the constant offset
+ // of the GEP step by step upwards by bottom-up DFS method, i.e. step by step
+ // down to the tail.
+ // A simple example is given:
+ /// %gep0 = getelementptr half, ptr addrspace(3) %ptr, i32 512
+ /// %gep1 = getelementptr half, ptr addrspace(3) %gep0, i32 %ofst0
+ /// %gep2 = getelementptr half, ptr addrspace(3) %gep1, i32 %ofst1
+ /// %data = load half, ptr addrspace(3) %gep2, align 2
+ /// ==>
+ /// %gep0 = getelementptr half, ptr addrspace(3) %ptr, i32 %ofst0
+ /// %gep1 = getelementptr half, ptr addrspace(3) %gep0, i32 %ofst1
+ /// %gep2 = getelementptr half, ptr addrspace(3) %gep1, i32 512
+ /// %data = load half, ptr addrspace(3) %gep2, align 2
+ GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Ptr);
+ if (!GEP)
+ return false;
+
+ bool BaseResult = sinkGEPConstantOffset(GEP->getPointerOperand(), Changed);
+
+ if (GEP->getNumIndices() != 1)
+ return false;
+
+ ConstantInt *C = nullptr;
+ Value *Idx = GEP->getOperand(1);
+ bool MatchConstant = match(Idx, m_ConstantInt(C));
+
+ if (!BaseResult)
+ return MatchConstant;
+
+ Type *ResTy = GEP->getResultElementType();
+ GetElementPtrInst *BaseGEP =
+ dyn_cast<GetElementPtrInst>(GEP->getPointerOperand());
+ assert(BaseGEP);
+ Value *BaseIdx = BaseGEP->getOperand(1);
+ Type *BaseResTy = BaseGEP->getResultElementType();
+
+ if (MatchConstant) {
+ // %gep0 = getelementptr half, ptr addrspace(3) %ptr, i32 8
+ // %gep1 = getelementptr half, ptr addrspace(3) %gep0, i32 4
+ // as:
+ // %gep1 = getelementptr half, ptr addrspace(3) %ptr, i32 12
+ Type *NewResTy = nullptr;
+ Constant *NewIdx = nullptr;
+ if (ResTy == BaseResTy) {
+ NewResTy = ResTy;
+ int64_t NewIdxValue = cast<ConstantInt>(BaseIdx)->getSExtValue() +
+ cast<ConstantInt>(Idx)->getSExtValue();
+ Type *NewIdxType = (NewIdxValue < std::numeric_limits<int32_t>::min() ||
+ NewIdxValue > std::numeric_limits<int32_t>::max())
+ ? Type::getInt64Ty(GEP->getContext())
+ : Type::getInt32Ty(GEP->getContext());
+ NewIdx = ConstantInt::get(NewIdxType, NewIdxValue);
+ } else {
+ NewResTy = Type::getInt8Ty(GEP->getContext());
+ int64_t NewIdxValue = (cast<ConstantInt>(BaseIdx)->getSExtValue() *
+ DL->getTypeAllocSize(BaseResTy)) +
+ (cast<ConstantInt>(Idx)->getSExtValue() *
+ DL->getTypeAllocSize(ResTy));
+ Type *NewIdxType = (NewIdxValue < std::numeric_limits<int32_t>::min() ||
+ NewIdxValue > std::numeric_limits<int32_t>::max())
+ ? Type::getInt64Ty(GEP->getContext())
+ : Type::getInt32Ty(GEP->getContext());
+ NewIdx = ConstantInt::get(NewIdxType, NewIdxValue);
+ }
+ assert(NewResTy);
+ assert(NewIdx);
+ auto *NewGEP = GetElementPtrInst::Create(
+ NewResTy, BaseGEP->getPointerOperand(), NewIdx);
+ NewGEP->setIsInBounds(GEP->isInBounds());
+ NewGEP->insertBefore(GEP->getIterator());
+ NewGEP->takeName(GEP);
+
+ GEP->replaceAllUsesWith(NewGEP);
+ GEP->eraseFromParent();
+
+ Changed = true;
+ return true;
+ }
+
+ // %gep0 = getelementptr half, ptr addrspace(3) %ptr, i32 8
+ // %gep1 = getelementptr half, ptr addrspace(3) %gep0, i32 %idx
+ // as:
+ // %gepx0 = getelementptr half, ptr addrspace(3) %ptr, i32 %idx
+ // %gepx1 = getelementptr half, ptr addrspace(3) %gepx0, i32 8
+ auto *GEPX0 =
+ GetElementPtrInst::Create(ResTy, BaseGEP->getPointerOperand(), Idx);
+ GEPX0->setIsInBounds(BaseGEP->isInBounds());
+ GEPX0->insertBefore(GEP->getIterator());
+ auto *GEPX1 = GetElementPtrInst::Create(BaseResTy, GEPX0, BaseIdx);
+ GEPX1->setIsInBounds(GEP->isInBounds());
+ GEPX1->insertBefore(GEP->getIterator());
+ GEPX1->takeName(GEP);
+
+ GEP->replaceAllUsesWith(GEPX1);
+ GEP->eraseFromParent();
+
+ Changed = true;
+ return true;
+}
+
+bool SeparateConstOffsetFromGEP::sinkGEPConstantOffset(Function &F) {
+ bool Changed = false;
+ SmallVector<Value *, 4> Candidates;
+ for (BasicBlock &B : F) {
+ for (Instruction &I : B) {
+ Value *Ptr = nullptr;
+ if (LoadInst *LI = dyn_cast<LoadInst>(&I)) {
+ Ptr = LI->getPointerOperand();
+ } else if (StoreInst *SI = dyn_cast<StoreInst>(&I)) {
+ Ptr = SI->getPointerOperand();
+ }
+ if (Ptr)
+ Candidates.push_back(Ptr);
+ }
+ }
+
+ for (Value *Ptr : Candidates)
+ sinkGEPConstantOffset(Ptr, Changed);
+
+ return Changed;
+}
+
void SeparateConstOffsetFromGEP::verifyNoDeadCode(Function &F) {
for (BasicBlock &B : F) {
for (Instruction &I : B) {
diff --git a/llvm/test/CodeGen/AArch64/aarch64-loop-gep-opt.ll b/llvm/test/CodeGen/AArch64/aarch64-loop-gep-opt.ll
index c32e0b9b3e7aa..9bf777c796c0f 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-loop-gep-opt.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-loop-gep-opt.ll
@@ -45,4 +45,3 @@ fooo.exit: ; preds = %do.body.i
store i32 %nb.1.i, ptr %k0, align 4
br label %do.body.i.backedge
}
-
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.ll
index 565ad295ebbb3..4f5d93d767a7a 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.ll
@@ -21,15 +21,15 @@ define amdgpu_kernel void @test_iglp_opt_mfma_gemm(ptr addrspace(3) noalias %in,
; GCN-NEXT: ; iglp_opt mask(0x00000000)
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: v_add_u32_e32 v1, s0, v0
-; GCN-NEXT: v_add_u32_e32 v2, 0x6000, v1
-; GCN-NEXT: ds_read_b128 a[28:31], v2 offset:57456
-; GCN-NEXT: ds_read_b128 a[24:27], v2 offset:57440
-; GCN-NEXT: ds_read_b128 a[20:23], v2 offset:57424
-; GCN-NEXT: ds_read_b128 a[16:19], v2 offset:57408
-; GCN-NEXT: ds_read_b128 a[0:3], v2 offset:57344
-; GCN-NEXT: ds_read_b128 a[4:7], v2 offset:57360
-; GCN-NEXT: ds_read_b128 a[8:11], v2 offset:57376
-; GCN-NEXT: ds_read_b128 a[12:15], v2 offset:57392
+; GCN-NEXT: v_add_u32_e32 v2, 0x14000, v1
+; GCN-NEXT: ds_read_b128 a[28:31], v2 offset:112
+; GCN-NEXT: ds_read_b128 a[24:27], v2 offset:96
+; GCN-NEXT: ds_read_b128 a[20:23], v2 offset:80
+; GCN-NEXT: ds_read_b128 a[16:19], v2 offset:64
+; GCN-NEXT: ds_read_b128 a[0:3], v2
+; GCN-NEXT: ds_read_b128 a[4:7], v2 offset:16
+; GCN-NEXT: ds_read_b128 a[8:11], v2 offset:32
+; GCN-NEXT: ds_read_b128 a[12:15], v2 offset:48
; GCN-NEXT: v_mov_b32_e32 v2, 1.0
; GCN-NEXT: ds_read_b128 a[60:63], v1 offset:49264
; GCN-NEXT: ds_read_b128 a[56:59], v1 offset:49248
@@ -199,17 +199,17 @@ define amdgpu_kernel void @test_iglp_opt_rev_mfma_gemm(ptr addrspace(3) noalias
; GCN-NEXT: ds_read_b128 a[72:75], v1 offset:49184
; GCN-NEXT: ds_read_b128 a[68:71], v1 offset:49168
; GCN-NEXT: ds_read_b128 a[64:67], v1 offset:49152
-; GCN-NEXT: v_add_u32_e32 v1, 0x6000, v1
+; GCN-NEXT: v_add_u32_e32 v1, 0x14000, v1
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: v_mfma_f32_32x32x1f32 a[64:95], v2, v3, a[64:95]
-; GCN-NEXT: ds_read_b128 a[60:63], v1 offset:57456
-; GCN-NEXT: ds_read_b128 a[56:59], v1 offset:57440
-; GCN-NEXT: ds_read_b128 a[52:55], v1 offset:57424
-; GCN-NEXT: ds_read_b128 a[48:51], v1 offset:57408
-; GCN-NEXT: ds_read_b128 a[32:35], v1 offset:57344
-; GCN-NEXT: ds_read_b128 a[36:39], v1 offset:57360
-; GCN-NEXT: ds_read_b128 a[40:43], v1 offset:57376
-; GCN-NEXT: ds_read_b128 a[44:47], v1 offset:57392
+; GCN-NEXT: ds_read_b128 a[60:63], v1 offset:112
+; GCN-NEXT: ds_read_b128 a[56:59], v1 offset:96
+; GCN-NEXT: ds_read_b128 a[52:55], v1 offset:80
+; GCN-NEXT: ds_read_b128 a[48:51], v1 offset:64
+; GCN-NEXT: ds_read_b128 a[32:35], v1
+; GCN-NEXT: ds_read_b128 a[36:39], v1 offset:16
+; GCN-NEXT: ds_read_b128 a[40:43], v1 offset:32
+; GCN-NEXT: ds_read_b128 a[44:47], v1 offset:48
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v2, v3, a[32:63]
; GCN-NEXT: ds_write_b128 v0, a[28:31] offset:112
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.iterative.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.iterative.ll
index 371b4f070094d..c80c42ce73efb 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.iterative.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.iterative.ll
@@ -25,7 +25,7 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_MFMA_interleave(ptr
; GCN-MINREG-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v1, a[0:31]
; GCN-MINREG-NEXT: v_add_u32_e32 v5, s1, v0
; GCN-MINREG-NEXT: v_mov_b32_e32 v0, s1
-; GCN-MINREG-NEXT: v_add_u32_e32 v3, 0x6000, v4
+; GCN-MINREG-NEXT: v_add_u32_e32 v3, 0x14000, v4
; GCN-MINREG-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
; GCN-MINREG-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
; GCN-MINREG-NEXT: s_nop 7
@@ -110,14 +110,14 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_MFMA_interleave(ptr
; GCN-MINREG-NEXT: ds_write_b128 v0, a[12:15] offset:24624
; GCN-MINREG-NEXT: ds_write_b128 v0, a[0:3] offset:24576
; GCN-MINREG-NEXT: ds_write_b128 v0, a[4:7] offset:24592
-; GCN-MINREG-NEXT: ds_read_b128 a[28:31], v3 offset:57456
-; GCN-MINREG-NEXT: ds_read_b128 a[24:27], v3 offset:57440
-; GCN-MINREG-NEXT: ds_read_b128 a[20:23], v3 offset:57424
-; GCN-MINREG-NEXT: ds_read_b128 a[16:19], v3 offset:57408
-; GCN-MINREG-NEXT: ds_read_b128 a[0:3], v3 offset:57344
-; GCN-MINREG-NEXT: ds_read_b128 a[4:7], v3 offset:57360
-; GCN-MINREG-NEXT: ds_read_b128 a[8:11], v3 offset:57376
-; GCN-MINREG-NEXT: ds_read_b128 a[12:15], v3 offset:57392
+; GCN-MINREG-NEXT: ds_read_b128 a[28:31], v3 offset:112
+; GCN-MINREG-NEXT: ds_read_b128 a[24:27], v3 offset:96
+; GCN-MINREG-NEXT: ds_read_b128 a[20:23], v3 offset:80
+; GCN-MINREG-NEXT: ds_read_b128 a[16:19], v3 offset:64
+; GCN-MINREG-NEXT: ds_read_b128 a[0:3], v3
+; GCN-MINREG-NEXT: ds_read_b128 a[4:7], v3 offset:16
+; GCN-MINREG-NEXT: ds_read_b128 a[8:11], v3 offset:32
+; GCN-MINREG-NEXT: ds_read_b128 a[12:15], v3 offset:48
; GCN-MINREG-NEXT: s_waitcnt lgkmcnt(0)
; GCN-MINREG-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v1, a[0:31]
; GCN-MINREG-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
@@ -229,7 +229,7 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_MFMA_interleave(ptr
; GCN-MAXOCC-NEXT: ds_read_b128 a[0:3], v0 offset:49152
; GCN-MAXOCC-NEXT: s_waitcnt lgkmcnt(0)
; GCN-MAXOCC-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v3, a[0:31]
-; GCN-MAXOCC-NEXT: v_add_u32_e32 v0, 0x6000, v0
+; GCN-MAXOCC-NEXT: v_add_u32_e32 v0, 0x14000, v0
; GCN-MAXOCC-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
; GCN-MAXOCC-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
; GCN-MAXOCC-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
@@ -244,14 +244,14 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_MFMA_interleave(ptr
; GCN-MAXOCC-NEXT: ds_write_b128 v1, a[12:15] offset:24624
; GCN-MAXOCC-NEXT: ds_write_b128 v1, a[0:3] offset:24576
; GCN-MAXOCC-NEXT: ds_write_b128 v1, a[4:7] offset:24592
-; GCN-MAXOCC-NEXT: ds_read_b128 a[28:31], v0 offset:57456
-; GCN-MAXOCC-NEXT: ds_read_b128 a[24:27], v0 offset:57440
-; GCN-MAXOCC-NEXT: ds_read_b128 a[20:23], v0 offset:57424
-; GCN-MAXOCC-NEXT: ds_read_b128 a[16:19], v0 offset:57408
-; GCN-MAXOCC-NEXT: ds_read_b128 a[0:3], v0 offset:57344
-; GCN-MAXOCC-NEXT: ds_read_b128 a[4:7], v0 offset:57360
-; GCN-MAXOCC-NEXT: ds_read_b128 a[8:11], v0 offset:57376
-; GCN-MAXOCC-NEXT: ds_read_b128 a[12:15], v0 offset:57392
+; GCN-MAXOCC-NEXT: ds_read_b128 a[28:31], v0 offset:112
+; GCN-MAXOCC-NEXT: ds_read_b128 a[24:27], v0 offset:96
+; GCN-MAXOCC-NEXT: ds_read_b128 a[20:23], v0 offset:80
+; GCN-MAXOCC-NEXT: ds_read_b128 a[16:19], v0 offset:64
+; GCN-MAXOCC-NEXT: ds_read_b128 a[0:3], v0
+; GCN-MAXOCC-NEXT: ds_read_b128 a[4:7], v0 offset:16
+; GCN-MAXOCC-NEXT: ds_read_b128 a[8:11], v0 offset:32
+; GCN-MAXOCC-NEXT: ds_read_b128 a[12:15], v0 offset:48
; GCN-MAXOCC-NEXT: s_waitcnt lgkmcnt(0)
; GCN-MAXOCC-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v3, a[0:31]
; GCN-MAXOCC-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
@@ -357,26 +357,26 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_MFMA_interleave(ptr
; GCN-ILP-NEXT: ds_read_b128 a[24:27], v3 offset:49248
; GCN-ILP-NEXT: s_waitcnt lgkmcnt(0)
; GCN-ILP-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v1, v2, a[0:31]
-; GCN-ILP-NEXT: v_add_u32_e32 v3, 0x6000, v3
+; GCN-ILP-NEXT: v_add_u32_e32 v3, 0x14000, v3
; GCN-ILP-NEXT: s_nop 7
; GCN-ILP-NEXT: s_nop 7
; GCN-ILP-NEXT: s_nop 1
; GCN-ILP-NEXT: ds_write_b128 v0, a[4:7] offset:24592
-; GCN-ILP-NEXT: ds_read_b128 a[4:7], v3 offset:57360
+; GCN-ILP-NEXT: ds_read_b128 a[4:7], v3 offset:16
; GCN-ILP-NEXT: ds_write_b128 v0, a[0:3] offset:24576
-; GCN-ILP-NEXT: ds_read_b128 a[0:3], v3 offset:57344
+; GCN-ILP-NEXT: ds_read_b128 a[0:3], v3
; GCN-ILP-NEXT: ds_write_b128 v0, a[12:15] offset:24624
-; GCN-ILP-NEXT: ds_read_b128 a[12:15], v3 offset:57392
+; GCN-ILP-NEXT: ds_read_b128 a[12:15], v3 offset:48
; GCN-ILP-NEXT: ds_write_b128 v0, a[8:11] offset:24608
-; GCN-ILP-NEXT: ds_read_b128 a[8:11], v3 offset:57376
+; GCN-ILP-NEXT: ds_read_b128 a[8:11], v3 offset:32
; GCN-ILP-NEXT: ds_write_b128 v0, a[20:23] offset:24656
-; GCN-ILP-NEXT: ds_read_b128 a[20:23], v3 offset:57424
+; GCN-ILP-NEXT: ds_read_b128 a[20:23], v3 offset:80
; GCN-ILP-NEXT: ds_write_b128 v0, a[16:19] offset:24640
-; GCN-ILP-NEXT: ds_read_b128 a[16:19], v3 offset:57408
+; GCN-ILP-NEXT: ds_read_b128 a[16:19], v3 offset:64
; GCN-ILP-NEXT: ds_write_b128 v0, a[28:31] offset:24688
-; GCN-ILP-NEXT: ds_read_b128 a[28:31], v3 offset:57456
+; GCN-ILP-NEXT: ds_read_b128 a[28:31], v3 offset:112
; GCN-ILP-NEXT: ds_write_b128 v0, a[24:27] offset:24672
-; GCN-ILP-NEXT: ds_read_b128 a[24:27], v3 offset:57440
+; GCN-ILP-NEXT: ds_read_b128 a[24:27], v3 offset:96
; GCN-ILP-NEXT: s_waitcnt lgkmcnt(0)
; GCN-ILP-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v1, v2, a[0:31]
; GCN-ILP-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
@@ -536,7 +536,7 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_MFMA_interleave_spl
; GCN-MINREG-NEXT: ds_read_b128 a[12:15], v3 offset:24624
; GCN-MINREG-NEXT: s_waitcnt lgkmcnt(0)
; GCN-MINREG-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v1, v0, a[0:31]
-; GCN-MINREG-NEXT: v_add_u32_e32 v4, 0x6000, v3
+; GCN-MINREG-NEXT: v_add_u32_e32 v4, 0x14000, v3
; GCN-MINREG-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
; GCN-MINREG-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
; GCN-MINREG-NEXT: s_nop 7
@@ -574,14 +574,14 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_MFMA_interleave_spl
; GCN-MINREG-NEXT: ds_write_b128 v2, a[8:11] offset:24608
; GCN-MINREG-NEXT: ds_write_b128 v2, a[4:7] offset:24592
; GCN-MINREG-NEXT: ds_write_b128 v2, a[0:3] offset:24576
-; GCN-MINREG-NEXT: ds_read_b128 a[28:31], v4 offset:57456
-; GCN-MINREG-NEXT: ds_read_b128 a[24:27], v4 offset:57440
-; GCN-MINREG-NEXT: ds_read_b128 a[20:23], v4 offset:57424
-; GCN-MINREG-NEXT: ds_read_b128 a[16:19], v4 offset:57408
-; GCN-MINREG-NEXT: ds_read_b128 a[0:3], v4 offset:57344
-; GCN-MINREG-NEXT: ds_read_b128 a[4:7], v4 offset:57360
-; GCN-MINREG-NEXT: ds_read_b128 a[8:11], v4 offset:57376
-; GCN-MINREG-NEXT: ds_read_b128 a[12:15], v4 offset:57392
+; GCN-MINREG-NEXT: ds_read_b128 a[28:31], v4 offset:112
+; GCN-MINREG-NEXT: ds_read_b128 a[24:27], v4 offset:96
+; GCN-MINREG-NEXT: ds_read_b128 a[20:23], v4 offset:80
+; GCN-MINREG-NEXT: ds_read_b128 a[16:19], v4 offset:64
+; GCN-MINREG-NEXT: ds_read_b128 a[0:3], v4
+; GCN-MINREG-NEXT: ds_read_b128 a[4:7], v4 offset:16
+; GCN-MINREG-NEXT: ds_read_b128 a[8:11], v4 offset:32
+; GCN-MINREG-NEXT: ds_read_b128 a[12:15], v4 offset:48
; GCN-MINREG-NEXT: s_waitcnt lgkmcnt(0)
; GCN-MINREG-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v1, v0, a[0:31]
; GCN-MINREG-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
@@ -694,7 +694,7 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_MFMA_interleave_spl
; GCN-MAXOCC-NEXT: ds_read_b128 a[0:3], v0 offset:49152
; GCN-MAXOCC-NEXT: s_waitcnt lgkmcnt(0)
; GCN-MAXOCC-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v1, v2, a[0:31]
-; GCN-MAXOCC-NEXT: v_add_u32_e32 v0, 0x6000, v0
+; GCN-MAXOCC-NEXT: v_add_u32_e32 v0, 0x14000, v0
; GCN-MAXOCC-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
; GCN-MAXOCC-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
; GCN-MAXOCC-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
@@ -709,14 +709,14 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_MFMA_interleave_spl
; GCN-MAXOCC-NEXT: ds_write_b128 v3, a[8:11] offset:24608
; GCN-MAXOCC-NEXT: ds_write_b128 v3, a[4:7] offset:24592
; GCN-MAXOCC-NEXT: ds_write_b128 v3, a[0:3] offset:24576
-; GCN-MAXOCC-NEXT: ds_read_b128 a[28:31], v0 offset:57456
-; GCN-MAXOCC-NEXT: ds_read_b128 a[24:27], v0 offset:57440
-; GCN-MAXOCC-NEXT: ds_read_b128 a[20:23], v0 offset:57424
-; GCN-MAXOCC-NEXT: ds_read_b128 a[16:19], v0 offset:57408
-; GCN-MAXOCC-NEXT: ds_read_b128 a[0:3], v0 offset:57344
-; GCN-MAXOCC-NEXT: ds_read_b128 a[4:7], v0 offset:57360
-; GCN-MAXOCC-NEXT: ds_read_b128 a[8:11], v0 offset:57376
-; GCN-MAXOCC-NEXT: ds_read_b128 a[12:15], v0 offset:57392
+; GCN-MAXOCC-NEXT: ds_read_b128 a[28:31], v0 offset:112
+; GCN-MAXOCC-NEXT: ds_read_b128 a[24:27], v0 offset:96
+; GCN-MAXOCC-NEXT: ds_read_b128 a[20:23], v0 offset:80
+; GCN-MAXOCC-NEXT: ds_read_b128 a[16:19], v0 offset:64
+; GCN-MAXOCC-NEXT: ds_read_b128 a[0:3], v0
+; GCN-MAXOCC-NEXT: ds_read_b128 a[4:7], v0 offset:16
+; GCN-MAXOCC-NEXT: ds_read_b128 a[8:11], v0 offset:32
+; GCN-MAXOCC-NEXT: ds_read_b128 a[12:15], v0 offset:48
; GCN-MAXOCC-NEXT: s_waitcnt lgkmcnt(0)
; GCN-MAXOCC-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v1, v2, a[0:31]
; GCN-MAXOCC-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
@@ -829,26 +829,26 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_MFMA_interleave_spl
; GCN-ILP-NEXT: ds_read_b128 a[28:31], v3 offset:49264
; GCN-ILP-NEXT: s_waitcnt lgkmcnt(0)
; GCN-ILP-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v0, v1, a[0:31]
-; GCN-ILP-NEXT: v_add_u32_e32 v3, 0x6000, v3
+; GCN-ILP-NEXT: v_add_u32_e32 v3, 0x14000, v3
; GCN-ILP-NEXT: s_nop 7
; GCN-ILP-NEXT: s_nop 7
; GCN-ILP-NEXT: s_nop 1
; GCN-ILP-NEXT: ds_write_b128 v2, a[0:3] offset:24576
-; GCN-ILP-NEXT: ds_read_b128 a[0:3], v3 offset:57344
+; GCN-ILP-NEXT: ds_read_b128 a[0:3], v3
; GCN-ILP-NEXT: ds_write_b128 v2, a[4:7] offset:24592
-; GCN-ILP-NEXT: ds_read_b128 a[4:7], v3 offset:57360
+; GCN-ILP-NEXT: ds_read_b128 a[4:7], v3 offset:16
; GCN-ILP-NEXT: ds_write_b128 v2, a[8:11] offset:24608
-; GCN-ILP-NEXT: ds_read_b128 a[8:11], v3 offset:57376
+; GCN-ILP-NEXT: ds_read_b128 a[8:11], v3 offset:32
; GCN-ILP-NEXT: ds_write_b128 v2, a[12:15] offset:24624
-; GCN-ILP-NEXT: ds_read_b128 a[12:15], v3 offset:57392
+; GCN-ILP-NEXT: ds_read_b128 a[12:15], v3 offset:48
; GCN-ILP-NEXT: ds_write_b128 v2, a[16:19] offset:24640
-; GCN-ILP-NEXT: ds_read_b128 a[16:19], v3 offset:57408
+; GCN-ILP-NEXT: ds_read_b128 a[16:19], v3 offset:64
; GCN-ILP-NEXT: ds_write_b128 v2, a[20:23] offset:24656
-; GCN-ILP-NEXT: ds_read_b128 a[20:23], v3 offset:57424
+; GCN-ILP-NEXT: ds_read_b128 a[20:23], v3 offset:80
; GCN-ILP-NEXT: ds_write_b128 v2, a[24:27] offset:24672
-; GCN-ILP-NEXT: ds_read_b128 a[24:27], v3 offset:57440
+; GCN-ILP-NEXT: ds_read_b128 a[24:27], v3 offset:96
; GCN-ILP-NEXT: ds_write_b128 v2, a[28:31] offset:24688
-; GCN-ILP-NEXT: ds_read_b128 a[28:31], v3 offset:57456
+; GCN-ILP-NEXT: ds_read_b128 a[28:31], v3 offset:112
; GCN-ILP-NEXT: s_waitcnt lgkmcnt(0)
; GCN-ILP-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v0, v1, a[0:31]
; GCN-ILP-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.ll
index 73586b1243376..218e1b513c097 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.ll
@@ -623,6 +623,7 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_MFMA_cluster(ptr ad
; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GCN-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GCN-NEXT: v_lshlrev_b32_e32 v0, 7, v0
+; GCN-NEXT: v_mov_b32_e32 v2, 1.0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: v_add_u32_e32 v1, s0, v0
; GCN-NEXT: ds_read_b128 a[156:159], v1 offset:112
@@ -633,51 +634,50 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_MFMA_cluster(ptr ad
; GCN-NEXT: ds_read_b128 a[132:135], v1 offset:16
; GCN-NEXT: ds_read_b128 a[136:139], v1 offset:32
; GCN-NEXT: ds_read_b128 a[140:143], v1 offset:48
-; GCN-NEXT: ds_read_b128 a[28:31], v1 offset:8304
-; GCN-NEXT: ds_read_b128 a[24:27], v1 offset:8288
-; GCN-NEXT: ds_read_b128 a[20:23], v1 offset:8272
-; GCN-NEXT: ds_read_b128 a[16:19], v1 offset:8256
-; GCN-NEXT: ds_read_b128 a[12:15], v1 offset:8240
-; GCN-NEXT: ds_read_b128 a[8:11], v1 offset:8224
-; GCN-NEXT: ds_read_b128 a[4:7], v1 offset:8208
-; GCN-NEXT: ds_read_b128 a[0:3], v1 offset:8192
-; GCN-NEXT: v_add_u32_e32 v2, 0x6000, v1
-; GCN-NEXT: ds_read_b128 a[124:127], v1 offset:24688
-; GCN-NEXT: ds_read_b128 a[120:123], v1 offset:24672
-; GCN-NEXT: ds_read_b128 a[116:119], v1 offset:24656
-; GCN-NEXT: ds_read_b128 a[112:115], v1 offset:24640
-; GCN-NEXT: ds_read_b128 a[108:111], v1 offset:24624
-; GCN-NEXT: ds_read_b128 a[104:107], v1 offset:24608
-; GCN-NEXT: ds_read_b128 a[100:103], v1 offset:24592
-; GCN-NEXT: ds_read_b128 a[96:99], v1 offset:24576
-; GCN-NEXT: ds_read_b128 a[92:95], v1 offset:49264
-; GCN-NEXT: ds_read_b128 a[88:91], v1 offset:49248
-; GCN-NEXT: ds_read_b128 a[84:87], v1 offset:49232
-; GCN-NEXT: ds_read_b128 a[80:83], v1 offset:49216
-; GCN-NEXT: ds_read_b128 a[76:79], v1 offset:49200
-; GCN-NEXT: ds_read_b128 a[72:75], v1 offset:49184
-; GCN-NEXT: ds_read_b128 a[68:71], v1 offset:49168
-; GCN-NEXT: ds_read_b128 a[64:67], v1 offset:49152
-; GCN-NEXT: v_mov_b32_e32 v1, 1.0
-; GCN-NEXT: ds_read_b128 a[60:63], v2 offset:57456
-; GCN-NEXT: ds_read_b128 a[56:59], v2 offset:57440
-; GCN-NEXT: ds_read_b128 a[52:55], v2 offset:57424
-; GCN-NEXT: ds_read_b128 a[48:51], v2 offset:57408
-; GCN-NEXT: ds_read_b128 a[32:35], v2 offset:57344
-; GCN-NEXT: ds_read_b128 a[36:39], v2 offset:57360
-; GCN-NEXT: ds_read_b128 a[40:43], v2 offset:57376
-; GCN-NEXT: ds_read_b128 a[44:47], v2 offset:57392
-; GCN-NEXT: v_mov_b32_e32 v2, 2.0
+; GCN-NEXT: ds_read_b128 a[92:95], v1 offset:8304
+; GCN-NEXT: ds_read_b128 a[88:91], v1 offset:8288
+; GCN-NEXT: ds_read_b128 a[84:87], v1 offset:8272
+; GCN-NEXT: ds_read_b128 a[80:83], v1 offset:8256
+; GCN-NEXT: ds_read_b128 a[76:79], v1 offset:8240
+; GCN-NEXT: ds_read_b128 a[72:75], v1 offset:8224
+; GCN-NEXT: ds_read_b128 a[68:71], v1 offset:8208
+; GCN-NEXT: ds_read_b128 a[64:67], v1 offset:8192
+; GCN-NEXT: ds_read_b128 a[60:63], v1 offset:24688
+; GCN-NEXT: ds_read_b128 a[56:59], v1 offset:24672
+; GCN-NEXT: ds_read_b128 a[52:55], v1 offset:24656
+; GCN-NEXT: ds_read_b128 a[48:51], v1 offset:24640
+; GCN-NEXT: ds_read_b128 a[44:47], v1 offset:24624
+; GCN-NEXT: ds_read_b128 a[40:43], v1 offset:24608
+; GCN-NEXT: ds_read_b128 a[36:39], v1 offset:24592
+; GCN-NEXT: ds_read_b128 a[32:35], v1 offset:24576
+; GCN-NEXT: ds_read_b128 a[28:31], v1 offset:49264
+; GCN-NEXT: ds_read_b128 a[24:27], v1 offset:49248
+; GCN-NEXT: ds_read_b128 a[20:23], v1 offset:49232
+; GCN-NEXT: ds_read_b128 a[16:19], v1 offset:49216
+; GCN-NEXT: ds_read_b128 a[12:15], v1 offset:49200
+; GCN-NEXT: ds_read_b128 a[8:11], v1 offset:49184
+; GCN-NEXT: ds_read_b128 a[4:7], v1 offset:49168
+; GCN-NEXT: ds_read_b128 a[0:3], v1 offset:49152
+; GCN-NEXT: v_add_u32_e32 v1, 0x14000, v1
+; GCN-NEXT: ds_read_b128 a[124:127], v1 offset:112
+; GCN-NEXT: ds_read_b128 a[120:123], v1 offset:96
+; GCN-NEXT: ds_read_b128 a[116:119], v1 offset:80
+; GCN-NEXT: ds_read_b128 a[112:115], v1 offset:64
+; GCN-NEXT: ds_read_b128 a[96:99], v1
+; GCN-NEXT: ds_read_b128 a[100:103], v1 offset:16
+; GCN-NEXT: ds_read_b128 a[104:107], v1 offset:32
+; GCN-NEXT: ds_read_b128 a[108:111], v1 offset:48
+; GCN-NEXT: v_mov_b32_e32 v1, 2.0
; GCN-NEXT: v_add_u32_e32 v0, s1, v0
; GCN-NEXT: ; sched_group_barrier mask(0x00000100) size(40) SyncID(0)
; GCN-NEXT: s_waitcnt lgkmcnt(14)
-; GCN-NEXT: v_mfma_f32_32x32x1f32 a[128:159], v1, v2, a[128:159]
-; GCN-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v1, v2, a[0:31]
-; GCN-NEXT: v_mfma_f32_32x32x1f32 a[96:127], v1, v2, a[96:127]
+; GCN-NEXT: v_mfma_f32_32x32x1f32 a[128:159], v2, v1, a[128:159]
+; GCN-NEXT: v_mfma_f32_32x32x1f32 a[64:95], v2, v1, a[64:95]
+; GCN-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v2, v1, a[32:63]
; GCN-NEXT: s_waitcnt lgkmcnt(8)
-; GCN-NEXT: v_mfma_f32_32x32x1f32 a[64:95], v1, v2, a[64:95]
+; GCN-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v1, a[0:31]
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v1, v2, a[32:63]
+; GCN-NEXT: v_mfma_f32_32x32x1f32 a[96:127], v2, v1, a[96:127]
; GCN-NEXT: s_nop 7
; GCN-NEXT: s_nop 4
; GCN-NEXT: ds_write_b128 v0, a[156:159] offset:112
@@ -689,38 +689,38 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_MFMA_cluster(ptr ad
; GCN-NEXT: ds_write_b128 v0, a[132:135] offset:16
; GCN-NEXT: ds_write_b128 v0, a[128:131]
; GCN-NEXT: v_mov_b32_e32 v0, s1
-; GCN-NEXT: ds_write_b128 v0, a[24:27] offset:8288
-; GCN-NEXT: ds_write_b128 v0, a[28:31] offset:8304
-; GCN-NEXT: ds_write_b128 v0, a[16:19] offset:8256
-; GCN-NEXT: ds_write_b128 v0, a[20:23] offset:8272
-; GCN-NEXT: ds_write_b128 v0, a[8:11] offset:8224
-; GCN-NEXT: ds_write_b128 v0, a[12:15] offset:8240
-; GCN-NEXT: ds_write_b128 v0, a[0:3] offset:8192
-; GCN-NEXT: ds_write_b128 v0, a[4:7] offset:8208
-; GCN-NEXT: ds_write_b128 v0, a[120:123] offset:16480
-; GCN-NEXT: ds_write_b128 v0, a[124:127] offset:16496
-; GCN-NEXT: ds_write_b128 v0, a[112:115] offset:16448
-; GCN-NEXT: ds_write_b128 v0, a[116:119] offset:16464
-; GCN-NEXT: ds_write_b128 v0, a[104:107] offset:16416
-; GCN-NEXT: ds_write_b128 v0, a[108:111] offset:16432
-; GCN-NEXT: ds_write_b128 v0, a[96:99] offset:16384
-; GCN-NEXT: ds_write_b128 v0, a[100:103] offset:16400
-; GCN-NEXT: ds_write_b128 v0, a[88:91] offset:24672
-; GCN-NEXT: ds_write_b128 v0, a[92:95] offset:24688
-; GCN-NEXT: ds_write_b128 v0, a[80:83] offset:24640
-; GCN-NEXT: ds_write_b128 v0, a[84:87] offset:24656
-; GCN-NEXT: ds_write_b128 v0, a[72:75] offset:24608
-; GCN-NEXT: ds_write_b128 v0, a[76:79] offset:24624
-; GCN-NEXT: ds_write_b128 v0, a[64:67] offset:24576
-; GCN-NEXT: ds_write_b128 v0, a[68:71] offset:24592
-; GCN-NEXT: ds_write_b128 v0, a[56:59] offset:32864
-; GCN-NEXT: ds_write_b128 v0, a[60:63] offset:32880
-; GCN-NEXT: ds_write_b128 v0, a[48:51] offset:32832
-; GCN-NEXT: ds_write_b128 v0, a[52:55] offset:32848
-; GCN-NEXT: ds_write_b128 v0, a[40:43] offset:32800
-; GCN-NEXT: ds_write_b128 v0, a[44:47] offset:32816
-; GCN-NEXT: ds_write_b128 v0, a[32:35] offset:32768
-; GCN-NEXT: ds_write_b128 v0, a[36:39] offset:32784
+; GCN-NEXT: ds_write_b128 v0, a[88:91] offset:8288
+; GCN-NEXT: ds_write_b128 v0, a[92:95] offset:8304
+; GCN-NEXT: ds_write_b128 v0, a[80:83] offset:8256
+; GCN-NEXT: ds_write_b128 v0, a[84:87] offset:8272
+; GCN-NEXT: ds_write_b128 v0, a[72:75] offset:8224
+; GCN-NEXT: ds_write_b128 v0, a[76:79] offset:8240
+; GCN-NEXT: ds_write_b128 v0, a[64:67] offset:8192
+; GCN-NEXT: ds_write_b128 v0, a[68:71] offset:8208
+; GCN-NEXT: ds_write_b128 v0, a[56:59] offset:16480
+; GCN-NEXT: ds_write_b128 v0, a[60:63] offset:16496
+; GCN-NEXT: ds_write_b128 v0, a[48:51] offset:16448
+; GCN-NEXT: ds_write_b128 v0, a[52:55] offset:16464
+; GCN-NEXT: ds_write_b128 v0, a[40:43] offset:16416
+; GCN-NEXT: ds_write_b128 v0, a[44:47] offset:16432
+; GCN-NEXT: ds_write_b128 v0, a[32:35] offset:16384
+; GCN-NEXT: ds_write_b128 v0, a[36:39] offset:16400
+; GCN-NEXT: ds_write_b128 v0, a[24:27] offset:24672
+; GCN-NEXT: ds_write_b128 v0, a[28:31] offset:24688
+; GCN-NEXT: ds_write_b128 v0, a[16:19] offset:24640
+; GCN-NEXT: ds_write_b128 v0, a[20:23] offset:24656
+; GCN-NEXT: ds_write_b128 v0, a[8:11] offset:24608
+; GCN-NEXT: ds_write_b128 v0, a[12:15] offset:24624
+; GCN-NEXT: ds_write_b128 v0, a[0:3] offset:24576
+; GCN-NEXT: ds_write_b128 v0, a[4:7] offset:24592
+; GCN-NEXT: ds_write_b128 v0, a[120:123] offset:32864
+; GCN-NEXT: ds_write_b128 v0, a[124:127] offset:32880
+; GCN-NEXT: ds_write_b128 v0, a[112:115] offset:32832
+; GCN-NEXT: ds_write_b128 v0, a[116:119] offset:32848
+; GCN-NEXT: ds_write_b128 v0, a[104:107] offset:32800
+; GCN-NEXT: ds_write_b128 v0, a[108:111] offset:32816
+; GCN-NEXT: ds_write_b128 v0, a[96:99] offset:32768
+; GCN-NEXT: ds_write_b128 v0, a[100:103] offset:32784
; GCN-NEXT: ; sched_group_barrier mask(0x00000008) size(5) SyncID(0)
; GCN-NEXT: ; sched_group_barrier mask(0x00000200) size(40) SyncID(0)
; GCN-NEXT: s_endpgm
@@ -730,6 +730,7 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_MFMA_cluster(ptr ad
; EXACTCUTOFF-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; EXACTCUTOFF-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; EXACTCUTOFF-NEXT: v_lshlrev_b32_e32 v0, 7, v0
+; EXACTCUTOFF-NEXT: v_mov_b32_e32 v2, 1.0
; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(0)
; EXACTCUTOFF-NEXT: v_add_u32_e32 v1, s0, v0
; EXACTCUTOFF-NEXT: ds_read_b128 a[156:159], v1 offset:112
@@ -740,51 +741,50 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_MFMA_cluster(ptr ad
; EXACTCUTOFF-NEXT: ds_read_b128 a[132:135], v1 offset:16
; EXACTCUTOFF-NEXT: ds_read_b128 a[136:139], v1 offset:32
; EXACTCUTOFF-NEXT: ds_read_b128 a[140:143], v1 offset:48
-; EXACTCUTOFF-NEXT: ds_read_b128 a[28:31], v1 offset:8304
-; EXACTCUTOFF-NEXT: ds_read_b128 a[24:27], v1 offset:8288
-; EXACTCUTOFF-NEXT: ds_read_b128 a[20:23], v1 offset:8272
-; EXACTCUTOFF-NEXT: ds_read_b128 a[16:19], v1 offset:8256
-; EXACTCUTOFF-NEXT: ds_read_b128 a[12:15], v1 offset:8240
-; EXACTCUTOFF-NEXT: ds_read_b128 a[8:11], v1 offset:8224
-; EXACTCUTOFF-NEXT: ds_read_b128 a[4:7], v1 offset:8208
-; EXACTCUTOFF-NEXT: ds_read_b128 a[0:3], v1 offset:8192
-; EXACTCUTOFF-NEXT: v_add_u32_e32 v2, 0x6000, v1
-; EXACTCUTOFF-NEXT: ds_read_b128 a[124:127], v1 offset:24688
-; EXACTCUTOFF-NEXT: ds_read_b128 a[120:123], v1 offset:24672
-; EXACTCUTOFF-NEXT: ds_read_b128 a[116:119], v1 offset:24656
-; EXACTCUTOFF-NEXT: ds_read_b128 a[112:115], v1 offset:24640
-; EXACTCUTOFF-NEXT: ds_read_b128 a[108:111], v1 offset:24624
-; EXACTCUTOFF-NEXT: ds_read_b128 a[104:107], v1 offset:24608
-; EXACTCUTOFF-NEXT: ds_read_b128 a[100:103], v1 offset:24592
-; EXACTCUTOFF-NEXT: ds_read_b128 a[96:99], v1 offset:24576
-; EXACTCUTOFF-NEXT: ds_read_b128 a[92:95], v1 offset:49264
-; EXACTCUTOFF-NEXT: ds_read_b128 a[88:91], v1 offset:49248
-; EXACTCUTOFF-NEXT: ds_read_b128 a[84:87], v1 offset:49232
-; EXACTCUTOFF-NEXT: ds_read_b128 a[80:83], v1 offset:49216
-; EXACTCUTOFF-NEXT: ds_read_b128 a[76:79], v1 offset:49200
-; EXACTCUTOFF-NEXT: ds_read_b128 a[72:75], v1 offset:49184
-; EXACTCUTOFF-NEXT: ds_read_b128 a[68:71], v1 offset:49168
-; EXACTCUTOFF-NEXT: ds_read_b128 a[64:67], v1 offset:49152
-; EXACTCUTOFF-NEXT: v_mov_b32_e32 v1, 1.0
-; EXACTCUTOFF-NEXT: ds_read_b128 a[60:63], v2 offset:57456
-; EXACTCUTOFF-NEXT: ds_read_b128 a[56:59], v2 offset:57440
-; EXACTCUTOFF-NEXT: ds_read_b128 a[52:55], v2 offset:57424
-; EXACTCUTOFF-NEXT: ds_read_b128 a[48:51], v2 offset:57408
-; EXACTCUTOFF-NEXT: ds_read_b128 a[32:35], v2 offset:57344
-; EXACTCUTOFF-NEXT: ds_read_b128 a[36:39], v2 offset:57360
-; EXACTCUTOFF-NEXT: ds_read_b128 a[40:43], v2 offset:57376
-; EXACTCUTOFF-NEXT: ds_read_b128 a[44:47], v2 offset:57392
-; EXACTCUTOFF-NEXT: v_mov_b32_e32 v2, 2.0
+; EXACTCUTOFF-NEXT: ds_read_b128 a[92:95], v1 offset:8304
+; EXACTCUTOFF-NEXT: ds_read_b128 a[88:91], v1 offset:8288
+; EXACTCUTOFF-NEXT: ds_read_b128 a[84:87], v1 offset:8272
+; EXACTCUTOFF-NEXT: ds_read_b128 a[80:83], v1 offset:8256
+; EXACTCUTOFF-NEXT: ds_read_b128 a[76:79], v1 offset:8240
+; EXACTCUTOFF-NEXT: ds_read_b128 a[72:75], v1 offset:8224
+; EXACTCUTOFF-NEXT: ds_read_b128 a[68:71], v1 offset:8208
+; EXACTCUTOFF-NEXT: ds_read_b128 a[64:67], v1 offset:8192
+; EXACTCUTOFF-NEXT: ds_read_b128 a[60:63], v1 offset:24688
+; EXACTCUTOFF-NEXT: ds_read_b128 a[56:59], v1 offset:24672
+; EXACTCUTOFF-NEXT: ds_read_b128 a[52:55], v1 offset:24656
+; EXACTCUTOFF-NEXT: ds_read_b128 a[48:51], v1 offset:24640
+; EXACTCUTOFF-NEXT: ds_read_b128 a[44:47], v1 offset:24624
+; EXACTCUTOFF-NEXT: ds_read_b128 a[40:43], v1 offset:24608
+; EXACTCUTOFF-NEXT: ds_read_b128 a[36:39], v1 offset:24592
+; EXACTCUTOFF-NEXT: ds_read_b128 a[32:35], v1 offset:24576
+; EXACTCUTOFF-NEXT: ds_read_b128 a[28:31], v1 offset:49264
+; EXACTCUTOFF-NEXT: ds_read_b128 a[24:27], v1 offset:49248
+; EXACTCUTOFF-NEXT: ds_read_b128 a[20:23], v1 offset:49232
+; EXACTCUTOFF-NEXT: ds_read_b128 a[16:19], v1 offset:49216
+; EXACTCUTOFF-NEXT: ds_read_b128 a[12:15], v1 offset:49200
+; EXACTCUTOFF-NEXT: ds_read_b128 a[8:11], v1 offset:49184
+; EXACTCUTOFF-NEXT: ds_read_b128 a[4:7], v1 offset:49168
+; EXACTCUTOFF-NEXT: ds_read_b128 a[0:3], v1 offset:49152
+; EXACTCUTOFF-NEXT: v_add_u32_e32 v1, 0x14000, v1
+; EXACTCUTOFF-NEXT: ds_read_b128 a[124:127], v1 offset:112
+; EXACTCUTOFF-NEXT: ds_read_b128 a[120:123], v1 offset:96
+; EXACTCUTOFF-NEXT: ds_read_b128 a[116:119], v1 offset:80
+; EXACTCUTOFF-NEXT: ds_read_b128 a[112:115], v1 offset:64
+; EXACTCUTOFF-NEXT: ds_read_b128 a[96:99], v1
+; EXACTCUTOFF-NEXT: ds_read_b128 a[100:103], v1 offset:16
+; EXACTCUTOFF-NEXT: ds_read_b128 a[104:107], v1 offset:32
+; EXACTCUTOFF-NEXT: ds_read_b128 a[108:111], v1 offset:48
+; EXACTCUTOFF-NEXT: v_mov_b32_e32 v1, 2.0
; EXACTCUTOFF-NEXT: v_add_u32_e32 v0, s1, v0
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000100) size(40) SyncID(0)
; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(14)
-; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[128:159], v1, v2, a[128:159]
-; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v1, v2, a[0:31]
-; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[96:127], v1, v2, a[96:127]
+; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[128:159], v2, v1, a[128:159]
+; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[64:95], v2, v1, a[64:95]
+; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v2, v1, a[32:63]
; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(8)
-; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[64:95], v1, v2, a[64:95]
+; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v1, a[0:31]
; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(0)
-; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v1, v2, a[32:63]
+; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[96:127], v2, v1, a[96:127]
; EXACTCUTOFF-NEXT: s_nop 7
; EXACTCUTOFF-NEXT: s_nop 4
; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[156:159] offset:112
@@ -796,38 +796,38 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_MFMA_cluster(ptr ad
; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[132:135] offset:16
; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[128:131]
; EXACTCUTOFF-NEXT: v_mov_b32_e32 v0, s1
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[24:27] offset:8288
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[28:31] offset:8304
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[16:19] offset:8256
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[20:23] offset:8272
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[8:11] offset:8224
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[12:15] offset:8240
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[0:3] offset:8192
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[4:7] offset:8208
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[120:123] offset:16480
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[124:127] offset:16496
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[112:115] offset:16448
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[116:119] offset:16464
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[104:107] offset:16416
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[108:111] offset:16432
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[96:99] offset:16384
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[100:103] offset:16400
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[88:91] offset:24672
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[92:95] offset:24688
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[80:83] offset:24640
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[84:87] offset:24656
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[72:75] offset:24608
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[76:79] offset:24624
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[64:67] offset:24576
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[68:71] offset:24592
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[56:59] offset:32864
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[60:63] offset:32880
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[48:51] offset:32832
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[52:55] offset:32848
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[40:43] offset:32800
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[44:47] offset:32816
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[32:35] offset:32768
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[36:39] offset:32784
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[88:91] offset:8288
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[92:95] offset:8304
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[80:83] offset:8256
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[84:87] offset:8272
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[72:75] offset:8224
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[76:79] offset:8240
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[64:67] offset:8192
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[68:71] offset:8208
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[56:59] offset:16480
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[60:63] offset:16496
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[48:51] offset:16448
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[52:55] offset:16464
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[40:43] offset:16416
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[44:47] offset:16432
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[32:35] offset:16384
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[36:39] offset:16400
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[24:27] offset:24672
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[28:31] offset:24688
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[16:19] offset:24640
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[20:23] offset:24656
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[8:11] offset:24608
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[12:15] offset:24624
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[0:3] offset:24576
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[4:7] offset:24592
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[120:123] offset:32864
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[124:127] offset:32880
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[112:115] offset:32832
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[116:119] offset:32848
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[104:107] offset:32800
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[108:111] offset:32816
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[96:99] offset:32768
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[100:103] offset:32784
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000008) size(5) SyncID(0)
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000200) size(40) SyncID(0)
; EXACTCUTOFF-NEXT: s_endpgm
@@ -960,7 +960,7 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_MFMA_interleave(ptr
; GCN-NEXT: ds_read_b128 a[0:3], v1 offset:49152
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v3, a[0:31]
-; GCN-NEXT: v_add_u32_e32 v1, 0x6000, v1
+; GCN-NEXT: v_add_u32_e32 v1, 0x14000, v1
; GCN-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
; GCN-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
; GCN-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
@@ -975,14 +975,14 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_MFMA_interleave(ptr
; GCN-NEXT: ds_write_b128 v0, a[12:15] offset:24624
; GCN-NEXT: ds_write_b128 v0, a[0:3] offset:24576
; GCN-NEXT: ds_write_b128 v0, a[4:7] offset:24592
-; GCN-NEXT: ds_read_b128 a[28:31], v1 offset:57456
-; GCN-NEXT: ds_read_b128 a[24:27], v1 offset:57440
-; GCN-NEXT: ds_read_b128 a[20:23], v1 offset:57424
-; GCN-NEXT: ds_read_b128 a[16:19], v1 offset:57408
-; GCN-NEXT: ds_read_b128 a[0:3], v1 offset:57344
-; GCN-NEXT: ds_read_b128 a[4:7], v1 offset:57360
-; GCN-NEXT: ds_read_b128 a[8:11], v1 offset:57376
-; GCN-NEXT: ds_read_b128 a[12:15], v1 offset:57392
+; GCN-NEXT: ds_read_b128 a[28:31], v1 offset:112
+; GCN-NEXT: ds_read_b128 a[24:27], v1 offset:96
+; GCN-NEXT: ds_read_b128 a[20:23], v1 offset:80
+; GCN-NEXT: ds_read_b128 a[16:19], v1 offset:64
+; GCN-NEXT: ds_read_b128 a[0:3], v1
+; GCN-NEXT: ds_read_b128 a[4:7], v1 offset:16
+; GCN-NEXT: ds_read_b128 a[8:11], v1 offset:32
+; GCN-NEXT: ds_read_b128 a[12:15], v1 offset:48
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v3, a[0:31]
; GCN-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
@@ -1094,7 +1094,7 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_MFMA_interleave(ptr
; EXACTCUTOFF-NEXT: ds_read_b128 a[0:3], v1 offset:49152
; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(0)
; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v3, a[0:31]
-; EXACTCUTOFF-NEXT: v_add_u32_e32 v1, 0x6000, v1
+; EXACTCUTOFF-NEXT: v_add_u32_e32 v1, 0x14000, v1
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
@@ -1109,14 +1109,14 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_MFMA_interleave(ptr
; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[12:15] offset:24624
; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[0:3] offset:24576
; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[4:7] offset:24592
-; EXACTCUTOFF-NEXT: ds_read_b128 a[28:31], v1 offset:57456
-; EXACTCUTOFF-NEXT: ds_read_b128 a[24:27], v1 offset:57440
-; EXACTCUTOFF-NEXT: ds_read_b128 a[20:23], v1 offset:57424
-; EXACTCUTOFF-NEXT: ds_read_b128 a[16:19], v1 offset:57408
-; EXACTCUTOFF-NEXT: ds_read_b128 a[0:3], v1 offset:57344
-; EXACTCUTOFF-NEXT: ds_read_b128 a[4:7], v1 offset:57360
-; EXACTCUTOFF-NEXT: ds_read_b128 a[8:11], v1 offset:57376
-; EXACTCUTOFF-NEXT: ds_read_b128 a[12:15], v1 offset:57392
+; EXACTCUTOFF-NEXT: ds_read_b128 a[28:31], v1 offset:112
+; EXACTCUTOFF-NEXT: ds_read_b128 a[24:27], v1 offset:96
+; EXACTCUTOFF-NEXT: ds_read_b128 a[20:23], v1 offset:80
+; EXACTCUTOFF-NEXT: ds_read_b128 a[16:19], v1 offset:64
+; EXACTCUTOFF-NEXT: ds_read_b128 a[0:3], v1
+; EXACTCUTOFF-NEXT: ds_read_b128 a[4:7], v1 offset:16
+; EXACTCUTOFF-NEXT: ds_read_b128 a[8:11], v1 offset:32
+; EXACTCUTOFF-NEXT: ds_read_b128 a[12:15], v1 offset:48
; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(0)
; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v3, a[0:31]
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
@@ -1199,19 +1199,19 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_interleave_EXP_MFMA
; GCN-LABEL: test_sched_group_barrier_pipeline_interleave_EXP_MFMA:
; GCN: ; %bb.0: ; %entry
; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x44
-; GCN-NEXT: v_mov_b32_e32 v3, 0x3fb8aa3b
+; GCN-NEXT: v_mov_b32_e32 v2, 0x3fb8aa3b
; GCN-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GCN-NEXT: v_mov_b32_e32 v7, 0x32a5705f
+; GCN-NEXT: v_mov_b32_e32 v6, 0x32a5705f
; GCN-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_mul_f32_e32 v4, s0, v3
-; GCN-NEXT: v_rndne_f32_e32 v5, v4
-; GCN-NEXT: v_sub_f32_e32 v6, v4, v5
-; GCN-NEXT: v_fma_f32 v4, s0, v3, -v4
-; GCN-NEXT: v_fmac_f32_e32 v4, s0, v7
-; GCN-NEXT: v_add_f32_e32 v4, v6, v4
-; GCN-NEXT: v_exp_f32_e32 v4, v4
-; GCN-NEXT: v_cvt_i32_f32_e32 v5, v5
+; GCN-NEXT: v_mul_f32_e32 v3, s0, v2
+; GCN-NEXT: v_rndne_f32_e32 v4, v3
+; GCN-NEXT: v_sub_f32_e32 v5, v3, v4
+; GCN-NEXT: v_fma_f32 v3, s0, v2, -v3
+; GCN-NEXT: v_fmac_f32_e32 v3, s0, v6
+; GCN-NEXT: v_add_f32_e32 v3, v5, v3
+; GCN-NEXT: v_exp_f32_e32 v3, v3
+; GCN-NEXT: v_cvt_i32_f32_e32 v4, v4
; GCN-NEXT: v_lshlrev_b32_e32 v0, 7, v0
; GCN-NEXT: v_add_u32_e32 v1, s6, v0
; GCN-NEXT: ds_read_b128 a[124:127], v1 offset:112
@@ -1222,112 +1222,113 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_interleave_EXP_MFMA
; GCN-NEXT: ds_read_b128 a[100:103], v1 offset:16
; GCN-NEXT: ds_read_b128 a[104:107], v1 offset:32
; GCN-NEXT: ds_read_b128 a[108:111], v1 offset:48
-; GCN-NEXT: v_mov_b32_e32 v9, 1.0
-; GCN-NEXT: v_ldexp_f32 v4, v4, v5
-; GCN-NEXT: v_mov_b32_e32 v5, 0xc2ce8ed0
-; GCN-NEXT: v_mul_f32_e32 v10, s1, v3
-; GCN-NEXT: v_cmp_nlt_f32_e32 vcc, s0, v5
-; GCN-NEXT: v_mov_b32_e32 v6, 0x42b17218
-; GCN-NEXT: v_rndne_f32_e32 v11, v10
-; GCN-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc
-; GCN-NEXT: v_cmp_ngt_f32_e32 vcc, s0, v6
-; GCN-NEXT: v_mov_b32_e32 v8, 0x7f800000
-; GCN-NEXT: v_sub_f32_e32 v12, v10, v11
-; GCN-NEXT: v_fma_f32 v10, s1, v3, -v10
-; GCN-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc
-; GCN-NEXT: v_fmac_f32_e32 v10, s1, v7
-; GCN-NEXT: ds_read_b128 a[28:31], v1 offset:8304
+; GCN-NEXT: v_mov_b32_e32 v8, 1.0
+; GCN-NEXT: v_ldexp_f32 v3, v3, v4
+; GCN-NEXT: v_mov_b32_e32 v4, 0xc2ce8ed0
+; GCN-NEXT: v_mul_f32_e32 v9, s1, v2
+; GCN-NEXT: v_cmp_nlt_f32_e32 vcc, s0, v4
+; GCN-NEXT: v_mov_b32_e32 v5, 0x42b17218
+; GCN-NEXT: v_rndne_f32_e32 v10, v9
+; GCN-NEXT: v_cndmask_b32_e32 v3, 0, v3, vcc
+; GCN-NEXT: v_cmp_ngt_f32_e32 vcc, s0, v5
+; GCN-NEXT: v_mov_b32_e32 v7, 0x7f800000
+; GCN-NEXT: v_sub_f32_e32 v11, v9, v10
+; GCN-NEXT: v_fma_f32 v9, s1, v2, -v9
+; GCN-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc
+; GCN-NEXT: v_fmac_f32_e32 v9, s1, v6
+; GCN-NEXT: ds_read_b128 a[92:95], v1 offset:8304
; GCN-NEXT: s_waitcnt lgkmcnt(1)
-; GCN-NEXT: v_mfma_f32_32x32x1f32 a[96:127], v9, v4, a[96:127]
-; GCN-NEXT: v_add_f32_e32 v4, v12, v10
-; GCN-NEXT: v_exp_f32_e32 v4, v4
-; GCN-NEXT: v_cvt_i32_f32_e32 v10, v11
-; GCN-NEXT: ds_read_b128 a[24:27], v1 offset:8288
-; GCN-NEXT: ds_read_b128 a[20:23], v1 offset:8272
-; GCN-NEXT: ds_read_b128 a[16:19], v1 offset:8256
-; GCN-NEXT: ds_read_b128 a[12:15], v1 offset:8240
-; GCN-NEXT: ds_read_b128 a[8:11], v1 offset:8224
-; GCN-NEXT: ds_read_b128 a[4:7], v1 offset:8208
-; GCN-NEXT: ds_read_b128 a[0:3], v1 offset:8192
-; GCN-NEXT: v_ldexp_f32 v4, v4, v10
-; GCN-NEXT: v_cmp_nlt_f32_e32 vcc, s1, v5
-; GCN-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc
-; GCN-NEXT: v_cmp_ngt_f32_e32 vcc, s1, v6
-; GCN-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc
-; GCN-NEXT: v_mul_f32_e32 v10, s2, v3
-; GCN-NEXT: v_rndne_f32_e32 v11, v10
+; GCN-NEXT: v_mfma_f32_32x32x1f32 a[96:127], v8, v3, a[96:127]
+; GCN-NEXT: v_add_f32_e32 v3, v11, v9
+; GCN-NEXT: v_exp_f32_e32 v3, v3
+; GCN-NEXT: v_cvt_i32_f32_e32 v9, v10
+; GCN-NEXT: ds_read_b128 a[88:91], v1 offset:8288
+; GCN-NEXT: ds_read_b128 a[84:87], v1 offset:8272
+; GCN-NEXT: ds_read_b128 a[80:83], v1 offset:8256
+; GCN-NEXT: ds_read_b128 a[76:79], v1 offset:8240
+; GCN-NEXT: ds_read_b128 a[72:75], v1 offset:8224
+; GCN-NEXT: ds_read_b128 a[68:71], v1 offset:8208
+; GCN-NEXT: ds_read_b128 a[64:67], v1 offset:8192
+; GCN-NEXT: v_ldexp_f32 v3, v3, v9
+; GCN-NEXT: v_cmp_nlt_f32_e32 vcc, s1, v4
+; GCN-NEXT: v_cndmask_b32_e32 v3, 0, v3, vcc
+; GCN-NEXT: v_cmp_ngt_f32_e32 vcc, s1, v5
+; GCN-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc
+; GCN-NEXT: v_mul_f32_e32 v9, s2, v2
+; GCN-NEXT: v_rndne_f32_e32 v10, v9
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v9, v4, a[0:31]
-; GCN-NEXT: v_fma_f32 v4, s2, v3, -v10
-; GCN-NEXT: v_sub_f32_e32 v12, v10, v11
-; GCN-NEXT: v_fmac_f32_e32 v4, s2, v7
-; GCN-NEXT: v_add_f32_e32 v4, v12, v4
-; GCN-NEXT: v_exp_f32_e32 v4, v4
-; GCN-NEXT: v_cvt_i32_f32_e32 v10, v11
-; GCN-NEXT: ds_read_b128 a[92:95], v1 offset:24688
-; GCN-NEXT: ds_read_b128 a[88:91], v1 offset:24672
-; GCN-NEXT: ds_read_b128 a[84:87], v1 offset:24656
-; GCN-NEXT: ds_read_b128 a[80:83], v1 offset:24640
-; GCN-NEXT: ds_read_b128 a[76:79], v1 offset:24624
-; GCN-NEXT: ds_read_b128 a[72:75], v1 offset:24608
-; GCN-NEXT: ds_read_b128 a[68:71], v1 offset:24592
-; GCN-NEXT: ds_read_b128 a[64:67], v1 offset:24576
-; GCN-NEXT: v_add_u32_e32 v2, 0x6000, v1
-; GCN-NEXT: ds_read_b128 a[60:63], v1 offset:49264
-; GCN-NEXT: ds_read_b128 a[56:59], v1 offset:49248
-; GCN-NEXT: ds_read_b128 a[52:55], v1 offset:49232
-; GCN-NEXT: ds_read_b128 a[48:51], v1 offset:49216
-; GCN-NEXT: ds_read_b128 a[44:47], v1 offset:49200
-; GCN-NEXT: ds_read_b128 a[40:43], v1 offset:49184
-; GCN-NEXT: ds_read_b128 a[36:39], v1 offset:49168
-; GCN-NEXT: ds_read_b128 a[32:35], v1 offset:49152
-; GCN-NEXT: v_ldexp_f32 v1, v4, v10
-; GCN-NEXT: v_cmp_nlt_f32_e32 vcc, s2, v5
-; GCN-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
-; GCN-NEXT: v_cmp_ngt_f32_e32 vcc, s2, v6
-; GCN-NEXT: v_mul_f32_e32 v4, s3, v3
-; GCN-NEXT: v_cndmask_b32_e32 v1, v8, v1, vcc
-; GCN-NEXT: v_rndne_f32_e32 v10, v4
+; GCN-NEXT: v_mfma_f32_32x32x1f32 a[64:95], v8, v3, a[64:95]
+; GCN-NEXT: v_fma_f32 v3, s2, v2, -v9
+; GCN-NEXT: v_sub_f32_e32 v11, v9, v10
+; GCN-NEXT: v_fmac_f32_e32 v3, s2, v6
+; GCN-NEXT: v_add_f32_e32 v3, v11, v3
+; GCN-NEXT: v_exp_f32_e32 v3, v3
+; GCN-NEXT: v_cvt_i32_f32_e32 v9, v10
+; GCN-NEXT: ds_read_b128 a[60:63], v1 offset:24688
+; GCN-NEXT: ds_read_b128 a[56:59], v1 offset:24672
+; GCN-NEXT: ds_read_b128 a[52:55], v1 offset:24656
+; GCN-NEXT: ds_read_b128 a[48:51], v1 offset:24640
+; GCN-NEXT: ds_read_b128 a[44:47], v1 offset:24624
+; GCN-NEXT: ds_read_b128 a[40:43], v1 offset:24608
+; GCN-NEXT: ds_read_b128 a[36:39], v1 offset:24592
+; GCN-NEXT: ds_read_b128 a[32:35], v1 offset:24576
+; GCN-NEXT: v_ldexp_f32 v3, v3, v9
+; GCN-NEXT: v_cmp_nlt_f32_e32 vcc, s2, v4
+; GCN-NEXT: v_cndmask_b32_e32 v3, 0, v3, vcc
+; GCN-NEXT: v_cmp_ngt_f32_e32 vcc, s2, v5
+; GCN-NEXT: v_mul_f32_e32 v9, s3, v2
+; GCN-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc
+; GCN-NEXT: v_rndne_f32_e32 v10, v9
; GCN-NEXT: s_load_dword s8, s[4:5], 0x54
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_mfma_f32_32x32x1f32 a[64:95], v9, v1, a[64:95]
-; GCN-NEXT: v_sub_f32_e32 v1, v4, v10
-; GCN-NEXT: v_fma_f32 v4, s3, v3, -v4
-; GCN-NEXT: v_fmac_f32_e32 v4, s3, v7
-; GCN-NEXT: v_add_f32_e32 v1, v1, v4
-; GCN-NEXT: v_exp_f32_e32 v1, v1
-; GCN-NEXT: v_cvt_i32_f32_e32 v4, v10
-; GCN-NEXT: v_cmp_nlt_f32_e32 vcc, s3, v5
-; GCN-NEXT: ds_read_b128 a[156:159], v2 offset:57456
-; GCN-NEXT: ds_read_b128 a[152:155], v2 offset:57440
-; GCN-NEXT: v_ldexp_f32 v1, v1, v4
-; GCN-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
-; GCN-NEXT: v_cmp_ngt_f32_e32 vcc, s3, v6
-; GCN-NEXT: v_cndmask_b32_e32 v1, v8, v1, vcc
-; GCN-NEXT: v_mul_f32_e32 v4, s8, v3
-; GCN-NEXT: v_fma_f32 v3, s8, v3, -v4
-; GCN-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v9, v1, a[32:63]
-; GCN-NEXT: v_rndne_f32_e32 v1, v4
-; GCN-NEXT: v_sub_f32_e32 v10, v4, v1
-; GCN-NEXT: v_fmac_f32_e32 v3, s8, v7
-; GCN-NEXT: v_add_f32_e32 v3, v10, v3
+; GCN-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v8, v3, a[32:63]
+; GCN-NEXT: v_sub_f32_e32 v3, v9, v10
+; GCN-NEXT: v_fma_f32 v9, s3, v2, -v9
+; GCN-NEXT: v_fmac_f32_e32 v9, s3, v6
+; GCN-NEXT: v_add_f32_e32 v3, v3, v9
; GCN-NEXT: v_exp_f32_e32 v3, v3
-; GCN-NEXT: v_cvt_i32_f32_e32 v1, v1
-; GCN-NEXT: ds_read_b128 a[148:151], v2 offset:57424
-; GCN-NEXT: ds_read_b128 a[144:147], v2 offset:57408
-; GCN-NEXT: ds_read_b128 a[128:131], v2 offset:57344
-; GCN-NEXT: ds_read_b128 a[132:135], v2 offset:57360
-; GCN-NEXT: ds_read_b128 a[136:139], v2 offset:57376
-; GCN-NEXT: ds_read_b128 a[140:143], v2 offset:57392
-; GCN-NEXT: v_ldexp_f32 v1, v3, v1
-; GCN-NEXT: v_cmp_nlt_f32_e32 vcc, s8, v5
+; GCN-NEXT: v_cvt_i32_f32_e32 v9, v10
+; GCN-NEXT: ds_read_b128 a[28:31], v1 offset:49264
+; GCN-NEXT: ds_read_b128 a[24:27], v1 offset:49248
+; GCN-NEXT: ds_read_b128 a[20:23], v1 offset:49232
+; GCN-NEXT: ds_read_b128 a[16:19], v1 offset:49216
+; GCN-NEXT: ds_read_b128 a[12:15], v1 offset:49200
+; GCN-NEXT: ds_read_b128 a[8:11], v1 offset:49184
+; GCN-NEXT: ds_read_b128 a[4:7], v1 offset:49168
+; GCN-NEXT: ds_read_b128 a[0:3], v1 offset:49152
+; GCN-NEXT: v_ldexp_f32 v3, v3, v9
+; GCN-NEXT: v_cmp_nlt_f32_e32 vcc, s3, v4
+; GCN-NEXT: v_cndmask_b32_e32 v3, 0, v3, vcc
+; GCN-NEXT: v_cmp_ngt_f32_e32 vcc, s3, v5
+; GCN-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc
+; GCN-NEXT: v_mul_f32_e32 v9, s8, v2
+; GCN-NEXT: v_fma_f32 v2, s8, v2, -v9
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v8, v3, a[0:31]
+; GCN-NEXT: v_rndne_f32_e32 v3, v9
+; GCN-NEXT: v_sub_f32_e32 v10, v9, v3
+; GCN-NEXT: v_fmac_f32_e32 v2, s8, v6
+; GCN-NEXT: v_add_f32_e32 v2, v10, v2
+; GCN-NEXT: v_exp_f32_e32 v2, v2
+; GCN-NEXT: v_cvt_i32_f32_e32 v3, v3
+; GCN-NEXT: v_add_u32_e32 v1, 0x14000, v1
+; GCN-NEXT: ds_read_b128 a[156:159], v1 offset:112
+; GCN-NEXT: ds_read_b128 a[152:155], v1 offset:96
+; GCN-NEXT: ds_read_b128 a[148:151], v1 offset:80
+; GCN-NEXT: ds_read_b128 a[144:147], v1 offset:64
+; GCN-NEXT: ds_read_b128 a[128:131], v1
+; GCN-NEXT: ds_read_b128 a[132:135], v1 offset:16
+; GCN-NEXT: ds_read_b128 a[136:139], v1 offset:32
+; GCN-NEXT: ds_read_b128 a[140:143], v1 offset:48
+; GCN-NEXT: v_ldexp_f32 v1, v2, v3
+; GCN-NEXT: v_cmp_nlt_f32_e32 vcc, s8, v4
; GCN-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
-; GCN-NEXT: v_cmp_ngt_f32_e32 vcc, s8, v6
-; GCN-NEXT: v_cndmask_b32_e32 v1, v8, v1, vcc
+; GCN-NEXT: v_cmp_ngt_f32_e32 vcc, s8, v5
+; GCN-NEXT: v_cndmask_b32_e32 v1, v7, v1, vcc
; GCN-NEXT: v_add_u32_e32 v0, s7, v0
; GCN-NEXT: ds_write_b128 v0, a[124:127] offset:112
; GCN-NEXT: s_waitcnt lgkmcnt(1)
-; GCN-NEXT: v_mfma_f32_32x32x1f32 a[128:159], v9, v1, a[128:159]
+; GCN-NEXT: v_mfma_f32_32x32x1f32 a[128:159], v8, v1, a[128:159]
; GCN-NEXT: ds_write_b128 v0, a[120:123] offset:96
; GCN-NEXT: ds_write_b128 v0, a[116:119] offset:80
; GCN-NEXT: ds_write_b128 v0, a[112:115] offset:64
@@ -1347,30 +1348,30 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_interleave_EXP_MFMA
; GCN-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
; GCN-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0)
; GCN-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
-; GCN-NEXT: ds_write_b128 v0, a[24:27] offset:8288
-; GCN-NEXT: ds_write_b128 v0, a[28:31] offset:8304
-; GCN-NEXT: ds_write_b128 v0, a[16:19] offset:8256
-; GCN-NEXT: ds_write_b128 v0, a[20:23] offset:8272
-; GCN-NEXT: ds_write_b128 v0, a[8:11] offset:8224
-; GCN-NEXT: ds_write_b128 v0, a[12:15] offset:8240
-; GCN-NEXT: ds_write_b128 v0, a[0:3] offset:8192
-; GCN-NEXT: ds_write_b128 v0, a[4:7] offset:8208
-; GCN-NEXT: ds_write_b128 v0, a[88:91] offset:16480
-; GCN-NEXT: ds_write_b128 v0, a[92:95] offset:16496
-; GCN-NEXT: ds_write_b128 v0, a[80:83] offset:16448
-; GCN-NEXT: ds_write_b128 v0, a[84:87] offset:16464
-; GCN-NEXT: ds_write_b128 v0, a[72:75] offset:16416
-; GCN-NEXT: ds_write_b128 v0, a[76:79] offset:16432
-; GCN-NEXT: ds_write_b128 v0, a[64:67] offset:16384
-; GCN-NEXT: ds_write_b128 v0, a[68:71] offset:16400
-; GCN-NEXT: ds_write_b128 v0, a[56:59] offset:24672
-; GCN-NEXT: ds_write_b128 v0, a[60:63] offset:24688
-; GCN-NEXT: ds_write_b128 v0, a[48:51] offset:24640
-; GCN-NEXT: ds_write_b128 v0, a[52:55] offset:24656
-; GCN-NEXT: ds_write_b128 v0, a[40:43] offset:24608
-; GCN-NEXT: ds_write_b128 v0, a[44:47] offset:24624
-; GCN-NEXT: ds_write_b128 v0, a[32:35] offset:24576
-; GCN-NEXT: ds_write_b128 v0, a[36:39] offset:24592
+; GCN-NEXT: ds_write_b128 v0, a[88:91] offset:8288
+; GCN-NEXT: ds_write_b128 v0, a[92:95] offset:8304
+; GCN-NEXT: ds_write_b128 v0, a[80:83] offset:8256
+; GCN-NEXT: ds_write_b128 v0, a[84:87] offset:8272
+; GCN-NEXT: ds_write_b128 v0, a[72:75] offset:8224
+; GCN-NEXT: ds_write_b128 v0, a[76:79] offset:8240
+; GCN-NEXT: ds_write_b128 v0, a[64:67] offset:8192
+; GCN-NEXT: ds_write_b128 v0, a[68:71] offset:8208
+; GCN-NEXT: ds_write_b128 v0, a[56:59] offset:16480
+; GCN-NEXT: ds_write_b128 v0, a[60:63] offset:16496
+; GCN-NEXT: ds_write_b128 v0, a[48:51] offset:16448
+; GCN-NEXT: ds_write_b128 v0, a[52:55] offset:16464
+; GCN-NEXT: ds_write_b128 v0, a[40:43] offset:16416
+; GCN-NEXT: ds_write_b128 v0, a[44:47] offset:16432
+; GCN-NEXT: ds_write_b128 v0, a[32:35] offset:16384
+; GCN-NEXT: ds_write_b128 v0, a[36:39] offset:16400
+; GCN-NEXT: ds_write_b128 v0, a[24:27] offset:24672
+; GCN-NEXT: ds_write_b128 v0, a[28:31] offset:24688
+; GCN-NEXT: ds_write_b128 v0, a[16:19] offset:24640
+; GCN-NEXT: ds_write_b128 v0, a[20:23] offset:24656
+; GCN-NEXT: ds_write_b128 v0, a[8:11] offset:24608
+; GCN-NEXT: ds_write_b128 v0, a[12:15] offset:24624
+; GCN-NEXT: ds_write_b128 v0, a[0:3] offset:24576
+; GCN-NEXT: ds_write_b128 v0, a[4:7] offset:24592
; GCN-NEXT: ds_write_b128 v0, a[152:155] offset:32864
; GCN-NEXT: ds_write_b128 v0, a[156:159] offset:32880
; GCN-NEXT: ds_write_b128 v0, a[144:147] offset:32832
@@ -1384,19 +1385,19 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_interleave_EXP_MFMA
; EXACTCUTOFF-LABEL: test_sched_group_barrier_pipeline_interleave_EXP_MFMA:
; EXACTCUTOFF: ; %bb.0: ; %entry
; EXACTCUTOFF-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x44
-; EXACTCUTOFF-NEXT: v_mov_b32_e32 v3, 0x3fb8aa3b
+; EXACTCUTOFF-NEXT: v_mov_b32_e32 v2, 0x3fb8aa3b
; EXACTCUTOFF-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; EXACTCUTOFF-NEXT: v_mov_b32_e32 v7, 0x32a5705f
+; EXACTCUTOFF-NEXT: v_mov_b32_e32 v6, 0x32a5705f
; EXACTCUTOFF-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(0)
-; EXACTCUTOFF-NEXT: v_mul_f32_e32 v4, s0, v3
-; EXACTCUTOFF-NEXT: v_rndne_f32_e32 v5, v4
-; EXACTCUTOFF-NEXT: v_sub_f32_e32 v6, v4, v5
-; EXACTCUTOFF-NEXT: v_fma_f32 v4, s0, v3, -v4
-; EXACTCUTOFF-NEXT: v_fmac_f32_e32 v4, s0, v7
-; EXACTCUTOFF-NEXT: v_add_f32_e32 v4, v6, v4
-; EXACTCUTOFF-NEXT: v_exp_f32_e32 v4, v4
-; EXACTCUTOFF-NEXT: v_cvt_i32_f32_e32 v5, v5
+; EXACTCUTOFF-NEXT: v_mul_f32_e32 v3, s0, v2
+; EXACTCUTOFF-NEXT: v_rndne_f32_e32 v4, v3
+; EXACTCUTOFF-NEXT: v_sub_f32_e32 v5, v3, v4
+; EXACTCUTOFF-NEXT: v_fma_f32 v3, s0, v2, -v3
+; EXACTCUTOFF-NEXT: v_fmac_f32_e32 v3, s0, v6
+; EXACTCUTOFF-NEXT: v_add_f32_e32 v3, v5, v3
+; EXACTCUTOFF-NEXT: v_exp_f32_e32 v3, v3
+; EXACTCUTOFF-NEXT: v_cvt_i32_f32_e32 v4, v4
; EXACTCUTOFF-NEXT: v_lshlrev_b32_e32 v0, 7, v0
; EXACTCUTOFF-NEXT: v_add_u32_e32 v1, s6, v0
; EXACTCUTOFF-NEXT: ds_read_b128 a[124:127], v1 offset:112
@@ -1407,112 +1408,113 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_interleave_EXP_MFMA
; EXACTCUTOFF-NEXT: ds_read_b128 a[100:103], v1 offset:16
; EXACTCUTOFF-NEXT: ds_read_b128 a[104:107], v1 offset:32
; EXACTCUTOFF-NEXT: ds_read_b128 a[108:111], v1 offset:48
-; EXACTCUTOFF-NEXT: v_mov_b32_e32 v9, 1.0
-; EXACTCUTOFF-NEXT: v_ldexp_f32 v4, v4, v5
-; EXACTCUTOFF-NEXT: v_mov_b32_e32 v5, 0xc2ce8ed0
-; EXACTCUTOFF-NEXT: v_mul_f32_e32 v10, s1, v3
-; EXACTCUTOFF-NEXT: v_cmp_nlt_f32_e32 vcc, s0, v5
-; EXACTCUTOFF-NEXT: v_mov_b32_e32 v6, 0x42b17218
-; EXACTCUTOFF-NEXT: v_rndne_f32_e32 v11, v10
-; EXACTCUTOFF-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc
-; EXACTCUTOFF-NEXT: v_cmp_ngt_f32_e32 vcc, s0, v6
-; EXACTCUTOFF-NEXT: v_mov_b32_e32 v8, 0x7f800000
-; EXACTCUTOFF-NEXT: v_sub_f32_e32 v12, v10, v11
-; EXACTCUTOFF-NEXT: v_fma_f32 v10, s1, v3, -v10
-; EXACTCUTOFF-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc
-; EXACTCUTOFF-NEXT: v_fmac_f32_e32 v10, s1, v7
-; EXACTCUTOFF-NEXT: ds_read_b128 a[28:31], v1 offset:8304
+; EXACTCUTOFF-NEXT: v_mov_b32_e32 v8, 1.0
+; EXACTCUTOFF-NEXT: v_ldexp_f32 v3, v3, v4
+; EXACTCUTOFF-NEXT: v_mov_b32_e32 v4, 0xc2ce8ed0
+; EXACTCUTOFF-NEXT: v_mul_f32_e32 v9, s1, v2
+; EXACTCUTOFF-NEXT: v_cmp_nlt_f32_e32 vcc, s0, v4
+; EXACTCUTOFF-NEXT: v_mov_b32_e32 v5, 0x42b17218
+; EXACTCUTOFF-NEXT: v_rndne_f32_e32 v10, v9
+; EXACTCUTOFF-NEXT: v_cndmask_b32_e32 v3, 0, v3, vcc
+; EXACTCUTOFF-NEXT: v_cmp_ngt_f32_e32 vcc, s0, v5
+; EXACTCUTOFF-NEXT: v_mov_b32_e32 v7, 0x7f800000
+; EXACTCUTOFF-NEXT: v_sub_f32_e32 v11, v9, v10
+; EXACTCUTOFF-NEXT: v_fma_f32 v9, s1, v2, -v9
+; EXACTCUTOFF-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc
+; EXACTCUTOFF-NEXT: v_fmac_f32_e32 v9, s1, v6
+; EXACTCUTOFF-NEXT: ds_read_b128 a[92:95], v1 offset:8304
; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(1)
-; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[96:127], v9, v4, a[96:127]
-; EXACTCUTOFF-NEXT: v_add_f32_e32 v4, v12, v10
-; EXACTCUTOFF-NEXT: v_exp_f32_e32 v4, v4
-; EXACTCUTOFF-NEXT: v_cvt_i32_f32_e32 v10, v11
-; EXACTCUTOFF-NEXT: ds_read_b128 a[24:27], v1 offset:8288
-; EXACTCUTOFF-NEXT: ds_read_b128 a[20:23], v1 offset:8272
-; EXACTCUTOFF-NEXT: ds_read_b128 a[16:19], v1 offset:8256
-; EXACTCUTOFF-NEXT: ds_read_b128 a[12:15], v1 offset:8240
-; EXACTCUTOFF-NEXT: ds_read_b128 a[8:11], v1 offset:8224
-; EXACTCUTOFF-NEXT: ds_read_b128 a[4:7], v1 offset:8208
-; EXACTCUTOFF-NEXT: ds_read_b128 a[0:3], v1 offset:8192
-; EXACTCUTOFF-NEXT: v_ldexp_f32 v4, v4, v10
-; EXACTCUTOFF-NEXT: v_cmp_nlt_f32_e32 vcc, s1, v5
-; EXACTCUTOFF-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc
-; EXACTCUTOFF-NEXT: v_cmp_ngt_f32_e32 vcc, s1, v6
-; EXACTCUTOFF-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc
-; EXACTCUTOFF-NEXT: v_mul_f32_e32 v10, s2, v3
-; EXACTCUTOFF-NEXT: v_rndne_f32_e32 v11, v10
+; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[96:127], v8, v3, a[96:127]
+; EXACTCUTOFF-NEXT: v_add_f32_e32 v3, v11, v9
+; EXACTCUTOFF-NEXT: v_exp_f32_e32 v3, v3
+; EXACTCUTOFF-NEXT: v_cvt_i32_f32_e32 v9, v10
+; EXACTCUTOFF-NEXT: ds_read_b128 a[88:91], v1 offset:8288
+; EXACTCUTOFF-NEXT: ds_read_b128 a[84:87], v1 offset:8272
+; EXACTCUTOFF-NEXT: ds_read_b128 a[80:83], v1 offset:8256
+; EXACTCUTOFF-NEXT: ds_read_b128 a[76:79], v1 offset:8240
+; EXACTCUTOFF-NEXT: ds_read_b128 a[72:75], v1 offset:8224
+; EXACTCUTOFF-NEXT: ds_read_b128 a[68:71], v1 offset:8208
+; EXACTCUTOFF-NEXT: ds_read_b128 a[64:67], v1 offset:8192
+; EXACTCUTOFF-NEXT: v_ldexp_f32 v3, v3, v9
+; EXACTCUTOFF-NEXT: v_cmp_nlt_f32_e32 vcc, s1, v4
+; EXACTCUTOFF-NEXT: v_cndmask_b32_e32 v3, 0, v3, vcc
+; EXACTCUTOFF-NEXT: v_cmp_ngt_f32_e32 vcc, s1, v5
+; EXACTCUTOFF-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc
+; EXACTCUTOFF-NEXT: v_mul_f32_e32 v9, s2, v2
+; EXACTCUTOFF-NEXT: v_rndne_f32_e32 v10, v9
; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(0)
-; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v9, v4, a[0:31]
-; EXACTCUTOFF-NEXT: v_fma_f32 v4, s2, v3, -v10
-; EXACTCUTOFF-NEXT: v_sub_f32_e32 v12, v10, v11
-; EXACTCUTOFF-NEXT: v_fmac_f32_e32 v4, s2, v7
-; EXACTCUTOFF-NEXT: v_add_f32_e32 v4, v12, v4
-; EXACTCUTOFF-NEXT: v_exp_f32_e32 v4, v4
-; EXACTCUTOFF-NEXT: v_cvt_i32_f32_e32 v10, v11
-; EXACTCUTOFF-NEXT: ds_read_b128 a[92:95], v1 offset:24688
-; EXACTCUTOFF-NEXT: ds_read_b128 a[88:91], v1 offset:24672
-; EXACTCUTOFF-NEXT: ds_read_b128 a[84:87], v1 offset:24656
-; EXACTCUTOFF-NEXT: ds_read_b128 a[80:83], v1 offset:24640
-; EXACTCUTOFF-NEXT: ds_read_b128 a[76:79], v1 offset:24624
-; EXACTCUTOFF-NEXT: ds_read_b128 a[72:75], v1 offset:24608
-; EXACTCUTOFF-NEXT: ds_read_b128 a[68:71], v1 offset:24592
-; EXACTCUTOFF-NEXT: ds_read_b128 a[64:67], v1 offset:24576
-; EXACTCUTOFF-NEXT: v_add_u32_e32 v2, 0x6000, v1
-; EXACTCUTOFF-NEXT: ds_read_b128 a[60:63], v1 offset:49264
-; EXACTCUTOFF-NEXT: ds_read_b128 a[56:59], v1 offset:49248
-; EXACTCUTOFF-NEXT: ds_read_b128 a[52:55], v1 offset:49232
-; EXACTCUTOFF-NEXT: ds_read_b128 a[48:51], v1 offset:49216
-; EXACTCUTOFF-NEXT: ds_read_b128 a[44:47], v1 offset:49200
-; EXACTCUTOFF-NEXT: ds_read_b128 a[40:43], v1 offset:49184
-; EXACTCUTOFF-NEXT: ds_read_b128 a[36:39], v1 offset:49168
-; EXACTCUTOFF-NEXT: ds_read_b128 a[32:35], v1 offset:49152
-; EXACTCUTOFF-NEXT: v_ldexp_f32 v1, v4, v10
-; EXACTCUTOFF-NEXT: v_cmp_nlt_f32_e32 vcc, s2, v5
-; EXACTCUTOFF-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
-; EXACTCUTOFF-NEXT: v_cmp_ngt_f32_e32 vcc, s2, v6
-; EXACTCUTOFF-NEXT: v_mul_f32_e32 v4, s3, v3
-; EXACTCUTOFF-NEXT: v_cndmask_b32_e32 v1, v8, v1, vcc
-; EXACTCUTOFF-NEXT: v_rndne_f32_e32 v10, v4
+; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[64:95], v8, v3, a[64:95]
+; EXACTCUTOFF-NEXT: v_fma_f32 v3, s2, v2, -v9
+; EXACTCUTOFF-NEXT: v_sub_f32_e32 v11, v9, v10
+; EXACTCUTOFF-NEXT: v_fmac_f32_e32 v3, s2, v6
+; EXACTCUTOFF-NEXT: v_add_f32_e32 v3, v11, v3
+; EXACTCUTOFF-NEXT: v_exp_f32_e32 v3, v3
+; EXACTCUTOFF-NEXT: v_cvt_i32_f32_e32 v9, v10
+; EXACTCUTOFF-NEXT: ds_read_b128 a[60:63], v1 offset:24688
+; EXACTCUTOFF-NEXT: ds_read_b128 a[56:59], v1 offset:24672
+; EXACTCUTOFF-NEXT: ds_read_b128 a[52:55], v1 offset:24656
+; EXACTCUTOFF-NEXT: ds_read_b128 a[48:51], v1 offset:24640
+; EXACTCUTOFF-NEXT: ds_read_b128 a[44:47], v1 offset:24624
+; EXACTCUTOFF-NEXT: ds_read_b128 a[40:43], v1 offset:24608
+; EXACTCUTOFF-NEXT: ds_read_b128 a[36:39], v1 offset:24592
+; EXACTCUTOFF-NEXT: ds_read_b128 a[32:35], v1 offset:24576
+; EXACTCUTOFF-NEXT: v_ldexp_f32 v3, v3, v9
+; EXACTCUTOFF-NEXT: v_cmp_nlt_f32_e32 vcc, s2, v4
+; EXACTCUTOFF-NEXT: v_cndmask_b32_e32 v3, 0, v3, vcc
+; EXACTCUTOFF-NEXT: v_cmp_ngt_f32_e32 vcc, s2, v5
+; EXACTCUTOFF-NEXT: v_mul_f32_e32 v9, s3, v2
+; EXACTCUTOFF-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc
+; EXACTCUTOFF-NEXT: v_rndne_f32_e32 v10, v9
; EXACTCUTOFF-NEXT: s_load_dword s8, s[4:5], 0x54
; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(0)
-; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[64:95], v9, v1, a[64:95]
-; EXACTCUTOFF-NEXT: v_sub_f32_e32 v1, v4, v10
-; EXACTCUTOFF-NEXT: v_fma_f32 v4, s3, v3, -v4
-; EXACTCUTOFF-NEXT: v_fmac_f32_e32 v4, s3, v7
-; EXACTCUTOFF-NEXT: v_add_f32_e32 v1, v1, v4
-; EXACTCUTOFF-NEXT: v_exp_f32_e32 v1, v1
-; EXACTCUTOFF-NEXT: v_cvt_i32_f32_e32 v4, v10
-; EXACTCUTOFF-NEXT: v_cmp_nlt_f32_e32 vcc, s3, v5
-; EXACTCUTOFF-NEXT: ds_read_b128 a[156:159], v2 offset:57456
-; EXACTCUTOFF-NEXT: ds_read_b128 a[152:155], v2 offset:57440
-; EXACTCUTOFF-NEXT: v_ldexp_f32 v1, v1, v4
-; EXACTCUTOFF-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
-; EXACTCUTOFF-NEXT: v_cmp_ngt_f32_e32 vcc, s3, v6
-; EXACTCUTOFF-NEXT: v_cndmask_b32_e32 v1, v8, v1, vcc
-; EXACTCUTOFF-NEXT: v_mul_f32_e32 v4, s8, v3
-; EXACTCUTOFF-NEXT: v_fma_f32 v3, s8, v3, -v4
-; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v9, v1, a[32:63]
-; EXACTCUTOFF-NEXT: v_rndne_f32_e32 v1, v4
-; EXACTCUTOFF-NEXT: v_sub_f32_e32 v10, v4, v1
-; EXACTCUTOFF-NEXT: v_fmac_f32_e32 v3, s8, v7
-; EXACTCUTOFF-NEXT: v_add_f32_e32 v3, v10, v3
+; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v8, v3, a[32:63]
+; EXACTCUTOFF-NEXT: v_sub_f32_e32 v3, v9, v10
+; EXACTCUTOFF-NEXT: v_fma_f32 v9, s3, v2, -v9
+; EXACTCUTOFF-NEXT: v_fmac_f32_e32 v9, s3, v6
+; EXACTCUTOFF-NEXT: v_add_f32_e32 v3, v3, v9
; EXACTCUTOFF-NEXT: v_exp_f32_e32 v3, v3
-; EXACTCUTOFF-NEXT: v_cvt_i32_f32_e32 v1, v1
-; EXACTCUTOFF-NEXT: ds_read_b128 a[148:151], v2 offset:57424
-; EXACTCUTOFF-NEXT: ds_read_b128 a[144:147], v2 offset:57408
-; EXACTCUTOFF-NEXT: ds_read_b128 a[128:131], v2 offset:57344
-; EXACTCUTOFF-NEXT: ds_read_b128 a[132:135], v2 offset:57360
-; EXACTCUTOFF-NEXT: ds_read_b128 a[136:139], v2 offset:57376
-; EXACTCUTOFF-NEXT: ds_read_b128 a[140:143], v2 offset:57392
-; EXACTCUTOFF-NEXT: v_ldexp_f32 v1, v3, v1
-; EXACTCUTOFF-NEXT: v_cmp_nlt_f32_e32 vcc, s8, v5
+; EXACTCUTOFF-NEXT: v_cvt_i32_f32_e32 v9, v10
+; EXACTCUTOFF-NEXT: ds_read_b128 a[28:31], v1 offset:49264
+; EXACTCUTOFF-NEXT: ds_read_b128 a[24:27], v1 offset:49248
+; EXACTCUTOFF-NEXT: ds_read_b128 a[20:23], v1 offset:49232
+; EXACTCUTOFF-NEXT: ds_read_b128 a[16:19], v1 offset:49216
+; EXACTCUTOFF-NEXT: ds_read_b128 a[12:15], v1 offset:49200
+; EXACTCUTOFF-NEXT: ds_read_b128 a[8:11], v1 offset:49184
+; EXACTCUTOFF-NEXT: ds_read_b128 a[4:7], v1 offset:49168
+; EXACTCUTOFF-NEXT: ds_read_b128 a[0:3], v1 offset:49152
+; EXACTCUTOFF-NEXT: v_ldexp_f32 v3, v3, v9
+; EXACTCUTOFF-NEXT: v_cmp_nlt_f32_e32 vcc, s3, v4
+; EXACTCUTOFF-NEXT: v_cndmask_b32_e32 v3, 0, v3, vcc
+; EXACTCUTOFF-NEXT: v_cmp_ngt_f32_e32 vcc, s3, v5
+; EXACTCUTOFF-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc
+; EXACTCUTOFF-NEXT: v_mul_f32_e32 v9, s8, v2
+; EXACTCUTOFF-NEXT: v_fma_f32 v2, s8, v2, -v9
+; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(0)
+; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v8, v3, a[0:31]
+; EXACTCUTOFF-NEXT: v_rndne_f32_e32 v3, v9
+; EXACTCUTOFF-NEXT: v_sub_f32_e32 v10, v9, v3
+; EXACTCUTOFF-NEXT: v_fmac_f32_e32 v2, s8, v6
+; EXACTCUTOFF-NEXT: v_add_f32_e32 v2, v10, v2
+; EXACTCUTOFF-NEXT: v_exp_f32_e32 v2, v2
+; EXACTCUTOFF-NEXT: v_cvt_i32_f32_e32 v3, v3
+; EXACTCUTOFF-NEXT: v_add_u32_e32 v1, 0x14000, v1
+; EXACTCUTOFF-NEXT: ds_read_b128 a[156:159], v1 offset:112
+; EXACTCUTOFF-NEXT: ds_read_b128 a[152:155], v1 offset:96
+; EXACTCUTOFF-NEXT: ds_read_b128 a[148:151], v1 offset:80
+; EXACTCUTOFF-NEXT: ds_read_b128 a[144:147], v1 offset:64
+; EXACTCUTOFF-NEXT: ds_read_b128 a[128:131], v1
+; EXACTCUTOFF-NEXT: ds_read_b128 a[132:135], v1 offset:16
+; EXACTCUTOFF-NEXT: ds_read_b128 a[136:139], v1 offset:32
+; EXACTCUTOFF-NEXT: ds_read_b128 a[140:143], v1 offset:48
+; EXACTCUTOFF-NEXT: v_ldexp_f32 v1, v2, v3
+; EXACTCUTOFF-NEXT: v_cmp_nlt_f32_e32 vcc, s8, v4
; EXACTCUTOFF-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
-; EXACTCUTOFF-NEXT: v_cmp_ngt_f32_e32 vcc, s8, v6
-; EXACTCUTOFF-NEXT: v_cndmask_b32_e32 v1, v8, v1, vcc
+; EXACTCUTOFF-NEXT: v_cmp_ngt_f32_e32 vcc, s8, v5
+; EXACTCUTOFF-NEXT: v_cndmask_b32_e32 v1, v7, v1, vcc
; EXACTCUTOFF-NEXT: v_add_u32_e32 v0, s7, v0
; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[124:127] offset:112
; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(1)
-; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[128:159], v9, v1, a[128:159]
+; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[128:159], v8, v1, a[128:159]
; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[120:123] offset:96
; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[116:119] offset:80
; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[112:115] offset:64
@@ -1532,30 +1534,30 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_interleave_EXP_MFMA
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0)
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[24:27] offset:8288
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[28:31] offset:8304
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[16:19] offset:8256
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[20:23] offset:8272
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[8:11] offset:8224
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[12:15] offset:8240
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[0:3] offset:8192
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[4:7] offset:8208
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[88:91] offset:16480
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[92:95] offset:16496
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[80:83] offset:16448
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[84:87] offset:16464
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[72:75] offset:16416
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[76:79] offset:16432
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[64:67] offset:16384
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[68:71] offset:16400
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[56:59] offset:24672
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[60:63] offset:24688
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[48:51] offset:24640
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[52:55] offset:24656
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[40:43] offset:24608
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[44:47] offset:24624
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[32:35] offset:24576
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[36:39] offset:24592
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[88:91] offset:8288
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[92:95] offset:8304
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[80:83] offset:8256
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[84:87] offset:8272
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[72:75] offset:8224
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[76:79] offset:8240
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[64:67] offset:8192
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[68:71] offset:8208
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[56:59] offset:16480
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[60:63] offset:16496
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[48:51] offset:16448
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[52:55] offset:16464
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[40:43] offset:16416
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[44:47] offset:16432
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[32:35] offset:16384
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[36:39] offset:16400
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[24:27] offset:24672
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[28:31] offset:24688
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[16:19] offset:24640
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[20:23] offset:24656
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[8:11] offset:24608
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[12:15] offset:24624
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[0:3] offset:24576
+; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[4:7] offset:24592
; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[152:155] offset:32864
; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[156:159] offset:32880
; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[144:147] offset:32832
diff --git a/llvm/test/CodeGen/AMDGPU/machine-sink-temporal-divergence-swdev407790.ll b/llvm/test/CodeGen/AMDGPU/machine-sink-temporal-divergence-swdev407790.ll
index 553d7e09390fd..680942fcb4d4b 100644
--- a/llvm/test/CodeGen/AMDGPU/machine-sink-temporal-divergence-swdev407790.ll
+++ b/llvm/test/CodeGen/AMDGPU/machine-sink-temporal-divergence-swdev407790.ll
@@ -279,11 +279,11 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
; CHECK-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; CHECK-NEXT: ds_write_b32 v0, v58
; CHECK-NEXT: s_branch .LBB0_7
-; CHECK-NEXT: .LBB0_16: ; %Flow45
+; CHECK-NEXT: .LBB0_16: ; %Flow43
; CHECK-NEXT: ; in Loop: Header=BB0_5 Depth=1
; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s69
; CHECK-NEXT: v_mov_b32_e32 v57, v0
-; CHECK-NEXT: .LBB0_17: ; %Flow46
+; CHECK-NEXT: .LBB0_17: ; %Flow44
; CHECK-NEXT: ; in Loop: Header=BB0_5 Depth=1
; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s68
; CHECK-NEXT: s_mov_b32 s55, exec_lo
@@ -330,11 +330,11 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
; CHECK-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; CHECK-NEXT: ds_write_b32 v0, v57
; CHECK-NEXT: s_branch .LBB0_19
-; CHECK-NEXT: .LBB0_22: ; %Flow43
+; CHECK-NEXT: .LBB0_22: ; %Flow41
; CHECK-NEXT: ; in Loop: Header=BB0_5 Depth=1
; CHECK-NEXT: s_inst_prefetch 0x2
; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s68
-; CHECK-NEXT: .LBB0_23: ; %Flow44
+; CHECK-NEXT: .LBB0_23: ; %Flow42
; CHECK-NEXT: ; in Loop: Header=BB0_5 Depth=1
; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s55
; CHECK-NEXT: ; %bb.24: ; in Loop: Header=BB0_5 Depth=1
@@ -347,7 +347,7 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
; CHECK-NEXT: s_or_b32 s53, s4, s53
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s53
; CHECK-NEXT: s_cbranch_execnz .LBB0_5
-; CHECK-NEXT: .LBB0_25: ; %Flow51
+; CHECK-NEXT: .LBB0_25: ; %Flow49
; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s52
; CHECK-NEXT: v_mov_b32_e32 v31, v40
; CHECK-NEXT: v_mov_b32_e32 v0, 1
diff --git a/llvm/test/CodeGen/PowerPC/split-index-tc.ll b/llvm/test/CodeGen/PowerPC/split-index-tc.ll
index e332428aa8e2b..a8d79b7fecbe5 100644
--- a/llvm/test/CodeGen/PowerPC/split-index-tc.ll
+++ b/llvm/test/CodeGen/PowerPC/split-index-tc.ll
@@ -77,5 +77,3 @@ if.end89: ; preds = %if.else59
}
attributes #0 = { nounwind }
-
-
diff --git a/llvm/test/Transforms/SeparateConstOffsetFromGEP/reorder-gep.ll b/llvm/test/Transforms/SeparateConstOffsetFromGEP/reorder-gep.ll
index 2e3b6ca3653fc..cdd47f890ef44 100644
--- a/llvm/test/Transforms/SeparateConstOffsetFromGEP/reorder-gep.ll
+++ b/llvm/test/Transforms/SeparateConstOffsetFromGEP/reorder-gep.ll
@@ -8,11 +8,14 @@ define void @illegal_addr_mode(ptr %in.ptr, i64 %in.idx0, i64 %in.idx1) {
; CHECK-NEXT: [[BASE:%.*]] = getelementptr i64, ptr [[IN_PTR]], i64 [[IN_IDX0]]
; CHECK-NEXT: [[IDX0:%.*]] = getelementptr i64, ptr [[BASE]], i64 [[IN_IDX1]]
; CHECK-NEXT: [[CONST1:%.*]] = getelementptr i64, ptr [[BASE]], i64 256
-; CHECK-NEXT: [[IDX1:%.*]] = getelementptr i64, ptr [[CONST1]], i64 [[IN_IDX1]]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i64, ptr [[BASE]], i64 [[IN_IDX1]]
+; CHECK-NEXT: [[IDX1:%.*]] = getelementptr i64, ptr [[TMP0]], i64 256
; CHECK-NEXT: [[CONST2:%.*]] = getelementptr i64, ptr [[BASE]], i64 512
-; CHECK-NEXT: [[IDX2:%.*]] = getelementptr i64, ptr [[CONST2]], i64 [[IN_IDX1]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[BASE]], i64 [[IN_IDX1]]
+; CHECK-NEXT: [[IDX2:%.*]] = getelementptr i64, ptr [[TMP1]], i64 512
; CHECK-NEXT: [[CONST3:%.*]] = getelementptr i64, ptr [[BASE]], i64 768
-; CHECK-NEXT: [[IDX3:%.*]] = getelementptr i64, ptr [[CONST3]], i64 [[IN_IDX1]]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i64, ptr [[BASE]], i64 [[IN_IDX1]]
+; CHECK-NEXT: [[IDX3:%.*]] = getelementptr i64, ptr [[TMP2]], i64 768
; CHECK-NEXT: [[CMP0:%.*]] = icmp eq i64 [[IN_IDX0]], 0
; CHECK-NEXT: br i1 [[CMP0]], label [[BB_1:%.*]], label [[END:%.*]]
; CHECK: bb.1:
@@ -132,11 +135,14 @@ define void @different_type_reorder(ptr %in.ptr, i64 %in.idx0, i64 %in.idx1) {
; CHECK-NEXT: [[BASE:%.*]] = getelementptr i64, ptr [[IN_PTR]], i64 [[IN_IDX0]]
; CHECK-NEXT: [[IDX0:%.*]] = getelementptr i64, ptr [[BASE]], i64 [[IN_IDX1]]
; CHECK-NEXT: [[CONST1:%.*]] = getelementptr i8, ptr [[BASE]], i64 256
-; CHECK-NEXT: [[IDX1:%.*]] = getelementptr i64, ptr [[CONST1]], i64 [[IN_IDX1]]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i64, ptr [[BASE]], i64 [[IN_IDX1]]
+; CHECK-NEXT: [[IDX1:%.*]] = getelementptr i8, ptr [[TMP0]], i64 256
; CHECK-NEXT: [[CONST2:%.*]] = getelementptr i8, ptr [[BASE]], i64 512
-; CHECK-NEXT: [[IDX2:%.*]] = getelementptr i64, ptr [[CONST2]], i64 [[IN_IDX1]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[BASE]], i64 [[IN_IDX1]]
+; CHECK-NEXT: [[IDX2:%.*]] = getelementptr i8, ptr [[TMP1]], i64 512
; CHECK-NEXT: [[CONST3:%.*]] = getelementptr i8, ptr [[BASE]], i64 768
-; CHECK-NEXT: [[IDX3:%.*]] = getelementptr i64, ptr [[CONST3]], i64 [[IN_IDX1]]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i64, ptr [[BASE]], i64 [[IN_IDX1]]
+; CHECK-NEXT: [[IDX3:%.*]] = getelementptr i8, ptr [[TMP2]], i64 768
; CHECK-NEXT: [[CMP0:%.*]] = icmp eq i64 [[IN_IDX0]], 0
; CHECK-NEXT: br i1 [[CMP0]], label [[BB_1:%.*]], label [[END:%.*]]
; CHECK: bb.1:
@@ -195,11 +201,14 @@ define void @different_type_reorder2(ptr %in.ptr, i64 %in.idx0, i64 %in.idx1) {
; CHECK-NEXT: [[BASE:%.*]] = getelementptr i8, ptr [[IN_PTR]], i64 [[IN_IDX0]]
; CHECK-NEXT: [[IDX0:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[IN_IDX1]]
; CHECK-NEXT: [[CONST1:%.*]] = getelementptr i64, ptr [[BASE]], i64 256
-; CHECK-NEXT: [[IDX1:%.*]] = getelementptr i8, ptr [[CONST1]], i64 [[IN_IDX1]]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[IN_IDX1]]
+; CHECK-NEXT: [[IDX1:%.*]] = getelementptr i64, ptr [[TMP0]], i64 256
; CHECK-NEXT: [[CONST2:%.*]] = getelementptr i64, ptr [[BASE]], i64 512
-; CHECK-NEXT: [[IDX2:%.*]] = getelementptr i8, ptr [[CONST2]], i64 [[IN_IDX1]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[IN_IDX1]]
+; CHECK-NEXT: [[IDX2:%.*]] = getelementptr i64, ptr [[TMP1]], i64 512
; CHECK-NEXT: [[CONST3:%.*]] = getelementptr i64, ptr [[BASE]], i64 768
-; CHECK-NEXT: [[IDX3:%.*]] = getelementptr i8, ptr [[CONST3]], i64 [[IN_IDX1]]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[IN_IDX1]]
+; CHECK-NEXT: [[IDX3:%.*]] = getelementptr i64, ptr [[TMP2]], i64 768
; CHECK-NEXT: [[CMP0:%.*]] = icmp eq i64 [[IN_IDX0]], 0
; CHECK-NEXT: br i1 [[CMP0]], label [[BB_1:%.*]], label [[END:%.*]]
; CHECK: bb.1:
More information about the llvm-commits
mailing list