[llvm] [AMDGPU] IGroupLP: Avoid DAG manipulation in greedyFind (PR #194827)
Frederik Harwath via llvm-commits
llvm-commits at lists.llvm.org
Thu Apr 30 05:28:38 PDT 2026
https://github.com/frederik-h updated https://github.com/llvm/llvm-project/pull/194827
>From 81ad853d4e91cca39a49a6c777fd4113d965921e Mon Sep 17 00:00:00 2001
From: Frederik Harwath <fharwath at amd.com>
Date: Wed, 29 Apr 2026 04:27:20 -0400
Subject: [PATCH 1/7] [AMDGPU] IGroupLP: Avoid DAG manipulations in greedyFind
The greedy pipeline solver needs to determine the edges that are
implied by assigning an SUnit to a SchedGroup and the cost of this
assignment. The cost is the number of edges that cannot be added without
introducing cycles. The current implementation (addEdges) adds the
edges to the DAG and uses the DAG reachability function for cycle
checking. This happens for each candidate SchedGroup and needs to be
undone before other candidates are considered. The DAG manipulations
become a significant performance bottleneck on bigger pipelines.
This commit implements an alternative function for computing the edges
and cost of an assignment. This function performs the reachability
analysis that is necessary for the cyclicity checks without modifying
the DAG.
The new function returns the same cost as addEdges. The concrete edge
set may show insignificant differences, because, for instance,
the link function called from addEdges chooses not to add transitive
edges (DAG manipulations are more expensive than the check to avoid
this), but the new function includes such edges since this means that
it can avoid to update reachability information in one direction.
The patch has been tested on pipelines originating from different
users and more than 10x performance improvements have been observed on
pipelines with thousands of SUnits. No significant performance
regressions on small pipelines were observed.
The schedule changes are caused due to the existing implementation not
resetting the DAG's topological order after removing the edges that
are added while examining the SchedGroup candidates. Hence it used
incorrect reachability information.
---
llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp | 171 ++++++++++-
.../AMDGPU/llvm.amdgcn.iglp.opt.single.2b.mir | 54 ++--
.../AMDGPU/llvm.amdgcn.iglp.opt.single.2c.mir | 16 +-
.../AMDGPU/llvm.amdgcn.sched.group.barrier.ll | 278 +++++++++---------
.../sched-group-barrier-pipeline-solver.mir | 6 +-
.../AMDGPU/sched-group-barrier-pre-RA.mir | 10 +-
6 files changed, 343 insertions(+), 192 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
index 6c95dc76f4f6f..5cb5b41eb9a96 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
@@ -222,7 +222,7 @@ class SchedGroup {
int getSyncID() { return SyncID; }
- int getSGID() { return SGID; }
+ int getSGID() const { return SGID; }
SchedGroupMask getMask() { return SGMask; }
@@ -322,6 +322,49 @@ class PipelineSolver {
// return the number of edges missed.
int addEdges(SmallVectorImpl<SchedGroup> &SyncPipeline, SUnit *SU, int SGID,
std::list<std::pair<SUnit *, SUnit *>> &AddedEdges);
+
+ /// This class is used to build the edge set implied by an
+ /// assignment of an SUnit to a SchedGroup and to compute the cost
+ /// (edges than cannot be assigned without introducing cycles) of
+ /// the assignment.
+ class EdgeSetBuilder {
+ SUnit *SU;
+ SmallVectorImpl<SchedGroup> &SyncPipeline;
+ bool IsBottomUp;
+ DenseSet<SUnit *> InitialPreds;
+ DenseSet<SUnit *> Succs;
+ bool Initialized = false;
+
+ /// Compute reachability via DFS. If ComputePreds is true, follows
+ /// predecessor edges; otherwise follows successor edges.
+ template <bool ComputePreds>
+ static void computeReachable(DenseSet<SUnit *> &Reachable, SUnit *Start);
+
+ /// Compute all nodes that can reach Start via predecessor edges, including
+ /// Start itself.
+ static void computePreds(DenseSet<SUnit *> &Preds, SUnit *Start);
+
+ /// Compute all nodes reachable from Start via successor edges, including
+ /// Start itself.
+ static void computeSuccs(DenseSet<SUnit *> &Succs, SUnit *Start);
+
+ public:
+ EdgeSetBuilder(SUnit *SU, SmallVectorImpl<SchedGroup> &SyncPipeline,
+ bool IsBottomUp)
+ : SU(SU), SyncPipeline(SyncPipeline), IsBottomUp(IsBottomUp) {}
+
+ /// Determine the edges implied by assigning SU to the SchedGroup
+ /// with ID SGID. Edges are added to NewEdges unless they
+ /// introduce cycles. Return the number of edges that cannot be
+ /// added.
+ int build(int SGID, std::list<std::pair<SUnit *, SUnit *>> &NewEdges);
+
+ private:
+ template <typename T>
+ int buildImpl(int SGID, const iterator_range<T> SchedGroups,
+ std::list<std::pair<SUnit *, SUnit *>> &NewEdges);
+ };
+
/// Link the pipeline as if \p SU was in the SchedGroup with ID \p SGID. It
/// returns the cost (in terms of missed pipeline edges), and tracks the edges
/// added in \p AddedEdges
@@ -469,6 +512,117 @@ int PipelineSolver::linkSUnit(
return AddedCost;
}
+template <bool ComputePreds>
+void PipelineSolver::EdgeSetBuilder::computeReachable(
+ DenseSet<SUnit *> &Reachable, SUnit *Start) {
+ if (!Reachable.insert(Start).second)
+ return;
+
+ SmallVector<SUnit *, 32> WorkList;
+ WorkList.push_back(Start);
+
+ while (!WorkList.empty()) {
+ SUnit *Current = WorkList.pop_back_val();
+
+ for (const SDep &Dep : ComputePreds ? Current->Preds : Current->Succs) {
+ if (Reachable.insert(Dep.getSUnit()).second)
+ WorkList.push_back(Dep.getSUnit());
+ }
+ }
+}
+
+void PipelineSolver::EdgeSetBuilder::computePreds(DenseSet<SUnit *> &Preds,
+ SUnit *Start) {
+ computeReachable<true>(Preds, Start);
+}
+
+void PipelineSolver::EdgeSetBuilder::computeSuccs(DenseSet<SUnit *> &Succs,
+ SUnit *Start) {
+ computeReachable<false>(Succs, Start);
+}
+
+int PipelineSolver::EdgeSetBuilder::build(
+ int SGID, std::list<std::pair<SUnit *, SUnit *>> &NewEdges) {
+ if (!Initialized) {
+ computePreds(InitialPreds, SU);
+ computeSuccs(Succs, SU);
+ Initialized = true;
+ }
+
+ // See comment in addEdges concerning the iterator direction.
+ return IsBottomUp ? buildImpl(SGID,
+ llvm::make_range(SyncPipeline.rbegin(),
+ SyncPipeline.rend()),
+ NewEdges)
+ : buildImpl(SGID,
+ llvm::make_range(SyncPipeline.begin(),
+ SyncPipeline.end()),
+ NewEdges);
+}
+
+template <typename T>
+int PipelineSolver::EdgeSetBuilder::buildImpl(
+ int SGID, iterator_range<T> SchedGroups,
+ std::list<std::pair<SUnit *, SUnit *>> &NewEdges) {
+
+ // Determine the edges that will be added to the DAG if SU is
+ // assigned to the SchedGroup SG with the given SGID. It might be
+ // impossible to add some edges because they would introduce
+ // cycles. The number of such edges is counted and returned, all
+ // other edges are added to NewEdges.
+ //
+ // SU is made a successor of SUnits in SchedGroups before SG, and a
+ // predecessor of SUnits after SG. In each case, the cycle check
+ // requires reachability information for the opposing direction.
+
+ // Nodes U reachable from SU (U ~> SU).
+ // Will be extended as new edges are added and hence cannot be
+ // shared between calls to this function, in contrast to Succs.
+ DenseSet<SUnit *> Preds = InitialPreds;
+
+ int MissedEdges = 0;
+ auto TryMakeSuccEdge = [&](SUnit *Pred) {
+ // If SU ~> Pred, adding Pred -> SU creates a cycle.
+ if (Succs.contains(Pred))
+ ++MissedEdges;
+ else {
+ NewEdges.emplace_back(Pred, SU);
+ computePreds(Preds, Pred);
+ }
+ };
+
+ auto TryMakePredEdge = [&](SUnit *Succ) {
+ // If Succ ~> SU, adding SU -> Succ creates a cycle.
+ if (Preds.contains(Succ))
+ ++MissedEdges;
+ else {
+ // Succs does not need to be updated, since it will not be
+ // queried after the loop below enters the MakePred case.
+ NewEdges.emplace_back(SU, Succ);
+ }
+ };
+
+ bool MakePred = false;
+ for (const SchedGroup &SG : SchedGroups) {
+ if (SG.getSGID() == SGID) {
+ MakePred = true;
+ continue;
+ }
+
+ for (auto *A : SG.Collection) {
+ if (A->getInstr()->getOpcode() == AMDGPU::SCHED_GROUP_BARRIER)
+ continue;
+
+ if (MakePred)
+ TryMakePredEdge(A);
+ else
+ TryMakeSuccEdge(A);
+ }
+ }
+
+ return MissedEdges;
+}
+
int PipelineSolver::addEdges(
SmallVectorImpl<SchedGroup> &SyncPipeline, SUnit *SU, int SGID,
std::list<std::pair<SUnit *, SUnit *>> &AddedEdges) {
@@ -495,12 +649,11 @@ void PipelineSolver::removeEdges(
SUnit *Pred = PredSuccPair.first;
SUnit *Succ = PredSuccPair.second;
- auto *Match = llvm::find_if(
- Succ->Preds, [&Pred](SDep &P) { return P.getSUnit() == Pred; });
- if (Match != Succ->Preds.end()) {
- assert(Match->isArtificial());
+ auto *Match = llvm::find_if(Succ->Preds, [&Pred](SDep &P) {
+ return P.getSUnit() == Pred && P.isArtificial();
+ });
+ if (Match != Succ->Preds.end())
Succ->removePred(*Match);
- }
}
}
@@ -707,6 +860,8 @@ void PipelineSolver::greedyFind(
LLVM_DEBUG(dbgs() << "Fitting SU(" << CurrSU.first->NodeNum
<< ") in Pipeline # " << CurrSyncGroupIdx << "\n");
+ EdgeSetBuilder Builder(CurrSU.first, SyncPipeline, IsBottomUp);
+
// Since we have added the potential SchedGroups from bottom up, but
// traversed the DAG from top down, parse over the groups from last to
// first. If we fail to do this for the greedy algorithm, the solution will
@@ -731,7 +886,7 @@ void PipelineSolver::greedyFind(
}
std::list<std::pair<SUnit *, SUnit *>> TempEdges;
- int TempCost = addEdges(SyncPipeline, CurrSU.first, CandSGID, TempEdges);
+ int TempCost = Builder.build(CandSGID, TempEdges);
LLVM_DEBUG(dbgs() << "Cost of Group " << TempCost << "\n");
if (!Best || TempCost < Best->Cost) {
@@ -739,8 +894,6 @@ void PipelineSolver::greedyFind(
if (Best->Cost == 0)
break;
}
-
- removeEdges(TempEdges);
}
if (Best) {
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.single.2b.mir b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.single.2b.mir
index e93595b9ef273..6c07e313eeeb5 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.single.2b.mir
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.single.2b.mir
@@ -81,39 +81,33 @@ body: |
; GCN-NEXT: [[DS_READ_B128_gfx9_3:%[0-9]+]]:av_128_align2 = DS_READ_B128_gfx9 [[DEF7]], 2064, 0, implicit $exec :: (load (s128) from %ir.in5, !alias.scope !0, addrspace 3)
; GCN-NEXT: [[DEF:%[0-9]+]]:areg_512_align2 = contract V_MFMA_F32_32X32X8F16_mac_e64 [[DS_READ_B128_gfx9_]].sub0_sub1, [[DS_READ_B128_gfx9_1]].sub0_sub1, [[DEF]], 0, 0, 0, implicit $mode, implicit $exec
; GCN-NEXT: [[DS_READ_B128_gfx9_4:%[0-9]+]]:av_128_align2 = DS_READ_B128_gfx9 [[DEF6]], 2080, 0, implicit $exec :: (load (s128) from %ir.in2, !alias.scope !0, addrspace 3)
- ; GCN-NEXT: [[V_ADD_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF42]], [[DEF8]], implicit $exec
- ; GCN-NEXT: [[V_ADD_U32_e32_1:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF42]], [[DEF9]], implicit $exec
; GCN-NEXT: dead [[V_MFMA_F32_32X32X8F16_mac_e64_:%[0-9]+]]:areg_512_align2 = contract V_MFMA_F32_32X32X8F16_mac_e64 [[DS_READ_B128_gfx9_]].sub2_sub3, [[DS_READ_B128_gfx9_1]].sub2_sub3, [[DEF]], 0, 0, 0, implicit $mode, implicit $exec
; GCN-NEXT: [[DS_READ_B128_gfx9_5:%[0-9]+]]:av_128_align2 = DS_READ_B128_gfx9 [[DEF6]], 3120, 0, implicit $exec :: (load (s128) from %ir.in3, !alias.scope !0, addrspace 3)
- ; GCN-NEXT: [[V_ADD_U32_e32_2:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF42]], [[DEF10]], implicit $exec
- ; GCN-NEXT: [[V_ADD_U32_e32_3:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF42]], [[DEF11]], implicit $exec
; GCN-NEXT: [[DEF:%[0-9]+]]:areg_512_align2 = contract V_MFMA_F32_32X32X8F16_mac_e64 [[DS_READ_B128_gfx9_2]].sub0_sub1, [[DS_READ_B128_gfx9_3]].sub0_sub1, [[DEF]], 0, 0, 0, implicit $mode, implicit $exec
; GCN-NEXT: [[DS_READ_B128_gfx9_6:%[0-9]+]]:av_128_align2 = DS_READ_B128_gfx9 [[DEF7]], 4128, 0, implicit $exec :: (load (s128) from %ir.in6, !alias.scope !0, addrspace 3)
- ; GCN-NEXT: [[V_ADD_U32_e32_4:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF42]], [[DEF12]], implicit $exec
- ; GCN-NEXT: [[V_ADD_U32_e32_5:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF42]], [[DEF13]], implicit $exec
; GCN-NEXT: [[DEF:%[0-9]+]]:areg_512_align2 = contract V_MFMA_F32_32X32X8F16_mac_e64 [[DS_READ_B128_gfx9_2]].sub2_sub3, [[DS_READ_B128_gfx9_3]].sub2_sub3, [[DEF]], 0, 0, 0, implicit $mode, implicit $exec
; GCN-NEXT: [[DS_READ_B128_gfx9_7:%[0-9]+]]:av_128_align2 = DS_READ_B128_gfx9 [[DEF7]], 6192, 0, implicit $exec :: (load (s128) from %ir.in7, !alias.scope !0, addrspace 3)
- ; GCN-NEXT: [[V_ADD_U32_e32_6:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF42]], [[DEF14]], implicit $exec
- ; GCN-NEXT: [[V_ADD_U32_e32_7:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF42]], [[DEF15]], implicit $exec
; GCN-NEXT: [[DEF:%[0-9]+]]:areg_512_align2 = contract V_MFMA_F32_32X32X8F16_mac_e64 [[DS_READ_B128_gfx9_4]].sub0_sub1, [[DS_READ_B128_gfx9_6]].sub0_sub1, [[DEF]], 0, 0, 0, implicit $mode, implicit $exec
; GCN-NEXT: [[DS_READ_B128_gfx9_8:%[0-9]+]]:av_128_align2 = DS_READ_B128_gfx9 [[DEF7]], 1024, 0, implicit $exec :: (load (s128) from %ir.in8, !alias.scope !0, addrspace 3)
- ; GCN-NEXT: [[V_ADD_U32_e32_8:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF42]], [[DEF16]], implicit $exec
- ; GCN-NEXT: [[V_ADD_U32_e32_9:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF42]], [[DEF17]], implicit $exec
; GCN-NEXT: [[DEF:%[0-9]+]]:areg_512_align2 = contract V_MFMA_F32_32X32X8F16_mac_e64 [[DS_READ_B128_gfx9_4]].sub2_sub3, [[DS_READ_B128_gfx9_6]].sub2_sub3, [[DEF]], 0, 0, 0, implicit $mode, implicit $exec
; GCN-NEXT: [[DS_READ_B128_gfx9_9:%[0-9]+]]:av_128_align2 = DS_READ_B128_gfx9 [[DEF7]], 3088, 0, implicit $exec :: (load (s128) from %ir.in9, !alias.scope !0, addrspace 3)
- ; GCN-NEXT: [[V_ADD_U32_e32_10:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF42]], [[DEF18]], implicit $exec
- ; GCN-NEXT: [[V_ADD_U32_e32_11:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF42]], [[DEF19]], implicit $exec
; GCN-NEXT: [[DEF:%[0-9]+]]:areg_512_align2 = contract V_MFMA_F32_32X32X8F16_mac_e64 [[DS_READ_B128_gfx9_5]].sub0_sub1, [[DS_READ_B128_gfx9_7]].sub0_sub1, [[DEF]], 0, 0, 0, implicit $mode, implicit $exec
; GCN-NEXT: [[DS_READ_B128_gfx9_10:%[0-9]+]]:av_128_align2 = DS_READ_B128_gfx9 [[DEF7]], 5152, 0, implicit $exec :: (load (s128) from %ir.in10, !alias.scope !0, addrspace 3)
- ; GCN-NEXT: [[V_ADD_U32_e32_12:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF42]], [[DEF20]], implicit $exec
- ; GCN-NEXT: [[V_ADD_U32_e32_13:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF42]], [[DEF21]], implicit $exec
; GCN-NEXT: [[DEF:%[0-9]+]]:areg_512_align2 = contract V_MFMA_F32_32X32X8F16_mac_e64 [[DS_READ_B128_gfx9_5]].sub2_sub3, [[DS_READ_B128_gfx9_7]].sub2_sub3, [[DEF]], 0, 0, 0, implicit $mode, implicit $exec
; GCN-NEXT: [[DS_READ_B128_gfx9_11:%[0-9]+]]:av_128_align2 = DS_READ_B128_gfx9 [[DEF7]], 7216, 0, implicit $exec :: (load (s128) from %ir.in11, !alias.scope !0, addrspace 3)
; GCN-NEXT: [[DEF1:%[0-9]+]]:areg_512_align2 = contract V_MFMA_F32_32X32X8F16_mac_e64 [[DS_READ_B128_gfx9_]].sub0_sub1, [[DS_READ_B128_gfx9_8]].sub0_sub1, [[DEF1]], 0, 0, 0, implicit $mode, implicit $exec
- ; GCN-NEXT: undef [[V_PERM_B32_e64_:%[0-9]+]].sub1:vreg_128_align2 = V_PERM_B32_e64 [[DEF28]], [[DEF29]], [[DEF44]], implicit $exec
- ; GCN-NEXT: [[V_PERM_B32_e64_:%[0-9]+]].sub0:vreg_128_align2 = V_PERM_B32_e64 [[DEF30]], [[DEF31]], [[DEF44]], implicit $exec
- ; GCN-NEXT: [[V_PERM_B32_e64_:%[0-9]+]].sub3:vreg_128_align2 = V_PERM_B32_e64 [[DEF24]], [[DEF25]], [[DEF44]], implicit $exec
+ ; GCN-NEXT: undef [[V_PERM_B32_e64_:%[0-9]+]].sub3:vreg_128_align2 = V_PERM_B32_e64 [[DEF24]], [[DEF25]], [[DEF44]], implicit $exec
; GCN-NEXT: [[V_PERM_B32_e64_:%[0-9]+]].sub2:vreg_128_align2 = V_PERM_B32_e64 [[DEF26]], [[DEF27]], [[DEF44]], implicit $exec
+ ; GCN-NEXT: [[V_PERM_B32_e64_:%[0-9]+]].sub1:vreg_128_align2 = V_PERM_B32_e64 [[DEF28]], [[DEF29]], [[DEF44]], implicit $exec
+ ; GCN-NEXT: [[V_PERM_B32_e64_:%[0-9]+]].sub0:vreg_128_align2 = V_PERM_B32_e64 [[DEF30]], [[DEF31]], [[DEF44]], implicit $exec
+ ; GCN-NEXT: [[V_ADD_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF42]], [[DEF8]], implicit $exec
+ ; GCN-NEXT: [[V_ADD_U32_e32_1:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF42]], [[DEF9]], implicit $exec
+ ; GCN-NEXT: [[V_ADD_U32_e32_2:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF42]], [[DEF10]], implicit $exec
+ ; GCN-NEXT: [[V_ADD_U32_e32_3:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF42]], [[DEF11]], implicit $exec
+ ; GCN-NEXT: [[V_ADD_U32_e32_4:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF42]], [[DEF12]], implicit $exec
+ ; GCN-NEXT: [[V_ADD_U32_e32_5:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF42]], [[DEF13]], implicit $exec
+ ; GCN-NEXT: [[V_ADD_U32_e32_6:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF42]], [[DEF14]], implicit $exec
+ ; GCN-NEXT: [[V_ADD_U32_e32_7:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF42]], [[DEF15]], implicit $exec
; GCN-NEXT: DS_WRITE_B128_gfx9 [[DEF40]], [[V_PERM_B32_e64_]], 0, 0, implicit $exec :: (store (s128) into %ir.in0, !alias.scope !0, addrspace 3)
; GCN-NEXT: [[DEF31:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_USHORT_OFFEN [[V_ADD_U32_e32_]], [[DEF47]], 0, 0, 0, 0, implicit $exec :: (load (s16) from %ir.in12, !alias.scope !0, addrspace 7)
; GCN-NEXT: [[DEF30:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_USHORT_OFFEN [[V_ADD_U32_e32_1]], [[DEF47]], 0, 0, 0, 0, implicit $exec :: (load (s16) from %ir.in13, !alias.scope !0, addrspace 7)
@@ -125,10 +119,16 @@ body: |
; GCN-NEXT: [[DEF25:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_USHORT_OFFEN [[V_ADD_U32_e32_6]], [[DEF47]], 0, 0, 0, 0, implicit $exec :: (load (s16) from %ir.in18, !alias.scope !0, addrspace 7)
; GCN-NEXT: [[DEF24:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_USHORT_OFFEN [[V_ADD_U32_e32_7]], [[DEF47]], 0, 0, 0, 0, implicit $exec :: (load (s16) from %ir.in19, !alias.scope !0, addrspace 7)
; GCN-NEXT: [[DEF1:%[0-9]+]]:areg_512_align2 = contract V_MFMA_F32_32X32X8F16_mac_e64 [[DS_READ_B128_gfx9_2]].sub0_sub1, [[DS_READ_B128_gfx9_9]].sub0_sub1, [[DEF1]], 0, 0, 0, implicit $mode, implicit $exec
- ; GCN-NEXT: undef [[V_PERM_B32_e64_1:%[0-9]+]].sub1:vreg_128_align2 = V_PERM_B32_e64 [[DEF36]], [[DEF37]], [[DEF44]], implicit $exec
- ; GCN-NEXT: [[V_PERM_B32_e64_1:%[0-9]+]].sub0:vreg_128_align2 = V_PERM_B32_e64 [[DEF38]], [[DEF39]], [[DEF44]], implicit $exec
- ; GCN-NEXT: [[V_PERM_B32_e64_1:%[0-9]+]].sub3:vreg_128_align2 = V_PERM_B32_e64 [[DEF32]], [[DEF33]], [[DEF44]], implicit $exec
+ ; GCN-NEXT: undef [[V_PERM_B32_e64_1:%[0-9]+]].sub3:vreg_128_align2 = V_PERM_B32_e64 [[DEF32]], [[DEF33]], [[DEF44]], implicit $exec
; GCN-NEXT: [[V_PERM_B32_e64_1:%[0-9]+]].sub2:vreg_128_align2 = V_PERM_B32_e64 [[DEF34]], [[DEF35]], [[DEF44]], implicit $exec
+ ; GCN-NEXT: [[V_PERM_B32_e64_1:%[0-9]+]].sub1:vreg_128_align2 = V_PERM_B32_e64 [[DEF36]], [[DEF37]], [[DEF44]], implicit $exec
+ ; GCN-NEXT: [[V_PERM_B32_e64_1:%[0-9]+]].sub0:vreg_128_align2 = V_PERM_B32_e64 [[DEF38]], [[DEF39]], [[DEF44]], implicit $exec
+ ; GCN-NEXT: [[V_ADD_U32_e32_8:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF42]], [[DEF16]], implicit $exec
+ ; GCN-NEXT: [[V_ADD_U32_e32_9:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF42]], [[DEF17]], implicit $exec
+ ; GCN-NEXT: [[V_ADD_U32_e32_10:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF42]], [[DEF18]], implicit $exec
+ ; GCN-NEXT: [[V_ADD_U32_e32_11:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF42]], [[DEF19]], implicit $exec
+ ; GCN-NEXT: [[V_ADD_U32_e32_12:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF42]], [[DEF20]], implicit $exec
+ ; GCN-NEXT: [[V_ADD_U32_e32_13:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF42]], [[DEF21]], implicit $exec
; GCN-NEXT: DS_WRITE_B128_gfx9 [[DEF40]], [[V_PERM_B32_e64_1]], 1040, 0, implicit $exec :: (store (s128) into %ir.in1, !alias.scope !0, addrspace 3)
; GCN-NEXT: [[DEF32:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_USHORT_OFFEN [[V_ADD_U32_e32_8]], [[DEF47]], 0, 0, 0, 0, implicit $exec :: (load (s16) from %ir.in20, !alias.scope !0, addrspace 7)
; GCN-NEXT: [[DEF33:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_USHORT_OFFEN [[V_ADD_U32_e32_9]], [[DEF47]], 0, 0, 0, 0, implicit $exec :: (load (s16) from %ir.in21, !alias.scope !0, addrspace 7)
@@ -150,11 +150,11 @@ body: |
; GCN-NEXT: DS_WRITE_B128_gfx9 [[DEF41]], [[DEF3]], 2064, 0, implicit $exec :: (store (s128) into %ir.in3, !alias.scope !0, addrspace 3)
; GCN-NEXT: [[DEF3:%[0-9]+]]:av_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN [[DEF45]], [[DEF48]], 0, 0, 0, 0, implicit $exec :: (load (s128) from %ir.in27, !alias.scope !0, addrspace 7)
; GCN-NEXT: [[DEF45:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 128, [[DEF45]], implicit $exec
- ; GCN-NEXT: [[DEF23:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF43]], [[DEF23]], implicit $exec
; GCN-NEXT: [[DEF1:%[0-9]+]]:areg_512_align2 = contract V_MFMA_F32_32X32X8F16_mac_e64 [[DS_READ_B128_gfx9_5]].sub0_sub1, [[DS_READ_B128_gfx9_11]].sub0_sub1, [[DEF1]], 0, 0, 0, implicit $mode, implicit $exec
; GCN-NEXT: DS_WRITE_B128_gfx9 [[DEF41]], [[DEF4]], 2080, 0, implicit $exec :: (store (s128) into %ir.in4, !alias.scope !0, addrspace 3)
; GCN-NEXT: [[DEF4:%[0-9]+]]:av_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN [[DEF46]], [[DEF48]], 0, 0, 0, 0, implicit $exec :: (load (s128) from %ir.in28, !alias.scope !0, addrspace 7)
; GCN-NEXT: [[DEF46:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 128, [[DEF46]], implicit $exec
+ ; GCN-NEXT: [[DEF23:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF43]], [[DEF23]], implicit $exec
; GCN-NEXT: [[DEF22:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF43]], [[DEF22]], implicit $exec
; GCN-NEXT: [[DEF1:%[0-9]+]]:areg_512_align2 = contract V_MFMA_F32_32X32X8F16_mac_e64 [[DS_READ_B128_gfx9_5]].sub2_sub3, [[DS_READ_B128_gfx9_11]].sub2_sub3, [[DEF1]], 0, 0, 0, implicit $mode, implicit $exec
; GCN-NEXT: DS_WRITE_B128_gfx9 [[DEF41]], [[DEF5]], 16, 0, implicit $exec :: (store (s128) into %ir.in5, !alias.scope !0, addrspace 3)
@@ -162,18 +162,18 @@ body: |
; GCN-NEXT: IGLP_OPT 1
; GCN-NEXT: [[DEF49:%[0-9]+]]:sreg_32 = nsw S_ADD_I32 [[DEF49]], -1, implicit-def dead $scc
; GCN-NEXT: S_CMP_LG_U32 [[DEF49]], 0, implicit-def $scc
- ; GCN-NEXT: [[DEF9:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF43]], [[DEF9]], implicit $exec
- ; GCN-NEXT: [[DEF8:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF43]], [[DEF8]], implicit $exec
+ ; GCN-NEXT: [[DEF15:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF43]], [[DEF15]], implicit $exec
+ ; GCN-NEXT: [[DEF14:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF43]], [[DEF14]], implicit $exec
; GCN-NEXT: [[DEF13:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF43]], [[DEF13]], implicit $exec
; GCN-NEXT: [[DEF12:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF43]], [[DEF12]], implicit $exec
; GCN-NEXT: [[DEF11:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF43]], [[DEF11]], implicit $exec
; GCN-NEXT: [[DEF10:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF43]], [[DEF10]], implicit $exec
- ; GCN-NEXT: [[DEF15:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF43]], [[DEF15]], implicit $exec
- ; GCN-NEXT: [[DEF14:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF43]], [[DEF14]], implicit $exec
- ; GCN-NEXT: [[DEF17:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF43]], [[DEF17]], implicit $exec
- ; GCN-NEXT: [[DEF16:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF43]], [[DEF16]], implicit $exec
+ ; GCN-NEXT: [[DEF9:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF43]], [[DEF9]], implicit $exec
+ ; GCN-NEXT: [[DEF8:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF43]], [[DEF8]], implicit $exec
; GCN-NEXT: [[DEF19:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF43]], [[DEF19]], implicit $exec
; GCN-NEXT: [[DEF18:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF43]], [[DEF18]], implicit $exec
+ ; GCN-NEXT: [[DEF17:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF43]], [[DEF17]], implicit $exec
+ ; GCN-NEXT: [[DEF16:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF43]], [[DEF16]], implicit $exec
; GCN-NEXT: [[DEF21:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF43]], [[DEF21]], implicit $exec
; GCN-NEXT: [[DEF20:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF43]], [[DEF20]], implicit $exec
; GCN-NEXT: S_CBRANCH_SCC1 %bb.1, implicit killed $scc
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.single.2c.mir b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.single.2c.mir
index a85478df10eb2..5f4dfc95ce8b3 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.single.2c.mir
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.single.2c.mir
@@ -64,31 +64,31 @@ body: |
; GCN-NEXT: [[COPY:%[0-9]+]]:areg_512_align2 = COPY [[DEF1]]
; GCN-NEXT: [[COPY:%[0-9]+]]:areg_512_align2 = contract V_MFMA_F32_32X32X8F16_mac_e64 [[DS_READ_B128_gfx9_]].sub0_sub1, [[DS_READ_B128_gfx9_1]].sub0_sub1, [[COPY]], 0, 0, 0, implicit $mode, implicit $exec
; GCN-NEXT: [[DS_READ_B128_gfx9_4:%[0-9]+]]:av_128_align2 = DS_READ_B128_gfx9 [[DEF3]], 1024, 0, implicit $exec :: (load (s128) from %ir.in4, !alias.scope !0, addrspace 3)
- ; GCN-NEXT: [[V_ADD_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF19]], [[DEF33]], implicit $exec
- ; GCN-NEXT: [[V_ADD_U32_e32_1:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF19]], [[DEF21]], implicit $exec
; GCN-NEXT: [[COPY:%[0-9]+]]:areg_512_align2 = contract V_MFMA_F32_32X32X8F16_mac_e64 [[DS_READ_B128_gfx9_]].sub2_sub3, [[DS_READ_B128_gfx9_1]].sub2_sub3, [[COPY]], 0, 0, 0, implicit $mode, implicit $exec
; GCN-NEXT: [[DS_READ_B128_gfx9_5:%[0-9]+]]:av_128_align2 = DS_READ_B128_gfx9 [[DEF3]], 3088, 0, implicit $exec :: (load (s128) from %ir.in5, !alias.scope !0, addrspace 3)
- ; GCN-NEXT: [[V_ADD_U32_e32_2:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF19]], [[DEF22]], implicit $exec
- ; GCN-NEXT: [[V_ADD_U32_e32_3:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF19]], [[DEF23]], implicit $exec
; GCN-NEXT: [[COPY:%[0-9]+]]:areg_512_align2 = contract V_MFMA_F32_32X32X8F16_mac_e64 [[DS_READ_B128_gfx9_2]].sub0_sub1, [[DS_READ_B128_gfx9_3]].sub0_sub1, [[COPY]], 0, 0, 0, implicit $mode, implicit $exec
; GCN-NEXT: DS_WRITE_B128_gfx9 [[DEF4]], [[DEF16]], 0, 0, implicit $exec :: (store (s128) into %ir.in6, !alias.scope !0, addrspace 3)
; GCN-NEXT: [[DEF16:%[0-9]+]]:av_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN [[DEF6]], [[DEF7]], 0, 0, 0, 0, implicit $exec :: (load (s128) from %ir.in7, !alias.scope !0, addrspace 7)
; GCN-NEXT: dead [[COPY:%[0-9]+]]:areg_512_align2 = contract V_MFMA_F32_32X32X8F16_mac_e64 [[DS_READ_B128_gfx9_2]].sub2_sub3, [[DS_READ_B128_gfx9_3]].sub2_sub3, [[COPY]], 0, 0, 0, implicit $mode, implicit $exec
; GCN-NEXT: [[COPY1:%[0-9]+]]:areg_512_align2 = COPY [[DEF]]
- ; GCN-NEXT: undef [[DEF17:%[0-9]+]].sub2:vreg_128_align2 = V_PERM_B32_e64 [[DEF13]], [[DEF12]], [[DEF30]], implicit $exec
- ; GCN-NEXT: [[DEF17:%[0-9]+]].sub3:vreg_128_align2 = V_PERM_B32_e64 [[DEF15]], [[DEF14]], [[DEF30]], implicit $exec
- ; GCN-NEXT: [[DEF17:%[0-9]+]].sub0:vreg_128_align2 = V_PERM_B32_e64 [[DEF8]], [[DEF9]], [[DEF30]], implicit $exec
+ ; GCN-NEXT: undef [[DEF17:%[0-9]+]].sub0:vreg_128_align2 = V_PERM_B32_e64 [[DEF8]], [[DEF9]], [[DEF30]], implicit $exec
; GCN-NEXT: [[DEF17:%[0-9]+]].sub1:vreg_128_align2 = V_PERM_B32_e64 [[DEF11]], [[DEF10]], [[DEF30]], implicit $exec
+ ; GCN-NEXT: [[DEF17:%[0-9]+]].sub2:vreg_128_align2 = V_PERM_B32_e64 [[DEF13]], [[DEF12]], [[DEF30]], implicit $exec
+ ; GCN-NEXT: [[DEF17:%[0-9]+]].sub3:vreg_128_align2 = V_PERM_B32_e64 [[DEF15]], [[DEF14]], [[DEF30]], implicit $exec
; GCN-NEXT: DS_WRITE_B128_gfx9 [[DEF5]], [[DEF17]], 0, 0, implicit $exec :: (store (s128) into %ir.in8, !alias.scope !0, addrspace 3)
; GCN-NEXT: [[COPY1:%[0-9]+]]:areg_512_align2 = contract V_MFMA_F32_32X32X8F16_mac_e64 [[DS_READ_B128_gfx9_]].sub0_sub1, [[DS_READ_B128_gfx9_4]].sub0_sub1, [[COPY1]], 0, 0, 0, implicit $mode, implicit $exec
; GCN-NEXT: undef [[DEF18:%[0-9]+]].sub0:vreg_128_align2 = V_PERM_B32_e64 [[DEF8]], [[DEF9]], [[DEF31]], implicit $exec
; GCN-NEXT: [[DEF18:%[0-9]+]].sub1:vreg_128_align2 = V_PERM_B32_e64 [[DEF11]], [[DEF10]], [[DEF31]], implicit $exec
; GCN-NEXT: [[DEF18:%[0-9]+]].sub2:vreg_128_align2 = V_PERM_B32_e64 [[DEF13]], [[DEF12]], [[DEF31]], implicit $exec
; GCN-NEXT: [[DEF18:%[0-9]+]].sub3:vreg_128_align2 = V_PERM_B32_e64 [[DEF15]], [[DEF14]], [[DEF31]], implicit $exec
+ ; GCN-NEXT: [[V_ADD_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF19]], [[DEF33]], implicit $exec
+ ; GCN-NEXT: [[V_ADD_U32_e32_1:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF19]], [[DEF21]], implicit $exec
; GCN-NEXT: DS_WRITE_B128_gfx9 [[DEF5]], [[DEF18]], 16, 0, implicit $exec :: (store (s128) into %ir.in9, !alias.scope !0, addrspace 3)
; GCN-NEXT: [[COPY1:%[0-9]+]]:areg_512_align2 = contract V_MFMA_F32_32X32X8F16_mac_e64 [[DS_READ_B128_gfx9_]].sub2_sub3, [[DS_READ_B128_gfx9_4]].sub2_sub3, [[COPY1]], 0, 0, 0, implicit $mode, implicit $exec
; GCN-NEXT: [[DEF9:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[V_ADD_U32_e32_]], [[DEF32]], 0, 0, 0, 0, implicit $exec :: (load (s128) from %ir.in10, !alias.scope !0, addrspace 7)
; GCN-NEXT: [[DEF8:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[V_ADD_U32_e32_1]], [[DEF32]], 0, 0, 0, 0, implicit $exec :: (load (s128) from %ir.in11, !alias.scope !0, addrspace 7)
+ ; GCN-NEXT: [[V_ADD_U32_e32_2:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF19]], [[DEF22]], implicit $exec
+ ; GCN-NEXT: [[V_ADD_U32_e32_3:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF19]], [[DEF23]], implicit $exec
; GCN-NEXT: [[DEF10:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[V_ADD_U32_e32_2]], [[DEF32]], 0, 0, 0, 0, implicit $exec :: (load (s128) from %ir.in12, !alias.scope !0, addrspace 7)
; GCN-NEXT: [[DEF11:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[V_ADD_U32_e32_3]], [[DEF32]], 0, 0, 0, 0, implicit $exec :: (load (s128) from %ir.in13, !alias.scope !0, addrspace 7)
; GCN-NEXT: [[COPY1:%[0-9]+]]:areg_512_align2 = contract V_MFMA_F32_32X32X8F16_mac_e64 [[DS_READ_B128_gfx9_2]].sub0_sub1, [[DS_READ_B128_gfx9_5]].sub0_sub1, [[COPY1]], 0, 0, 0, implicit $mode, implicit $exec
@@ -104,11 +104,11 @@ body: |
; GCN-NEXT: IGLP_OPT 1
; GCN-NEXT: [[DEF29:%[0-9]+]]:sreg_32 = nsw S_ADD_I32 [[DEF29]], -1, implicit-def dead $scc
; GCN-NEXT: S_CMP_LG_U32 [[DEF29]], 0, implicit-def $scc
+ ; GCN-NEXT: [[DEF6:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 64, [[DEF6]], implicit $exec
; GCN-NEXT: [[DEF21:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF28]], [[DEF21]], implicit $exec
; GCN-NEXT: [[DEF33:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF28]], [[DEF33]], implicit $exec
; GCN-NEXT: [[DEF23:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF28]], [[DEF23]], implicit $exec
; GCN-NEXT: [[DEF22:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF28]], [[DEF22]], implicit $exec
- ; GCN-NEXT: [[DEF6:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 64, [[DEF6]], implicit $exec
; GCN-NEXT: [[DEF27:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF28]], [[DEF27]], implicit $exec
; GCN-NEXT: [[DEF26:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF28]], [[DEF26]], implicit $exec
; GCN-NEXT: [[DEF25:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF28]], [[DEF25]], implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.ll
index 996c76488080f..bb8888b450629 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.ll
@@ -184,13 +184,13 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_alternating_READ_VA
; GCN-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GCN-NEXT: v_lshlrev_b32_e32 v32, 7, v0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: global_load_dwordx4 v[28:31], v32, s[0:1] offset:16
+; GCN-NEXT: global_load_dwordx4 v[28:31], v32, s[0:1] offset:48
; GCN-NEXT: global_load_dwordx4 v[0:3], v32, s[0:1]
; GCN-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
; GCN-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
; GCN-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
; GCN-NEXT: s_waitcnt vmcnt(1)
-; GCN-NEXT: v_mul_lo_u32 v29, v29, v29
+; GCN-NEXT: v_mul_lo_u32 v31, v31, v31
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: v_mul_lo_u32 v3, v3, v3
; GCN-NEXT: v_mul_lo_u32 v2, v2, v2
@@ -198,9 +198,9 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_alternating_READ_VA
; GCN-NEXT: v_mul_lo_u32 v1, v1, v1
; GCN-NEXT: v_mul_lo_u32 v0, v0, v0
; GCN-NEXT: global_load_dwordx4 v[8:11], v32, s[0:1] offset:96
-; GCN-NEXT: v_mul_lo_u32 v28, v28, v28
-; GCN-NEXT: v_mul_lo_u32 v31, v31, v31
; GCN-NEXT: v_mul_lo_u32 v30, v30, v30
+; GCN-NEXT: v_mul_lo_u32 v29, v29, v29
+; GCN-NEXT: v_mul_lo_u32 v28, v28, v28
; GCN-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
; GCN-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
; GCN-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
@@ -211,7 +211,7 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_alternating_READ_VA
; GCN-NEXT: global_load_dwordx4 v[12:15], v32, s[0:1] offset:80
; GCN-NEXT: v_mul_lo_u32 v5, v5, v5
; GCN-NEXT: v_mul_lo_u32 v4, v4, v4
-; GCN-NEXT: global_load_dwordx4 v[16:19], v32, s[0:1] offset:48
+; GCN-NEXT: global_load_dwordx4 v[16:19], v32, s[0:1] offset:16
; GCN-NEXT: s_waitcnt vmcnt(2)
; GCN-NEXT: v_mul_lo_u32 v11, v11, v11
; GCN-NEXT: v_mul_lo_u32 v10, v10, v10
@@ -222,40 +222,40 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_alternating_READ_VA
; GCN-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
; GCN-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
; GCN-NEXT: s_waitcnt vmcnt(1)
-; GCN-NEXT: v_mul_lo_u32 v15, v15, v15
-; GCN-NEXT: v_mul_lo_u32 v14, v14, v14
; GCN-NEXT: v_mul_lo_u32 v13, v13, v13
+; GCN-NEXT: v_mul_lo_u32 v12, v12, v12
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_mul_lo_u32 v19, v19, v19
-; GCN-NEXT: v_mul_lo_u32 v18, v18, v18
-; GCN-NEXT: global_load_dwordx4 v[20:23], v32, s[0:1] offset:64
-; GCN-NEXT: global_load_dwordx4 v[24:27], v32, s[0:1] offset:32
; GCN-NEXT: v_mul_lo_u32 v17, v17, v17
; GCN-NEXT: v_mul_lo_u32 v16, v16, v16
-; GCN-NEXT: v_mul_lo_u32 v12, v12, v12
+; GCN-NEXT: global_load_dwordx4 v[20:23], v32, s[0:1] offset:64
+; GCN-NEXT: v_mul_lo_u32 v15, v15, v15
+; GCN-NEXT: v_mul_lo_u32 v14, v14, v14
+; GCN-NEXT: global_load_dwordx4 v[24:27], v32, s[0:1] offset:32
+; GCN-NEXT: v_mul_lo_u32 v19, v19, v19
+; GCN-NEXT: v_mul_lo_u32 v18, v18, v18
+; GCN-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
+; GCN-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
; GCN-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
; GCN-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
; GCN-NEXT: s_waitcnt vmcnt(1)
; GCN-NEXT: v_mul_lo_u32 v23, v23, v23
+; GCN-NEXT: v_mul_lo_u32 v22, v22, v22
+; GCN-NEXT: v_mul_lo_u32 v21, v21, v21
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_mul_lo_u32 v27, v27, v27
-; GCN-NEXT: v_mul_lo_u32 v26, v26, v26
; GCN-NEXT: v_mul_lo_u32 v25, v25, v25
; GCN-NEXT: v_mul_lo_u32 v24, v24, v24
-; GCN-NEXT: v_mul_lo_u32 v22, v22, v22
-; GCN-NEXT: v_mul_lo_u32 v21, v21, v21
; GCN-NEXT: v_mul_lo_u32 v20, v20, v20
+; GCN-NEXT: v_mul_lo_u32 v27, v27, v27
+; GCN-NEXT: v_mul_lo_u32 v26, v26, v26
; GCN-NEXT: global_store_dwordx4 v32, v[4:7], s[2:3] offset:112
; GCN-NEXT: global_store_dwordx4 v32, v[8:11], s[2:3] offset:96
; GCN-NEXT: global_store_dwordx4 v32, v[12:15], s[2:3] offset:80
; GCN-NEXT: global_store_dwordx4 v32, v[20:23], s[2:3] offset:64
-; GCN-NEXT: global_store_dwordx4 v32, v[16:19], s[2:3] offset:48
+; GCN-NEXT: global_store_dwordx4 v32, v[28:31], s[2:3] offset:48
; GCN-NEXT: global_store_dwordx4 v32, v[24:27], s[2:3] offset:32
-; GCN-NEXT: global_store_dwordx4 v32, v[28:31], s[2:3] offset:16
+; GCN-NEXT: global_store_dwordx4 v32, v[16:19], s[2:3] offset:16
; GCN-NEXT: global_store_dwordx4 v32, v[0:3], s[2:3]
; GCN-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
-; GCN-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
-; GCN-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
; GCN-NEXT: ; sched_group_barrier mask(0x00000040) size(8) SyncID(0)
; GCN-NEXT: s_endpgm
;
@@ -265,13 +265,13 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_alternating_READ_VA
; EXACTCUTOFF-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; EXACTCUTOFF-NEXT: v_lshlrev_b32_e32 v32, 7, v0
; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(0)
-; EXACTCUTOFF-NEXT: global_load_dwordx4 v[28:31], v32, s[0:1] offset:16
+; EXACTCUTOFF-NEXT: global_load_dwordx4 v[28:31], v32, s[0:1] offset:48
; EXACTCUTOFF-NEXT: global_load_dwordx4 v[0:3], v32, s[0:1]
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
; EXACTCUTOFF-NEXT: s_waitcnt vmcnt(1)
-; EXACTCUTOFF-NEXT: v_mul_lo_u32 v29, v29, v29
+; EXACTCUTOFF-NEXT: v_mul_lo_u32 v31, v31, v31
; EXACTCUTOFF-NEXT: s_waitcnt vmcnt(0)
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v3, v3, v3
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v2, v2, v2
@@ -279,9 +279,9 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_alternating_READ_VA
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v1, v1, v1
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v0, v0, v0
; EXACTCUTOFF-NEXT: global_load_dwordx4 v[8:11], v32, s[0:1] offset:96
-; EXACTCUTOFF-NEXT: v_mul_lo_u32 v28, v28, v28
-; EXACTCUTOFF-NEXT: v_mul_lo_u32 v31, v31, v31
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v30, v30, v30
+; EXACTCUTOFF-NEXT: v_mul_lo_u32 v29, v29, v29
+; EXACTCUTOFF-NEXT: v_mul_lo_u32 v28, v28, v28
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
@@ -292,7 +292,7 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_alternating_READ_VA
; EXACTCUTOFF-NEXT: global_load_dwordx4 v[12:15], v32, s[0:1] offset:80
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v5, v5, v5
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v4, v4, v4
-; EXACTCUTOFF-NEXT: global_load_dwordx4 v[16:19], v32, s[0:1] offset:48
+; EXACTCUTOFF-NEXT: global_load_dwordx4 v[16:19], v32, s[0:1] offset:16
; EXACTCUTOFF-NEXT: s_waitcnt vmcnt(2)
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v11, v11, v11
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v10, v10, v10
@@ -303,40 +303,40 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_alternating_READ_VA
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
; EXACTCUTOFF-NEXT: s_waitcnt vmcnt(1)
-; EXACTCUTOFF-NEXT: v_mul_lo_u32 v15, v15, v15
-; EXACTCUTOFF-NEXT: v_mul_lo_u32 v14, v14, v14
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v13, v13, v13
+; EXACTCUTOFF-NEXT: v_mul_lo_u32 v12, v12, v12
; EXACTCUTOFF-NEXT: s_waitcnt vmcnt(0)
-; EXACTCUTOFF-NEXT: v_mul_lo_u32 v19, v19, v19
-; EXACTCUTOFF-NEXT: v_mul_lo_u32 v18, v18, v18
-; EXACTCUTOFF-NEXT: global_load_dwordx4 v[20:23], v32, s[0:1] offset:64
-; EXACTCUTOFF-NEXT: global_load_dwordx4 v[24:27], v32, s[0:1] offset:32
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v17, v17, v17
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v16, v16, v16
-; EXACTCUTOFF-NEXT: v_mul_lo_u32 v12, v12, v12
+; EXACTCUTOFF-NEXT: global_load_dwordx4 v[20:23], v32, s[0:1] offset:64
+; EXACTCUTOFF-NEXT: v_mul_lo_u32 v15, v15, v15
+; EXACTCUTOFF-NEXT: v_mul_lo_u32 v14, v14, v14
+; EXACTCUTOFF-NEXT: global_load_dwordx4 v[24:27], v32, s[0:1] offset:32
+; EXACTCUTOFF-NEXT: v_mul_lo_u32 v19, v19, v19
+; EXACTCUTOFF-NEXT: v_mul_lo_u32 v18, v18, v18
+; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
+; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
; EXACTCUTOFF-NEXT: s_waitcnt vmcnt(1)
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v23, v23, v23
+; EXACTCUTOFF-NEXT: v_mul_lo_u32 v22, v22, v22
+; EXACTCUTOFF-NEXT: v_mul_lo_u32 v21, v21, v21
; EXACTCUTOFF-NEXT: s_waitcnt vmcnt(0)
-; EXACTCUTOFF-NEXT: v_mul_lo_u32 v27, v27, v27
-; EXACTCUTOFF-NEXT: v_mul_lo_u32 v26, v26, v26
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v25, v25, v25
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v24, v24, v24
-; EXACTCUTOFF-NEXT: v_mul_lo_u32 v22, v22, v22
-; EXACTCUTOFF-NEXT: v_mul_lo_u32 v21, v21, v21
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v20, v20, v20
+; EXACTCUTOFF-NEXT: v_mul_lo_u32 v27, v27, v27
+; EXACTCUTOFF-NEXT: v_mul_lo_u32 v26, v26, v26
; EXACTCUTOFF-NEXT: global_store_dwordx4 v32, v[4:7], s[2:3] offset:112
; EXACTCUTOFF-NEXT: global_store_dwordx4 v32, v[8:11], s[2:3] offset:96
; EXACTCUTOFF-NEXT: global_store_dwordx4 v32, v[12:15], s[2:3] offset:80
; EXACTCUTOFF-NEXT: global_store_dwordx4 v32, v[20:23], s[2:3] offset:64
-; EXACTCUTOFF-NEXT: global_store_dwordx4 v32, v[16:19], s[2:3] offset:48
+; EXACTCUTOFF-NEXT: global_store_dwordx4 v32, v[28:31], s[2:3] offset:48
; EXACTCUTOFF-NEXT: global_store_dwordx4 v32, v[24:27], s[2:3] offset:32
-; EXACTCUTOFF-NEXT: global_store_dwordx4 v32, v[28:31], s[2:3] offset:16
+; EXACTCUTOFF-NEXT: global_store_dwordx4 v32, v[16:19], s[2:3] offset:16
; EXACTCUTOFF-NEXT: global_store_dwordx4 v32, v[0:3], s[2:3]
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
-; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
-; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000040) size(8) SyncID(0)
; EXACTCUTOFF-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x() #2
@@ -387,19 +387,32 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_alternating_READ_VA
; GCN: ; %bb.0:
; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GCN-NEXT: v_lshlrev_b32_e32 v16, 7, v0
-; GCN-NEXT: ; kill: killed $sgpr0_sgpr1
+; GCN-NEXT: v_lshlrev_b32_e32 v12, 7, v0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: global_load_dwordx4 v[12:15], v16, s[0:1] offset:32
+; GCN-NEXT: global_load_dwordx4 v[8:11], v12, s[0:1] offset:16
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_lo_u32 v9, v9, v9
+; GCN-NEXT: v_mul_lo_u32 v8, v8, v8
+; GCN-NEXT: v_mul_lo_u32 v11, v11, v11
+; GCN-NEXT: v_mul_lo_u32 v10, v10, v10
+; GCN-NEXT: global_store_dwordx4 v12, v[8:11], s[2:3] offset:16
+; GCN-NEXT: global_load_dwordx4 v[8:11], v12, s[0:1] offset:32
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_lo_u32 v9, v9, v9
+; GCN-NEXT: v_mul_lo_u32 v8, v8, v8
+; GCN-NEXT: v_mul_lo_u32 v11, v11, v11
+; GCN-NEXT: v_mul_lo_u32 v10, v10, v10
+; GCN-NEXT: global_store_dwordx4 v12, v[8:11], s[2:3] offset:32
+; GCN-NEXT: global_load_dwordx4 v[8:11], v12, s[0:1] offset:48
; GCN-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
; GCN-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_mul_lo_u32 v13, v13, v13
-; GCN-NEXT: v_mul_lo_u32 v12, v12, v12
-; GCN-NEXT: v_mul_lo_u32 v15, v15, v15
-; GCN-NEXT: v_mul_lo_u32 v14, v14, v14
-; GCN-NEXT: global_store_dwordx4 v16, v[12:15], s[2:3] offset:32
-; GCN-NEXT: global_load_dwordx4 v[0:3], v16, s[0:1]
+; GCN-NEXT: v_mul_lo_u32 v11, v11, v11
+; GCN-NEXT: v_mul_lo_u32 v10, v10, v10
+; GCN-NEXT: v_mul_lo_u32 v9, v9, v9
+; GCN-NEXT: v_mul_lo_u32 v8, v8, v8
+; GCN-NEXT: global_store_dwordx4 v12, v[8:11], s[2:3] offset:48
+; GCN-NEXT: global_load_dwordx4 v[0:3], v12, s[0:1]
; GCN-NEXT: ; sched_group_barrier mask(0x00000040) size(1) SyncID(0)
; GCN-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
; GCN-NEXT: s_waitcnt vmcnt(0)
@@ -407,68 +420,54 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_alternating_READ_VA
; GCN-NEXT: v_mul_lo_u32 v2, v2, v2
; GCN-NEXT: v_mul_lo_u32 v1, v1, v1
; GCN-NEXT: v_mul_lo_u32 v0, v0, v0
-; GCN-NEXT: global_store_dwordx4 v16, v[0:3], s[2:3]
-; GCN-NEXT: global_load_dwordx4 v[0:3], v16, s[0:1] offset:112
-; GCN-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_mul_lo_u32 v3, v3, v3
-; GCN-NEXT: v_mul_lo_u32 v2, v2, v2
-; GCN-NEXT: v_mul_lo_u32 v1, v1, v1
-; GCN-NEXT: v_mul_lo_u32 v0, v0, v0
-; GCN-NEXT: global_store_dwordx4 v16, v[0:3], s[2:3] offset:112
-; GCN-NEXT: global_load_dwordx4 v[0:3], v16, s[0:1] offset:96
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_mul_lo_u32 v3, v3, v3
-; GCN-NEXT: v_mul_lo_u32 v2, v2, v2
-; GCN-NEXT: v_mul_lo_u32 v1, v1, v1
-; GCN-NEXT: v_mul_lo_u32 v0, v0, v0
-; GCN-NEXT: global_store_dwordx4 v16, v[0:3], s[2:3] offset:96
-; GCN-NEXT: global_load_dwordx4 v[0:3], v16, s[0:1] offset:80
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_mul_lo_u32 v3, v3, v3
-; GCN-NEXT: v_mul_lo_u32 v2, v2, v2
-; GCN-NEXT: v_mul_lo_u32 v1, v1, v1
-; GCN-NEXT: v_mul_lo_u32 v0, v0, v0
-; GCN-NEXT: global_store_dwordx4 v16, v[0:3], s[2:3] offset:80
-; GCN-NEXT: global_load_dwordx4 v[4:7], v16, s[0:1] offset:48
-; GCN-NEXT: ; sched_group_barrier mask(0x00000040) size(1) SyncID(0)
-; GCN-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
+; GCN-NEXT: global_store_dwordx4 v12, v[0:3], s[2:3]
+; GCN-NEXT: global_load_dwordx4 v[4:7], v12, s[0:1] offset:64
; GCN-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
; GCN-NEXT: ; sched_group_barrier mask(0x00000040) size(1) SyncID(0)
; GCN-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_lo_u32 v7, v7, v7
+; GCN-NEXT: v_mul_lo_u32 v6, v6, v6
+; GCN-NEXT: v_mul_lo_u32 v5, v5, v5
+; GCN-NEXT: v_mul_lo_u32 v4, v4, v4
+; GCN-NEXT: global_store_dwordx4 v12, v[4:7], s[2:3] offset:64
+; GCN-NEXT: global_load_dwordx4 v[4:7], v12, s[0:1] offset:112
; GCN-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: v_mul_lo_u32 v7, v7, v7
; GCN-NEXT: v_mul_lo_u32 v6, v6, v6
; GCN-NEXT: v_mul_lo_u32 v5, v5, v5
; GCN-NEXT: v_mul_lo_u32 v4, v4, v4
-; GCN-NEXT: global_store_dwordx4 v16, v[4:7], s[2:3] offset:48
-; GCN-NEXT: global_load_dwordx4 v[8:11], v16, s[0:1] offset:16
+; GCN-NEXT: global_store_dwordx4 v12, v[4:7], s[2:3] offset:112
+; GCN-NEXT: global_load_dwordx4 v[4:7], v12, s[0:1] offset:96
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_lo_u32 v7, v7, v7
+; GCN-NEXT: v_mul_lo_u32 v6, v6, v6
+; GCN-NEXT: v_mul_lo_u32 v5, v5, v5
+; GCN-NEXT: v_mul_lo_u32 v4, v4, v4
+; GCN-NEXT: global_store_dwordx4 v12, v[4:7], s[2:3] offset:96
+; GCN-NEXT: global_load_dwordx4 v[4:7], v12, s[0:1] offset:80
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_lo_u32 v7, v7, v7
+; GCN-NEXT: v_mul_lo_u32 v6, v6, v6
+; GCN-NEXT: v_mul_lo_u32 v5, v5, v5
+; GCN-NEXT: v_mul_lo_u32 v4, v4, v4
+; GCN-NEXT: global_store_dwordx4 v12, v[4:7], s[2:3] offset:80
+; GCN-NEXT: ; sched_group_barrier mask(0x00000040) size(1) SyncID(0)
+; GCN-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
+; GCN-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
; GCN-NEXT: ; sched_group_barrier mask(0x00000040) size(1) SyncID(0)
; GCN-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
; GCN-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
; GCN-NEXT: ; sched_group_barrier mask(0x00000040) size(1) SyncID(0)
; GCN-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
; GCN-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_mul_lo_u32 v9, v9, v9
-; GCN-NEXT: v_mul_lo_u32 v8, v8, v8
-; GCN-NEXT: v_mul_lo_u32 v11, v11, v11
-; GCN-NEXT: v_mul_lo_u32 v10, v10, v10
-; GCN-NEXT: global_store_dwordx4 v16, v[8:11], s[2:3] offset:16
-; GCN-NEXT: global_load_dwordx4 v[8:11], v16, s[0:1] offset:64
; GCN-NEXT: ; sched_group_barrier mask(0x00000040) size(1) SyncID(0)
; GCN-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
; GCN-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
; GCN-NEXT: ; sched_group_barrier mask(0x00000040) size(1) SyncID(0)
; GCN-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
; GCN-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_mul_lo_u32 v11, v11, v11
-; GCN-NEXT: v_mul_lo_u32 v10, v10, v10
-; GCN-NEXT: v_mul_lo_u32 v9, v9, v9
-; GCN-NEXT: v_mul_lo_u32 v8, v8, v8
-; GCN-NEXT: global_store_dwordx4 v16, v[8:11], s[2:3] offset:64
; GCN-NEXT: ; sched_group_barrier mask(0x00000040) size(1) SyncID(0)
; GCN-NEXT: s_endpgm
;
@@ -476,19 +475,32 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_alternating_READ_VA
; EXACTCUTOFF: ; %bb.0:
; EXACTCUTOFF-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; EXACTCUTOFF-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; EXACTCUTOFF-NEXT: v_lshlrev_b32_e32 v16, 7, v0
-; EXACTCUTOFF-NEXT: ; kill: killed $sgpr0_sgpr1
+; EXACTCUTOFF-NEXT: v_lshlrev_b32_e32 v12, 7, v0
; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(0)
-; EXACTCUTOFF-NEXT: global_load_dwordx4 v[12:15], v16, s[0:1] offset:32
+; EXACTCUTOFF-NEXT: global_load_dwordx4 v[8:11], v12, s[0:1] offset:16
+; EXACTCUTOFF-NEXT: s_waitcnt vmcnt(0)
+; EXACTCUTOFF-NEXT: v_mul_lo_u32 v9, v9, v9
+; EXACTCUTOFF-NEXT: v_mul_lo_u32 v8, v8, v8
+; EXACTCUTOFF-NEXT: v_mul_lo_u32 v11, v11, v11
+; EXACTCUTOFF-NEXT: v_mul_lo_u32 v10, v10, v10
+; EXACTCUTOFF-NEXT: global_store_dwordx4 v12, v[8:11], s[2:3] offset:16
+; EXACTCUTOFF-NEXT: global_load_dwordx4 v[8:11], v12, s[0:1] offset:32
+; EXACTCUTOFF-NEXT: s_waitcnt vmcnt(0)
+; EXACTCUTOFF-NEXT: v_mul_lo_u32 v9, v9, v9
+; EXACTCUTOFF-NEXT: v_mul_lo_u32 v8, v8, v8
+; EXACTCUTOFF-NEXT: v_mul_lo_u32 v11, v11, v11
+; EXACTCUTOFF-NEXT: v_mul_lo_u32 v10, v10, v10
+; EXACTCUTOFF-NEXT: global_store_dwordx4 v12, v[8:11], s[2:3] offset:32
+; EXACTCUTOFF-NEXT: global_load_dwordx4 v[8:11], v12, s[0:1] offset:48
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
; EXACTCUTOFF-NEXT: s_waitcnt vmcnt(0)
-; EXACTCUTOFF-NEXT: v_mul_lo_u32 v13, v13, v13
-; EXACTCUTOFF-NEXT: v_mul_lo_u32 v12, v12, v12
-; EXACTCUTOFF-NEXT: v_mul_lo_u32 v15, v15, v15
-; EXACTCUTOFF-NEXT: v_mul_lo_u32 v14, v14, v14
-; EXACTCUTOFF-NEXT: global_store_dwordx4 v16, v[12:15], s[2:3] offset:32
-; EXACTCUTOFF-NEXT: global_load_dwordx4 v[0:3], v16, s[0:1]
+; EXACTCUTOFF-NEXT: v_mul_lo_u32 v11, v11, v11
+; EXACTCUTOFF-NEXT: v_mul_lo_u32 v10, v10, v10
+; EXACTCUTOFF-NEXT: v_mul_lo_u32 v9, v9, v9
+; EXACTCUTOFF-NEXT: v_mul_lo_u32 v8, v8, v8
+; EXACTCUTOFF-NEXT: global_store_dwordx4 v12, v[8:11], s[2:3] offset:48
+; EXACTCUTOFF-NEXT: global_load_dwordx4 v[0:3], v12, s[0:1]
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000040) size(1) SyncID(0)
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
; EXACTCUTOFF-NEXT: s_waitcnt vmcnt(0)
@@ -496,68 +508,54 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_alternating_READ_VA
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v2, v2, v2
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v1, v1, v1
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v0, v0, v0
-; EXACTCUTOFF-NEXT: global_store_dwordx4 v16, v[0:3], s[2:3]
-; EXACTCUTOFF-NEXT: global_load_dwordx4 v[0:3], v16, s[0:1] offset:112
-; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
-; EXACTCUTOFF-NEXT: s_waitcnt vmcnt(0)
-; EXACTCUTOFF-NEXT: v_mul_lo_u32 v3, v3, v3
-; EXACTCUTOFF-NEXT: v_mul_lo_u32 v2, v2, v2
-; EXACTCUTOFF-NEXT: v_mul_lo_u32 v1, v1, v1
-; EXACTCUTOFF-NEXT: v_mul_lo_u32 v0, v0, v0
-; EXACTCUTOFF-NEXT: global_store_dwordx4 v16, v[0:3], s[2:3] offset:112
-; EXACTCUTOFF-NEXT: global_load_dwordx4 v[0:3], v16, s[0:1] offset:96
-; EXACTCUTOFF-NEXT: s_waitcnt vmcnt(0)
-; EXACTCUTOFF-NEXT: v_mul_lo_u32 v3, v3, v3
-; EXACTCUTOFF-NEXT: v_mul_lo_u32 v2, v2, v2
-; EXACTCUTOFF-NEXT: v_mul_lo_u32 v1, v1, v1
-; EXACTCUTOFF-NEXT: v_mul_lo_u32 v0, v0, v0
-; EXACTCUTOFF-NEXT: global_store_dwordx4 v16, v[0:3], s[2:3] offset:96
-; EXACTCUTOFF-NEXT: global_load_dwordx4 v[0:3], v16, s[0:1] offset:80
-; EXACTCUTOFF-NEXT: s_waitcnt vmcnt(0)
-; EXACTCUTOFF-NEXT: v_mul_lo_u32 v3, v3, v3
-; EXACTCUTOFF-NEXT: v_mul_lo_u32 v2, v2, v2
-; EXACTCUTOFF-NEXT: v_mul_lo_u32 v1, v1, v1
-; EXACTCUTOFF-NEXT: v_mul_lo_u32 v0, v0, v0
-; EXACTCUTOFF-NEXT: global_store_dwordx4 v16, v[0:3], s[2:3] offset:80
-; EXACTCUTOFF-NEXT: global_load_dwordx4 v[4:7], v16, s[0:1] offset:48
-; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000040) size(1) SyncID(0)
-; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
+; EXACTCUTOFF-NEXT: global_store_dwordx4 v12, v[0:3], s[2:3]
+; EXACTCUTOFF-NEXT: global_load_dwordx4 v[4:7], v12, s[0:1] offset:64
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000040) size(1) SyncID(0)
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
+; EXACTCUTOFF-NEXT: s_waitcnt vmcnt(0)
+; EXACTCUTOFF-NEXT: v_mul_lo_u32 v7, v7, v7
+; EXACTCUTOFF-NEXT: v_mul_lo_u32 v6, v6, v6
+; EXACTCUTOFF-NEXT: v_mul_lo_u32 v5, v5, v5
+; EXACTCUTOFF-NEXT: v_mul_lo_u32 v4, v4, v4
+; EXACTCUTOFF-NEXT: global_store_dwordx4 v12, v[4:7], s[2:3] offset:64
+; EXACTCUTOFF-NEXT: global_load_dwordx4 v[4:7], v12, s[0:1] offset:112
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
; EXACTCUTOFF-NEXT: s_waitcnt vmcnt(0)
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v7, v7, v7
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v6, v6, v6
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v5, v5, v5
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v4, v4, v4
-; EXACTCUTOFF-NEXT: global_store_dwordx4 v16, v[4:7], s[2:3] offset:48
-; EXACTCUTOFF-NEXT: global_load_dwordx4 v[8:11], v16, s[0:1] offset:16
+; EXACTCUTOFF-NEXT: global_store_dwordx4 v12, v[4:7], s[2:3] offset:112
+; EXACTCUTOFF-NEXT: global_load_dwordx4 v[4:7], v12, s[0:1] offset:96
+; EXACTCUTOFF-NEXT: s_waitcnt vmcnt(0)
+; EXACTCUTOFF-NEXT: v_mul_lo_u32 v7, v7, v7
+; EXACTCUTOFF-NEXT: v_mul_lo_u32 v6, v6, v6
+; EXACTCUTOFF-NEXT: v_mul_lo_u32 v5, v5, v5
+; EXACTCUTOFF-NEXT: v_mul_lo_u32 v4, v4, v4
+; EXACTCUTOFF-NEXT: global_store_dwordx4 v12, v[4:7], s[2:3] offset:96
+; EXACTCUTOFF-NEXT: global_load_dwordx4 v[4:7], v12, s[0:1] offset:80
+; EXACTCUTOFF-NEXT: s_waitcnt vmcnt(0)
+; EXACTCUTOFF-NEXT: v_mul_lo_u32 v7, v7, v7
+; EXACTCUTOFF-NEXT: v_mul_lo_u32 v6, v6, v6
+; EXACTCUTOFF-NEXT: v_mul_lo_u32 v5, v5, v5
+; EXACTCUTOFF-NEXT: v_mul_lo_u32 v4, v4, v4
+; EXACTCUTOFF-NEXT: global_store_dwordx4 v12, v[4:7], s[2:3] offset:80
+; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000040) size(1) SyncID(0)
+; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
+; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000040) size(1) SyncID(0)
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000040) size(1) SyncID(0)
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
-; EXACTCUTOFF-NEXT: s_waitcnt vmcnt(0)
-; EXACTCUTOFF-NEXT: v_mul_lo_u32 v9, v9, v9
-; EXACTCUTOFF-NEXT: v_mul_lo_u32 v8, v8, v8
-; EXACTCUTOFF-NEXT: v_mul_lo_u32 v11, v11, v11
-; EXACTCUTOFF-NEXT: v_mul_lo_u32 v10, v10, v10
-; EXACTCUTOFF-NEXT: global_store_dwordx4 v16, v[8:11], s[2:3] offset:16
-; EXACTCUTOFF-NEXT: global_load_dwordx4 v[8:11], v16, s[0:1] offset:64
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000040) size(1) SyncID(0)
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000040) size(1) SyncID(0)
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
-; EXACTCUTOFF-NEXT: s_waitcnt vmcnt(0)
-; EXACTCUTOFF-NEXT: v_mul_lo_u32 v11, v11, v11
-; EXACTCUTOFF-NEXT: v_mul_lo_u32 v10, v10, v10
-; EXACTCUTOFF-NEXT: v_mul_lo_u32 v9, v9, v9
-; EXACTCUTOFF-NEXT: v_mul_lo_u32 v8, v8, v8
-; EXACTCUTOFF-NEXT: global_store_dwordx4 v16, v[8:11], s[2:3] offset:64
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000040) size(1) SyncID(0)
; EXACTCUTOFF-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x() #2
diff --git a/llvm/test/CodeGen/AMDGPU/sched-group-barrier-pipeline-solver.mir b/llvm/test/CodeGen/AMDGPU/sched-group-barrier-pipeline-solver.mir
index d6774bb39dca7..ff57a3e34f06e 100644
--- a/llvm/test/CodeGen/AMDGPU/sched-group-barrier-pipeline-solver.mir
+++ b/llvm/test/CodeGen/AMDGPU/sched-group-barrier-pipeline-solver.mir
@@ -134,16 +134,16 @@ body: |
; EXACT-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; EXACT-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR [[DEF]], [[DEF1]], 0, 0, implicit $exec :: (load (s32) from %ir.in, !alias.scope !0, addrspace 1)
; EXACT-NEXT: [[V_MUL_LO_U32_e64_:%[0-9]+]]:vgpr_32 = nsw V_MUL_LO_U32_e64 [[GLOBAL_LOAD_DWORD_SADDR]], [[GLOBAL_LOAD_DWORD_SADDR]], implicit $exec
- ; EXACT-NEXT: [[DEF2:%[0-9]+]]:areg_128 = IMPLICIT_DEF
; EXACT-NEXT: GLOBAL_STORE_DWORD_SADDR [[DEF1]], [[V_MUL_LO_U32_e64_]], [[DEF]], 0, 0, implicit $exec :: (store (s32) into %ir.out, !noalias !0, addrspace 1)
+ ; EXACT-NEXT: S_NOP 0
+ ; EXACT-NEXT: [[GLOBAL_LOAD_DWORD_SADDR1:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR [[DEF]], [[DEF1]], 512, 0, implicit $exec :: (load (s32) from %ir.in, !alias.scope !0, addrspace 1)
+ ; EXACT-NEXT: [[DEF2:%[0-9]+]]:areg_128 = IMPLICIT_DEF
; EXACT-NEXT: [[V_MFMA_F32_4X4X1F32_e64_:%[0-9]+]]:areg_128 = V_MFMA_F32_4X4X1F32_e64 [[DEF1]], [[GLOBAL_LOAD_DWORD_SADDR]], [[DEF2]], 0, 0, 0, implicit $mode, implicit $exec
; EXACT-NEXT: [[V_MFMA_F32_4X4X1F32_e64_1:%[0-9]+]]:areg_128 = V_MFMA_F32_4X4X1F32_e64 [[DEF1]], [[GLOBAL_LOAD_DWORD_SADDR]], [[V_MFMA_F32_4X4X1F32_e64_]], 0, 0, 0, implicit $mode, implicit $exec
; EXACT-NEXT: [[V_MUL_LO_U32_e64_1:%[0-9]+]]:vgpr_32 = nsw V_MUL_LO_U32_e64 [[GLOBAL_LOAD_DWORD_SADDR]], [[DEF1]], implicit $exec
; EXACT-NEXT: [[V_MFMA_F32_4X4X1F32_e64_2:%[0-9]+]]:areg_128 = V_MFMA_F32_4X4X1F32_e64 [[DEF1]], [[GLOBAL_LOAD_DWORD_SADDR]], [[V_MFMA_F32_4X4X1F32_e64_1]], 0, 0, 0, implicit $mode, implicit $exec
; EXACT-NEXT: [[V_MUL_LO_U32_e64_2:%[0-9]+]]:vgpr_32 = nsw V_MUL_LO_U32_e64 [[GLOBAL_LOAD_DWORD_SADDR]], [[DEF1]], implicit $exec
; EXACT-NEXT: [[V_MFMA_F32_4X4X1F32_e64_3:%[0-9]+]]:areg_128 = V_MFMA_F32_4X4X1F32_e64 [[DEF1]], [[GLOBAL_LOAD_DWORD_SADDR]], [[V_MFMA_F32_4X4X1F32_e64_2]], 0, 0, 0, implicit $mode, implicit $exec
- ; EXACT-NEXT: S_NOP 0
- ; EXACT-NEXT: [[GLOBAL_LOAD_DWORD_SADDR1:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR [[DEF]], [[DEF1]], 512, 0, implicit $exec :: (load (s32) from %ir.in, !alias.scope !0, addrspace 1)
; EXACT-NEXT: [[V_MFMA_F32_4X4X1F32_e64_4:%[0-9]+]]:areg_128 = V_MFMA_F32_4X4X1F32_e64 [[DEF1]], [[GLOBAL_LOAD_DWORD_SADDR]], [[V_MFMA_F32_4X4X1F32_e64_3]], 0, 0, 0, implicit $mode, implicit $exec
; EXACT-NEXT: [[V_MUL_LO_U32_e64_3:%[0-9]+]]:vgpr_32 = nsw V_MUL_LO_U32_e64 [[GLOBAL_LOAD_DWORD_SADDR1]], [[GLOBAL_LOAD_DWORD_SADDR1]], implicit $exec
; EXACT-NEXT: GLOBAL_STORE_DWORD_SADDR [[DEF1]], [[V_MUL_LO_U32_e64_3]], [[DEF]], 512, 0, implicit $exec :: (store (s32) into %ir.out, !noalias !0, addrspace 1)
diff --git a/llvm/test/CodeGen/AMDGPU/sched-group-barrier-pre-RA.mir b/llvm/test/CodeGen/AMDGPU/sched-group-barrier-pre-RA.mir
index 4f844762b24e3..4cafc90d1d070 100644
--- a/llvm/test/CodeGen/AMDGPU/sched-group-barrier-pre-RA.mir
+++ b/llvm/test/CodeGen/AMDGPU/sched-group-barrier-pre-RA.mir
@@ -198,11 +198,11 @@ body: |
; EXACT-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; EXACT-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR [[DEF]], [[DEF1]], 0, 0, implicit $exec :: (load (s32) from %ir.in, !alias.scope !0, addrspace 1)
; EXACT-NEXT: [[V_MUL_LO_U32_e64_:%[0-9]+]]:vgpr_32 = nsw V_MUL_LO_U32_e64 [[GLOBAL_LOAD_DWORD_SADDR]], [[GLOBAL_LOAD_DWORD_SADDR]], implicit $exec
- ; EXACT-NEXT: [[DEF2:%[0-9]+]]:areg_128 = IMPLICIT_DEF
; EXACT-NEXT: GLOBAL_STORE_DWORD_SADDR [[DEF1]], [[V_MUL_LO_U32_e64_]], [[DEF]], 0, 0, implicit $exec :: (store (s32) into %ir.out, !noalias !0, addrspace 1)
; EXACT-NEXT: S_NOP 0
- ; EXACT-NEXT: [[V_MFMA_F32_4X4X1F32_e64_:%[0-9]+]]:areg_128 = V_MFMA_F32_4X4X1F32_e64 [[DEF1]], [[GLOBAL_LOAD_DWORD_SADDR]], [[DEF2]], 0, 0, 0, implicit $mode, implicit $exec
; EXACT-NEXT: [[GLOBAL_LOAD_DWORD_SADDR1:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR [[DEF]], [[DEF1]], 512, 0, implicit $exec :: (load (s32) from %ir.in, !alias.scope !0, addrspace 1)
+ ; EXACT-NEXT: [[DEF2:%[0-9]+]]:areg_128 = IMPLICIT_DEF
+ ; EXACT-NEXT: [[V_MFMA_F32_4X4X1F32_e64_:%[0-9]+]]:areg_128 = V_MFMA_F32_4X4X1F32_e64 [[DEF1]], [[GLOBAL_LOAD_DWORD_SADDR]], [[DEF2]], 0, 0, 0, implicit $mode, implicit $exec
; EXACT-NEXT: [[V_MFMA_F32_4X4X1F32_e64_1:%[0-9]+]]:areg_128 = V_MFMA_F32_4X4X1F32_e64 [[DEF1]], [[GLOBAL_LOAD_DWORD_SADDR]], [[V_MFMA_F32_4X4X1F32_e64_]], 0, 0, 0, implicit $mode, implicit $exec
; EXACT-NEXT: [[V_MUL_LO_U32_e64_1:%[0-9]+]]:vgpr_32 = nsw V_MUL_LO_U32_e64 [[GLOBAL_LOAD_DWORD_SADDR]], [[DEF1]], implicit $exec
; EXACT-NEXT: [[V_MFMA_F32_4X4X1F32_e64_2:%[0-9]+]]:areg_128 = V_MFMA_F32_4X4X1F32_e64 [[DEF1]], [[GLOBAL_LOAD_DWORD_SADDR]], [[V_MFMA_F32_4X4X1F32_e64_1]], 0, 0, 0, implicit $mode, implicit $exec
@@ -286,16 +286,16 @@ body: |
; EXACT-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; EXACT-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR [[DEF]], [[DEF1]], 0, 0, implicit $exec :: (load (s32) from %ir.in, !alias.scope !0, addrspace 1)
; EXACT-NEXT: [[V_MUL_LO_U32_e64_:%[0-9]+]]:vgpr_32 = nsw V_MUL_LO_U32_e64 [[GLOBAL_LOAD_DWORD_SADDR]], [[GLOBAL_LOAD_DWORD_SADDR]], implicit $exec
- ; EXACT-NEXT: [[DEF2:%[0-9]+]]:areg_128 = IMPLICIT_DEF
; EXACT-NEXT: GLOBAL_STORE_DWORD_SADDR [[DEF1]], [[V_MUL_LO_U32_e64_]], [[DEF]], 0, 0, implicit $exec :: (store (s32) into %ir.out, !noalias !0, addrspace 1)
+ ; EXACT-NEXT: S_NOP 0
+ ; EXACT-NEXT: [[GLOBAL_LOAD_DWORD_SADDR1:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR [[DEF]], [[DEF1]], 512, 0, implicit $exec :: (load (s32) from %ir.in, !alias.scope !0, addrspace 1)
+ ; EXACT-NEXT: [[DEF2:%[0-9]+]]:areg_128 = IMPLICIT_DEF
; EXACT-NEXT: [[V_MFMA_F32_4X4X1F32_e64_:%[0-9]+]]:areg_128 = V_MFMA_F32_4X4X1F32_e64 [[DEF1]], [[GLOBAL_LOAD_DWORD_SADDR]], [[DEF2]], 0, 0, 0, implicit $mode, implicit $exec
; EXACT-NEXT: [[V_MFMA_F32_4X4X1F32_e64_1:%[0-9]+]]:areg_128 = V_MFMA_F32_4X4X1F32_e64 [[DEF1]], [[GLOBAL_LOAD_DWORD_SADDR]], [[V_MFMA_F32_4X4X1F32_e64_]], 0, 0, 0, implicit $mode, implicit $exec
; EXACT-NEXT: [[V_MUL_LO_U32_e64_1:%[0-9]+]]:vgpr_32 = nsw V_MUL_LO_U32_e64 [[GLOBAL_LOAD_DWORD_SADDR]], [[DEF1]], implicit $exec
; EXACT-NEXT: [[V_MFMA_F32_4X4X1F32_e64_2:%[0-9]+]]:areg_128 = V_MFMA_F32_4X4X1F32_e64 [[DEF1]], [[GLOBAL_LOAD_DWORD_SADDR]], [[V_MFMA_F32_4X4X1F32_e64_1]], 0, 0, 0, implicit $mode, implicit $exec
; EXACT-NEXT: [[V_MUL_LO_U32_e64_2:%[0-9]+]]:vgpr_32 = nsw V_MUL_LO_U32_e64 [[GLOBAL_LOAD_DWORD_SADDR]], [[DEF1]], implicit $exec
; EXACT-NEXT: [[V_MFMA_F32_4X4X1F32_e64_3:%[0-9]+]]:areg_128 = V_MFMA_F32_4X4X1F32_e64 [[DEF1]], [[GLOBAL_LOAD_DWORD_SADDR]], [[V_MFMA_F32_4X4X1F32_e64_2]], 0, 0, 0, implicit $mode, implicit $exec
- ; EXACT-NEXT: S_NOP 0
- ; EXACT-NEXT: [[GLOBAL_LOAD_DWORD_SADDR1:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR [[DEF]], [[DEF1]], 512, 0, implicit $exec :: (load (s32) from %ir.in, !alias.scope !0, addrspace 1)
; EXACT-NEXT: [[V_MFMA_F32_4X4X1F32_e64_4:%[0-9]+]]:areg_128 = V_MFMA_F32_4X4X1F32_e64 [[DEF1]], [[GLOBAL_LOAD_DWORD_SADDR]], [[V_MFMA_F32_4X4X1F32_e64_3]], 0, 0, 0, implicit $mode, implicit $exec
; EXACT-NEXT: [[V_MUL_LO_U32_e64_3:%[0-9]+]]:vgpr_32 = nsw V_MUL_LO_U32_e64 [[GLOBAL_LOAD_DWORD_SADDR1]], [[GLOBAL_LOAD_DWORD_SADDR1]], implicit $exec
; EXACT-NEXT: GLOBAL_STORE_DWORD_SADDR [[DEF1]], [[V_MUL_LO_U32_e64_3]], [[DEF]], 512, 0, implicit $exec :: (store (s32) into %ir.out, !noalias !0, addrspace 1)
>From 3f6237c940e49050a67464e68880b47a641eded1 Mon Sep 17 00:00:00 2001
From: Frederik Harwath <fharwath at amd.com>
Date: Wed, 29 Apr 2026 06:05:18 -0400
Subject: [PATCH 2/7] Add temporary validation check
Adds validation check that the optimized EdgeSetBuilder::build
function returns the same cost as addEdges.
---
llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
index 5cb5b41eb9a96..10510310498dc 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
@@ -889,6 +889,12 @@ void PipelineSolver::greedyFind(
int TempCost = Builder.build(CandSGID, TempEdges);
LLVM_DEBUG(dbgs() << "Cost of Group " << TempCost << "\n");
+ std::list<std::pair<SUnit *, SUnit *>> AddEdgesEdges;
+ int AddEdgesCost = addEdges(SyncPipeline, CurrSU.first, CandSGID, AddEdgesEdges);
+ if (TempCost != AddEdgesCost)
+ llvm_unreachable("Cost mismatch between EdgeSetBuilder and addEdges");
+ removeEdges(AddEdgesEdges);
+
if (!Best || TempCost < Best->Cost) {
Best = {Match, TempEdges, TempCost};
if (Best->Cost == 0)
>From 87dcdbf610bab843891e32828a454efc70f42d16 Mon Sep 17 00:00:00 2001
From: Frederik Harwath <fharwath at amd.com>
Date: Wed, 29 Apr 2026 06:10:02 -0400
Subject: [PATCH 3/7] Revert "Add temporary validation check"
This reverts commit dbb16ef12f95c9e927bb8e0a9919c0f3a5c57ea2.
---
llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp | 6 ------
1 file changed, 6 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
index 10510310498dc..5cb5b41eb9a96 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
@@ -889,12 +889,6 @@ void PipelineSolver::greedyFind(
int TempCost = Builder.build(CandSGID, TempEdges);
LLVM_DEBUG(dbgs() << "Cost of Group " << TempCost << "\n");
- std::list<std::pair<SUnit *, SUnit *>> AddEdgesEdges;
- int AddEdgesCost = addEdges(SyncPipeline, CurrSU.first, CandSGID, AddEdgesEdges);
- if (TempCost != AddEdgesCost)
- llvm_unreachable("Cost mismatch between EdgeSetBuilder and addEdges");
- removeEdges(AddEdgesEdges);
-
if (!Best || TempCost < Best->Cost) {
Best = {Match, TempEdges, TempCost};
if (Best->Cost == 0)
>From dd2c5c4f8b3a54c2f4460a169ee445e489513cb1 Mon Sep 17 00:00:00 2001
From: Frederik Harwath <fharwath at amd.com>
Date: Wed, 29 Apr 2026 07:01:00 -0400
Subject: [PATCH 4/7] Review changes
---
llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
index 5cb5b41eb9a96..78ab2c4dffa9b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
@@ -325,7 +325,7 @@ class PipelineSolver {
/// This class is used to build the edge set implied by an
/// assignment of an SUnit to a SchedGroup and to compute the cost
- /// (edges than cannot be assigned without introducing cycles) of
+ /// (edges that cannot be assigned without introducing cycles) of
/// the assignment.
class EdgeSetBuilder {
SUnit *SU;
@@ -575,7 +575,7 @@ int PipelineSolver::EdgeSetBuilder::buildImpl(
// predecessor of SUnits after SG. In each case, the cycle check
// requires reachability information for the opposing direction.
- // Nodes U reachable from SU (U ~> SU).
+ // Nodes U that can reach SU (U ~> SU).
// Will be extended as new edges are added and hence cannot be
// shared between calls to this function, in contrast to Succs.
DenseSet<SUnit *> Preds = InitialPreds;
>From 139aeb80c0a02fd983f63ae989eb86b661f362e7 Mon Sep 17 00:00:00 2001
From: Frederik Harwath <frederik at harwath.name>
Date: Thu, 30 Apr 2026 08:48:38 +0200
Subject: [PATCH 5/7] Update llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
Co-authored-by: Matt Arsenault <arsenm2 at gmail.com>
---
llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
index 78ab2c4dffa9b..4f1ed1ad2dd11 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
@@ -518,8 +518,7 @@ void PipelineSolver::EdgeSetBuilder::computeReachable(
if (!Reachable.insert(Start).second)
return;
- SmallVector<SUnit *, 32> WorkList;
- WorkList.push_back(Start);
+ SmallVector<SUnit *, 32> WorkList = {Start};
while (!WorkList.empty()) {
SUnit *Current = WorkList.pop_back_val();
>From 5c3bb25327ffd131388b06a144d1ebf94ed9716e Mon Sep 17 00:00:00 2001
From: Frederik Harwath <fharwath at amd.com>
Date: Thu, 30 Apr 2026 04:08:36 -0400
Subject: [PATCH 6/7] Review changes
- Inline single use lambdas
---
llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp | 43 ++++++++++-------------
1 file changed, 18 insertions(+), 25 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
index 4f1ed1ad2dd11..8fbf393d371e5 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
@@ -580,27 +580,6 @@ int PipelineSolver::EdgeSetBuilder::buildImpl(
DenseSet<SUnit *> Preds = InitialPreds;
int MissedEdges = 0;
- auto TryMakeSuccEdge = [&](SUnit *Pred) {
- // If SU ~> Pred, adding Pred -> SU creates a cycle.
- if (Succs.contains(Pred))
- ++MissedEdges;
- else {
- NewEdges.emplace_back(Pred, SU);
- computePreds(Preds, Pred);
- }
- };
-
- auto TryMakePredEdge = [&](SUnit *Succ) {
- // If Succ ~> SU, adding SU -> Succ creates a cycle.
- if (Preds.contains(Succ))
- ++MissedEdges;
- else {
- // Succs does not need to be updated, since it will not be
- // queried after the loop below enters the MakePred case.
- NewEdges.emplace_back(SU, Succ);
- }
- };
-
bool MakePred = false;
for (const SchedGroup &SG : SchedGroups) {
if (SG.getSGID() == SGID) {
@@ -612,10 +591,24 @@ int PipelineSolver::EdgeSetBuilder::buildImpl(
if (A->getInstr()->getOpcode() == AMDGPU::SCHED_GROUP_BARRIER)
continue;
- if (MakePred)
- TryMakePredEdge(A);
- else
- TryMakeSuccEdge(A);
+ if (MakePred) {
+ // Try add SU -> A.
+ if (Preds.contains(A)) { // Would add cycle since A ~> SU.
+ ++MissedEdges;
+ continue;
+ }
+ // Succs does not need to be updated, since it will not be
+ // queried after entering the MakePred case.
+ NewEdges.emplace_back(SU, A);
+ } else {
+ // Try add Pred -> SU.
+ if (Succs.contains(A)) { // Would add cycle since SU ~> A.
+ ++MissedEdges;
+ continue;
+ }
+ NewEdges.emplace_back(A, SU);
+ computePreds(Preds, A);
+ }
}
}
>From a4c7937b6eaa4f121129d5c1b4275fce2f1c0b91 Mon Sep 17 00:00:00 2001
From: Frederik Harwath <fharwath at amd.com>
Date: Thu, 30 Apr 2026 08:22:16 -0400
Subject: [PATCH 7/7] Review changes
---
llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp | 9 +++------
1 file changed, 3 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
index 8fbf393d371e5..d407ddea60414 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
@@ -222,7 +222,7 @@ class SchedGroup {
int getSyncID() { return SyncID; }
- int getSGID() const { return SGID; }
+ int getSGID() { return SGID; }
SchedGroupMask getMask() { return SGMask; }
@@ -549,10 +549,7 @@ int PipelineSolver::EdgeSetBuilder::build(
}
// See comment in addEdges concerning the iterator direction.
- return IsBottomUp ? buildImpl(SGID,
- llvm::make_range(SyncPipeline.rbegin(),
- SyncPipeline.rend()),
- NewEdges)
+ return IsBottomUp ? buildImpl(SGID, reverse(SyncPipeline), NewEdges)
: buildImpl(SGID,
llvm::make_range(SyncPipeline.begin(),
SyncPipeline.end()),
@@ -581,7 +578,7 @@ int PipelineSolver::EdgeSetBuilder::buildImpl(
int MissedEdges = 0;
bool MakePred = false;
- for (const SchedGroup &SG : SchedGroups) {
+ for (SchedGroup &SG : SchedGroups) {
if (SG.getSGID() == SGID) {
MakePred = true;
continue;
More information about the llvm-commits
mailing list