[llvm] [AMDGPU] Skip spill slot elimination when reload isn't dominated by stores (PR #215953)
Arseniy Obolenskiy via llvm-commits
llvm-commits at lists.llvm.org
Wed Aug 12 23:33:11 PDT 2026
https://github.com/aobolensk updated https://github.com/llvm/llvm-project/pull/215953
>From 55af73a8163249512fabcd5245b18758cf81965f Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Thu, 13 Aug 2026 07:33:45 +0200
Subject: [PATCH] [AMDGPU] Skip spill slot elimination when reload isn't
dominated by stores
A memory slot can be safely read even if no single store dominates the
reload, but a register replacing it cannot
Detect that case and skip it
---
.../AMDGPU/AMDGPURewriteAGPRCopyMFMA.cpp | 69 +++++++
...mfma-to-agpr-spill-reload-not-dominated.ll | 168 ++++++++++++++++++
2 files changed, 237 insertions(+)
create mode 100644 llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-reload-not-dominated.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURewriteAGPRCopyMFMA.cpp b/llvm/lib/Target/AMDGPU/AMDGPURewriteAGPRCopyMFMA.cpp
index 6510c07358e01..0934c6faf07bf 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURewriteAGPRCopyMFMA.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURewriteAGPRCopyMFMA.cpp
@@ -26,6 +26,7 @@
#include "GCNSubtarget.h"
#include "SIMachineFunctionInfo.h"
#include "SIRegisterInfo.h"
+#include "llvm/ADT/DepthFirstIterator.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/CodeGen/LiveIntervals.h"
#include "llvm/CodeGen/LiveRegMatrix.h"
@@ -127,6 +128,14 @@ class AMDGPURewriteAGPRCopyMFMAImpl {
void collectSpillIndexUses(ArrayRef<LiveInterval *> StackIntervals,
SpillReferenceMap &Map) const;
+ /// Return true if every path to \p LoadMI passes through a store to the
+ /// same stack slot first.
+ bool isJointlyDominatedByStores(
+ const MachineInstr &LoadMI, const LiveInterval &SlotLI,
+ const SmallPtrSetImpl<const MachineBasicBlock *> &Reachable,
+ const SmallPtrSetImpl<const MachineBasicBlock *> &StoreFreeReachable)
+ const;
+
/// Attempt to unspill VGPRs by finding a free register and replacing the
/// spill instructions with copies.
void eliminateSpillsOfReassignedVGPRs() const;
@@ -475,6 +484,26 @@ void AMDGPURewriteAGPRCopyMFMAImpl::collectSpillIndexUses(
}
}
+bool AMDGPURewriteAGPRCopyMFMAImpl::isJointlyDominatedByStores(
+ const MachineInstr &LoadMI, const LiveInterval &SlotLI,
+ const SmallPtrSetImpl<const MachineBasicBlock *> &Reachable,
+ const SmallPtrSetImpl<const MachineBasicBlock *> &StoreFreeReachable)
+ const {
+ // An unreachable reload has no reaching definition; keep it conservatively.
+ const MachineBasicBlock *LoadMBB = LoadMI.getParent();
+ if (!Reachable.contains(LoadMBB))
+ return false;
+
+ if (!StoreFreeReachable.contains(LoadMBB))
+ return true;
+
+ // A store-free path reaches this block, so any preceding store must be
+ // inside it. The slot being live at the load but not live-in means its
+ // reaching def is a store in this block, not the store-free path.
+ SlotIndex LoadIdx = LIS.getInstructionIndex(LoadMI);
+ return SlotLI.liveAt(LoadIdx) && !LIS.isLiveInToMBB(SlotLI, LoadMBB);
+}
+
void AMDGPURewriteAGPRCopyMFMAImpl::eliminateSpillsOfReassignedVGPRs() const {
unsigned NumSlots = LSS.getNumIntervals();
if (NumSlots == 0)
@@ -522,12 +551,52 @@ void AMDGPURewriteAGPRCopyMFMAImpl::eliminateSpillsOfReassignedVGPRs() const {
DenseMap<int, SmallVector<MachineInstr *, 4>> SpillSlotReferences;
collectSpillIndexUses(StackIntervals, SpillSlotReferences);
+ const MachineBasicBlock *Entry = &MF.front();
+ df_iterator_default_set<const MachineBasicBlock *, 16> Reachable;
+ for (const MachineBasicBlock *MBB : depth_first_ext(Entry, Reachable))
+ (void)MBB;
+
for (LiveInterval *LI : StackIntervals) {
+ if (LI->empty())
+ continue;
+
int Slot = LI->reg().stackSlotIndex();
auto SpillReferences = SpillSlotReferences.find(Slot);
if (SpillReferences == SpillSlotReferences.end())
continue;
+ // A reload with no dominating store is fine for a memory slot, but not
+ // once replaced by a register: its uses would not be dominated by its defs.
+ SmallPtrSet<const MachineBasicBlock *, 4> StoreBlocks;
+ for (const MachineInstr *MI : SpillReferences->second) {
+ if (MI->mayStore() && Reachable.contains(MI->getParent()))
+ StoreBlocks.insert(MI->getParent());
+ }
+
+ if (StoreBlocks.empty())
+ continue;
+
+ using ReachableSet = df_iterator_default_set<const MachineBasicBlock *, 16>;
+ ReachableSet StoreFreeReachable;
+ for (df_ext_iterator<const MachineBasicBlock *, ReachableSet>
+ I = df_ext_begin(Entry, StoreFreeReachable),
+ E = df_ext_end(Entry, StoreFreeReachable);
+ I != E;) {
+ if (StoreBlocks.contains(*I))
+ I.skipChildren();
+ else
+ ++I;
+ }
+
+ if (any_of(SpillReferences->second, [&](const MachineInstr *MI) {
+ return MI->mayLoad() && !isJointlyDominatedByStores(
+ *MI, *LI, Reachable, StoreFreeReachable);
+ })) {
+ LLVM_DEBUG(dbgs() << "Skipping SS#" << Slot
+ << ": reload not jointly dominated by stores\n");
+ continue;
+ }
+
const TargetRegisterClass *RC = LSS.getIntervalRegClass(Slot);
LLVM_DEBUG(dbgs() << "Trying to eliminate " << printReg(Slot, &TRI)
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-reload-not-dominated.ll b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-reload-not-dominated.ll
new file mode 100644
index 0000000000000..eda4140c9db85
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-reload-not-dominated.ll
@@ -0,0 +1,168 @@
+; REQUIRES: asserts
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 -O3 -verify-machineinstrs \
+; RUN: -stop-after=amdgpu-rewrite-agpr-copy-mfma \
+; RUN: -debug-only=amdgpu-rewrite-agpr-copy-mfma -filetype=null %s 2>&1 \
+; RUN: | FileCheck %s
+
+; Reduced from a bf16 multi-head-attention kernel (issue #196671). Register
+; allocation spills some of the <16 x float> MFMA accumulators around the
+; if/else diamond in the loop: the spill stores end up in the two conditional
+; arms, while the reload
+; sits in the join block, which is also reachable through a path that bypasses
+; both arms. When the rewrite pass frees registers by switching MFMAs to their
+; AGPR form and then tries to replace such a spill slot with a register, the
+; replacement register's reload is not dominated by its defs and computing its
+; live interval fails a machine verifier check ("Virtual register defs don't
+; dominate all uses"). Check that the slot is skipped instead.
+
+; CHECK: Skipping SS#{{[0-9]+}}: reload not jointly dominated by stores
+
+target triple = "amdgcn-amd-amdhsa"
+
+define amdgpu_kernel void @reload_not_jointly_dominated_by_stores(i1 %arg, <16 x float> %.sroa.366.2) #0 {
+.lr.ph.i:
+ br label %bb
+
+bb: ; preds = %bb49, %.lr.ph.i
+ %.sroa.01121.2 = phi <16 x float> [ zeroinitializer, %.lr.ph.i ], [ %i78, %bb49 ]
+ %.sroa.54.2 = phi <16 x float> [ zeroinitializer, %.lr.ph.i ], [ %i80, %bb49 ]
+ %.sroa.106.2 = phi <16 x float> [ zeroinitializer, %.lr.ph.i ], [ %i82, %bb49 ]
+ %.sroa.1581182.2 = phi <16 x float> [ zeroinitializer, %.lr.ph.i ], [ %i83, %bb49 ]
+ %.sroa.210.2 = phi <16 x float> [ zeroinitializer, %.lr.ph.i ], [ %i84, %bb49 ]
+ %.sroa.262.2 = phi <16 x float> [ zeroinitializer, %.lr.ph.i ], [ %i85, %bb49 ]
+ %.sroa.314.2 = phi <16 x float> [ zeroinitializer, %.lr.ph.i ], [ %i86, %bb49 ]
+ %.sroa.366.21 = phi <16 x float> [ zeroinitializer, %.lr.ph.i ], [ %i70, %bb49 ]
+ %.sroa.418.2 = phi <16 x float> [ zeroinitializer, %.lr.ph.i ], [ %i87, %bb49 ]
+ %.sroa.470.2 = phi <16 x float> [ zeroinitializer, %.lr.ph.i ], [ %i72, %bb49 ]
+ %.sroa.522.2 = phi <16 x float> [ zeroinitializer, %.lr.ph.i ], [ %i73, %bb49 ]
+ %.sroa.574.2 = phi <16 x float> [ zeroinitializer, %.lr.ph.i ], [ %i88, %bb49 ]
+ %.sroa.626.2 = phi <16 x float> [ zeroinitializer, %.lr.ph.i ], [ %i74, %bb49 ]
+ %.sroa.678.2 = phi <16 x float> [ zeroinitializer, %.lr.ph.i ], [ %i89, %bb49 ]
+ %.sroa.730.2 = phi <16 x float> [ zeroinitializer, %.lr.ph.i ], [ %i90, %bb49 ]
+ %.sroa.782.2 = phi <16 x float> [ zeroinitializer, %.lr.ph.i ], [ %i77, %bb49 ]
+ %i = phi i64 [ 0, %.lr.ph.i ], [ 1, %bb49 ]
+ br i1 %arg, label %bb1, label %bb2
+
+bb1: ; preds = %bb
+ store <4 x i32> zeroinitializer, ptr addrspace(5) null, align 16
+ br label %bb49
+
+bb2: ; preds = %bb
+ %i3 = fmul <16 x float> %.sroa.01121.2, zeroinitializer
+ %i4 = fmul <16 x float> %.sroa.54.2, zeroinitializer
+ %i5 = fmul <16 x float> %.sroa.106.2, zeroinitializer
+ %i6 = fmul <16 x float> %.sroa.1581182.2, zeroinitializer
+ %i7 = fmul <16 x float> %.sroa.210.2, zeroinitializer
+ %i8 = fmul <16 x float> %.sroa.262.2, zeroinitializer
+ %i9 = fmul <16 x float> %.sroa.314.2, zeroinitializer
+ %i10 = fmul <16 x float> %.sroa.366.21, zeroinitializer
+ %i11 = fmul <16 x float> %.sroa.418.2, zeroinitializer
+ %i12 = fmul <16 x float> %.sroa.470.2, zeroinitializer
+ %i13 = fmul <16 x float> %.sroa.522.2, zeroinitializer
+ %i14 = fmul <16 x float> %.sroa.574.2, zeroinitializer
+ %i15 = fmul <16 x float> %.sroa.626.2, zeroinitializer
+ %i16 = fmul <16 x float> %.sroa.678.2, zeroinitializer
+ %i17 = fmul <16 x float> %.sroa.730.2, zeroinitializer
+ %i18 = fmul <16 x float> %.sroa.782.2, zeroinitializer
+ %i19 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i3, i32 0, i32 0, i32 0)
+ %i20 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i4, i32 0, i32 0, i32 0)
+ %i21 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i5, i32 0, i32 0, i32 0)
+ %i22 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i6, i32 0, i32 0, i32 0)
+ %i23 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i7, i32 0, i32 0, i32 0)
+ %i24 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i8, i32 0, i32 0, i32 0)
+ %i25 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i9, i32 0, i32 0, i32 0)
+ %i26 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i10, i32 0, i32 0, i32 0)
+ %i27 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i11, i32 0, i32 0, i32 0)
+ %i28 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i12, i32 0, i32 0, i32 0)
+ %i29 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i13, i32 0, i32 0, i32 0)
+ %i30 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i15, i32 0, i32 0, i32 0)
+ %i31 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i16, i32 0, i32 0, i32 0)
+ %i32 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i17, i32 0, i32 0, i32 0)
+ %i33 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i18, i32 0, i32 0, i32 0)
+ %i34 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i19, i32 0, i32 0, i32 0)
+ %i35 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i20, i32 0, i32 0, i32 0)
+ %i36 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i21, i32 0, i32 0, i32 0)
+ %i37 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i22, i32 0, i32 0, i32 0)
+ %i38 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i23, i32 0, i32 0, i32 0)
+ %i39 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i24, i32 0, i32 0, i32 0)
+ %i40 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i25, i32 0, i32 0, i32 0)
+ %i41 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i26, i32 0, i32 0, i32 0)
+ %i42 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i27, i32 0, i32 0, i32 0)
+ %i43 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i28, i32 0, i32 0, i32 0)
+ %i44 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i29, i32 0, i32 0, i32 0)
+ %i45 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i30, i32 0, i32 0, i32 0)
+ %i46 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i31, i32 0, i32 0, i32 0)
+ %i47 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i32, i32 0, i32 0, i32 0)
+ %i48 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i33, i32 0, i32 0, i32 0)
+ br label %bb49
+
+bb49: ; preds = %bb2, %bb1
+ %.sroa.01121.3 = phi <16 x float> [ %.sroa.01121.2, %bb1 ], [ %i34, %bb2 ]
+ %.sroa.54.3 = phi <16 x float> [ %.sroa.54.2, %bb1 ], [ %i35, %bb2 ]
+ %.sroa.106.3 = phi <16 x float> [ %.sroa.106.2, %bb1 ], [ %i36, %bb2 ]
+ %.sroa.1581182.3 = phi <16 x float> [ %.sroa.1581182.2, %bb1 ], [ %i37, %bb2 ]
+ %.sroa.210.3 = phi <16 x float> [ %.sroa.210.2, %bb1 ], [ %i38, %bb2 ]
+ %.sroa.262.3 = phi <16 x float> [ %.sroa.262.2, %bb1 ], [ %i39, %bb2 ]
+ %.sroa.314.3 = phi <16 x float> [ %.sroa.314.2, %bb1 ], [ %i40, %bb2 ]
+ %.sroa.366.3 = phi <16 x float> [ %.sroa.366.2, %bb1 ], [ %i41, %bb2 ]
+ %.sroa.418.3 = phi <16 x float> [ %.sroa.418.2, %bb1 ], [ %i42, %bb2 ]
+ %.sroa.470.3 = phi <16 x float> [ %.sroa.470.2, %bb1 ], [ %i43, %bb2 ]
+ %.sroa.522.3 = phi <16 x float> [ %.sroa.522.2, %bb1 ], [ %i44, %bb2 ]
+ %.sroa.574.3 = phi <16 x float> [ %.sroa.574.2, %bb1 ], [ %i14, %bb2 ]
+ %.sroa.626.3 = phi <16 x float> [ zeroinitializer, %bb1 ], [ %i45, %bb2 ]
+ %.sroa.678.3 = phi <16 x float> [ %.sroa.678.2, %bb1 ], [ %i46, %bb2 ]
+ %.sroa.730.3 = phi <16 x float> [ %.sroa.730.2, %bb1 ], [ %i47, %bb2 ]
+ %.sroa.782.3 = phi <16 x float> [ %.sroa.782.2, %bb1 ], [ %i48, %bb2 ]
+ %i50 = fmul <16 x float> %.sroa.01121.3, zeroinitializer
+ %i51 = fmul <16 x float> %.sroa.54.3, zeroinitializer
+ %i52 = fmul <16 x float> %.sroa.106.3, zeroinitializer
+ %i53 = fmul <16 x float> %.sroa.1581182.3, zeroinitializer
+ %i54 = fmul <16 x float> %.sroa.210.3, zeroinitializer
+ %i55 = fmul <16 x float> %.sroa.262.3, zeroinitializer
+ %i56 = fmul <16 x float> %.sroa.314.3, zeroinitializer
+ %i57 = fmul <16 x float> %.sroa.366.3, zeroinitializer
+ %i58 = fmul <16 x float> %.sroa.418.3, zeroinitializer
+ %i59 = fmul <16 x float> %.sroa.470.3, zeroinitializer
+ %i60 = fmul <16 x float> %.sroa.522.3, zeroinitializer
+ %i61 = fmul <16 x float> %.sroa.574.3, zeroinitializer
+ %i62 = fmul <16 x float> %.sroa.626.3, zeroinitializer
+ %i63 = fmul <16 x float> %.sroa.678.3, zeroinitializer
+ %i64 = fmul <16 x float> %.sroa.730.3, zeroinitializer
+ %i65 = fmul <16 x float> %.sroa.782.3, zeroinitializer
+ %i66 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i53, i32 0, i32 0, i32 0)
+ %i67 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i54, i32 0, i32 0, i32 0)
+ %i68 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i55, i32 0, i32 0, i32 0)
+ %i69 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i56, i32 0, i32 0, i32 0)
+ %i70 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i57, i32 0, i32 0, i32 0)
+ %i71 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i58, i32 0, i32 0, i32 0)
+ %i72 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i59, i32 0, i32 0, i32 0)
+ %i73 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i60, i32 0, i32 0, i32 0)
+ %i74 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i62, i32 0, i32 0, i32 0)
+ %i75 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i63, i32 0, i32 0, i32 0)
+ %i76 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i64, i32 0, i32 0, i32 0)
+ %i77 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i65, i32 0, i32 0, i32 0)
+ %i78 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i50, i32 0, i32 0, i32 0)
+ %i79 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i51, i32 0, i32 0, i32 0)
+ %i80 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i79, i32 0, i32 0, i32 0)
+ %i81 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i52, i32 0, i32 0, i32 0)
+ %i82 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i81, i32 0, i32 0, i32 0)
+ %i83 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i66, i32 0, i32 0, i32 0)
+ %i84 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i67, i32 0, i32 0, i32 0)
+ %i85 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i68, i32 0, i32 0, i32 0)
+ %i86 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i69, i32 0, i32 0, i32 0)
+ %i87 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i71, i32 0, i32 0, i32 0)
+ %i88 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %.sroa.574.2, i32 0, i32 0, i32 0)
+ %i89 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i75, i32 0, i32 0, i32 0)
+ %i90 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i76, i32 0, i32 0, i32 0)
+ %exitcond.not.i = icmp eq i64 %i, 0
+ br i1 %exitcond.not.i, label %._crit_edge.i.loopexit, label %bb
+
+._crit_edge.i.loopexit: ; preds = %bb49
+ ret void
+}
+
+; Function Attrs: convergent nocallback nocreateundeforpoison nofree nosync nounwind willreturn memory(none)
+declare <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat>, <8 x bfloat>, <16 x float>, i32 immarg, i32 immarg, i32 immarg) #1
+
+attributes #0 = { "amdgpu-flat-work-group-size"="1,256" }
+attributes #1 = { convergent nocallback nocreateundeforpoison nofree nosync nounwind willreturn memory(none) }
More information about the llvm-commits
mailing list