[llvm] [1/2][AMDGPU] Fixed crash due to virtual register defs not dominating uses (PR #198472)
Dhruva Chakrabarti via llvm-commits
llvm-commits at lists.llvm.org
Mon Aug 17 11:08:24 PDT 2026
https://github.com/dhruvachak updated https://github.com/llvm/llvm-project/pull/198472
>From 1488669dd37cdd2f2891e5389b9b4c83d0667323 Mon Sep 17 00:00:00 2001
From: Dhruva Chakrabarti <Dhruva.Chakrabarti at amd.com>
Date: Tue, 19 May 2026 02:48:30 -0500
Subject: [PATCH 01/15] [AMDGPU] Fixed crash due to virtual register defs not
dominating uses.
Fixes https://github.com/llvm/llvm-project/issues/196671.
Fixes duplicate ROCM-24494.
In Rewrite AGPR-Copy-MFMA pass, a spill reload may not have a dominating
spill store. If such a slot is unspilled into a vreg, the elimination
phase crashes because virtual register defs do not dominate all uses.
This patch fixes this problem by inserting IMPLICIT_DEFs on paths that
do not have a spill store.
This patch is based on suggestion by @ruiling on
https://github.com/llvm/llvm-project/pull/167347.
Compared to https://github.com/llvm/llvm-project/pull/167347 which bails
out if dominating defs are not found, this patch continues with spill
elimination by inserting implicit defs.
2 tests are added that check insertion of implicit defs.
Assisted-by: Cursor/Claude Opus
---
.../AMDGPU/AMDGPURewriteAGPRCopyMFMA.cpp | 91 +
...pr-mfma-to-agpr-spill-implicit-def-mir.mir | 1775 +++++++++++++++++
...te-vgpr-mfma-to-agpr-spill-implicit-def.ll | 159 ++
3 files changed, 2025 insertions(+)
create mode 100644 llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-implicit-def-mir.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-implicit-def.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURewriteAGPRCopyMFMA.cpp b/llvm/lib/Target/AMDGPU/AMDGPURewriteAGPRCopyMFMA.cpp
index 6510c07358e01..7f60a09b4fa14 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURewriteAGPRCopyMFMA.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURewriteAGPRCopyMFMA.cpp
@@ -35,6 +35,7 @@
#include "llvm/CodeGen/SlotIndexes.h"
#include "llvm/CodeGen/VirtRegMap.h"
#include "llvm/InitializePasses.h"
+#include "llvm/Support/CommandLine.h"
#include "llvm/Support/DebugCounter.h"
using namespace llvm;
@@ -44,6 +45,12 @@ using namespace llvm;
DEBUG_COUNTER(RewriteAGPRCopyMFMACounter, DEBUG_TYPE,
"Controls which MFMA chains are rewritten to AGPR form");
+static cl::opt<unsigned> ImplicitDefScanLimit(
+ "amdgpu-mfma-vgpr-to-agpr-spill-scan-limit", cl::Hidden, cl::init(4),
+ cl::desc("Maximum number of instructions to scan forward from a live "
+ "segment start when searching for a spill store before "
+ "inserting an IMPLICIT_DEF"));
+
namespace {
STATISTIC(NumMFMAsRewrittenToAGPR,
@@ -127,6 +134,15 @@ class AMDGPURewriteAGPRCopyMFMAImpl {
void collectSpillIndexUses(ArrayRef<LiveInterval *> StackIntervals,
SpillReferenceMap &Map) const;
+ /// For each segment in the stack slot's LiveInterval whose start does not
+ /// correspond to a spill store, insert an IMPLICIT_DEF of \p NewVReg.
+ /// This handles a control flow path where there is no spill store dominating
+ /// a spill reload. Returns false if any segment start cannot be resolved
+ /// (caller should bail out and not unspill).
+ bool insertImplicitDefsForLiveSegments(LiveInterval &StackLI, int Slot,
+ ArrayRef<MachineInstr *> SpillStores,
+ Register NewVReg) const;
+
/// Attempt to unspill VGPRs by finding a free register and replacing the
/// spill instructions with copies.
void eliminateSpillsOfReassignedVGPRs() const;
@@ -475,6 +491,69 @@ void AMDGPURewriteAGPRCopyMFMAImpl::collectSpillIndexUses(
}
}
+bool AMDGPURewriteAGPRCopyMFMAImpl::insertImplicitDefsForLiveSegments(
+ LiveInterval &StackLI, int Slot, ArrayRef<MachineInstr *> SpillStores,
+ Register NewVReg) const {
+ const unsigned ScanLimit = ImplicitDefScanLimit;
+
+ SmallPtrSet<MachineInstr *, 4> StoreSet(SpillStores.begin(),
+ SpillStores.end());
+
+ SmallVector<MachineInstr *, 2> InsertedDefs;
+
+ for (const LiveRange::Segment &Seg : StackLI) {
+ MachineInstr *MI = LIS.getInstructionFromIndex(Seg.start);
+ // Fall back to LIS query in case the segment start does not correspond
+ // to an instruction.
+ MachineBasicBlock *MBB =
+ MI ? MI->getParent() : LIS.getMBBFromIndex(Seg.start);
+
+ // Bail-out: we could not resolve any MBB for this segment start.
+ // Roll back any defs already inserted for earlier segments -- the
+ // caller must skip unspilling this slot entirely.
+ if (!MBB) {
+ for (MachineInstr *Def : InsertedDefs) {
+ LIS.RemoveMachineInstrFromMaps(*Def);
+ Def->eraseFromParent();
+ }
+ return false;
+ }
+
+ // Scan forward from the segment start looking for a spill store to this
+ // slot. LiveStacks liveness for a store can start a few instructions
+ // before the store itself, so we allow a small window controlled by
+ // ImplicitDefScanLimit. Redundant IMPLICIT_DEFs (e.g. because of a small
+ // scan limit) are harmless, since they will be cleaned up by downstream
+ // transformations.
+ MachineBasicBlock::iterator It = MI ? MI->getIterator() : MBB->begin();
+ bool FoundStore = false;
+ for (unsigned I = 0; I < ScanLimit && It != MBB->end(); ++I, ++It) {
+ if (StoreSet.count(&*It)) {
+ FoundStore = true;
+ break;
+ }
+ }
+
+ // A spill store is found for this segment, so no IMPLICIT_DEF is needed.
+ if (FoundStore)
+ continue;
+
+ // Insert an IMPLICIT_DEF to provide a reaching definition for NewVReg.
+ MachineBasicBlock::iterator InsertPt =
+ MI ? MI->getIterator() : MBB->begin();
+ MachineInstr *ImpDef =
+ BuildMI(*MBB, InsertPt, DebugLoc(), TII.get(TargetOpcode::IMPLICIT_DEF),
+ NewVReg);
+ LIS.InsertMachineInstrInMaps(*ImpDef);
+ InsertedDefs.push_back(ImpDef);
+
+ LLVM_DEBUG(dbgs() << "Inserted IMPLICIT_DEF for " << printReg(NewVReg)
+ << " in " << printMBBReference(*MBB) << '\n');
+ }
+
+ return true;
+}
+
void AMDGPURewriteAGPRCopyMFMAImpl::eliminateSpillsOfReassignedVGPRs() const {
unsigned NumSlots = LSS.getNumIntervals();
if (NumSlots == 0)
@@ -545,6 +624,18 @@ void AMDGPURewriteAGPRCopyMFMAImpl::eliminateSpillsOfReassignedVGPRs() const {
const TargetRegisterClass *RC = LSS.getIntervalRegClass(Slot);
Register NewVReg = MRI.createVirtualRegister(RC);
+ // It is legal for a spill reload to not have a dominating spill store.
+ // But after un-spilling, the replacement vreg must have reaching defs
+ // on all paths. Ensure this condition by inserting IMPLICIT_DEFs if
+ // required. Bail out if segment starts cannot be resolved.
+ SmallVector<MachineInstr *, 4> SpillStores;
+ for (MachineInstr *MI : SpillReferences->second)
+ if (MI->mayStore())
+ SpillStores.push_back(MI);
+
+ if (!insertImplicitDefsForLiveSegments(*LI, Slot, SpillStores, NewVReg))
+ continue;
+
for (MachineInstr *SpillMI : SpillReferences->second)
replaceSpillWithCopyToVReg(*SpillMI, Slot, NewVReg);
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-implicit-def-mir.mir b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-implicit-def-mir.mir
new file mode 100644
index 0000000000000..5858b29b5bbe8
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-implicit-def-mir.mir
@@ -0,0 +1,1775 @@
+# REQUIRES: asserts
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 \
+# RUN: -start-before=register-coalescer \
+# RUN: -stop-after=amdgpu-rewrite-agpr-copy-mfma \
+# RUN: -debug-only=amdgpu-rewrite-agpr-copy-mfma -filetype=null %s 2>&1 \
+# RUN: | FileCheck %s
+
+# It is legal for a spill reload to not have a dominating spill store.
+# When the AGPR rewrite pass unspills such a slot into a vreg, it must insert
+# IMPLICIT_DEF so the vreg has defs on all paths.
+
+# CHECK: Inserted IMPLICIT_DEF for %{{[0-9]+}} in %bb.{{[0-9]+}}
+
+--- |
+ define amdgpu_kernel void @rewrite_vgpr_mfma_to_agpr_spill_implicit_def(i1 %0, <16 x float> %.sroa.366.2) #0 {
+ %kernarg.segment = call ptr addrspace(4) @llvm.amdgcn.kernarg.segment.ptr()
+ %.kernarg.offset.align.down66 = bitcast ptr addrspace(4) %kernarg.segment to ptr addrspace(4)
+ %.sroa.366.2.kernarg.offset = getelementptr inbounds i8, ptr addrspace(4) %kernarg.segment, i64 64
+ unreachable
+ }
+
+ declare ptr addrspace(4) @llvm.amdgcn.kernarg.segment.ptr()
+
+ attributes #0 = { "amdgpu-flat-work-group-size"="1, 256" "target-cpu"="gfx950" }
+...
+---
+name: rewrite_vgpr_mfma_to_agpr_spill_implicit_def
+tracksRegLiveness: true
+liveins:
+ - { reg: '$sgpr4_sgpr5', virtual-reg: '%0' }
+body: |
+ bb.0:
+ successors: %bb.1(0x80000000)
+ liveins: $sgpr4_sgpr5
+
+ %0:sgpr_64 = COPY killed $sgpr4_sgpr5
+ %1:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM %0, 0, 0 :: (dereferenceable invariant load (s32) from %ir..kernarg.offset.align.down66, align 16, addrspace 4)
+ S_BITCMP1_B32 killed %1, 0, implicit-def $scc
+ %2:sreg_64_xexec = S_CSELECT_B64 -1, 0, implicit killed $scc
+ %3:sreg_64 = S_MOV_B64 -1
+ %4:sreg_64 = S_XOR_B64 killed %2, -1, implicit-def dead $scc
+ %5:sgpr_32 = S_MOV_B32 0
+ %6:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ undef %7.sub0:av_512_align2 = COPY %6
+ %7.sub1:av_512_align2 = COPY %6
+ %7.sub2:av_512_align2 = COPY %6
+ %7.sub3:av_512_align2 = COPY %6
+ %7.sub4:av_512_align2 = COPY %6
+ %7.sub5:av_512_align2 = COPY %6
+ %7.sub6:av_512_align2 = COPY %6
+ %7.sub7:av_512_align2 = COPY %6
+ %7.sub8:av_512_align2 = COPY %6
+ %7.sub9:av_512_align2 = COPY %6
+ %7.sub10:av_512_align2 = COPY %6
+ %7.sub11:av_512_align2 = COPY %6
+ %7.sub12:av_512_align2 = COPY %6
+ %7.sub13:av_512_align2 = COPY %6
+ %7.sub14:av_512_align2 = COPY %6
+ %7.sub15:av_512_align2 = COPY %6
+ early-clobber %8:sgpr_512 = S_LOAD_DWORDX16_IMM_ec killed %0, 64, 0 :: (dereferenceable invariant load (s512) from %ir..sroa.366.2.kernarg.offset, align 16, addrspace 4)
+ %9:sreg_64 = S_MOV_B64 0
+ %10:vreg_512_align2 = COPY %7, implicit $exec
+ %11:sreg_64 = S_AND_B64 $exec, killed %4, implicit-def dead $scc
+ undef %12.sub0:sgpr_128 = COPY %5
+ %12.sub1:sgpr_128 = COPY %5
+ %12.sub2:sgpr_128 = COPY %5
+ %12.sub3:sgpr_128 = COPY %5
+ %13:av_128_align2 = COPY killed %12
+ undef %14.sub0:av_128_align2 = COPY %6
+ %14.sub1:av_128_align2 = COPY %6
+ %14.sub2:av_128_align2 = COPY %6
+ %14.sub3:av_128_align2 = COPY %6
+ %15:vreg_512_align2 = COPY %7
+ %16:vreg_512_align2 = COPY %7
+ %17:vreg_512_align2 = COPY %7
+ %18:vreg_512_align2 = COPY %7
+ %19:vreg_512_align2 = COPY %7
+ %20:vreg_512_align2 = COPY %7
+ %21:vreg_512_align2 = COPY %7
+ %22:vreg_512_align2 = COPY %7
+ %23:vreg_512_align2 = COPY %7
+ %24:vreg_512_align2 = COPY %7
+ %25:vreg_512_align2 = COPY killed %10
+ %26:vreg_512_align2 = COPY %7
+ %27:vreg_512_align2 = COPY %7
+ %28:vreg_512_align2 = COPY killed %7
+ %29:sreg_64 = COPY killed %9
+ %30:av_32 = COPY %6
+ %31:av_32 = COPY %6
+ %32:av_32 = COPY %6
+ %33:av_32 = COPY %6
+ %34:av_32 = COPY %6
+ %35:av_32 = COPY %6
+ %36:av_32 = COPY %6
+ %37:av_32 = COPY %6
+ %38:av_32 = COPY %6
+ %39:av_32 = COPY %6
+ %40:av_32 = COPY %6
+ %41:av_32 = COPY %6
+ %42:av_32 = COPY %6
+ %43:av_32 = COPY %6
+ %44:av_32 = COPY %6
+ %45:av_32 = COPY %6
+ %46:av_32 = COPY %6
+ %47:av_32 = COPY %6
+ %48:av_32 = COPY %6
+ %49:av_32 = COPY %6
+ %50:av_32 = COPY %6
+ %51:av_32 = COPY %6
+ %52:av_32 = COPY %6
+ %53:av_32 = COPY %6
+ %54:av_32 = COPY %6
+ %55:av_32 = COPY %6
+ %56:av_32 = COPY %6
+ %57:av_32 = COPY %6
+ %58:av_32 = COPY %6
+ %59:av_32 = COPY %6
+ %60:av_32 = COPY %6
+ %61:av_32 = COPY killed %6
+
+ bb.1:
+ successors: %bb.4(0x40000000), %bb.2(0x40000000)
+
+ %62:av_32 = COPY killed %61
+ %63:av_32 = COPY killed %60
+ %64:av_32 = COPY killed %59
+ %65:av_32 = COPY killed %58
+ %66:av_32 = COPY killed %57
+ %67:av_32 = COPY killed %56
+ %68:av_32 = COPY killed %55
+ %69:av_32 = COPY killed %54
+ %70:av_32 = COPY killed %53
+ %71:av_32 = COPY killed %52
+ %72:av_32 = COPY killed %51
+ %73:av_32 = COPY killed %50
+ %74:av_32 = COPY killed %49
+ %75:av_32 = COPY killed %48
+ %76:av_32 = COPY killed %47
+ %77:av_32 = COPY killed %46
+ %78:av_32 = COPY killed %45
+ %79:av_32 = COPY killed %44
+ %80:av_32 = COPY killed %43
+ %81:av_32 = COPY killed %42
+ %82:av_32 = COPY killed %41
+ %83:av_32 = COPY killed %40
+ %84:av_32 = COPY killed %39
+ %85:av_32 = COPY killed %38
+ %86:av_32 = COPY killed %37
+ %87:av_32 = COPY killed %36
+ %88:av_32 = COPY killed %35
+ %89:av_32 = COPY killed %34
+ %90:av_32 = COPY killed %33
+ %91:av_32 = COPY killed %32
+ %92:av_32 = COPY killed %31
+ %93:av_32 = COPY killed %30
+ %94:sreg_64 = COPY killed %29
+ %95:vreg_512_align2 = COPY killed %28
+ %96:vreg_512_align2 = COPY killed %27
+ %97:vreg_512_align2 = COPY killed %26
+ %98:vreg_512_align2 = COPY killed %25
+ %99:vreg_512_align2 = COPY killed %24
+ %100:vreg_512_align2 = COPY killed %23
+ %101:vreg_512_align2 = COPY killed %22
+ %102:vreg_512_align2 = COPY killed %21
+ %103:vreg_512_align2 = COPY killed %20
+ %104:vreg_512_align2 = COPY killed %19
+ %105:vreg_512_align2 = COPY killed %18
+ %106:vreg_512_align2 = COPY killed %17
+ %107:vreg_512_align2 = COPY killed %16
+ %108:vreg_512_align2 = COPY killed %15
+ $vcc = COPY %11
+ %109:av_32 = IMPLICIT_DEF
+ %110:av_32 = IMPLICIT_DEF
+ %111:av_32 = IMPLICIT_DEF
+ %112:av_32 = IMPLICIT_DEF
+ %113:av_32 = IMPLICIT_DEF
+ %114:av_32 = IMPLICIT_DEF
+ %115:av_32 = IMPLICIT_DEF
+ %116:av_32 = IMPLICIT_DEF
+ %117:av_32 = IMPLICIT_DEF
+ %118:av_32 = IMPLICIT_DEF
+ %119:av_32 = IMPLICIT_DEF
+ %120:av_32 = IMPLICIT_DEF
+ %121:av_32 = IMPLICIT_DEF
+ %122:av_32 = IMPLICIT_DEF
+ %123:av_32 = IMPLICIT_DEF
+ %124:av_32 = IMPLICIT_DEF
+ %125:av_512_align2 = IMPLICIT_DEF
+ %126:av_512_align2 = IMPLICIT_DEF
+ %127:av_512_align2 = IMPLICIT_DEF
+ %128:av_512_align2 = IMPLICIT_DEF
+ %129:av_512_align2 = IMPLICIT_DEF
+ %130:av_512_align2 = IMPLICIT_DEF
+ %131:av_512_align2 = IMPLICIT_DEF
+ %132:av_512_align2 = IMPLICIT_DEF
+ %133:av_512_align2 = IMPLICIT_DEF
+ %134:av_512_align2 = IMPLICIT_DEF
+ %135:av_512_align2 = IMPLICIT_DEF
+ %136:av_512_align2 = IMPLICIT_DEF
+ %137:av_512_align2 = IMPLICIT_DEF
+ %138:av_512_align2 = IMPLICIT_DEF
+ %139:sreg_64_xexec = COPY %3
+ S_CBRANCH_VCCNZ %bb.4, implicit killed $vcc
+ S_BRANCH %bb.2
+
+ bb.2:
+ successors: %bb.3(0x40000000), %bb.5(0x40000000)
+
+ %140:av_512_align2 = COPY killed %138
+ %141:av_512_align2 = COPY killed %137
+ %142:av_512_align2 = COPY killed %136
+ %143:av_512_align2 = COPY killed %135
+ %144:av_512_align2 = COPY killed %134
+ %145:av_512_align2 = COPY killed %133
+ %146:av_512_align2 = COPY killed %132
+ %147:av_512_align2 = COPY killed %131
+ %148:av_512_align2 = COPY killed %130
+ %149:av_512_align2 = COPY killed %129
+ %150:av_512_align2 = COPY killed %128
+ %151:av_512_align2 = COPY killed %127
+ %152:av_512_align2 = COPY killed %126
+ %153:av_512_align2 = COPY killed %125
+ %154:av_32 = COPY killed %124
+ %155:av_32 = COPY killed %123
+ %156:av_32 = COPY killed %122
+ %157:av_32 = COPY killed %121
+ %158:av_32 = COPY killed %120
+ %159:av_32 = COPY killed %119
+ %160:av_32 = COPY killed %118
+ %161:av_32 = COPY killed %117
+ %162:av_32 = COPY killed %116
+ %163:av_32 = COPY killed %115
+ %164:av_32 = COPY killed %114
+ %165:av_32 = COPY killed %113
+ %166:av_32 = COPY killed %112
+ %167:av_32 = COPY killed %111
+ %168:av_32 = COPY killed %110
+ %169:av_32 = COPY killed %109
+ %170:sreg_64_xexec = COPY killed %139
+ %171:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, killed %170, implicit $exec
+ %172:sreg_64_xexec = V_CMP_NE_U32_e64 1, killed %171, implicit $exec
+ $vcc = S_AND_B64 $exec, killed %172, implicit-def dead $scc
+ %173:vreg_512_align2 = COPY killed %140
+ %174:vreg_512_align2 = COPY killed %141
+ %175:vreg_512_align2 = COPY killed %142
+ %176:vreg_512_align2 = COPY killed %143
+ %177:vreg_512_align2 = COPY killed %144
+ %178:vreg_512_align2 = COPY killed %145
+ %179:vreg_512_align2 = COPY killed %146
+ %180:vreg_512_align2 = COPY killed %147
+ %181:vreg_512_align2 = COPY killed %148
+ %182:vreg_512_align2 = COPY killed %149
+ %183:vreg_512_align2 = COPY killed %150
+ %184:vreg_512_align2 = COPY killed %151
+ %185:vreg_512_align2 = COPY killed %152
+ %186:vreg_512_align2 = COPY killed %153
+ %187:av_32 = COPY killed %154
+ %188:av_32 = COPY killed %155
+ %189:av_32 = COPY killed %156
+ %190:av_32 = COPY killed %157
+ %191:av_32 = COPY killed %158
+ %192:av_32 = COPY killed %159
+ %193:av_32 = COPY killed %160
+ %194:av_32 = COPY killed %161
+ %195:av_32 = COPY killed %162
+ %196:av_32 = COPY killed %163
+ %197:av_32 = COPY killed %164
+ %198:av_32 = COPY killed %165
+ %199:av_32 = COPY killed %166
+ %200:av_32 = COPY killed %167
+ %201:av_32 = COPY killed %168
+ %202:av_32 = COPY killed %169
+ S_CBRANCH_VCCNZ %bb.5, implicit killed $vcc
+ S_BRANCH %bb.3
+
+ bb.3:
+ successors: %bb.5(0x80000000)
+
+ SCRATCH_STORE_DWORDX4_SADDR %14, %5, 0, 0, implicit $exec, implicit $flat_scr :: (store (s128) into `ptr addrspace(5) null`, addrspace 5)
+ %203:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ %204:av_512_align2 = COPY %8, implicit $exec
+ %173:vreg_512_align2 = COPY killed %108
+ %174:vreg_512_align2 = COPY killed %107
+ %175:vreg_512_align2 = COPY killed %106
+ %176:vreg_512_align2 = COPY killed %105
+ %177:vreg_512_align2 = COPY killed %104
+ %178:vreg_512_align2 = COPY killed %103
+ %179:vreg_512_align2 = COPY killed %102
+ %180:vreg_512_align2 = COPY killed %204
+ %181:vreg_512_align2 = COPY killed %101
+ %182:vreg_512_align2 = COPY killed %100
+ %183:vreg_512_align2 = COPY killed %99
+ %184:vreg_512_align2 = COPY killed %97
+ %185:vreg_512_align2 = COPY killed %96
+ %186:vreg_512_align2 = COPY killed %95
+ %187:av_32 = COPY %203
+ %188:av_32 = COPY %203
+ %189:av_32 = COPY %203
+ %190:av_32 = COPY %203
+ %191:av_32 = COPY %203
+ %192:av_32 = COPY %203
+ %193:av_32 = COPY %203
+ %194:av_32 = COPY %203
+ %195:av_32 = COPY %203
+ %196:av_32 = COPY %203
+ %197:av_32 = COPY %203
+ %198:av_32 = COPY %203
+ %199:av_32 = COPY %203
+ %200:av_32 = COPY %203
+ %201:av_32 = COPY %203
+ %202:av_32 = COPY killed %203
+ S_BRANCH %bb.5
+
+ bb.4:
+ successors: %bb.2(0x80000000)
+
+ undef %205.sub0:vreg_64_align2 = COPY %108.sub14
+ %205.sub1:vreg_64_align2 = COPY %108.sub15
+ %206:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %205, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %207.sub0:vreg_64_align2 = COPY %108.sub12
+ %207.sub1:vreg_64_align2 = COPY %108.sub13
+ %208:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %207, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %209.sub0:vreg_64_align2 = COPY %108.sub10
+ %209.sub1:vreg_64_align2 = COPY %108.sub11
+ %210:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %209, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %211.sub0:vreg_64_align2 = COPY %108.sub8
+ %211.sub1:vreg_64_align2 = COPY %108.sub9
+ %212:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %211, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %213.sub0:vreg_64_align2 = COPY %108.sub6
+ %213.sub1:vreg_64_align2 = COPY %108.sub7
+ %214:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %213, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %215.sub0:vreg_64_align2 = COPY %108.sub4
+ %215.sub1:vreg_64_align2 = COPY %108.sub5
+ %216:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %215, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %217.sub0:vreg_64_align2 = COPY %108.sub2
+ %217.sub1:vreg_64_align2 = COPY %108.sub3
+ %218:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %217, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ %219:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, %108.sub0_sub1, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %220.sub0:vreg_512_align2 = COPY %219.sub0
+ %220.sub1:vreg_512_align2 = COPY killed %219.sub1
+ %220.sub2:vreg_512_align2 = COPY %218.sub0
+ %220.sub3:vreg_512_align2 = COPY killed %218.sub1
+ %220.sub4:vreg_512_align2 = COPY %216.sub0
+ %220.sub5:vreg_512_align2 = COPY killed %216.sub1
+ %220.sub6:vreg_512_align2 = COPY %214.sub0
+ %220.sub7:vreg_512_align2 = COPY killed %214.sub1
+ %220.sub8:vreg_512_align2 = COPY %212.sub0
+ %220.sub9:vreg_512_align2 = COPY killed %212.sub1
+ %220.sub10:vreg_512_align2 = COPY %210.sub0
+ %220.sub11:vreg_512_align2 = COPY killed %210.sub1
+ %220.sub12:vreg_512_align2 = COPY %208.sub0
+ %220.sub13:vreg_512_align2 = COPY killed %208.sub1
+ %220.sub14:vreg_512_align2 = COPY %206.sub0
+ %220.sub15:vreg_512_align2 = COPY killed %206.sub1
+ undef %221.sub0:vreg_64_align2 = COPY %107.sub14
+ %221.sub1:vreg_64_align2 = COPY %107.sub15
+ %222:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %221, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %223.sub0:vreg_64_align2 = COPY %107.sub12
+ %223.sub1:vreg_64_align2 = COPY %107.sub13
+ %224:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %223, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %225.sub0:vreg_64_align2 = COPY %107.sub10
+ %225.sub1:vreg_64_align2 = COPY %107.sub11
+ %226:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %225, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %227.sub0:vreg_64_align2 = COPY %107.sub8
+ %227.sub1:vreg_64_align2 = COPY %107.sub9
+ %228:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %227, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %229.sub0:vreg_64_align2 = COPY %107.sub6
+ %229.sub1:vreg_64_align2 = COPY %107.sub7
+ %230:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %229, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %231.sub0:vreg_64_align2 = COPY %107.sub4
+ %231.sub1:vreg_64_align2 = COPY %107.sub5
+ %232:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %231, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %233.sub0:vreg_64_align2 = COPY %107.sub2
+ %233.sub1:vreg_64_align2 = COPY %107.sub3
+ %234:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %233, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ %235:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, %107.sub0_sub1, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %236.sub0:vreg_512_align2 = COPY %235.sub0
+ %236.sub1:vreg_512_align2 = COPY killed %235.sub1
+ %236.sub2:vreg_512_align2 = COPY %234.sub0
+ %236.sub3:vreg_512_align2 = COPY killed %234.sub1
+ %236.sub4:vreg_512_align2 = COPY %232.sub0
+ %236.sub5:vreg_512_align2 = COPY killed %232.sub1
+ %236.sub6:vreg_512_align2 = COPY %230.sub0
+ %236.sub7:vreg_512_align2 = COPY killed %230.sub1
+ %236.sub8:vreg_512_align2 = COPY %228.sub0
+ %236.sub9:vreg_512_align2 = COPY killed %228.sub1
+ %236.sub10:vreg_512_align2 = COPY %226.sub0
+ %236.sub11:vreg_512_align2 = COPY killed %226.sub1
+ %236.sub12:vreg_512_align2 = COPY %224.sub0
+ %236.sub13:vreg_512_align2 = COPY killed %224.sub1
+ %236.sub14:vreg_512_align2 = COPY %222.sub0
+ %236.sub15:vreg_512_align2 = COPY killed %222.sub1
+ undef %237.sub0:vreg_64_align2 = COPY %106.sub14
+ %237.sub1:vreg_64_align2 = COPY %106.sub15
+ %238:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %237, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %239.sub0:vreg_64_align2 = COPY %106.sub12
+ %239.sub1:vreg_64_align2 = COPY %106.sub13
+ %240:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %239, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %241.sub0:vreg_64_align2 = COPY %106.sub10
+ %241.sub1:vreg_64_align2 = COPY %106.sub11
+ %242:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %241, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %243.sub0:vreg_64_align2 = COPY %106.sub8
+ %243.sub1:vreg_64_align2 = COPY %106.sub9
+ %244:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %243, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %245.sub0:vreg_64_align2 = COPY %106.sub6
+ %245.sub1:vreg_64_align2 = COPY %106.sub7
+ %246:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %245, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %247.sub0:vreg_64_align2 = COPY %106.sub4
+ %247.sub1:vreg_64_align2 = COPY %106.sub5
+ %248:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %247, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %249.sub0:vreg_64_align2 = COPY %106.sub2
+ %249.sub1:vreg_64_align2 = COPY %106.sub3
+ %250:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %249, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ %251:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, %106.sub0_sub1, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %252.sub0:vreg_512_align2 = COPY %251.sub0
+ %252.sub1:vreg_512_align2 = COPY killed %251.sub1
+ %252.sub2:vreg_512_align2 = COPY %250.sub0
+ %252.sub3:vreg_512_align2 = COPY killed %250.sub1
+ %252.sub4:vreg_512_align2 = COPY %248.sub0
+ %252.sub5:vreg_512_align2 = COPY killed %248.sub1
+ %252.sub6:vreg_512_align2 = COPY %246.sub0
+ %252.sub7:vreg_512_align2 = COPY killed %246.sub1
+ %252.sub8:vreg_512_align2 = COPY %244.sub0
+ %252.sub9:vreg_512_align2 = COPY killed %244.sub1
+ %252.sub10:vreg_512_align2 = COPY %242.sub0
+ %252.sub11:vreg_512_align2 = COPY killed %242.sub1
+ %252.sub12:vreg_512_align2 = COPY %240.sub0
+ %252.sub13:vreg_512_align2 = COPY killed %240.sub1
+ %252.sub14:vreg_512_align2 = COPY %238.sub0
+ %252.sub15:vreg_512_align2 = COPY killed %238.sub1
+ undef %253.sub0:vreg_64_align2 = COPY %105.sub14
+ %253.sub1:vreg_64_align2 = COPY %105.sub15
+ %254:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %253, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %255.sub0:vreg_64_align2 = COPY %105.sub12
+ %255.sub1:vreg_64_align2 = COPY %105.sub13
+ %256:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %255, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %257.sub0:vreg_64_align2 = COPY %105.sub10
+ %257.sub1:vreg_64_align2 = COPY %105.sub11
+ %258:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %257, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %259.sub0:vreg_64_align2 = COPY %105.sub8
+ %259.sub1:vreg_64_align2 = COPY %105.sub9
+ %260:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %259, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %261.sub0:vreg_64_align2 = COPY %105.sub6
+ %261.sub1:vreg_64_align2 = COPY %105.sub7
+ %262:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %261, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %263.sub0:vreg_64_align2 = COPY %105.sub4
+ %263.sub1:vreg_64_align2 = COPY %105.sub5
+ %264:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %263, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %265.sub0:vreg_64_align2 = COPY %105.sub2
+ %265.sub1:vreg_64_align2 = COPY %105.sub3
+ %266:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %265, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ %267:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, %105.sub0_sub1, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %268.sub0:vreg_512_align2 = COPY %267.sub0
+ %268.sub1:vreg_512_align2 = COPY killed %267.sub1
+ %268.sub2:vreg_512_align2 = COPY %266.sub0
+ %268.sub3:vreg_512_align2 = COPY killed %266.sub1
+ %268.sub4:vreg_512_align2 = COPY %264.sub0
+ %268.sub5:vreg_512_align2 = COPY killed %264.sub1
+ %268.sub6:vreg_512_align2 = COPY %262.sub0
+ %268.sub7:vreg_512_align2 = COPY killed %262.sub1
+ %268.sub8:vreg_512_align2 = COPY %260.sub0
+ %268.sub9:vreg_512_align2 = COPY killed %260.sub1
+ %268.sub10:vreg_512_align2 = COPY %258.sub0
+ %268.sub11:vreg_512_align2 = COPY killed %258.sub1
+ %268.sub12:vreg_512_align2 = COPY %256.sub0
+ %268.sub13:vreg_512_align2 = COPY killed %256.sub1
+ %268.sub14:vreg_512_align2 = COPY %254.sub0
+ %268.sub15:vreg_512_align2 = COPY killed %254.sub1
+ undef %269.sub0:vreg_64_align2 = COPY %104.sub14
+ %269.sub1:vreg_64_align2 = COPY %104.sub15
+ %270:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %269, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %271.sub0:vreg_64_align2 = COPY %104.sub12
+ %271.sub1:vreg_64_align2 = COPY %104.sub13
+ %272:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %271, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %273.sub0:vreg_64_align2 = COPY %104.sub10
+ %273.sub1:vreg_64_align2 = COPY %104.sub11
+ %274:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %273, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %275.sub0:vreg_64_align2 = COPY %104.sub8
+ %275.sub1:vreg_64_align2 = COPY %104.sub9
+ %276:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %275, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %277.sub0:vreg_64_align2 = COPY %104.sub6
+ %277.sub1:vreg_64_align2 = COPY %104.sub7
+ %278:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %277, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %279.sub0:vreg_64_align2 = COPY %104.sub4
+ %279.sub1:vreg_64_align2 = COPY %104.sub5
+ %280:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %279, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %281.sub0:vreg_64_align2 = COPY %104.sub2
+ %281.sub1:vreg_64_align2 = COPY %104.sub3
+ %282:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %281, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ %283:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, %104.sub0_sub1, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %284.sub0:vreg_512_align2 = COPY %283.sub0
+ %284.sub1:vreg_512_align2 = COPY killed %283.sub1
+ %284.sub2:vreg_512_align2 = COPY %282.sub0
+ %284.sub3:vreg_512_align2 = COPY killed %282.sub1
+ %284.sub4:vreg_512_align2 = COPY %280.sub0
+ %284.sub5:vreg_512_align2 = COPY killed %280.sub1
+ %284.sub6:vreg_512_align2 = COPY %278.sub0
+ %284.sub7:vreg_512_align2 = COPY killed %278.sub1
+ %284.sub8:vreg_512_align2 = COPY %276.sub0
+ %284.sub9:vreg_512_align2 = COPY killed %276.sub1
+ %284.sub10:vreg_512_align2 = COPY %274.sub0
+ %284.sub11:vreg_512_align2 = COPY killed %274.sub1
+ %284.sub12:vreg_512_align2 = COPY %272.sub0
+ %284.sub13:vreg_512_align2 = COPY killed %272.sub1
+ %284.sub14:vreg_512_align2 = COPY %270.sub0
+ %284.sub15:vreg_512_align2 = COPY killed %270.sub1
+ undef %285.sub0:vreg_64_align2 = COPY %103.sub14
+ %285.sub1:vreg_64_align2 = COPY %103.sub15
+ %286:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %285, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %287.sub0:vreg_64_align2 = COPY %103.sub12
+ %287.sub1:vreg_64_align2 = COPY %103.sub13
+ %288:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %287, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %289.sub0:vreg_64_align2 = COPY %103.sub10
+ %289.sub1:vreg_64_align2 = COPY %103.sub11
+ %290:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %289, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %291.sub0:vreg_64_align2 = COPY %103.sub8
+ %291.sub1:vreg_64_align2 = COPY %103.sub9
+ %292:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %291, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %293.sub0:vreg_64_align2 = COPY %103.sub6
+ %293.sub1:vreg_64_align2 = COPY %103.sub7
+ %294:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %293, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %295.sub0:vreg_64_align2 = COPY %103.sub4
+ %295.sub1:vreg_64_align2 = COPY %103.sub5
+ %296:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %295, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %297.sub0:vreg_64_align2 = COPY %103.sub2
+ %297.sub1:vreg_64_align2 = COPY %103.sub3
+ %298:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %297, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ %299:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, %103.sub0_sub1, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %300.sub0:vreg_512_align2 = COPY %299.sub0
+ %300.sub1:vreg_512_align2 = COPY killed %299.sub1
+ %300.sub2:vreg_512_align2 = COPY %298.sub0
+ %300.sub3:vreg_512_align2 = COPY killed %298.sub1
+ %300.sub4:vreg_512_align2 = COPY %296.sub0
+ %300.sub5:vreg_512_align2 = COPY killed %296.sub1
+ %300.sub6:vreg_512_align2 = COPY %294.sub0
+ %300.sub7:vreg_512_align2 = COPY killed %294.sub1
+ %300.sub8:vreg_512_align2 = COPY %292.sub0
+ %300.sub9:vreg_512_align2 = COPY killed %292.sub1
+ %300.sub10:vreg_512_align2 = COPY %290.sub0
+ %300.sub11:vreg_512_align2 = COPY killed %290.sub1
+ %300.sub12:vreg_512_align2 = COPY %288.sub0
+ %300.sub13:vreg_512_align2 = COPY killed %288.sub1
+ %300.sub14:vreg_512_align2 = COPY %286.sub0
+ %300.sub15:vreg_512_align2 = COPY killed %286.sub1
+ undef %301.sub0:vreg_64_align2 = COPY %102.sub14
+ %301.sub1:vreg_64_align2 = COPY %102.sub15
+ %302:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %301, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %303.sub0:vreg_64_align2 = COPY %102.sub12
+ %303.sub1:vreg_64_align2 = COPY %102.sub13
+ %304:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %303, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %305.sub0:vreg_64_align2 = COPY %102.sub10
+ %305.sub1:vreg_64_align2 = COPY %102.sub11
+ %306:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %305, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %307.sub0:vreg_64_align2 = COPY %102.sub8
+ %307.sub1:vreg_64_align2 = COPY %102.sub9
+ %308:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %307, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %309.sub0:vreg_64_align2 = COPY %102.sub6
+ %309.sub1:vreg_64_align2 = COPY %102.sub7
+ %310:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %309, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %311.sub0:vreg_64_align2 = COPY %102.sub4
+ %311.sub1:vreg_64_align2 = COPY %102.sub5
+ %312:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %311, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %313.sub0:vreg_64_align2 = COPY %102.sub2
+ %313.sub1:vreg_64_align2 = COPY %102.sub3
+ %314:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %313, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ %315:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, %102.sub0_sub1, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %316.sub0:vreg_512_align2 = COPY %315.sub0
+ %316.sub1:vreg_512_align2 = COPY killed %315.sub1
+ %316.sub2:vreg_512_align2 = COPY %314.sub0
+ %316.sub3:vreg_512_align2 = COPY killed %314.sub1
+ %316.sub4:vreg_512_align2 = COPY %312.sub0
+ %316.sub5:vreg_512_align2 = COPY killed %312.sub1
+ %316.sub6:vreg_512_align2 = COPY %310.sub0
+ %316.sub7:vreg_512_align2 = COPY killed %310.sub1
+ %316.sub8:vreg_512_align2 = COPY %308.sub0
+ %316.sub9:vreg_512_align2 = COPY killed %308.sub1
+ %316.sub10:vreg_512_align2 = COPY %306.sub0
+ %316.sub11:vreg_512_align2 = COPY killed %306.sub1
+ %316.sub12:vreg_512_align2 = COPY %304.sub0
+ %316.sub13:vreg_512_align2 = COPY killed %304.sub1
+ %316.sub14:vreg_512_align2 = COPY %302.sub0
+ %316.sub15:vreg_512_align2 = COPY killed %302.sub1
+ undef %317.sub0:vreg_64_align2 = COPY killed %63
+ %317.sub1:vreg_64_align2 = COPY killed %62
+ %318:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %317, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %319.sub0:vreg_64_align2 = COPY killed %65
+ %319.sub1:vreg_64_align2 = COPY killed %64
+ %320:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %319, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %321.sub0:vreg_64_align2 = COPY killed %67
+ %321.sub1:vreg_64_align2 = COPY killed %66
+ %322:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %321, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %323.sub0:vreg_64_align2 = COPY killed %69
+ %323.sub1:vreg_64_align2 = COPY killed %68
+ %324:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %323, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %325.sub0:vreg_64_align2 = COPY killed %71
+ %325.sub1:vreg_64_align2 = COPY killed %70
+ %326:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %325, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %327.sub0:vreg_64_align2 = COPY killed %73
+ %327.sub1:vreg_64_align2 = COPY killed %72
+ %328:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %327, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %329.sub0:vreg_64_align2 = COPY killed %75
+ %329.sub1:vreg_64_align2 = COPY killed %74
+ %330:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %329, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %331.sub0:vreg_64_align2 = COPY killed %77
+ %331.sub1:vreg_64_align2 = COPY killed %76
+ %332:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %331, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %333.sub0:vreg_512_align2 = COPY %332.sub0
+ %333.sub1:vreg_512_align2 = COPY killed %332.sub1
+ %333.sub2:vreg_512_align2 = COPY %330.sub0
+ %333.sub3:vreg_512_align2 = COPY killed %330.sub1
+ %333.sub4:vreg_512_align2 = COPY %328.sub0
+ %333.sub5:vreg_512_align2 = COPY killed %328.sub1
+ %333.sub6:vreg_512_align2 = COPY %326.sub0
+ %333.sub7:vreg_512_align2 = COPY killed %326.sub1
+ %333.sub8:vreg_512_align2 = COPY %324.sub0
+ %333.sub9:vreg_512_align2 = COPY killed %324.sub1
+ %333.sub10:vreg_512_align2 = COPY %322.sub0
+ %333.sub11:vreg_512_align2 = COPY killed %322.sub1
+ %333.sub12:vreg_512_align2 = COPY %320.sub0
+ %333.sub13:vreg_512_align2 = COPY killed %320.sub1
+ %333.sub14:vreg_512_align2 = COPY %318.sub0
+ %333.sub15:vreg_512_align2 = COPY killed %318.sub1
+ undef %334.sub0:vreg_64_align2 = COPY %101.sub14
+ %334.sub1:vreg_64_align2 = COPY %101.sub15
+ %335:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %334, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %336.sub0:vreg_64_align2 = COPY %101.sub12
+ %336.sub1:vreg_64_align2 = COPY %101.sub13
+ %337:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %336, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %338.sub0:vreg_64_align2 = COPY %101.sub10
+ %338.sub1:vreg_64_align2 = COPY %101.sub11
+ %339:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %338, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %340.sub0:vreg_64_align2 = COPY %101.sub8
+ %340.sub1:vreg_64_align2 = COPY %101.sub9
+ %341:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %340, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %342.sub0:vreg_64_align2 = COPY %101.sub6
+ %342.sub1:vreg_64_align2 = COPY %101.sub7
+ %343:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %342, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %344.sub0:vreg_64_align2 = COPY %101.sub4
+ %344.sub1:vreg_64_align2 = COPY %101.sub5
+ %345:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %344, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %346.sub0:vreg_64_align2 = COPY %101.sub2
+ %346.sub1:vreg_64_align2 = COPY %101.sub3
+ %347:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %346, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ %348:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, %101.sub0_sub1, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %349.sub0:vreg_512_align2 = COPY %348.sub0
+ %349.sub1:vreg_512_align2 = COPY killed %348.sub1
+ %349.sub2:vreg_512_align2 = COPY %347.sub0
+ %349.sub3:vreg_512_align2 = COPY killed %347.sub1
+ %349.sub4:vreg_512_align2 = COPY %345.sub0
+ %349.sub5:vreg_512_align2 = COPY killed %345.sub1
+ %349.sub6:vreg_512_align2 = COPY %343.sub0
+ %349.sub7:vreg_512_align2 = COPY killed %343.sub1
+ %349.sub8:vreg_512_align2 = COPY %341.sub0
+ %349.sub9:vreg_512_align2 = COPY killed %341.sub1
+ %349.sub10:vreg_512_align2 = COPY %339.sub0
+ %349.sub11:vreg_512_align2 = COPY killed %339.sub1
+ %349.sub12:vreg_512_align2 = COPY %337.sub0
+ %349.sub13:vreg_512_align2 = COPY killed %337.sub1
+ %349.sub14:vreg_512_align2 = COPY %335.sub0
+ %349.sub15:vreg_512_align2 = COPY killed %335.sub1
+ undef %350.sub0:vreg_64_align2 = COPY %100.sub14
+ %350.sub1:vreg_64_align2 = COPY %100.sub15
+ %351:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %350, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %352.sub0:vreg_64_align2 = COPY %100.sub12
+ %352.sub1:vreg_64_align2 = COPY %100.sub13
+ %353:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %352, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %354.sub0:vreg_64_align2 = COPY %100.sub10
+ %354.sub1:vreg_64_align2 = COPY %100.sub11
+ %355:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %354, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %356.sub0:vreg_64_align2 = COPY %100.sub8
+ %356.sub1:vreg_64_align2 = COPY %100.sub9
+ %357:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %356, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %358.sub0:vreg_64_align2 = COPY %100.sub6
+ %358.sub1:vreg_64_align2 = COPY %100.sub7
+ %359:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %358, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %360.sub0:vreg_64_align2 = COPY %100.sub4
+ %360.sub1:vreg_64_align2 = COPY %100.sub5
+ %361:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %360, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %362.sub0:vreg_64_align2 = COPY %100.sub2
+ %362.sub1:vreg_64_align2 = COPY %100.sub3
+ %363:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %362, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ %364:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, %100.sub0_sub1, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %365.sub0:vreg_512_align2 = COPY %364.sub0
+ %365.sub1:vreg_512_align2 = COPY killed %364.sub1
+ %365.sub2:vreg_512_align2 = COPY %363.sub0
+ %365.sub3:vreg_512_align2 = COPY killed %363.sub1
+ %365.sub4:vreg_512_align2 = COPY %361.sub0
+ %365.sub5:vreg_512_align2 = COPY killed %361.sub1
+ %365.sub6:vreg_512_align2 = COPY %359.sub0
+ %365.sub7:vreg_512_align2 = COPY killed %359.sub1
+ %365.sub8:vreg_512_align2 = COPY %357.sub0
+ %365.sub9:vreg_512_align2 = COPY killed %357.sub1
+ %365.sub10:vreg_512_align2 = COPY %355.sub0
+ %365.sub11:vreg_512_align2 = COPY killed %355.sub1
+ %365.sub12:vreg_512_align2 = COPY %353.sub0
+ %365.sub13:vreg_512_align2 = COPY killed %353.sub1
+ %365.sub14:vreg_512_align2 = COPY %351.sub0
+ %365.sub15:vreg_512_align2 = COPY killed %351.sub1
+ undef %366.sub0:vreg_64_align2 = COPY %99.sub14
+ %366.sub1:vreg_64_align2 = COPY %99.sub15
+ %367:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %366, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %368.sub0:vreg_64_align2 = COPY %99.sub12
+ %368.sub1:vreg_64_align2 = COPY %99.sub13
+ %369:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %368, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %370.sub0:vreg_64_align2 = COPY %99.sub10
+ %370.sub1:vreg_64_align2 = COPY %99.sub11
+ %371:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %370, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %372.sub0:vreg_64_align2 = COPY %99.sub8
+ %372.sub1:vreg_64_align2 = COPY %99.sub9
+ %373:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %372, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %374.sub0:vreg_64_align2 = COPY %99.sub6
+ %374.sub1:vreg_64_align2 = COPY %99.sub7
+ %375:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %374, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %376.sub0:vreg_64_align2 = COPY %99.sub4
+ %376.sub1:vreg_64_align2 = COPY %99.sub5
+ %377:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %376, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %378.sub0:vreg_64_align2 = COPY %99.sub2
+ %378.sub1:vreg_64_align2 = COPY %99.sub3
+ %379:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %378, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ %380:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, %99.sub0_sub1, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %381.sub0:vreg_512_align2 = COPY %380.sub0
+ %381.sub1:vreg_512_align2 = COPY killed %380.sub1
+ %381.sub2:vreg_512_align2 = COPY %379.sub0
+ %381.sub3:vreg_512_align2 = COPY killed %379.sub1
+ %381.sub4:vreg_512_align2 = COPY %377.sub0
+ %381.sub5:vreg_512_align2 = COPY killed %377.sub1
+ %381.sub6:vreg_512_align2 = COPY %375.sub0
+ %381.sub7:vreg_512_align2 = COPY killed %375.sub1
+ %381.sub8:vreg_512_align2 = COPY %373.sub0
+ %381.sub9:vreg_512_align2 = COPY killed %373.sub1
+ %381.sub10:vreg_512_align2 = COPY %371.sub0
+ %381.sub11:vreg_512_align2 = COPY killed %371.sub1
+ %381.sub12:vreg_512_align2 = COPY %369.sub0
+ %381.sub13:vreg_512_align2 = COPY killed %369.sub1
+ %381.sub14:vreg_512_align2 = COPY %367.sub0
+ %381.sub15:vreg_512_align2 = COPY killed %367.sub1
+ undef %382.sub0:vreg_64_align2 = COPY killed %79
+ %382.sub1:vreg_64_align2 = COPY killed %78
+ %383:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %382, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %384.sub0:vreg_64_align2 = COPY killed %81
+ %384.sub1:vreg_64_align2 = COPY killed %80
+ %385:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %384, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %386.sub0:vreg_64_align2 = COPY killed %83
+ %386.sub1:vreg_64_align2 = COPY killed %82
+ %387:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %386, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %388.sub0:vreg_64_align2 = COPY killed %85
+ %388.sub1:vreg_64_align2 = COPY killed %84
+ %389:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %388, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %390.sub0:vreg_64_align2 = COPY killed %87
+ %390.sub1:vreg_64_align2 = COPY killed %86
+ %391:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %390, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %392.sub0:vreg_64_align2 = COPY killed %89
+ %392.sub1:vreg_64_align2 = COPY killed %88
+ %393:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %392, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %394.sub0:vreg_64_align2 = COPY killed %91
+ %394.sub1:vreg_64_align2 = COPY killed %90
+ %395:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %394, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %396.sub0:vreg_64_align2 = COPY killed %93
+ %396.sub1:vreg_64_align2 = COPY killed %92
+ %397:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %396, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %398.sub0:vreg_512_align2 = COPY %397.sub0
+ %398.sub1:vreg_512_align2 = COPY killed %397.sub1
+ %398.sub2:vreg_512_align2 = COPY %395.sub0
+ %398.sub3:vreg_512_align2 = COPY killed %395.sub1
+ %398.sub4:vreg_512_align2 = COPY %393.sub0
+ %398.sub5:vreg_512_align2 = COPY killed %393.sub1
+ %398.sub6:vreg_512_align2 = COPY %391.sub0
+ %398.sub7:vreg_512_align2 = COPY killed %391.sub1
+ %398.sub8:vreg_512_align2 = COPY %389.sub0
+ %398.sub9:vreg_512_align2 = COPY killed %389.sub1
+ %398.sub10:vreg_512_align2 = COPY %387.sub0
+ %398.sub11:vreg_512_align2 = COPY killed %387.sub1
+ %398.sub12:vreg_512_align2 = COPY %385.sub0
+ %398.sub13:vreg_512_align2 = COPY killed %385.sub1
+ %398.sub14:vreg_512_align2 = COPY %383.sub0
+ %398.sub15:vreg_512_align2 = COPY killed %383.sub1
+ undef %399.sub0:vreg_64_align2 = COPY %97.sub14
+ %399.sub1:vreg_64_align2 = COPY %97.sub15
+ %400:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %399, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %401.sub0:vreg_64_align2 = COPY %97.sub12
+ %401.sub1:vreg_64_align2 = COPY %97.sub13
+ %402:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %401, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %403.sub0:vreg_64_align2 = COPY %97.sub10
+ %403.sub1:vreg_64_align2 = COPY %97.sub11
+ %404:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %403, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %405.sub0:vreg_64_align2 = COPY %97.sub8
+ %405.sub1:vreg_64_align2 = COPY %97.sub9
+ %406:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %405, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %407.sub0:vreg_64_align2 = COPY %97.sub6
+ %407.sub1:vreg_64_align2 = COPY %97.sub7
+ %408:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %407, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %409.sub0:vreg_64_align2 = COPY %97.sub4
+ %409.sub1:vreg_64_align2 = COPY %97.sub5
+ %410:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %409, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %411.sub0:vreg_64_align2 = COPY %97.sub2
+ %411.sub1:vreg_64_align2 = COPY %97.sub3
+ %412:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %411, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ %413:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, %97.sub0_sub1, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %414.sub0:vreg_512_align2 = COPY %413.sub0
+ %414.sub1:vreg_512_align2 = COPY killed %413.sub1
+ %414.sub2:vreg_512_align2 = COPY %412.sub0
+ %414.sub3:vreg_512_align2 = COPY killed %412.sub1
+ %414.sub4:vreg_512_align2 = COPY %410.sub0
+ %414.sub5:vreg_512_align2 = COPY killed %410.sub1
+ %414.sub6:vreg_512_align2 = COPY %408.sub0
+ %414.sub7:vreg_512_align2 = COPY killed %408.sub1
+ %414.sub8:vreg_512_align2 = COPY %406.sub0
+ %414.sub9:vreg_512_align2 = COPY killed %406.sub1
+ %414.sub10:vreg_512_align2 = COPY %404.sub0
+ %414.sub11:vreg_512_align2 = COPY killed %404.sub1
+ %414.sub12:vreg_512_align2 = COPY %402.sub0
+ %414.sub13:vreg_512_align2 = COPY killed %402.sub1
+ %414.sub14:vreg_512_align2 = COPY %400.sub0
+ %414.sub15:vreg_512_align2 = COPY killed %400.sub1
+ undef %415.sub0:vreg_64_align2 = COPY %96.sub14
+ %415.sub1:vreg_64_align2 = COPY %96.sub15
+ %416:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %415, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %417.sub0:vreg_64_align2 = COPY %96.sub12
+ %417.sub1:vreg_64_align2 = COPY %96.sub13
+ %418:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %417, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %419.sub0:vreg_64_align2 = COPY %96.sub10
+ %419.sub1:vreg_64_align2 = COPY %96.sub11
+ %420:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %419, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %421.sub0:vreg_64_align2 = COPY %96.sub8
+ %421.sub1:vreg_64_align2 = COPY %96.sub9
+ %422:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %421, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %423.sub0:vreg_64_align2 = COPY %96.sub6
+ %423.sub1:vreg_64_align2 = COPY %96.sub7
+ %424:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %423, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %425.sub0:vreg_64_align2 = COPY %96.sub4
+ %425.sub1:vreg_64_align2 = COPY %96.sub5
+ %426:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %425, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %427.sub0:vreg_64_align2 = COPY %96.sub2
+ %427.sub1:vreg_64_align2 = COPY %96.sub3
+ %428:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %427, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ %429:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, %96.sub0_sub1, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %430.sub0:vreg_512_align2 = COPY %429.sub0
+ %430.sub1:vreg_512_align2 = COPY killed %429.sub1
+ %430.sub2:vreg_512_align2 = COPY %428.sub0
+ %430.sub3:vreg_512_align2 = COPY killed %428.sub1
+ %430.sub4:vreg_512_align2 = COPY %426.sub0
+ %430.sub5:vreg_512_align2 = COPY killed %426.sub1
+ %430.sub6:vreg_512_align2 = COPY %424.sub0
+ %430.sub7:vreg_512_align2 = COPY killed %424.sub1
+ %430.sub8:vreg_512_align2 = COPY %422.sub0
+ %430.sub9:vreg_512_align2 = COPY killed %422.sub1
+ %430.sub10:vreg_512_align2 = COPY %420.sub0
+ %430.sub11:vreg_512_align2 = COPY killed %420.sub1
+ %430.sub12:vreg_512_align2 = COPY %418.sub0
+ %430.sub13:vreg_512_align2 = COPY killed %418.sub1
+ %430.sub14:vreg_512_align2 = COPY %416.sub0
+ %430.sub15:vreg_512_align2 = COPY killed %416.sub1
+ undef %431.sub0:vreg_64_align2 = COPY %95.sub14
+ %431.sub1:vreg_64_align2 = COPY %95.sub15
+ %432:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %431, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %433.sub0:vreg_64_align2 = COPY %95.sub12
+ %433.sub1:vreg_64_align2 = COPY %95.sub13
+ %434:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %433, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %435.sub0:vreg_64_align2 = COPY %95.sub10
+ %435.sub1:vreg_64_align2 = COPY %95.sub11
+ %436:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %435, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %437.sub0:vreg_64_align2 = COPY %95.sub8
+ %437.sub1:vreg_64_align2 = COPY %95.sub9
+ %438:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %437, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %439.sub0:vreg_64_align2 = COPY %95.sub6
+ %439.sub1:vreg_64_align2 = COPY %95.sub7
+ %440:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %439, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %441.sub0:vreg_64_align2 = COPY %95.sub4
+ %441.sub1:vreg_64_align2 = COPY %95.sub5
+ %442:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %441, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %443.sub0:vreg_64_align2 = COPY %95.sub2
+ %443.sub1:vreg_64_align2 = COPY %95.sub3
+ %444:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %443, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ %445:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, %95.sub0_sub1, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %446.sub0:vreg_512_align2 = COPY %445.sub0
+ %446.sub1:vreg_512_align2 = COPY killed %445.sub1
+ %446.sub2:vreg_512_align2 = COPY %444.sub0
+ %446.sub3:vreg_512_align2 = COPY killed %444.sub1
+ %446.sub4:vreg_512_align2 = COPY %442.sub0
+ %446.sub5:vreg_512_align2 = COPY killed %442.sub1
+ %446.sub6:vreg_512_align2 = COPY %440.sub0
+ %446.sub7:vreg_512_align2 = COPY killed %440.sub1
+ %446.sub8:vreg_512_align2 = COPY %438.sub0
+ %446.sub9:vreg_512_align2 = COPY killed %438.sub1
+ %446.sub10:vreg_512_align2 = COPY %436.sub0
+ %446.sub11:vreg_512_align2 = COPY killed %436.sub1
+ %446.sub12:vreg_512_align2 = COPY %434.sub0
+ %446.sub13:vreg_512_align2 = COPY killed %434.sub1
+ %446.sub14:vreg_512_align2 = COPY %432.sub0
+ %446.sub15:vreg_512_align2 = COPY killed %432.sub1
+ undef %447.sub0:sgpr_128 = COPY %5
+ %447.sub1:sgpr_128 = COPY %5
+ %447.sub2:sgpr_128 = COPY %5
+ %447.sub3:sgpr_128 = COPY %5
+ %448:av_128_align2 = COPY killed %447
+ %449:vreg_512_align2 = COPY killed %220
+ %449:vreg_512_align2 = V_MFMA_F32_32X32X16_BF16_mac_vgprcd_e64 %448, %448, %449, 0, 0, 0, implicit $mode, implicit $exec
+ %450:vreg_512_align2 = COPY killed %236
+ %450:vreg_512_align2 = V_MFMA_F32_32X32X16_BF16_mac_vgprcd_e64 %448, %448, %450, 0, 0, 0, implicit $mode, implicit $exec
+ %451:vreg_512_align2 = COPY killed %252
+ %451:vreg_512_align2 = V_MFMA_F32_32X32X16_BF16_mac_vgprcd_e64 %448, %448, %451, 0, 0, 0, implicit $mode, implicit $exec
+ %452:vreg_512_align2 = COPY killed %268
+ %452:vreg_512_align2 = V_MFMA_F32_32X32X16_BF16_mac_vgprcd_e64 %448, %448, %452, 0, 0, 0, implicit $mode, implicit $exec
+ %453:vreg_512_align2 = COPY killed %284
+ %453:vreg_512_align2 = V_MFMA_F32_32X32X16_BF16_mac_vgprcd_e64 %448, %448, %453, 0, 0, 0, implicit $mode, implicit $exec
+ %454:vreg_512_align2 = COPY killed %300
+ %454:vreg_512_align2 = V_MFMA_F32_32X32X16_BF16_mac_vgprcd_e64 %448, %448, %454, 0, 0, 0, implicit $mode, implicit $exec
+ %455:vreg_512_align2 = COPY killed %316
+ %455:vreg_512_align2 = V_MFMA_F32_32X32X16_BF16_mac_vgprcd_e64 %448, %448, %455, 0, 0, 0, implicit $mode, implicit $exec
+ %456:vreg_512_align2 = COPY killed %333
+ %456:vreg_512_align2 = V_MFMA_F32_32X32X16_BF16_mac_vgprcd_e64 %448, %448, %456, 0, 0, 0, implicit $mode, implicit $exec
+ %457:vreg_512_align2 = COPY killed %349
+ %457:vreg_512_align2 = V_MFMA_F32_32X32X16_BF16_mac_vgprcd_e64 %448, %448, %457, 0, 0, 0, implicit $mode, implicit $exec
+ %458:vreg_512_align2 = COPY killed %365
+ %458:vreg_512_align2 = V_MFMA_F32_32X32X16_BF16_mac_vgprcd_e64 %448, %448, %458, 0, 0, 0, implicit $mode, implicit $exec
+ %459:vreg_512_align2 = COPY killed %381
+ %459:vreg_512_align2 = V_MFMA_F32_32X32X16_BF16_mac_vgprcd_e64 %448, %448, %459, 0, 0, 0, implicit $mode, implicit $exec
+ %460:vreg_512_align2 = COPY killed %398
+ %460:vreg_512_align2 = V_MFMA_F32_32X32X16_BF16_mac_vgprcd_e64 %448, %448, %460, 0, 0, 0, implicit $mode, implicit $exec
+ %461:vreg_512_align2 = COPY killed %414
+ %461:vreg_512_align2 = V_MFMA_F32_32X32X16_BF16_mac_vgprcd_e64 %448, %448, %461, 0, 0, 0, implicit $mode, implicit $exec
+ %462:vreg_512_align2 = COPY killed %430
+ %462:vreg_512_align2 = V_MFMA_F32_32X32X16_BF16_mac_vgprcd_e64 %448, %448, %462, 0, 0, 0, implicit $mode, implicit $exec
+ %463:vreg_512_align2 = COPY killed %446
+ %463:vreg_512_align2 = V_MFMA_F32_32X32X16_BF16_mac_vgprcd_e64 %448, %448, %463, 0, 0, 0, implicit $mode, implicit $exec
+ %464:vreg_512_align2 = COPY killed %449
+ %464:vreg_512_align2 = V_MFMA_F32_32X32X16_BF16_mac_vgprcd_e64 %448, %448, %464, 0, 0, 0, implicit $mode, implicit $exec
+ %465:av_512_align2 = COPY killed %464
+ %466:vreg_512_align2 = COPY killed %450
+ %466:vreg_512_align2 = V_MFMA_F32_32X32X16_BF16_mac_vgprcd_e64 %448, %448, %466, 0, 0, 0, implicit $mode, implicit $exec
+ %467:av_512_align2 = COPY killed %466
+ %468:vreg_512_align2 = COPY killed %451
+ %468:vreg_512_align2 = V_MFMA_F32_32X32X16_BF16_mac_vgprcd_e64 %448, %448, %468, 0, 0, 0, implicit $mode, implicit $exec
+ %469:av_512_align2 = COPY killed %468
+ %470:vreg_512_align2 = COPY killed %452
+ %470:vreg_512_align2 = V_MFMA_F32_32X32X16_BF16_mac_vgprcd_e64 %448, %448, %470, 0, 0, 0, implicit $mode, implicit $exec
+ %471:av_512_align2 = COPY killed %470
+ %472:vreg_512_align2 = COPY killed %453
+ %472:vreg_512_align2 = V_MFMA_F32_32X32X16_BF16_mac_vgprcd_e64 %448, %448, %472, 0, 0, 0, implicit $mode, implicit $exec
+ %473:av_512_align2 = COPY killed %472
+ %474:vreg_512_align2 = COPY killed %454
+ %474:vreg_512_align2 = V_MFMA_F32_32X32X16_BF16_mac_vgprcd_e64 %448, %448, %474, 0, 0, 0, implicit $mode, implicit $exec
+ %475:av_512_align2 = COPY killed %474
+ %476:vreg_512_align2 = COPY killed %455
+ %476:vreg_512_align2 = V_MFMA_F32_32X32X16_BF16_mac_vgprcd_e64 %448, %448, %476, 0, 0, 0, implicit $mode, implicit $exec
+ %477:av_512_align2 = COPY killed %476
+ %478:vreg_512_align2 = COPY killed %456
+ %478:vreg_512_align2 = V_MFMA_F32_32X32X16_BF16_mac_vgprcd_e64 %448, %448, %478, 0, 0, 0, implicit $mode, implicit $exec
+ %479:av_512_align2 = COPY killed %478
+ %480:vreg_512_align2 = COPY killed %457
+ %480:vreg_512_align2 = V_MFMA_F32_32X32X16_BF16_mac_vgprcd_e64 %448, %448, %480, 0, 0, 0, implicit $mode, implicit $exec
+ %481:av_512_align2 = COPY killed %480
+ %482:vreg_512_align2 = COPY killed %458
+ %482:vreg_512_align2 = V_MFMA_F32_32X32X16_BF16_mac_vgprcd_e64 %448, %448, %482, 0, 0, 0, implicit $mode, implicit $exec
+ %483:av_512_align2 = COPY killed %482
+ %484:vreg_512_align2 = COPY killed %459
+ %484:vreg_512_align2 = V_MFMA_F32_32X32X16_BF16_mac_vgprcd_e64 %448, %448, %484, 0, 0, 0, implicit $mode, implicit $exec
+ %485:av_512_align2 = COPY killed %484
+ %486:vreg_512_align2 = COPY killed %460
+ %486:vreg_512_align2 = V_MFMA_F32_32X32X16_BF16_mac_vgprcd_e64 %448, %448, %486, 0, 0, 0, implicit $mode, implicit $exec
+ %487:vreg_512_align2 = COPY killed %461
+ %487:vreg_512_align2 = V_MFMA_F32_32X32X16_BF16_mac_vgprcd_e64 %448, %448, %487, 0, 0, 0, implicit $mode, implicit $exec
+ %488:av_512_align2 = COPY killed %487
+ %489:vreg_512_align2 = COPY killed %462
+ %489:vreg_512_align2 = V_MFMA_F32_32X32X16_BF16_mac_vgprcd_e64 %448, %448, %489, 0, 0, 0, implicit $mode, implicit $exec
+ %490:av_512_align2 = COPY killed %489
+ %491:vreg_512_align2 = COPY killed %463
+ %491:vreg_512_align2 = V_MFMA_F32_32X32X16_BF16_mac_vgprcd_e64 killed %448, %448, %491, 0, 0, 0, implicit $mode, implicit $exec
+ %492:av_512_align2 = COPY killed %491
+ %493:av_32 = COPY %486.sub0
+ %494:av_32 = COPY %486.sub1
+ %495:av_32 = COPY %486.sub2
+ %496:av_32 = COPY %486.sub3
+ %497:av_32 = COPY %486.sub4
+ %498:av_32 = COPY %486.sub5
+ %499:av_32 = COPY %486.sub6
+ %500:av_32 = COPY %486.sub7
+ %501:av_32 = COPY %486.sub8
+ %502:av_32 = COPY %486.sub9
+ %503:av_32 = COPY %486.sub10
+ %504:av_32 = COPY %486.sub11
+ %505:av_32 = COPY %486.sub12
+ %506:av_32 = COPY %486.sub13
+ %507:av_32 = COPY %486.sub14
+ %508:av_32 = COPY killed %486.sub15
+ %509:sreg_64 = S_MOV_B64 0
+ %109:av_32 = COPY killed %508
+ %110:av_32 = COPY killed %507
+ %111:av_32 = COPY killed %506
+ %112:av_32 = COPY killed %505
+ %113:av_32 = COPY killed %504
+ %114:av_32 = COPY killed %503
+ %115:av_32 = COPY killed %502
+ %116:av_32 = COPY killed %501
+ %117:av_32 = COPY killed %500
+ %118:av_32 = COPY killed %499
+ %119:av_32 = COPY killed %498
+ %120:av_32 = COPY killed %497
+ %121:av_32 = COPY killed %496
+ %122:av_32 = COPY killed %495
+ %123:av_32 = COPY killed %494
+ %124:av_32 = COPY killed %493
+ %125:av_512_align2 = COPY killed %492
+ %126:av_512_align2 = COPY killed %490
+ %127:av_512_align2 = COPY killed %488
+ %128:av_512_align2 = COPY killed %485
+ %129:av_512_align2 = COPY killed %483
+ %130:av_512_align2 = COPY killed %481
+ %131:av_512_align2 = COPY killed %479
+ %132:av_512_align2 = COPY killed %477
+ %133:av_512_align2 = COPY killed %475
+ %134:av_512_align2 = COPY killed %473
+ %135:av_512_align2 = COPY killed %471
+ %136:av_512_align2 = COPY killed %469
+ %137:av_512_align2 = COPY killed %467
+ %138:av_512_align2 = COPY killed %465
+ %139:sreg_64_xexec = COPY killed %509
+ S_BRANCH %bb.2
+
+ bb.5:
+ successors: %bb.6(0x04000000), %bb.1(0x7c000000)
+
+ %510:av_32 = COPY killed %201
+ %511:av_32 = COPY killed %200
+ %512:av_32 = COPY killed %199
+ %513:av_32 = COPY killed %198
+ %514:av_32 = COPY killed %197
+ %515:av_32 = COPY killed %196
+ %516:av_32 = COPY killed %195
+ %517:av_32 = COPY killed %194
+ %518:av_32 = COPY killed %193
+ %519:av_32 = COPY killed %192
+ %520:av_32 = COPY killed %191
+ %521:av_32 = COPY killed %190
+ %522:av_32 = COPY killed %189
+ %523:av_32 = COPY killed %188
+ %524:av_32 = COPY killed %187
+ %525:vreg_512_align2 = COPY killed %186
+ %526:vreg_512_align2 = COPY killed %185
+ %527:vreg_512_align2 = COPY killed %184
+ %528:vreg_512_align2 = COPY killed %183
+ %529:vreg_512_align2 = COPY killed %182
+ %530:vreg_512_align2 = COPY killed %181
+ %531:vreg_512_align2 = COPY killed %180
+ %532:vreg_512_align2 = COPY killed %179
+ %533:vreg_512_align2 = COPY killed %178
+ %534:vreg_512_align2 = COPY killed %177
+ %535:vreg_512_align2 = COPY killed %176
+ %536:vreg_512_align2 = COPY killed %175
+ %537:vreg_512_align2 = COPY killed %174
+ %538:vreg_512_align2 = COPY killed %173
+ undef %539.sub0:vreg_64_align2 = COPY %538.sub14
+ %539.sub1:vreg_64_align2 = COPY %538.sub15
+ %540:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %539, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %541.sub0:vreg_64_align2 = COPY %538.sub12
+ %541.sub1:vreg_64_align2 = COPY %538.sub13
+ %542:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %541, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %543.sub0:vreg_64_align2 = COPY %538.sub10
+ %543.sub1:vreg_64_align2 = COPY %538.sub11
+ %544:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %543, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %545.sub0:vreg_64_align2 = COPY %538.sub8
+ %545.sub1:vreg_64_align2 = COPY %538.sub9
+ %546:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %545, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %547.sub0:vreg_64_align2 = COPY %538.sub6
+ %547.sub1:vreg_64_align2 = COPY %538.sub7
+ %548:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %547, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %549.sub0:vreg_64_align2 = COPY %538.sub4
+ %549.sub1:vreg_64_align2 = COPY %538.sub5
+ %550:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %549, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %551.sub0:vreg_64_align2 = COPY %538.sub2
+ %551.sub1:vreg_64_align2 = COPY %538.sub3
+ %552:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %551, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ %553:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %538.sub0_sub1, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %554.sub0:vreg_512_align2 = COPY %553.sub0
+ %554.sub1:vreg_512_align2 = COPY killed %553.sub1
+ %554.sub2:vreg_512_align2 = COPY %552.sub0
+ %554.sub3:vreg_512_align2 = COPY killed %552.sub1
+ %554.sub4:vreg_512_align2 = COPY %550.sub0
+ %554.sub5:vreg_512_align2 = COPY killed %550.sub1
+ %554.sub6:vreg_512_align2 = COPY %548.sub0
+ %554.sub7:vreg_512_align2 = COPY killed %548.sub1
+ %554.sub8:vreg_512_align2 = COPY %546.sub0
+ %554.sub9:vreg_512_align2 = COPY killed %546.sub1
+ %554.sub10:vreg_512_align2 = COPY %544.sub0
+ %554.sub11:vreg_512_align2 = COPY killed %544.sub1
+ %554.sub12:vreg_512_align2 = COPY %542.sub0
+ %554.sub13:vreg_512_align2 = COPY killed %542.sub1
+ %554.sub14:vreg_512_align2 = COPY %540.sub0
+ %554.sub15:vreg_512_align2 = COPY killed %540.sub1
+ undef %555.sub0:vreg_64_align2 = COPY %537.sub14
+ %555.sub1:vreg_64_align2 = COPY %537.sub15
+ %556:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %555, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %557.sub0:vreg_64_align2 = COPY %537.sub12
+ %557.sub1:vreg_64_align2 = COPY %537.sub13
+ %558:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %557, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %559.sub0:vreg_64_align2 = COPY %537.sub10
+ %559.sub1:vreg_64_align2 = COPY %537.sub11
+ %560:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %559, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %561.sub0:vreg_64_align2 = COPY %537.sub8
+ %561.sub1:vreg_64_align2 = COPY %537.sub9
+ %562:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %561, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %563.sub0:vreg_64_align2 = COPY %537.sub6
+ %563.sub1:vreg_64_align2 = COPY %537.sub7
+ %564:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %563, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %565.sub0:vreg_64_align2 = COPY %537.sub4
+ %565.sub1:vreg_64_align2 = COPY %537.sub5
+ %566:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %565, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %567.sub0:vreg_64_align2 = COPY %537.sub2
+ %567.sub1:vreg_64_align2 = COPY %537.sub3
+ %568:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %567, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ %569:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %537.sub0_sub1, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %570.sub0:vreg_512_align2 = COPY %569.sub0
+ %570.sub1:vreg_512_align2 = COPY killed %569.sub1
+ %570.sub2:vreg_512_align2 = COPY %568.sub0
+ %570.sub3:vreg_512_align2 = COPY killed %568.sub1
+ %570.sub4:vreg_512_align2 = COPY %566.sub0
+ %570.sub5:vreg_512_align2 = COPY killed %566.sub1
+ %570.sub6:vreg_512_align2 = COPY %564.sub0
+ %570.sub7:vreg_512_align2 = COPY killed %564.sub1
+ %570.sub8:vreg_512_align2 = COPY %562.sub0
+ %570.sub9:vreg_512_align2 = COPY killed %562.sub1
+ %570.sub10:vreg_512_align2 = COPY %560.sub0
+ %570.sub11:vreg_512_align2 = COPY killed %560.sub1
+ %570.sub12:vreg_512_align2 = COPY %558.sub0
+ %570.sub13:vreg_512_align2 = COPY killed %558.sub1
+ %570.sub14:vreg_512_align2 = COPY %556.sub0
+ %570.sub15:vreg_512_align2 = COPY killed %556.sub1
+ undef %571.sub0:vreg_64_align2 = COPY %536.sub14
+ %571.sub1:vreg_64_align2 = COPY %536.sub15
+ %572:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %571, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %573.sub0:vreg_64_align2 = COPY %536.sub12
+ %573.sub1:vreg_64_align2 = COPY %536.sub13
+ %574:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %573, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %575.sub0:vreg_64_align2 = COPY %536.sub10
+ %575.sub1:vreg_64_align2 = COPY %536.sub11
+ %576:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %575, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %577.sub0:vreg_64_align2 = COPY %536.sub8
+ %577.sub1:vreg_64_align2 = COPY %536.sub9
+ %578:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %577, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %579.sub0:vreg_64_align2 = COPY %536.sub6
+ %579.sub1:vreg_64_align2 = COPY %536.sub7
+ %580:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %579, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %581.sub0:vreg_64_align2 = COPY %536.sub4
+ %581.sub1:vreg_64_align2 = COPY %536.sub5
+ %582:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %581, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %583.sub0:vreg_64_align2 = COPY %536.sub2
+ %583.sub1:vreg_64_align2 = COPY %536.sub3
+ %584:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %583, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ %585:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %536.sub0_sub1, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %586.sub0:vreg_512_align2 = COPY %585.sub0
+ %586.sub1:vreg_512_align2 = COPY killed %585.sub1
+ %586.sub2:vreg_512_align2 = COPY %584.sub0
+ %586.sub3:vreg_512_align2 = COPY killed %584.sub1
+ %586.sub4:vreg_512_align2 = COPY %582.sub0
+ %586.sub5:vreg_512_align2 = COPY killed %582.sub1
+ %586.sub6:vreg_512_align2 = COPY %580.sub0
+ %586.sub7:vreg_512_align2 = COPY killed %580.sub1
+ %586.sub8:vreg_512_align2 = COPY %578.sub0
+ %586.sub9:vreg_512_align2 = COPY killed %578.sub1
+ %586.sub10:vreg_512_align2 = COPY %576.sub0
+ %586.sub11:vreg_512_align2 = COPY killed %576.sub1
+ %586.sub12:vreg_512_align2 = COPY %574.sub0
+ %586.sub13:vreg_512_align2 = COPY killed %574.sub1
+ %586.sub14:vreg_512_align2 = COPY %572.sub0
+ %586.sub15:vreg_512_align2 = COPY killed %572.sub1
+ undef %587.sub0:vreg_64_align2 = COPY %535.sub14
+ %587.sub1:vreg_64_align2 = COPY %535.sub15
+ %588:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %587, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %589.sub0:vreg_64_align2 = COPY %535.sub12
+ %589.sub1:vreg_64_align2 = COPY %535.sub13
+ %590:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %589, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %591.sub0:vreg_64_align2 = COPY %535.sub10
+ %591.sub1:vreg_64_align2 = COPY %535.sub11
+ %592:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %591, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %593.sub0:vreg_64_align2 = COPY %535.sub8
+ %593.sub1:vreg_64_align2 = COPY %535.sub9
+ %594:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %593, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %595.sub0:vreg_64_align2 = COPY %535.sub6
+ %595.sub1:vreg_64_align2 = COPY %535.sub7
+ %596:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %595, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %597.sub0:vreg_64_align2 = COPY %535.sub4
+ %597.sub1:vreg_64_align2 = COPY %535.sub5
+ %598:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %597, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %599.sub0:vreg_64_align2 = COPY %535.sub2
+ %599.sub1:vreg_64_align2 = COPY %535.sub3
+ %600:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %599, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ %601:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %535.sub0_sub1, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %602.sub0:vreg_512_align2 = COPY %601.sub0
+ %602.sub1:vreg_512_align2 = COPY killed %601.sub1
+ %602.sub2:vreg_512_align2 = COPY %600.sub0
+ %602.sub3:vreg_512_align2 = COPY killed %600.sub1
+ %602.sub4:vreg_512_align2 = COPY %598.sub0
+ %602.sub5:vreg_512_align2 = COPY killed %598.sub1
+ %602.sub6:vreg_512_align2 = COPY %596.sub0
+ %602.sub7:vreg_512_align2 = COPY killed %596.sub1
+ %602.sub8:vreg_512_align2 = COPY %594.sub0
+ %602.sub9:vreg_512_align2 = COPY killed %594.sub1
+ %602.sub10:vreg_512_align2 = COPY %592.sub0
+ %602.sub11:vreg_512_align2 = COPY killed %592.sub1
+ %602.sub12:vreg_512_align2 = COPY %590.sub0
+ %602.sub13:vreg_512_align2 = COPY killed %590.sub1
+ %602.sub14:vreg_512_align2 = COPY %588.sub0
+ %602.sub15:vreg_512_align2 = COPY killed %588.sub1
+ undef %603.sub0:vreg_64_align2 = COPY %534.sub14
+ %603.sub1:vreg_64_align2 = COPY %534.sub15
+ %604:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %603, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %605.sub0:vreg_64_align2 = COPY %534.sub12
+ %605.sub1:vreg_64_align2 = COPY %534.sub13
+ %606:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %605, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %607.sub0:vreg_64_align2 = COPY %534.sub10
+ %607.sub1:vreg_64_align2 = COPY %534.sub11
+ %608:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %607, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %609.sub0:vreg_64_align2 = COPY %534.sub8
+ %609.sub1:vreg_64_align2 = COPY %534.sub9
+ %610:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %609, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %611.sub0:vreg_64_align2 = COPY %534.sub6
+ %611.sub1:vreg_64_align2 = COPY %534.sub7
+ %612:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %611, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %613.sub0:vreg_64_align2 = COPY %534.sub4
+ %613.sub1:vreg_64_align2 = COPY %534.sub5
+ %614:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %613, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %615.sub0:vreg_64_align2 = COPY %534.sub2
+ %615.sub1:vreg_64_align2 = COPY %534.sub3
+ %616:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %615, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ %617:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %534.sub0_sub1, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %618.sub0:vreg_512_align2 = COPY %617.sub0
+ %618.sub1:vreg_512_align2 = COPY killed %617.sub1
+ %618.sub2:vreg_512_align2 = COPY %616.sub0
+ %618.sub3:vreg_512_align2 = COPY killed %616.sub1
+ %618.sub4:vreg_512_align2 = COPY %614.sub0
+ %618.sub5:vreg_512_align2 = COPY killed %614.sub1
+ %618.sub6:vreg_512_align2 = COPY %612.sub0
+ %618.sub7:vreg_512_align2 = COPY killed %612.sub1
+ %618.sub8:vreg_512_align2 = COPY %610.sub0
+ %618.sub9:vreg_512_align2 = COPY killed %610.sub1
+ %618.sub10:vreg_512_align2 = COPY %608.sub0
+ %618.sub11:vreg_512_align2 = COPY killed %608.sub1
+ %618.sub12:vreg_512_align2 = COPY %606.sub0
+ %618.sub13:vreg_512_align2 = COPY killed %606.sub1
+ %618.sub14:vreg_512_align2 = COPY %604.sub0
+ %618.sub15:vreg_512_align2 = COPY killed %604.sub1
+ undef %619.sub0:vreg_64_align2 = COPY %533.sub14
+ %619.sub1:vreg_64_align2 = COPY %533.sub15
+ %620:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %619, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %621.sub0:vreg_64_align2 = COPY %533.sub12
+ %621.sub1:vreg_64_align2 = COPY %533.sub13
+ %622:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %621, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %623.sub0:vreg_64_align2 = COPY %533.sub10
+ %623.sub1:vreg_64_align2 = COPY %533.sub11
+ %624:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %623, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %625.sub0:vreg_64_align2 = COPY %533.sub8
+ %625.sub1:vreg_64_align2 = COPY %533.sub9
+ %626:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %625, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %627.sub0:vreg_64_align2 = COPY %533.sub6
+ %627.sub1:vreg_64_align2 = COPY %533.sub7
+ %628:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %627, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %629.sub0:vreg_64_align2 = COPY %533.sub4
+ %629.sub1:vreg_64_align2 = COPY %533.sub5
+ %630:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %629, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %631.sub0:vreg_64_align2 = COPY %533.sub2
+ %631.sub1:vreg_64_align2 = COPY %533.sub3
+ %632:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %631, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ %633:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %533.sub0_sub1, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %634.sub0:vreg_512_align2 = COPY %633.sub0
+ %634.sub1:vreg_512_align2 = COPY killed %633.sub1
+ %634.sub2:vreg_512_align2 = COPY %632.sub0
+ %634.sub3:vreg_512_align2 = COPY killed %632.sub1
+ %634.sub4:vreg_512_align2 = COPY %630.sub0
+ %634.sub5:vreg_512_align2 = COPY killed %630.sub1
+ %634.sub6:vreg_512_align2 = COPY %628.sub0
+ %634.sub7:vreg_512_align2 = COPY killed %628.sub1
+ %634.sub8:vreg_512_align2 = COPY %626.sub0
+ %634.sub9:vreg_512_align2 = COPY killed %626.sub1
+ %634.sub10:vreg_512_align2 = COPY %624.sub0
+ %634.sub11:vreg_512_align2 = COPY killed %624.sub1
+ %634.sub12:vreg_512_align2 = COPY %622.sub0
+ %634.sub13:vreg_512_align2 = COPY killed %622.sub1
+ %634.sub14:vreg_512_align2 = COPY %620.sub0
+ %634.sub15:vreg_512_align2 = COPY killed %620.sub1
+ undef %635.sub0:vreg_64_align2 = COPY %532.sub14
+ %635.sub1:vreg_64_align2 = COPY %532.sub15
+ %636:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %635, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %637.sub0:vreg_64_align2 = COPY %532.sub12
+ %637.sub1:vreg_64_align2 = COPY %532.sub13
+ %638:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %637, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %639.sub0:vreg_64_align2 = COPY %532.sub10
+ %639.sub1:vreg_64_align2 = COPY %532.sub11
+ %640:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %639, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %641.sub0:vreg_64_align2 = COPY %532.sub8
+ %641.sub1:vreg_64_align2 = COPY %532.sub9
+ %642:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %641, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %643.sub0:vreg_64_align2 = COPY %532.sub6
+ %643.sub1:vreg_64_align2 = COPY %532.sub7
+ %644:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %643, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %645.sub0:vreg_64_align2 = COPY %532.sub4
+ %645.sub1:vreg_64_align2 = COPY %532.sub5
+ %646:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %645, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %647.sub0:vreg_64_align2 = COPY %532.sub2
+ %647.sub1:vreg_64_align2 = COPY %532.sub3
+ %648:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %647, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ %649:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %532.sub0_sub1, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %650.sub0:vreg_512_align2 = COPY %649.sub0
+ %650.sub1:vreg_512_align2 = COPY killed %649.sub1
+ %650.sub2:vreg_512_align2 = COPY %648.sub0
+ %650.sub3:vreg_512_align2 = COPY killed %648.sub1
+ %650.sub4:vreg_512_align2 = COPY %646.sub0
+ %650.sub5:vreg_512_align2 = COPY killed %646.sub1
+ %650.sub6:vreg_512_align2 = COPY %644.sub0
+ %650.sub7:vreg_512_align2 = COPY killed %644.sub1
+ %650.sub8:vreg_512_align2 = COPY %642.sub0
+ %650.sub9:vreg_512_align2 = COPY killed %642.sub1
+ %650.sub10:vreg_512_align2 = COPY %640.sub0
+ %650.sub11:vreg_512_align2 = COPY killed %640.sub1
+ %650.sub12:vreg_512_align2 = COPY %638.sub0
+ %650.sub13:vreg_512_align2 = COPY killed %638.sub1
+ %650.sub14:vreg_512_align2 = COPY %636.sub0
+ %650.sub15:vreg_512_align2 = COPY killed %636.sub1
+ undef %651.sub0:vreg_64_align2 = COPY %531.sub14
+ %651.sub1:vreg_64_align2 = COPY %531.sub15
+ %652:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %651, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %653.sub0:vreg_64_align2 = COPY %531.sub12
+ %653.sub1:vreg_64_align2 = COPY %531.sub13
+ %654:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %653, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %655.sub0:vreg_64_align2 = COPY %531.sub10
+ %655.sub1:vreg_64_align2 = COPY %531.sub11
+ %656:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %655, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %657.sub0:vreg_64_align2 = COPY %531.sub8
+ %657.sub1:vreg_64_align2 = COPY %531.sub9
+ %658:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %657, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %659.sub0:vreg_64_align2 = COPY %531.sub6
+ %659.sub1:vreg_64_align2 = COPY %531.sub7
+ %660:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %659, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %661.sub0:vreg_64_align2 = COPY %531.sub4
+ %661.sub1:vreg_64_align2 = COPY %531.sub5
+ %662:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %661, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %663.sub0:vreg_64_align2 = COPY %531.sub2
+ %663.sub1:vreg_64_align2 = COPY %531.sub3
+ %664:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %663, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ %665:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %531.sub0_sub1, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %666.sub0:vreg_512_align2 = COPY %665.sub0
+ %666.sub1:vreg_512_align2 = COPY killed %665.sub1
+ %666.sub2:vreg_512_align2 = COPY %664.sub0
+ %666.sub3:vreg_512_align2 = COPY killed %664.sub1
+ %666.sub4:vreg_512_align2 = COPY %662.sub0
+ %666.sub5:vreg_512_align2 = COPY killed %662.sub1
+ %666.sub6:vreg_512_align2 = COPY %660.sub0
+ %666.sub7:vreg_512_align2 = COPY killed %660.sub1
+ %666.sub8:vreg_512_align2 = COPY %658.sub0
+ %666.sub9:vreg_512_align2 = COPY killed %658.sub1
+ %666.sub10:vreg_512_align2 = COPY %656.sub0
+ %666.sub11:vreg_512_align2 = COPY killed %656.sub1
+ %666.sub12:vreg_512_align2 = COPY %654.sub0
+ %666.sub13:vreg_512_align2 = COPY killed %654.sub1
+ %666.sub14:vreg_512_align2 = COPY %652.sub0
+ %666.sub15:vreg_512_align2 = COPY killed %652.sub1
+ undef %667.sub0:vreg_64_align2 = COPY %530.sub14
+ %667.sub1:vreg_64_align2 = COPY %530.sub15
+ %668:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %667, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %669.sub0:vreg_64_align2 = COPY %530.sub12
+ %669.sub1:vreg_64_align2 = COPY %530.sub13
+ %670:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %669, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %671.sub0:vreg_64_align2 = COPY %530.sub10
+ %671.sub1:vreg_64_align2 = COPY %530.sub11
+ %672:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %671, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %673.sub0:vreg_64_align2 = COPY %530.sub8
+ %673.sub1:vreg_64_align2 = COPY %530.sub9
+ %674:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %673, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %675.sub0:vreg_64_align2 = COPY %530.sub6
+ %675.sub1:vreg_64_align2 = COPY %530.sub7
+ %676:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %675, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %677.sub0:vreg_64_align2 = COPY %530.sub4
+ %677.sub1:vreg_64_align2 = COPY %530.sub5
+ %678:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %677, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %679.sub0:vreg_64_align2 = COPY %530.sub2
+ %679.sub1:vreg_64_align2 = COPY %530.sub3
+ %680:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %679, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ %681:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %530.sub0_sub1, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %682.sub0:vreg_512_align2 = COPY %681.sub0
+ %682.sub1:vreg_512_align2 = COPY killed %681.sub1
+ %682.sub2:vreg_512_align2 = COPY %680.sub0
+ %682.sub3:vreg_512_align2 = COPY killed %680.sub1
+ %682.sub4:vreg_512_align2 = COPY %678.sub0
+ %682.sub5:vreg_512_align2 = COPY killed %678.sub1
+ %682.sub6:vreg_512_align2 = COPY %676.sub0
+ %682.sub7:vreg_512_align2 = COPY killed %676.sub1
+ %682.sub8:vreg_512_align2 = COPY %674.sub0
+ %682.sub9:vreg_512_align2 = COPY killed %674.sub1
+ %682.sub10:vreg_512_align2 = COPY %672.sub0
+ %682.sub11:vreg_512_align2 = COPY killed %672.sub1
+ %682.sub12:vreg_512_align2 = COPY %670.sub0
+ %682.sub13:vreg_512_align2 = COPY killed %670.sub1
+ %682.sub14:vreg_512_align2 = COPY %668.sub0
+ %682.sub15:vreg_512_align2 = COPY killed %668.sub1
+ undef %683.sub0:vreg_64_align2 = COPY %529.sub14
+ %683.sub1:vreg_64_align2 = COPY %529.sub15
+ %684:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %683, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %685.sub0:vreg_64_align2 = COPY %529.sub12
+ %685.sub1:vreg_64_align2 = COPY %529.sub13
+ %686:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %685, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %687.sub0:vreg_64_align2 = COPY %529.sub10
+ %687.sub1:vreg_64_align2 = COPY %529.sub11
+ %688:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %687, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %689.sub0:vreg_64_align2 = COPY %529.sub8
+ %689.sub1:vreg_64_align2 = COPY %529.sub9
+ %690:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %689, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %691.sub0:vreg_64_align2 = COPY %529.sub6
+ %691.sub1:vreg_64_align2 = COPY %529.sub7
+ %692:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %691, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %693.sub0:vreg_64_align2 = COPY %529.sub4
+ %693.sub1:vreg_64_align2 = COPY %529.sub5
+ %694:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %693, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %695.sub0:vreg_64_align2 = COPY %529.sub2
+ %695.sub1:vreg_64_align2 = COPY %529.sub3
+ %696:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %695, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ %697:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %529.sub0_sub1, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %698.sub0:vreg_512_align2 = COPY %697.sub0
+ %698.sub1:vreg_512_align2 = COPY killed %697.sub1
+ %698.sub2:vreg_512_align2 = COPY %696.sub0
+ %698.sub3:vreg_512_align2 = COPY killed %696.sub1
+ %698.sub4:vreg_512_align2 = COPY %694.sub0
+ %698.sub5:vreg_512_align2 = COPY killed %694.sub1
+ %698.sub6:vreg_512_align2 = COPY %692.sub0
+ %698.sub7:vreg_512_align2 = COPY killed %692.sub1
+ %698.sub8:vreg_512_align2 = COPY %690.sub0
+ %698.sub9:vreg_512_align2 = COPY killed %690.sub1
+ %698.sub10:vreg_512_align2 = COPY %688.sub0
+ %698.sub11:vreg_512_align2 = COPY killed %688.sub1
+ %698.sub12:vreg_512_align2 = COPY %686.sub0
+ %698.sub13:vreg_512_align2 = COPY killed %686.sub1
+ %698.sub14:vreg_512_align2 = COPY %684.sub0
+ %698.sub15:vreg_512_align2 = COPY killed %684.sub1
+ undef %699.sub0:vreg_64_align2 = COPY %528.sub14
+ %699.sub1:vreg_64_align2 = COPY %528.sub15
+ %700:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %699, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %701.sub0:vreg_64_align2 = COPY %528.sub12
+ %701.sub1:vreg_64_align2 = COPY %528.sub13
+ %702:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %701, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %703.sub0:vreg_64_align2 = COPY %528.sub10
+ %703.sub1:vreg_64_align2 = COPY %528.sub11
+ %704:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %703, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %705.sub0:vreg_64_align2 = COPY %528.sub8
+ %705.sub1:vreg_64_align2 = COPY %528.sub9
+ %706:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %705, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %707.sub0:vreg_64_align2 = COPY %528.sub6
+ %707.sub1:vreg_64_align2 = COPY %528.sub7
+ %708:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %707, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %709.sub0:vreg_64_align2 = COPY %528.sub4
+ %709.sub1:vreg_64_align2 = COPY %528.sub5
+ %710:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %709, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %711.sub0:vreg_64_align2 = COPY %528.sub2
+ %711.sub1:vreg_64_align2 = COPY %528.sub3
+ %712:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %711, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ %713:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %528.sub0_sub1, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %714.sub0:vreg_512_align2 = COPY %713.sub0
+ %714.sub1:vreg_512_align2 = COPY killed %713.sub1
+ %714.sub2:vreg_512_align2 = COPY %712.sub0
+ %714.sub3:vreg_512_align2 = COPY killed %712.sub1
+ %714.sub4:vreg_512_align2 = COPY %710.sub0
+ %714.sub5:vreg_512_align2 = COPY killed %710.sub1
+ %714.sub6:vreg_512_align2 = COPY %708.sub0
+ %714.sub7:vreg_512_align2 = COPY killed %708.sub1
+ %714.sub8:vreg_512_align2 = COPY %706.sub0
+ %714.sub9:vreg_512_align2 = COPY killed %706.sub1
+ %714.sub10:vreg_512_align2 = COPY %704.sub0
+ %714.sub11:vreg_512_align2 = COPY killed %704.sub1
+ %714.sub12:vreg_512_align2 = COPY %702.sub0
+ %714.sub13:vreg_512_align2 = COPY killed %702.sub1
+ %714.sub14:vreg_512_align2 = COPY %700.sub0
+ %714.sub15:vreg_512_align2 = COPY killed %700.sub1
+ %715:av_32 = COPY killed %202
+ undef %716.sub0:vreg_64_align2 = COPY killed %510
+ %716.sub1:vreg_64_align2 = COPY killed %715
+ %717:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %716, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %718.sub0:vreg_64_align2 = COPY killed %512
+ %718.sub1:vreg_64_align2 = COPY killed %511
+ %719:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %718, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %720.sub0:vreg_64_align2 = COPY killed %514
+ %720.sub1:vreg_64_align2 = COPY killed %513
+ %721:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %720, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %722.sub0:vreg_64_align2 = COPY killed %516
+ %722.sub1:vreg_64_align2 = COPY killed %515
+ %723:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %722, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %724.sub0:vreg_64_align2 = COPY killed %518
+ %724.sub1:vreg_64_align2 = COPY killed %517
+ %725:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %724, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %726.sub0:vreg_64_align2 = COPY killed %520
+ %726.sub1:vreg_64_align2 = COPY killed %519
+ %727:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %726, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %728.sub0:vreg_64_align2 = COPY killed %522
+ %728.sub1:vreg_64_align2 = COPY killed %521
+ %729:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %728, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %730.sub0:vreg_64_align2 = COPY killed %524
+ %730.sub1:vreg_64_align2 = COPY killed %523
+ %731:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %730, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %732.sub0:vreg_512_align2 = COPY %731.sub0
+ %732.sub1:vreg_512_align2 = COPY killed %731.sub1
+ %732.sub2:vreg_512_align2 = COPY %729.sub0
+ %732.sub3:vreg_512_align2 = COPY killed %729.sub1
+ %732.sub4:vreg_512_align2 = COPY %727.sub0
+ %732.sub5:vreg_512_align2 = COPY killed %727.sub1
+ %732.sub6:vreg_512_align2 = COPY %725.sub0
+ %732.sub7:vreg_512_align2 = COPY killed %725.sub1
+ %732.sub8:vreg_512_align2 = COPY %723.sub0
+ %732.sub9:vreg_512_align2 = COPY killed %723.sub1
+ %732.sub10:vreg_512_align2 = COPY %721.sub0
+ %732.sub11:vreg_512_align2 = COPY killed %721.sub1
+ %732.sub12:vreg_512_align2 = COPY %719.sub0
+ %732.sub13:vreg_512_align2 = COPY killed %719.sub1
+ %732.sub14:vreg_512_align2 = COPY %717.sub0
+ %732.sub15:vreg_512_align2 = COPY killed %717.sub1
+ undef %733.sub0:vreg_64_align2 = COPY %527.sub14
+ %733.sub1:vreg_64_align2 = COPY %527.sub15
+ %734:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %733, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %735.sub0:vreg_64_align2 = COPY %527.sub12
+ %735.sub1:vreg_64_align2 = COPY %527.sub13
+ %736:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %735, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %737.sub0:vreg_64_align2 = COPY %527.sub10
+ %737.sub1:vreg_64_align2 = COPY %527.sub11
+ %738:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %737, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %739.sub0:vreg_64_align2 = COPY %527.sub8
+ %739.sub1:vreg_64_align2 = COPY %527.sub9
+ %740:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %739, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %741.sub0:vreg_64_align2 = COPY %527.sub6
+ %741.sub1:vreg_64_align2 = COPY %527.sub7
+ %742:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %741, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %743.sub0:vreg_64_align2 = COPY %527.sub4
+ %743.sub1:vreg_64_align2 = COPY %527.sub5
+ %744:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %743, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %745.sub0:vreg_64_align2 = COPY %527.sub2
+ %745.sub1:vreg_64_align2 = COPY %527.sub3
+ %746:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %745, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ %747:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %527.sub0_sub1, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %748.sub0:vreg_512_align2 = COPY %747.sub0
+ %748.sub1:vreg_512_align2 = COPY killed %747.sub1
+ %748.sub2:vreg_512_align2 = COPY %746.sub0
+ %748.sub3:vreg_512_align2 = COPY killed %746.sub1
+ %748.sub4:vreg_512_align2 = COPY %744.sub0
+ %748.sub5:vreg_512_align2 = COPY killed %744.sub1
+ %748.sub6:vreg_512_align2 = COPY %742.sub0
+ %748.sub7:vreg_512_align2 = COPY killed %742.sub1
+ %748.sub8:vreg_512_align2 = COPY %740.sub0
+ %748.sub9:vreg_512_align2 = COPY killed %740.sub1
+ %748.sub10:vreg_512_align2 = COPY %738.sub0
+ %748.sub11:vreg_512_align2 = COPY killed %738.sub1
+ %748.sub12:vreg_512_align2 = COPY %736.sub0
+ %748.sub13:vreg_512_align2 = COPY killed %736.sub1
+ %748.sub14:vreg_512_align2 = COPY %734.sub0
+ %748.sub15:vreg_512_align2 = COPY killed %734.sub1
+ undef %749.sub0:vreg_64_align2 = COPY %526.sub14
+ %749.sub1:vreg_64_align2 = COPY %526.sub15
+ %750:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %749, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %751.sub0:vreg_64_align2 = COPY %526.sub12
+ %751.sub1:vreg_64_align2 = COPY %526.sub13
+ %752:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %751, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %753.sub0:vreg_64_align2 = COPY %526.sub10
+ %753.sub1:vreg_64_align2 = COPY %526.sub11
+ %754:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %753, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %755.sub0:vreg_64_align2 = COPY %526.sub8
+ %755.sub1:vreg_64_align2 = COPY %526.sub9
+ %756:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %755, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %757.sub0:vreg_64_align2 = COPY %526.sub6
+ %757.sub1:vreg_64_align2 = COPY %526.sub7
+ %758:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %757, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %759.sub0:vreg_64_align2 = COPY %526.sub4
+ %759.sub1:vreg_64_align2 = COPY %526.sub5
+ %760:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %759, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %761.sub0:vreg_64_align2 = COPY %526.sub2
+ %761.sub1:vreg_64_align2 = COPY %526.sub3
+ %762:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %761, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ %763:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %526.sub0_sub1, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %764.sub0:vreg_512_align2 = COPY %763.sub0
+ %764.sub1:vreg_512_align2 = COPY killed %763.sub1
+ %764.sub2:vreg_512_align2 = COPY %762.sub0
+ %764.sub3:vreg_512_align2 = COPY killed %762.sub1
+ %764.sub4:vreg_512_align2 = COPY %760.sub0
+ %764.sub5:vreg_512_align2 = COPY killed %760.sub1
+ %764.sub6:vreg_512_align2 = COPY %758.sub0
+ %764.sub7:vreg_512_align2 = COPY killed %758.sub1
+ %764.sub8:vreg_512_align2 = COPY %756.sub0
+ %764.sub9:vreg_512_align2 = COPY killed %756.sub1
+ %764.sub10:vreg_512_align2 = COPY %754.sub0
+ %764.sub11:vreg_512_align2 = COPY killed %754.sub1
+ %764.sub12:vreg_512_align2 = COPY %752.sub0
+ %764.sub13:vreg_512_align2 = COPY killed %752.sub1
+ %764.sub14:vreg_512_align2 = COPY %750.sub0
+ %764.sub15:vreg_512_align2 = COPY killed %750.sub1
+ undef %765.sub0:vreg_64_align2 = COPY %525.sub14
+ %765.sub1:vreg_64_align2 = COPY %525.sub15
+ %766:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %765, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %767.sub0:vreg_64_align2 = COPY %525.sub12
+ %767.sub1:vreg_64_align2 = COPY %525.sub13
+ %768:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %767, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %769.sub0:vreg_64_align2 = COPY %525.sub10
+ %769.sub1:vreg_64_align2 = COPY %525.sub11
+ %770:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %769, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %771.sub0:vreg_64_align2 = COPY %525.sub8
+ %771.sub1:vreg_64_align2 = COPY %525.sub9
+ %772:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %771, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %773.sub0:vreg_64_align2 = COPY %525.sub6
+ %773.sub1:vreg_64_align2 = COPY %525.sub7
+ %774:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %773, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %775.sub0:vreg_64_align2 = COPY %525.sub4
+ %775.sub1:vreg_64_align2 = COPY %525.sub5
+ %776:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %775, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %777.sub0:vreg_64_align2 = COPY %525.sub2
+ %777.sub1:vreg_64_align2 = COPY %525.sub3
+ %778:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %777, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ %779:vreg_64_align2 = nofpexcept V_PK_MUL_F32 8, killed %525.sub0_sub1, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ undef %780.sub0:vreg_512_align2 = COPY %779.sub0
+ %780.sub1:vreg_512_align2 = COPY killed %779.sub1
+ %780.sub2:vreg_512_align2 = COPY %778.sub0
+ %780.sub3:vreg_512_align2 = COPY killed %778.sub1
+ %780.sub4:vreg_512_align2 = COPY %776.sub0
+ %780.sub5:vreg_512_align2 = COPY killed %776.sub1
+ %780.sub6:vreg_512_align2 = COPY %774.sub0
+ %780.sub7:vreg_512_align2 = COPY killed %774.sub1
+ %780.sub8:vreg_512_align2 = COPY %772.sub0
+ %780.sub9:vreg_512_align2 = COPY killed %772.sub1
+ %780.sub10:vreg_512_align2 = COPY %770.sub0
+ %780.sub11:vreg_512_align2 = COPY killed %770.sub1
+ %780.sub12:vreg_512_align2 = COPY %768.sub0
+ %780.sub13:vreg_512_align2 = COPY killed %768.sub1
+ %780.sub14:vreg_512_align2 = COPY %766.sub0
+ %780.sub15:vreg_512_align2 = COPY killed %766.sub1
+ %781:vreg_512_align2 = COPY killed %602
+ %781:vreg_512_align2 = V_MFMA_F32_32X32X16_BF16_mac_vgprcd_e64 %13, %13, %781, 0, 0, 0, implicit $mode, implicit $exec
+ %782:vreg_512_align2 = COPY killed %618
+ %782:vreg_512_align2 = V_MFMA_F32_32X32X16_BF16_mac_vgprcd_e64 %13, %13, %782, 0, 0, 0, implicit $mode, implicit $exec
+ %783:vreg_512_align2 = COPY killed %634
+ %783:vreg_512_align2 = V_MFMA_F32_32X32X16_BF16_mac_vgprcd_e64 %13, %13, %783, 0, 0, 0, implicit $mode, implicit $exec
+ %784:vreg_512_align2 = COPY killed %650
+ %784:vreg_512_align2 = V_MFMA_F32_32X32X16_BF16_mac_vgprcd_e64 %13, %13, %784, 0, 0, 0, implicit $mode, implicit $exec
+ %785:vreg_512_align2 = COPY killed %666
+ %785:vreg_512_align2 = V_MFMA_F32_32X32X16_BF16_mac_vgprcd_e64 %13, %13, %785, 0, 0, 0, implicit $mode, implicit $exec
+ %786:vreg_512_align2 = COPY killed %682
+ %786:vreg_512_align2 = V_MFMA_F32_32X32X16_BF16_mac_vgprcd_e64 %13, %13, %786, 0, 0, 0, implicit $mode, implicit $exec
+ %787:vreg_512_align2 = COPY killed %698
+ %787:vreg_512_align2 = V_MFMA_F32_32X32X16_BF16_mac_vgprcd_e64 %13, %13, %787, 0, 0, 0, implicit $mode, implicit $exec
+ %788:av_512_align2 = COPY killed %787
+ %789:vreg_512_align2 = COPY killed %714
+ %789:vreg_512_align2 = V_MFMA_F32_32X32X16_BF16_mac_vgprcd_e64 %13, %13, %789, 0, 0, 0, implicit $mode, implicit $exec
+ %790:av_512_align2 = COPY killed %789
+ %791:vreg_512_align2 = COPY killed %732
+ %791:vreg_512_align2 = V_MFMA_F32_32X32X16_BF16_mac_vgprcd_e64 %13, %13, %791, 0, 0, 0, implicit $mode, implicit $exec
+ %792:vreg_512_align2 = COPY killed %748
+ %792:vreg_512_align2 = V_MFMA_F32_32X32X16_BF16_mac_vgprcd_e64 %13, %13, %792, 0, 0, 0, implicit $mode, implicit $exec
+ %793:vreg_512_align2 = COPY killed %764
+ %793:vreg_512_align2 = V_MFMA_F32_32X32X16_BF16_mac_vgprcd_e64 %13, %13, %793, 0, 0, 0, implicit $mode, implicit $exec
+ %794:vreg_512_align2 = COPY killed %780
+ %794:vreg_512_align2 = V_MFMA_F32_32X32X16_BF16_mac_vgprcd_e64 %13, %13, %794, 0, 0, 0, implicit $mode, implicit $exec
+ %795:av_512_align2 = COPY killed %794
+ %796:vreg_512_align2 = COPY killed %554
+ %796:vreg_512_align2 = V_MFMA_F32_32X32X16_BF16_mac_vgprcd_e64 %13, %13, %796, 0, 0, 0, implicit $mode, implicit $exec
+ %797:av_512_align2 = COPY killed %796
+ %798:vreg_512_align2 = COPY killed %570
+ %798:vreg_512_align2 = V_MFMA_F32_32X32X16_BF16_mac_vgprcd_e64 %13, %13, %798, 0, 0, 0, implicit $mode, implicit $exec
+ %799:vreg_512_align2 = COPY killed %798
+ %799:vreg_512_align2 = V_MFMA_F32_32X32X16_BF16_mac_vgprcd_e64 %13, %13, %799, 0, 0, 0, implicit $mode, implicit $exec
+ %800:av_512_align2 = COPY killed %799
+ %801:vreg_512_align2 = COPY killed %586
+ %801:vreg_512_align2 = V_MFMA_F32_32X32X16_BF16_mac_vgprcd_e64 %13, %13, %801, 0, 0, 0, implicit $mode, implicit $exec
+ %802:vreg_512_align2 = COPY killed %801
+ %802:vreg_512_align2 = V_MFMA_F32_32X32X16_BF16_mac_vgprcd_e64 %13, %13, %802, 0, 0, 0, implicit $mode, implicit $exec
+ %803:av_512_align2 = COPY killed %802
+ %804:vreg_512_align2 = COPY killed %781
+ %804:vreg_512_align2 = V_MFMA_F32_32X32X16_BF16_mac_vgprcd_e64 %13, %13, %804, 0, 0, 0, implicit $mode, implicit $exec
+ %805:av_512_align2 = COPY killed %804
+ %806:vreg_512_align2 = COPY killed %782
+ %806:vreg_512_align2 = V_MFMA_F32_32X32X16_BF16_mac_vgprcd_e64 %13, %13, %806, 0, 0, 0, implicit $mode, implicit $exec
+ %807:av_512_align2 = COPY killed %806
+ %808:vreg_512_align2 = COPY killed %783
+ %808:vreg_512_align2 = V_MFMA_F32_32X32X16_BF16_mac_vgprcd_e64 %13, %13, %808, 0, 0, 0, implicit $mode, implicit $exec
+ %809:av_512_align2 = COPY killed %808
+ %810:vreg_512_align2 = COPY killed %784
+ %810:vreg_512_align2 = V_MFMA_F32_32X32X16_BF16_mac_vgprcd_e64 %13, %13, %810, 0, 0, 0, implicit $mode, implicit $exec
+ %811:av_512_align2 = COPY killed %810
+ %812:vreg_512_align2 = COPY killed %786
+ %812:vreg_512_align2 = V_MFMA_F32_32X32X16_BF16_mac_vgprcd_e64 %13, %13, %812, 0, 0, 0, implicit $mode, implicit $exec
+ %813:av_512_align2 = COPY killed %812
+ %814:vreg_512_align2 = COPY killed %98
+ %814:vreg_512_align2 = V_MFMA_F32_32X32X16_BF16_mac_vgprcd_e64 %13, %13, %814, 0, 0, 0, implicit $mode, implicit $exec
+ %815:vreg_512_align2 = COPY killed %792
+ %815:vreg_512_align2 = V_MFMA_F32_32X32X16_BF16_mac_vgprcd_e64 %13, %13, %815, 0, 0, 0, implicit $mode, implicit $exec
+ %816:av_512_align2 = COPY killed %815
+ %817:vreg_512_align2 = COPY killed %793
+ %817:vreg_512_align2 = V_MFMA_F32_32X32X16_BF16_mac_vgprcd_e64 %13, %13, %817, 0, 0, 0, implicit $mode, implicit $exec
+ %818:av_512_align2 = COPY killed %817
+ S_CMP_LG_U64 killed %94, 0, implicit-def $scc
+ %819:av_32 = COPY %791.sub0
+ %820:av_32 = COPY %791.sub1
+ %821:av_32 = COPY %791.sub2
+ %822:av_32 = COPY %791.sub3
+ %823:av_32 = COPY %791.sub4
+ %824:av_32 = COPY %791.sub5
+ %825:av_32 = COPY %791.sub6
+ %826:av_32 = COPY %791.sub7
+ %827:av_32 = COPY %791.sub8
+ %828:av_32 = COPY %791.sub9
+ %829:av_32 = COPY %791.sub10
+ %830:av_32 = COPY %791.sub11
+ %831:av_32 = COPY %791.sub12
+ %832:av_32 = COPY %791.sub13
+ %833:av_32 = COPY %791.sub14
+ %834:av_32 = COPY killed %791.sub15
+ %835:av_32 = COPY %785.sub0
+ %836:av_32 = COPY %785.sub1
+ %837:av_32 = COPY %785.sub2
+ %838:av_32 = COPY %785.sub3
+ %839:av_32 = COPY %785.sub4
+ %840:av_32 = COPY %785.sub5
+ %841:av_32 = COPY %785.sub6
+ %842:av_32 = COPY %785.sub7
+ %843:av_32 = COPY %785.sub8
+ %844:av_32 = COPY %785.sub9
+ %845:av_32 = COPY %785.sub10
+ %846:av_32 = COPY %785.sub11
+ %847:av_32 = COPY %785.sub12
+ %848:av_32 = COPY %785.sub13
+ %849:av_32 = COPY %785.sub14
+ %850:av_32 = COPY killed %785.sub15
+ %851:sreg_64 = S_MOV_B64 1
+ %852:vreg_512_align2 = COPY killed %814, implicit $exec
+ %15:vreg_512_align2 = COPY killed %797
+ %16:vreg_512_align2 = COPY killed %800
+ %17:vreg_512_align2 = COPY killed %803
+ %18:vreg_512_align2 = COPY killed %805
+ %19:vreg_512_align2 = COPY killed %807
+ %20:vreg_512_align2 = COPY killed %809
+ %21:vreg_512_align2 = COPY killed %811
+ %22:vreg_512_align2 = COPY killed %813
+ %23:vreg_512_align2 = COPY killed %788
+ %24:vreg_512_align2 = COPY killed %790
+ %25:vreg_512_align2 = COPY killed %852
+ %26:vreg_512_align2 = COPY killed %816
+ %27:vreg_512_align2 = COPY killed %818
+ %28:vreg_512_align2 = COPY killed %795
+ %29:sreg_64 = COPY killed %851
+ %30:av_32 = COPY killed %819
+ %31:av_32 = COPY killed %820
+ %32:av_32 = COPY killed %821
+ %33:av_32 = COPY killed %822
+ %34:av_32 = COPY killed %823
+ %35:av_32 = COPY killed %824
+ %36:av_32 = COPY killed %825
+ %37:av_32 = COPY killed %826
+ %38:av_32 = COPY killed %827
+ %39:av_32 = COPY killed %828
+ %40:av_32 = COPY killed %829
+ %41:av_32 = COPY killed %830
+ %42:av_32 = COPY killed %831
+ %43:av_32 = COPY killed %832
+ %44:av_32 = COPY killed %833
+ %45:av_32 = COPY killed %834
+ %46:av_32 = COPY killed %835
+ %47:av_32 = COPY killed %836
+ %48:av_32 = COPY killed %837
+ %49:av_32 = COPY killed %838
+ %50:av_32 = COPY killed %839
+ %51:av_32 = COPY killed %840
+ %52:av_32 = COPY killed %841
+ %53:av_32 = COPY killed %842
+ %54:av_32 = COPY killed %843
+ %55:av_32 = COPY killed %844
+ %56:av_32 = COPY killed %845
+ %57:av_32 = COPY killed %846
+ %58:av_32 = COPY killed %847
+ %59:av_32 = COPY killed %848
+ %60:av_32 = COPY killed %849
+ %61:av_32 = COPY killed %850
+ S_CBRANCH_SCC1 %bb.1, implicit killed $scc
+ S_BRANCH %bb.6
+
+ bb.6:
+ S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-implicit-def.ll b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-implicit-def.ll
new file mode 100644
index 0000000000000..9b848866e11b1
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-implicit-def.ll
@@ -0,0 +1,159 @@
+; REQUIRES: asserts
+; RUN: llc -O3 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 \
+; RUN: -stop-after=amdgpu-rewrite-agpr-copy-mfma \
+; RUN: -debug-only=amdgpu-rewrite-agpr-copy-mfma -filetype=null %s 2>&1 \
+; RUN: | FileCheck %s
+
+; Regression test from https://github.com/llvm/llvm-project/issues/196671
+
+; It is legal for a spill reload to not have a dominating spill store.
+; When the AGPR rewrite pass unspills such a slot into a vreg, it must insert
+; IMPLICIT_DEF so the vreg has defs on all paths.
+
+; CHECK: Inserted IMPLICIT_DEF for %{{[0-9]+}} in %bb.{{[0-9]+}}
+
+define amdgpu_kernel void @rewrite_vgpr_mfma_to_agpr_spill_implicit_def(i1 %0, <16 x float> %.sroa.366.2) #0 {
+.lr.ph.i:
+ br label %1
+
+1: ; preds = %51, %.lr.ph.i
+ %.sroa.01121.2 = phi <16 x float> [ zeroinitializer, %.lr.ph.i ], [ %80, %51 ]
+ %.sroa.54.2 = phi <16 x float> [ zeroinitializer, %.lr.ph.i ], [ %82, %51 ]
+ %.sroa.106.2 = phi <16 x float> [ zeroinitializer, %.lr.ph.i ], [ %84, %51 ]
+ %.sroa.1581182.2 = phi <16 x float> [ zeroinitializer, %.lr.ph.i ], [ %85, %51 ]
+ %.sroa.210.2 = phi <16 x float> [ zeroinitializer, %.lr.ph.i ], [ %86, %51 ]
+ %.sroa.262.2 = phi <16 x float> [ zeroinitializer, %.lr.ph.i ], [ %87, %51 ]
+ %.sroa.314.2 = phi <16 x float> [ zeroinitializer, %.lr.ph.i ], [ %88, %51 ]
+ %.sroa.366.21 = phi <16 x float> [ zeroinitializer, %.lr.ph.i ], [ %72, %51 ]
+ %.sroa.418.2 = phi <16 x float> [ zeroinitializer, %.lr.ph.i ], [ %89, %51 ]
+ %.sroa.470.2 = phi <16 x float> [ zeroinitializer, %.lr.ph.i ], [ %74, %51 ]
+ %.sroa.522.2 = phi <16 x float> [ zeroinitializer, %.lr.ph.i ], [ %75, %51 ]
+ %.sroa.574.2 = phi <16 x float> [ zeroinitializer, %.lr.ph.i ], [ %90, %51 ]
+ %.sroa.626.2 = phi <16 x float> [ zeroinitializer, %.lr.ph.i ], [ %76, %51 ]
+ %.sroa.678.2 = phi <16 x float> [ zeroinitializer, %.lr.ph.i ], [ %91, %51 ]
+ %.sroa.730.2 = phi <16 x float> [ zeroinitializer, %.lr.ph.i ], [ %92, %51 ]
+ %.sroa.782.2 = phi <16 x float> [ zeroinitializer, %.lr.ph.i ], [ %79, %51 ]
+ %2 = phi i64 [ 0, %.lr.ph.i ], [ 1, %51 ]
+ br i1 %0, label %3, label %4
+
+3: ; preds = %1
+ store <4 x i32> zeroinitializer, ptr addrspace(5) null, align 16
+ br label %51
+
+4: ; preds = %1
+ %5 = fmul <16 x float> %.sroa.01121.2, zeroinitializer
+ %6 = fmul <16 x float> %.sroa.54.2, zeroinitializer
+ %7 = fmul <16 x float> %.sroa.106.2, zeroinitializer
+ %8 = fmul <16 x float> %.sroa.1581182.2, zeroinitializer
+ %9 = fmul <16 x float> %.sroa.210.2, zeroinitializer
+ %10 = fmul <16 x float> %.sroa.262.2, zeroinitializer
+ %11 = fmul <16 x float> %.sroa.314.2, zeroinitializer
+ %12 = fmul <16 x float> %.sroa.366.21, zeroinitializer
+ %13 = fmul <16 x float> %.sroa.418.2, zeroinitializer
+ %14 = fmul <16 x float> %.sroa.470.2, zeroinitializer
+ %15 = fmul <16 x float> %.sroa.522.2, zeroinitializer
+ %16 = fmul <16 x float> %.sroa.574.2, zeroinitializer
+ %17 = fmul <16 x float> %.sroa.626.2, zeroinitializer
+ %18 = fmul <16 x float> %.sroa.678.2, zeroinitializer
+ %19 = fmul <16 x float> %.sroa.730.2, zeroinitializer
+ %20 = fmul <16 x float> %.sroa.782.2, zeroinitializer
+ %21 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %5, i32 0, i32 0, i32 0)
+ %22 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %6, i32 0, i32 0, i32 0)
+ %23 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %7, i32 0, i32 0, i32 0)
+ %24 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %8, i32 0, i32 0, i32 0)
+ %25 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %9, i32 0, i32 0, i32 0)
+ %26 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %10, i32 0, i32 0, i32 0)
+ %27 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %11, i32 0, i32 0, i32 0)
+ %28 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %12, i32 0, i32 0, i32 0)
+ %29 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %13, i32 0, i32 0, i32 0)
+ %30 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %14, i32 0, i32 0, i32 0)
+ %31 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %15, i32 0, i32 0, i32 0)
+ %32 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %17, i32 0, i32 0, i32 0)
+ %33 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %18, i32 0, i32 0, i32 0)
+ %34 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %19, i32 0, i32 0, i32 0)
+ %35 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %20, i32 0, i32 0, i32 0)
+ %36 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %21, i32 0, i32 0, i32 0)
+ %37 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %22, i32 0, i32 0, i32 0)
+ %38 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %23, i32 0, i32 0, i32 0)
+ %39 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %24, i32 0, i32 0, i32 0)
+ %40 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %25, i32 0, i32 0, i32 0)
+ %41 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %26, i32 0, i32 0, i32 0)
+ %42 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %27, i32 0, i32 0, i32 0)
+ %43 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %28, i32 0, i32 0, i32 0)
+ %44 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %29, i32 0, i32 0, i32 0)
+ %45 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %30, i32 0, i32 0, i32 0)
+ %46 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %31, i32 0, i32 0, i32 0)
+ %47 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %32, i32 0, i32 0, i32 0)
+ %48 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %33, i32 0, i32 0, i32 0)
+ %49 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %34, i32 0, i32 0, i32 0)
+ %50 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %35, i32 0, i32 0, i32 0)
+ br label %51
+
+51: ; preds = %4, %3
+ %.sroa.01121.3 = phi <16 x float> [ %.sroa.01121.2, %3 ], [ %36, %4 ]
+ %.sroa.54.3 = phi <16 x float> [ %.sroa.54.2, %3 ], [ %37, %4 ]
+ %.sroa.106.3 = phi <16 x float> [ %.sroa.106.2, %3 ], [ %38, %4 ]
+ %.sroa.1581182.3 = phi <16 x float> [ %.sroa.1581182.2, %3 ], [ %39, %4 ]
+ %.sroa.210.3 = phi <16 x float> [ %.sroa.210.2, %3 ], [ %40, %4 ]
+ %.sroa.262.3 = phi <16 x float> [ %.sroa.262.2, %3 ], [ %41, %4 ]
+ %.sroa.314.3 = phi <16 x float> [ %.sroa.314.2, %3 ], [ %42, %4 ]
+ %.sroa.366.3 = phi <16 x float> [ %.sroa.366.2, %3 ], [ %43, %4 ]
+ %.sroa.418.3 = phi <16 x float> [ %.sroa.418.2, %3 ], [ %44, %4 ]
+ %.sroa.470.3 = phi <16 x float> [ %.sroa.470.2, %3 ], [ %45, %4 ]
+ %.sroa.522.3 = phi <16 x float> [ %.sroa.522.2, %3 ], [ %46, %4 ]
+ %.sroa.574.3 = phi <16 x float> [ %.sroa.574.2, %3 ], [ %16, %4 ]
+ %.sroa.626.3 = phi <16 x float> [ zeroinitializer, %3 ], [ %47, %4 ]
+ %.sroa.678.3 = phi <16 x float> [ %.sroa.678.2, %3 ], [ %48, %4 ]
+ %.sroa.730.3 = phi <16 x float> [ %.sroa.730.2, %3 ], [ %49, %4 ]
+ %.sroa.782.3 = phi <16 x float> [ %.sroa.782.2, %3 ], [ %50, %4 ]
+ %52 = fmul <16 x float> %.sroa.01121.3, zeroinitializer
+ %53 = fmul <16 x float> %.sroa.54.3, zeroinitializer
+ %54 = fmul <16 x float> %.sroa.106.3, zeroinitializer
+ %55 = fmul <16 x float> %.sroa.1581182.3, zeroinitializer
+ %56 = fmul <16 x float> %.sroa.210.3, zeroinitializer
+ %57 = fmul <16 x float> %.sroa.262.3, zeroinitializer
+ %58 = fmul <16 x float> %.sroa.314.3, zeroinitializer
+ %59 = fmul <16 x float> %.sroa.366.3, zeroinitializer
+ %60 = fmul <16 x float> %.sroa.418.3, zeroinitializer
+ %61 = fmul <16 x float> %.sroa.470.3, zeroinitializer
+ %62 = fmul <16 x float> %.sroa.522.3, zeroinitializer
+ %63 = fmul <16 x float> %.sroa.574.3, zeroinitializer
+ %64 = fmul <16 x float> %.sroa.626.3, zeroinitializer
+ %65 = fmul <16 x float> %.sroa.678.3, zeroinitializer
+ %66 = fmul <16 x float> %.sroa.730.3, zeroinitializer
+ %67 = fmul <16 x float> %.sroa.782.3, zeroinitializer
+ %68 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %55, i32 0, i32 0, i32 0)
+ %69 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %56, i32 0, i32 0, i32 0)
+ %70 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %57, i32 0, i32 0, i32 0)
+ %71 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %58, i32 0, i32 0, i32 0)
+ %72 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %59, i32 0, i32 0, i32 0)
+ %73 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %60, i32 0, i32 0, i32 0)
+ %74 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %61, i32 0, i32 0, i32 0)
+ %75 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %62, i32 0, i32 0, i32 0)
+ %76 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %64, i32 0, i32 0, i32 0)
+ %77 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %65, i32 0, i32 0, i32 0)
+ %78 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %66, i32 0, i32 0, i32 0)
+ %79 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %67, i32 0, i32 0, i32 0)
+ %80 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %52, i32 0, i32 0, i32 0)
+ %81 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %53, i32 0, i32 0, i32 0)
+ %82 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %81, i32 0, i32 0, i32 0)
+ %83 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %54, i32 0, i32 0, i32 0)
+ %84 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %83, i32 0, i32 0, i32 0)
+ %85 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %68, i32 0, i32 0, i32 0)
+ %86 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %69, i32 0, i32 0, i32 0)
+ %87 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %70, i32 0, i32 0, i32 0)
+ %88 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %71, i32 0, i32 0, i32 0)
+ %89 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %73, i32 0, i32 0, i32 0)
+ %90 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %.sroa.574.2, i32 0, i32 0, i32 0)
+ %91 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %77, i32 0, i32 0, i32 0)
+ %92 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %78, i32 0, i32 0, i32 0)
+ %exitcond.not.i = icmp eq i64 %2, 0
+ br i1 %exitcond.not.i, label %._crit_edge.i.loopexit, label %1
+
+._crit_edge.i.loopexit: ; preds = %51
+ ret void
+}
+
+declare <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat>, <8 x bfloat>, <16 x float>, i32 immarg, i32 immarg, i32 immarg)
+
+attributes #0 = { "amdgpu-flat-work-group-size"="1, 256" "target-cpu"="gfx950" }
>From f13a8a3ffef0cc22a82f0dc1be19138d8ac4d299 Mon Sep 17 00:00:00 2001
From: Dhruva Chakrabarti <Dhruva.Chakrabarti at amd.com>
Date: Sat, 23 May 2026 13:30:25 -0500
Subject: [PATCH 02/15] Revert LSS-based implicit def insertion for missing
dominating spill store.
---
.../AMDGPU/AMDGPURewriteAGPRCopyMFMA.cpp | 91 -------------------
1 file changed, 91 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURewriteAGPRCopyMFMA.cpp b/llvm/lib/Target/AMDGPU/AMDGPURewriteAGPRCopyMFMA.cpp
index 7f60a09b4fa14..6510c07358e01 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURewriteAGPRCopyMFMA.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURewriteAGPRCopyMFMA.cpp
@@ -35,7 +35,6 @@
#include "llvm/CodeGen/SlotIndexes.h"
#include "llvm/CodeGen/VirtRegMap.h"
#include "llvm/InitializePasses.h"
-#include "llvm/Support/CommandLine.h"
#include "llvm/Support/DebugCounter.h"
using namespace llvm;
@@ -45,12 +44,6 @@ using namespace llvm;
DEBUG_COUNTER(RewriteAGPRCopyMFMACounter, DEBUG_TYPE,
"Controls which MFMA chains are rewritten to AGPR form");
-static cl::opt<unsigned> ImplicitDefScanLimit(
- "amdgpu-mfma-vgpr-to-agpr-spill-scan-limit", cl::Hidden, cl::init(4),
- cl::desc("Maximum number of instructions to scan forward from a live "
- "segment start when searching for a spill store before "
- "inserting an IMPLICIT_DEF"));
-
namespace {
STATISTIC(NumMFMAsRewrittenToAGPR,
@@ -134,15 +127,6 @@ class AMDGPURewriteAGPRCopyMFMAImpl {
void collectSpillIndexUses(ArrayRef<LiveInterval *> StackIntervals,
SpillReferenceMap &Map) const;
- /// For each segment in the stack slot's LiveInterval whose start does not
- /// correspond to a spill store, insert an IMPLICIT_DEF of \p NewVReg.
- /// This handles a control flow path where there is no spill store dominating
- /// a spill reload. Returns false if any segment start cannot be resolved
- /// (caller should bail out and not unspill).
- bool insertImplicitDefsForLiveSegments(LiveInterval &StackLI, int Slot,
- ArrayRef<MachineInstr *> SpillStores,
- Register NewVReg) const;
-
/// Attempt to unspill VGPRs by finding a free register and replacing the
/// spill instructions with copies.
void eliminateSpillsOfReassignedVGPRs() const;
@@ -491,69 +475,6 @@ void AMDGPURewriteAGPRCopyMFMAImpl::collectSpillIndexUses(
}
}
-bool AMDGPURewriteAGPRCopyMFMAImpl::insertImplicitDefsForLiveSegments(
- LiveInterval &StackLI, int Slot, ArrayRef<MachineInstr *> SpillStores,
- Register NewVReg) const {
- const unsigned ScanLimit = ImplicitDefScanLimit;
-
- SmallPtrSet<MachineInstr *, 4> StoreSet(SpillStores.begin(),
- SpillStores.end());
-
- SmallVector<MachineInstr *, 2> InsertedDefs;
-
- for (const LiveRange::Segment &Seg : StackLI) {
- MachineInstr *MI = LIS.getInstructionFromIndex(Seg.start);
- // Fall back to LIS query in case the segment start does not correspond
- // to an instruction.
- MachineBasicBlock *MBB =
- MI ? MI->getParent() : LIS.getMBBFromIndex(Seg.start);
-
- // Bail-out: we could not resolve any MBB for this segment start.
- // Roll back any defs already inserted for earlier segments -- the
- // caller must skip unspilling this slot entirely.
- if (!MBB) {
- for (MachineInstr *Def : InsertedDefs) {
- LIS.RemoveMachineInstrFromMaps(*Def);
- Def->eraseFromParent();
- }
- return false;
- }
-
- // Scan forward from the segment start looking for a spill store to this
- // slot. LiveStacks liveness for a store can start a few instructions
- // before the store itself, so we allow a small window controlled by
- // ImplicitDefScanLimit. Redundant IMPLICIT_DEFs (e.g. because of a small
- // scan limit) are harmless, since they will be cleaned up by downstream
- // transformations.
- MachineBasicBlock::iterator It = MI ? MI->getIterator() : MBB->begin();
- bool FoundStore = false;
- for (unsigned I = 0; I < ScanLimit && It != MBB->end(); ++I, ++It) {
- if (StoreSet.count(&*It)) {
- FoundStore = true;
- break;
- }
- }
-
- // A spill store is found for this segment, so no IMPLICIT_DEF is needed.
- if (FoundStore)
- continue;
-
- // Insert an IMPLICIT_DEF to provide a reaching definition for NewVReg.
- MachineBasicBlock::iterator InsertPt =
- MI ? MI->getIterator() : MBB->begin();
- MachineInstr *ImpDef =
- BuildMI(*MBB, InsertPt, DebugLoc(), TII.get(TargetOpcode::IMPLICIT_DEF),
- NewVReg);
- LIS.InsertMachineInstrInMaps(*ImpDef);
- InsertedDefs.push_back(ImpDef);
-
- LLVM_DEBUG(dbgs() << "Inserted IMPLICIT_DEF for " << printReg(NewVReg)
- << " in " << printMBBReference(*MBB) << '\n');
- }
-
- return true;
-}
-
void AMDGPURewriteAGPRCopyMFMAImpl::eliminateSpillsOfReassignedVGPRs() const {
unsigned NumSlots = LSS.getNumIntervals();
if (NumSlots == 0)
@@ -624,18 +545,6 @@ void AMDGPURewriteAGPRCopyMFMAImpl::eliminateSpillsOfReassignedVGPRs() const {
const TargetRegisterClass *RC = LSS.getIntervalRegClass(Slot);
Register NewVReg = MRI.createVirtualRegister(RC);
- // It is legal for a spill reload to not have a dominating spill store.
- // But after un-spilling, the replacement vreg must have reaching defs
- // on all paths. Ensure this condition by inserting IMPLICIT_DEFs if
- // required. Bail out if segment starts cannot be resolved.
- SmallVector<MachineInstr *, 4> SpillStores;
- for (MachineInstr *MI : SpillReferences->second)
- if (MI->mayStore())
- SpillStores.push_back(MI);
-
- if (!insertImplicitDefsForLiveSegments(*LI, Slot, SpillStores, NewVReg))
- continue;
-
for (MachineInstr *SpillMI : SpillReferences->second)
replaceSpillWithCopyToVReg(*SpillMI, Slot, NewVReg);
>From abd33ef00fc27e202d08e5cacfcecac764624837 Mon Sep 17 00:00:00 2001
From: theRonShark <rlieberm at amd.com>
Date: Fri, 21 Nov 2025 06:07:52 -0500
Subject: [PATCH 03/15] Verify dominance when rewriting spills to registers
Co-authored by: Austin Kerbow <Austin.Kerbow at amd.com>
Cherry-picked https://github.com/ROCm/llvm-project/commit/e5d02ddb
---
.../AMDGPU/AMDGPURewriteAGPRCopyMFMA.cpp | 92 ++++++++++++++++++-
1 file changed, 88 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURewriteAGPRCopyMFMA.cpp b/llvm/lib/Target/AMDGPU/AMDGPURewriteAGPRCopyMFMA.cpp
index 6510c07358e01..1c236619987aa 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURewriteAGPRCopyMFMA.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURewriteAGPRCopyMFMA.cpp
@@ -30,6 +30,7 @@
#include "llvm/CodeGen/LiveIntervals.h"
#include "llvm/CodeGen/LiveRegMatrix.h"
#include "llvm/CodeGen/LiveStacks.h"
+#include "llvm/CodeGen/MachineDominators.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
#include "llvm/CodeGen/SlotIndexes.h"
@@ -63,6 +64,7 @@ class AMDGPURewriteAGPRCopyMFMAImpl {
LiveIntervals &LIS;
LiveStacks &LSS;
const RegisterClassInfo &RegClassInfo;
+ MachineDominatorTree &MDT;
bool attemptReassignmentsToAGPR(SmallSetVector<Register, 4> &InterferingRegs,
MCPhysReg PrefPhysReg) const;
@@ -71,10 +73,11 @@ class AMDGPURewriteAGPRCopyMFMAImpl {
AMDGPURewriteAGPRCopyMFMAImpl(MachineFunction &MF, VirtRegMap &VRM,
LiveRegMatrix &LRM, LiveIntervals &LIS,
LiveStacks &LSS,
- const RegisterClassInfo &RegClassInfo)
+ const RegisterClassInfo &RegClassInfo,
+ MachineDominatorTree &MDT)
: MF(MF), ST(MF.getSubtarget<GCNSubtarget>()), TII(*ST.getInstrInfo()),
TRI(*ST.getRegisterInfo()), MRI(MF.getRegInfo()), VRM(VRM), LRM(LRM),
- LIS(LIS), LSS(LSS), RegClassInfo(RegClassInfo) {}
+ LIS(LIS), LSS(LSS), RegClassInfo(RegClassInfo), MDT(MDT) {}
bool isRewriteCandidate(const MachineInstr &MI) const {
return TII.isMAI(MI) && AMDGPU::getAGPRFormOp(MI.getOpcode()) != -1;
@@ -528,6 +531,82 @@ void AMDGPURewriteAGPRCopyMFMAImpl::eliminateSpillsOfReassignedVGPRs() const {
if (SpillReferences == SpillSlotReferences.end())
continue;
+ // For each spill reload, every path from entry to the reload must pass
+ // through at least one spill store to the same stack slot.
+ SmallVector<MachineInstr *, 4> Stores, Loads;
+ Stores.reserve(SpillReferences->second.size());
+ Loads.reserve(SpillReferences->second.size());
+ for (MachineInstr *MI : SpillReferences->second) {
+ if (MI->mayStore())
+ Stores.push_back(MI);
+ else if (MI->mayLoad())
+ Loads.push_back(MI);
+ }
+
+ SmallPtrSet<MachineBasicBlock *, 4> StoreBlocks;
+ for (MachineInstr *S : Stores)
+ if (MDT.isReachableFromEntry(S->getParent()))
+ StoreBlocks.insert(S->getParent());
+
+ if (StoreBlocks.empty()) {
+ LLVM_DEBUG(dbgs() << "Skipping " << printReg(Slot, &TRI)
+ << ": no reachable stores\n");
+ continue;
+ }
+
+ // Compute blocks reachable from entry without passing through a store
+ // block.
+ SmallPtrSet<MachineBasicBlock *, 16> StoreFreeReachable;
+ SmallVector<MachineBasicBlock *, 16> Worklist;
+
+ MachineBasicBlock &EntryMBB = MF.front();
+ Worklist.push_back(&EntryMBB);
+ StoreFreeReachable.insert(&EntryMBB);
+
+ while (!Worklist.empty()) {
+ MachineBasicBlock *MBB = Worklist.pop_back_val();
+ if (StoreBlocks.contains(MBB))
+ continue;
+
+ for (MachineBasicBlock *Succ : MBB->successors()) {
+ if (StoreFreeReachable.insert(Succ).second)
+ Worklist.push_back(Succ);
+ }
+ }
+
+ auto IsLoadJointlyDominatedByStores = [&](MachineInstr *LoadMI) -> bool {
+ MachineBasicBlock *LoadMBB = LoadMI->getParent();
+ if (!MDT.isReachableFromEntry(LoadMBB))
+ return true;
+
+ // Check if every path passed through a store block.
+ if (!StoreFreeReachable.contains(LoadMBB))
+ return true;
+
+ // Otherwise, there exists a path to this block that has not seen any
+ // store yet. We must ensure that within this block there is a store to
+ // this slot before the load.
+ for (MachineInstr &MI : *LoadMBB) {
+ if (&MI == LoadMI)
+ break;
+ if (MI.mayStore()) {
+ for (MachineOperand &MO : MI.operands()) {
+ if (MO.isFI() && MO.getIndex() == Slot)
+ return true;
+ }
+ }
+ }
+
+ return false;
+ };
+
+ if (!llvm::all_of(Loads, IsLoadJointlyDominatedByStores)) {
+ LLVM_DEBUG(
+ dbgs() << "Skipping " << printReg(Slot, &TRI)
+ << ": some reachable load not jointly dominated by stores\n");
+ continue;
+ }
+
const TargetRegisterClass *RC = LSS.getIntervalRegClass(Slot);
LLVM_DEBUG(dbgs() << "Trying to eliminate " << printReg(Slot, &TRI)
@@ -632,11 +711,13 @@ class AMDGPURewriteAGPRCopyMFMALegacy : public MachineFunctionPass {
AU.addRequired<VirtRegMapWrapperLegacy>();
AU.addRequired<LiveRegMatrixWrapperLegacy>();
AU.addRequired<LiveStacksWrapperLegacy>();
+ AU.addRequired<MachineDominatorTreeWrapperPass>();
AU.addPreserved<LiveIntervalsWrapperPass>();
AU.addPreserved<VirtRegMapWrapperLegacy>();
AU.addPreserved<LiveRegMatrixWrapperLegacy>();
AU.addPreserved<LiveStacksWrapperLegacy>();
+ AU.addPreserved<MachineDominatorTreeWrapperPass>();
AU.setPreservesAll();
MachineFunctionPass::getAnalysisUsage(AU);
@@ -651,6 +732,7 @@ INITIALIZE_PASS_DEPENDENCY(LiveIntervalsWrapperPass)
INITIALIZE_PASS_DEPENDENCY(VirtRegMapWrapperLegacy)
INITIALIZE_PASS_DEPENDENCY(LiveRegMatrixWrapperLegacy)
INITIALIZE_PASS_DEPENDENCY(LiveStacksWrapperLegacy)
+INITIALIZE_PASS_DEPENDENCY(MachineDominatorTreeWrapperPass)
INITIALIZE_PASS_END(AMDGPURewriteAGPRCopyMFMALegacy, DEBUG_TYPE,
"AMDGPU Rewrite AGPR-Copy-MFMA", false, false)
@@ -670,7 +752,8 @@ bool AMDGPURewriteAGPRCopyMFMALegacy::runOnMachineFunction(
auto &LRM = getAnalysis<LiveRegMatrixWrapperLegacy>().getLRM();
auto &LIS = getAnalysis<LiveIntervalsWrapperPass>().getLIS();
auto &LSS = getAnalysis<LiveStacksWrapperLegacy>().getLS();
- AMDGPURewriteAGPRCopyMFMAImpl Impl(MF, VRM, LRM, LIS, LSS, RegClassInfo);
+ auto &MDT = getAnalysis<MachineDominatorTreeWrapperPass>().getDomTree();
+ AMDGPURewriteAGPRCopyMFMAImpl Impl(MF, VRM, LRM, LIS, LSS, RegClassInfo, MDT);
return Impl.run(MF);
}
@@ -681,10 +764,11 @@ AMDGPURewriteAGPRCopyMFMAPass::run(MachineFunction &MF,
LiveRegMatrix &LRM = MFAM.getResult<LiveRegMatrixAnalysis>(MF);
LiveIntervals &LIS = MFAM.getResult<LiveIntervalsAnalysis>(MF);
LiveStacks &LSS = MFAM.getResult<LiveStacksAnalysis>(MF);
+ MachineDominatorTree &MDT = MFAM.getResult<MachineDominatorTreeAnalysis>(MF);
RegisterClassInfo RegClassInfo;
RegClassInfo.runOnMachineFunction(MF);
- AMDGPURewriteAGPRCopyMFMAImpl Impl(MF, VRM, LRM, LIS, LSS, RegClassInfo);
+ AMDGPURewriteAGPRCopyMFMAImpl Impl(MF, VRM, LRM, LIS, LSS, RegClassInfo, MDT);
if (!Impl.run(MF))
return PreservedAnalyses::all();
auto PA = getMachineFunctionPassPreservedAnalyses();
>From ed952916e0a3d01bf0f8ae47b762ccbf30b5cd6d Mon Sep 17 00:00:00 2001
From: Dhruva Chakrabarti <Dhruva.Chakrabarti at amd.com>
Date: Sat, 23 May 2026 14:33:57 -0500
Subject: [PATCH 04/15] Changed test debug message to reflect dominance check.
---
.../AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-implicit-def-mir.mir | 2 +-
.../AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-implicit-def.ll | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-implicit-def-mir.mir b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-implicit-def-mir.mir
index 5858b29b5bbe8..27cd709c2cae6 100644
--- a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-implicit-def-mir.mir
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-implicit-def-mir.mir
@@ -9,7 +9,7 @@
# When the AGPR rewrite pass unspills such a slot into a vreg, it must insert
# IMPLICIT_DEF so the vreg has defs on all paths.
-# CHECK: Inserted IMPLICIT_DEF for %{{[0-9]+}} in %bb.{{[0-9]+}}
+# CHECK: some reachable load not jointly dominated by stores
--- |
define amdgpu_kernel void @rewrite_vgpr_mfma_to_agpr_spill_implicit_def(i1 %0, <16 x float> %.sroa.366.2) #0 {
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-implicit-def.ll b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-implicit-def.ll
index 9b848866e11b1..6da770eb4e7dd 100644
--- a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-implicit-def.ll
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-implicit-def.ll
@@ -10,7 +10,7 @@
; When the AGPR rewrite pass unspills such a slot into a vreg, it must insert
; IMPLICIT_DEF so the vreg has defs on all paths.
-; CHECK: Inserted IMPLICIT_DEF for %{{[0-9]+}} in %bb.{{[0-9]+}}
+; CHECK: some reachable load not jointly dominated by stores
define amdgpu_kernel void @rewrite_vgpr_mfma_to_agpr_spill_implicit_def(i1 %0, <16 x float> %.sroa.366.2) #0 {
.lr.ph.i:
>From f32b6ec43f9ec59778571b54839665a5a187d897 Mon Sep 17 00:00:00 2001
From: Dhruva Chakrabarti <Dhruva.Chakrabarti at amd.com>
Date: Tue, 26 May 2026 13:02:14 -0500
Subject: [PATCH 05/15] Ran instnamer to use named values.
---
...pr-mfma-to-agpr-spill-implicit-def-mir.mir | 2 +-
...te-vgpr-mfma-to-agpr-spill-implicit-def.ll | 264 +++++++++---------
2 files changed, 133 insertions(+), 133 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-implicit-def-mir.mir b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-implicit-def-mir.mir
index 27cd709c2cae6..c7b0b962e382f 100644
--- a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-implicit-def-mir.mir
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-implicit-def-mir.mir
@@ -12,7 +12,7 @@
# CHECK: some reachable load not jointly dominated by stores
--- |
- define amdgpu_kernel void @rewrite_vgpr_mfma_to_agpr_spill_implicit_def(i1 %0, <16 x float> %.sroa.366.2) #0 {
+ define amdgpu_kernel void @rewrite_vgpr_mfma_to_agpr_spill_implicit_def(i1 %arg, <16 x float> %.sroa.366.2) #0 {
%kernarg.segment = call ptr addrspace(4) @llvm.amdgcn.kernarg.segment.ptr()
%.kernarg.offset.align.down66 = bitcast ptr addrspace(4) %kernarg.segment to ptr addrspace(4)
%.sroa.366.2.kernarg.offset = getelementptr inbounds i8, ptr addrspace(4) %kernarg.segment, i64 64
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-implicit-def.ll b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-implicit-def.ll
index 6da770eb4e7dd..d25826121f100 100644
--- a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-implicit-def.ll
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-implicit-def.ll
@@ -12,145 +12,145 @@
; CHECK: some reachable load not jointly dominated by stores
-define amdgpu_kernel void @rewrite_vgpr_mfma_to_agpr_spill_implicit_def(i1 %0, <16 x float> %.sroa.366.2) #0 {
+define amdgpu_kernel void @rewrite_vgpr_mfma_to_agpr_spill_implicit_def(i1 %arg, <16 x float> %.sroa.366.2) #0 {
.lr.ph.i:
- br label %1
+ br label %bb
-1: ; preds = %51, %.lr.ph.i
- %.sroa.01121.2 = phi <16 x float> [ zeroinitializer, %.lr.ph.i ], [ %80, %51 ]
- %.sroa.54.2 = phi <16 x float> [ zeroinitializer, %.lr.ph.i ], [ %82, %51 ]
- %.sroa.106.2 = phi <16 x float> [ zeroinitializer, %.lr.ph.i ], [ %84, %51 ]
- %.sroa.1581182.2 = phi <16 x float> [ zeroinitializer, %.lr.ph.i ], [ %85, %51 ]
- %.sroa.210.2 = phi <16 x float> [ zeroinitializer, %.lr.ph.i ], [ %86, %51 ]
- %.sroa.262.2 = phi <16 x float> [ zeroinitializer, %.lr.ph.i ], [ %87, %51 ]
- %.sroa.314.2 = phi <16 x float> [ zeroinitializer, %.lr.ph.i ], [ %88, %51 ]
- %.sroa.366.21 = phi <16 x float> [ zeroinitializer, %.lr.ph.i ], [ %72, %51 ]
- %.sroa.418.2 = phi <16 x float> [ zeroinitializer, %.lr.ph.i ], [ %89, %51 ]
- %.sroa.470.2 = phi <16 x float> [ zeroinitializer, %.lr.ph.i ], [ %74, %51 ]
- %.sroa.522.2 = phi <16 x float> [ zeroinitializer, %.lr.ph.i ], [ %75, %51 ]
- %.sroa.574.2 = phi <16 x float> [ zeroinitializer, %.lr.ph.i ], [ %90, %51 ]
- %.sroa.626.2 = phi <16 x float> [ zeroinitializer, %.lr.ph.i ], [ %76, %51 ]
- %.sroa.678.2 = phi <16 x float> [ zeroinitializer, %.lr.ph.i ], [ %91, %51 ]
- %.sroa.730.2 = phi <16 x float> [ zeroinitializer, %.lr.ph.i ], [ %92, %51 ]
- %.sroa.782.2 = phi <16 x float> [ zeroinitializer, %.lr.ph.i ], [ %79, %51 ]
- %2 = phi i64 [ 0, %.lr.ph.i ], [ 1, %51 ]
- br i1 %0, label %3, label %4
+bb: ; preds = %bb49, %.lr.ph.i
+ %.sroa.01121.2 = phi <16 x float> [ zeroinitializer, %.lr.ph.i ], [ %i78, %bb49 ]
+ %.sroa.54.2 = phi <16 x float> [ zeroinitializer, %.lr.ph.i ], [ %i80, %bb49 ]
+ %.sroa.106.2 = phi <16 x float> [ zeroinitializer, %.lr.ph.i ], [ %i82, %bb49 ]
+ %.sroa.1581182.2 = phi <16 x float> [ zeroinitializer, %.lr.ph.i ], [ %i83, %bb49 ]
+ %.sroa.210.2 = phi <16 x float> [ zeroinitializer, %.lr.ph.i ], [ %i84, %bb49 ]
+ %.sroa.262.2 = phi <16 x float> [ zeroinitializer, %.lr.ph.i ], [ %i85, %bb49 ]
+ %.sroa.314.2 = phi <16 x float> [ zeroinitializer, %.lr.ph.i ], [ %i86, %bb49 ]
+ %.sroa.366.21 = phi <16 x float> [ zeroinitializer, %.lr.ph.i ], [ %i70, %bb49 ]
+ %.sroa.418.2 = phi <16 x float> [ zeroinitializer, %.lr.ph.i ], [ %i87, %bb49 ]
+ %.sroa.470.2 = phi <16 x float> [ zeroinitializer, %.lr.ph.i ], [ %i72, %bb49 ]
+ %.sroa.522.2 = phi <16 x float> [ zeroinitializer, %.lr.ph.i ], [ %i73, %bb49 ]
+ %.sroa.574.2 = phi <16 x float> [ zeroinitializer, %.lr.ph.i ], [ %i88, %bb49 ]
+ %.sroa.626.2 = phi <16 x float> [ zeroinitializer, %.lr.ph.i ], [ %i74, %bb49 ]
+ %.sroa.678.2 = phi <16 x float> [ zeroinitializer, %.lr.ph.i ], [ %i89, %bb49 ]
+ %.sroa.730.2 = phi <16 x float> [ zeroinitializer, %.lr.ph.i ], [ %i90, %bb49 ]
+ %.sroa.782.2 = phi <16 x float> [ zeroinitializer, %.lr.ph.i ], [ %i77, %bb49 ]
+ %i = phi i64 [ 0, %.lr.ph.i ], [ 1, %bb49 ]
+ br i1 %arg, label %bb1, label %bb2
-3: ; preds = %1
+bb1: ; preds = %bb
store <4 x i32> zeroinitializer, ptr addrspace(5) null, align 16
- br label %51
+ br label %bb49
-4: ; preds = %1
- %5 = fmul <16 x float> %.sroa.01121.2, zeroinitializer
- %6 = fmul <16 x float> %.sroa.54.2, zeroinitializer
- %7 = fmul <16 x float> %.sroa.106.2, zeroinitializer
- %8 = fmul <16 x float> %.sroa.1581182.2, zeroinitializer
- %9 = fmul <16 x float> %.sroa.210.2, zeroinitializer
- %10 = fmul <16 x float> %.sroa.262.2, zeroinitializer
- %11 = fmul <16 x float> %.sroa.314.2, zeroinitializer
- %12 = fmul <16 x float> %.sroa.366.21, zeroinitializer
- %13 = fmul <16 x float> %.sroa.418.2, zeroinitializer
- %14 = fmul <16 x float> %.sroa.470.2, zeroinitializer
- %15 = fmul <16 x float> %.sroa.522.2, zeroinitializer
- %16 = fmul <16 x float> %.sroa.574.2, zeroinitializer
- %17 = fmul <16 x float> %.sroa.626.2, zeroinitializer
- %18 = fmul <16 x float> %.sroa.678.2, zeroinitializer
- %19 = fmul <16 x float> %.sroa.730.2, zeroinitializer
- %20 = fmul <16 x float> %.sroa.782.2, zeroinitializer
- %21 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %5, i32 0, i32 0, i32 0)
- %22 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %6, i32 0, i32 0, i32 0)
- %23 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %7, i32 0, i32 0, i32 0)
- %24 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %8, i32 0, i32 0, i32 0)
- %25 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %9, i32 0, i32 0, i32 0)
- %26 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %10, i32 0, i32 0, i32 0)
- %27 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %11, i32 0, i32 0, i32 0)
- %28 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %12, i32 0, i32 0, i32 0)
- %29 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %13, i32 0, i32 0, i32 0)
- %30 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %14, i32 0, i32 0, i32 0)
- %31 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %15, i32 0, i32 0, i32 0)
- %32 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %17, i32 0, i32 0, i32 0)
- %33 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %18, i32 0, i32 0, i32 0)
- %34 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %19, i32 0, i32 0, i32 0)
- %35 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %20, i32 0, i32 0, i32 0)
- %36 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %21, i32 0, i32 0, i32 0)
- %37 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %22, i32 0, i32 0, i32 0)
- %38 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %23, i32 0, i32 0, i32 0)
- %39 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %24, i32 0, i32 0, i32 0)
- %40 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %25, i32 0, i32 0, i32 0)
- %41 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %26, i32 0, i32 0, i32 0)
- %42 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %27, i32 0, i32 0, i32 0)
- %43 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %28, i32 0, i32 0, i32 0)
- %44 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %29, i32 0, i32 0, i32 0)
- %45 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %30, i32 0, i32 0, i32 0)
- %46 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %31, i32 0, i32 0, i32 0)
- %47 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %32, i32 0, i32 0, i32 0)
- %48 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %33, i32 0, i32 0, i32 0)
- %49 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %34, i32 0, i32 0, i32 0)
- %50 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %35, i32 0, i32 0, i32 0)
- br label %51
+bb2: ; preds = %bb
+ %i3 = fmul <16 x float> %.sroa.01121.2, zeroinitializer
+ %i4 = fmul <16 x float> %.sroa.54.2, zeroinitializer
+ %i5 = fmul <16 x float> %.sroa.106.2, zeroinitializer
+ %i6 = fmul <16 x float> %.sroa.1581182.2, zeroinitializer
+ %i7 = fmul <16 x float> %.sroa.210.2, zeroinitializer
+ %i8 = fmul <16 x float> %.sroa.262.2, zeroinitializer
+ %i9 = fmul <16 x float> %.sroa.314.2, zeroinitializer
+ %i10 = fmul <16 x float> %.sroa.366.21, zeroinitializer
+ %i11 = fmul <16 x float> %.sroa.418.2, zeroinitializer
+ %i12 = fmul <16 x float> %.sroa.470.2, zeroinitializer
+ %i13 = fmul <16 x float> %.sroa.522.2, zeroinitializer
+ %i14 = fmul <16 x float> %.sroa.574.2, zeroinitializer
+ %i15 = fmul <16 x float> %.sroa.626.2, zeroinitializer
+ %i16 = fmul <16 x float> %.sroa.678.2, zeroinitializer
+ %i17 = fmul <16 x float> %.sroa.730.2, zeroinitializer
+ %i18 = fmul <16 x float> %.sroa.782.2, zeroinitializer
+ %i19 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i3, i32 0, i32 0, i32 0)
+ %i20 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i4, i32 0, i32 0, i32 0)
+ %i21 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i5, i32 0, i32 0, i32 0)
+ %i22 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i6, i32 0, i32 0, i32 0)
+ %i23 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i7, i32 0, i32 0, i32 0)
+ %i24 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i8, i32 0, i32 0, i32 0)
+ %i25 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i9, i32 0, i32 0, i32 0)
+ %i26 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i10, i32 0, i32 0, i32 0)
+ %i27 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i11, i32 0, i32 0, i32 0)
+ %i28 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i12, i32 0, i32 0, i32 0)
+ %i29 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i13, i32 0, i32 0, i32 0)
+ %i30 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i15, i32 0, i32 0, i32 0)
+ %i31 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i16, i32 0, i32 0, i32 0)
+ %i32 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i17, i32 0, i32 0, i32 0)
+ %i33 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i18, i32 0, i32 0, i32 0)
+ %i34 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i19, i32 0, i32 0, i32 0)
+ %i35 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i20, i32 0, i32 0, i32 0)
+ %i36 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i21, i32 0, i32 0, i32 0)
+ %i37 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i22, i32 0, i32 0, i32 0)
+ %i38 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i23, i32 0, i32 0, i32 0)
+ %i39 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i24, i32 0, i32 0, i32 0)
+ %i40 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i25, i32 0, i32 0, i32 0)
+ %i41 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i26, i32 0, i32 0, i32 0)
+ %i42 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i27, i32 0, i32 0, i32 0)
+ %i43 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i28, i32 0, i32 0, i32 0)
+ %i44 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i29, i32 0, i32 0, i32 0)
+ %i45 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i30, i32 0, i32 0, i32 0)
+ %i46 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i31, i32 0, i32 0, i32 0)
+ %i47 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i32, i32 0, i32 0, i32 0)
+ %i48 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i33, i32 0, i32 0, i32 0)
+ br label %bb49
-51: ; preds = %4, %3
- %.sroa.01121.3 = phi <16 x float> [ %.sroa.01121.2, %3 ], [ %36, %4 ]
- %.sroa.54.3 = phi <16 x float> [ %.sroa.54.2, %3 ], [ %37, %4 ]
- %.sroa.106.3 = phi <16 x float> [ %.sroa.106.2, %3 ], [ %38, %4 ]
- %.sroa.1581182.3 = phi <16 x float> [ %.sroa.1581182.2, %3 ], [ %39, %4 ]
- %.sroa.210.3 = phi <16 x float> [ %.sroa.210.2, %3 ], [ %40, %4 ]
- %.sroa.262.3 = phi <16 x float> [ %.sroa.262.2, %3 ], [ %41, %4 ]
- %.sroa.314.3 = phi <16 x float> [ %.sroa.314.2, %3 ], [ %42, %4 ]
- %.sroa.366.3 = phi <16 x float> [ %.sroa.366.2, %3 ], [ %43, %4 ]
- %.sroa.418.3 = phi <16 x float> [ %.sroa.418.2, %3 ], [ %44, %4 ]
- %.sroa.470.3 = phi <16 x float> [ %.sroa.470.2, %3 ], [ %45, %4 ]
- %.sroa.522.3 = phi <16 x float> [ %.sroa.522.2, %3 ], [ %46, %4 ]
- %.sroa.574.3 = phi <16 x float> [ %.sroa.574.2, %3 ], [ %16, %4 ]
- %.sroa.626.3 = phi <16 x float> [ zeroinitializer, %3 ], [ %47, %4 ]
- %.sroa.678.3 = phi <16 x float> [ %.sroa.678.2, %3 ], [ %48, %4 ]
- %.sroa.730.3 = phi <16 x float> [ %.sroa.730.2, %3 ], [ %49, %4 ]
- %.sroa.782.3 = phi <16 x float> [ %.sroa.782.2, %3 ], [ %50, %4 ]
- %52 = fmul <16 x float> %.sroa.01121.3, zeroinitializer
- %53 = fmul <16 x float> %.sroa.54.3, zeroinitializer
- %54 = fmul <16 x float> %.sroa.106.3, zeroinitializer
- %55 = fmul <16 x float> %.sroa.1581182.3, zeroinitializer
- %56 = fmul <16 x float> %.sroa.210.3, zeroinitializer
- %57 = fmul <16 x float> %.sroa.262.3, zeroinitializer
- %58 = fmul <16 x float> %.sroa.314.3, zeroinitializer
- %59 = fmul <16 x float> %.sroa.366.3, zeroinitializer
- %60 = fmul <16 x float> %.sroa.418.3, zeroinitializer
- %61 = fmul <16 x float> %.sroa.470.3, zeroinitializer
- %62 = fmul <16 x float> %.sroa.522.3, zeroinitializer
- %63 = fmul <16 x float> %.sroa.574.3, zeroinitializer
- %64 = fmul <16 x float> %.sroa.626.3, zeroinitializer
- %65 = fmul <16 x float> %.sroa.678.3, zeroinitializer
- %66 = fmul <16 x float> %.sroa.730.3, zeroinitializer
- %67 = fmul <16 x float> %.sroa.782.3, zeroinitializer
- %68 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %55, i32 0, i32 0, i32 0)
- %69 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %56, i32 0, i32 0, i32 0)
- %70 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %57, i32 0, i32 0, i32 0)
- %71 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %58, i32 0, i32 0, i32 0)
- %72 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %59, i32 0, i32 0, i32 0)
- %73 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %60, i32 0, i32 0, i32 0)
- %74 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %61, i32 0, i32 0, i32 0)
- %75 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %62, i32 0, i32 0, i32 0)
- %76 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %64, i32 0, i32 0, i32 0)
- %77 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %65, i32 0, i32 0, i32 0)
- %78 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %66, i32 0, i32 0, i32 0)
- %79 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %67, i32 0, i32 0, i32 0)
- %80 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %52, i32 0, i32 0, i32 0)
- %81 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %53, i32 0, i32 0, i32 0)
- %82 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %81, i32 0, i32 0, i32 0)
- %83 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %54, i32 0, i32 0, i32 0)
- %84 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %83, i32 0, i32 0, i32 0)
- %85 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %68, i32 0, i32 0, i32 0)
- %86 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %69, i32 0, i32 0, i32 0)
- %87 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %70, i32 0, i32 0, i32 0)
- %88 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %71, i32 0, i32 0, i32 0)
- %89 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %73, i32 0, i32 0, i32 0)
- %90 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %.sroa.574.2, i32 0, i32 0, i32 0)
- %91 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %77, i32 0, i32 0, i32 0)
- %92 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %78, i32 0, i32 0, i32 0)
- %exitcond.not.i = icmp eq i64 %2, 0
- br i1 %exitcond.not.i, label %._crit_edge.i.loopexit, label %1
+bb49: ; preds = %bb2, %bb1
+ %.sroa.01121.3 = phi <16 x float> [ %.sroa.01121.2, %bb1 ], [ %i34, %bb2 ]
+ %.sroa.54.3 = phi <16 x float> [ %.sroa.54.2, %bb1 ], [ %i35, %bb2 ]
+ %.sroa.106.3 = phi <16 x float> [ %.sroa.106.2, %bb1 ], [ %i36, %bb2 ]
+ %.sroa.1581182.3 = phi <16 x float> [ %.sroa.1581182.2, %bb1 ], [ %i37, %bb2 ]
+ %.sroa.210.3 = phi <16 x float> [ %.sroa.210.2, %bb1 ], [ %i38, %bb2 ]
+ %.sroa.262.3 = phi <16 x float> [ %.sroa.262.2, %bb1 ], [ %i39, %bb2 ]
+ %.sroa.314.3 = phi <16 x float> [ %.sroa.314.2, %bb1 ], [ %i40, %bb2 ]
+ %.sroa.366.3 = phi <16 x float> [ %.sroa.366.2, %bb1 ], [ %i41, %bb2 ]
+ %.sroa.418.3 = phi <16 x float> [ %.sroa.418.2, %bb1 ], [ %i42, %bb2 ]
+ %.sroa.470.3 = phi <16 x float> [ %.sroa.470.2, %bb1 ], [ %i43, %bb2 ]
+ %.sroa.522.3 = phi <16 x float> [ %.sroa.522.2, %bb1 ], [ %i44, %bb2 ]
+ %.sroa.574.3 = phi <16 x float> [ %.sroa.574.2, %bb1 ], [ %i14, %bb2 ]
+ %.sroa.626.3 = phi <16 x float> [ zeroinitializer, %bb1 ], [ %i45, %bb2 ]
+ %.sroa.678.3 = phi <16 x float> [ %.sroa.678.2, %bb1 ], [ %i46, %bb2 ]
+ %.sroa.730.3 = phi <16 x float> [ %.sroa.730.2, %bb1 ], [ %i47, %bb2 ]
+ %.sroa.782.3 = phi <16 x float> [ %.sroa.782.2, %bb1 ], [ %i48, %bb2 ]
+ %i50 = fmul <16 x float> %.sroa.01121.3, zeroinitializer
+ %i51 = fmul <16 x float> %.sroa.54.3, zeroinitializer
+ %i52 = fmul <16 x float> %.sroa.106.3, zeroinitializer
+ %i53 = fmul <16 x float> %.sroa.1581182.3, zeroinitializer
+ %i54 = fmul <16 x float> %.sroa.210.3, zeroinitializer
+ %i55 = fmul <16 x float> %.sroa.262.3, zeroinitializer
+ %i56 = fmul <16 x float> %.sroa.314.3, zeroinitializer
+ %i57 = fmul <16 x float> %.sroa.366.3, zeroinitializer
+ %i58 = fmul <16 x float> %.sroa.418.3, zeroinitializer
+ %i59 = fmul <16 x float> %.sroa.470.3, zeroinitializer
+ %i60 = fmul <16 x float> %.sroa.522.3, zeroinitializer
+ %i61 = fmul <16 x float> %.sroa.574.3, zeroinitializer
+ %i62 = fmul <16 x float> %.sroa.626.3, zeroinitializer
+ %i63 = fmul <16 x float> %.sroa.678.3, zeroinitializer
+ %i64 = fmul <16 x float> %.sroa.730.3, zeroinitializer
+ %i65 = fmul <16 x float> %.sroa.782.3, zeroinitializer
+ %i66 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i53, i32 0, i32 0, i32 0)
+ %i67 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i54, i32 0, i32 0, i32 0)
+ %i68 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i55, i32 0, i32 0, i32 0)
+ %i69 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i56, i32 0, i32 0, i32 0)
+ %i70 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i57, i32 0, i32 0, i32 0)
+ %i71 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i58, i32 0, i32 0, i32 0)
+ %i72 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i59, i32 0, i32 0, i32 0)
+ %i73 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i60, i32 0, i32 0, i32 0)
+ %i74 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i62, i32 0, i32 0, i32 0)
+ %i75 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i63, i32 0, i32 0, i32 0)
+ %i76 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i64, i32 0, i32 0, i32 0)
+ %i77 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i65, i32 0, i32 0, i32 0)
+ %i78 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i50, i32 0, i32 0, i32 0)
+ %i79 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i51, i32 0, i32 0, i32 0)
+ %i80 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i79, i32 0, i32 0, i32 0)
+ %i81 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i52, i32 0, i32 0, i32 0)
+ %i82 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i81, i32 0, i32 0, i32 0)
+ %i83 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i66, i32 0, i32 0, i32 0)
+ %i84 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i67, i32 0, i32 0, i32 0)
+ %i85 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i68, i32 0, i32 0, i32 0)
+ %i86 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i69, i32 0, i32 0, i32 0)
+ %i87 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i71, i32 0, i32 0, i32 0)
+ %i88 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %.sroa.574.2, i32 0, i32 0, i32 0)
+ %i89 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i75, i32 0, i32 0, i32 0)
+ %i90 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> zeroinitializer, <8 x bfloat> zeroinitializer, <16 x float> %i76, i32 0, i32 0, i32 0)
+ %exitcond.not.i = icmp eq i64 %i, 0
+ br i1 %exitcond.not.i, label %._crit_edge.i.loopexit, label %bb
-._crit_edge.i.loopexit: ; preds = %51
+._crit_edge.i.loopexit: ; preds = %bb49
ret void
}
>From e3d13115447b0a5549ac1ea8e646a31c9d91bfd6 Mon Sep 17 00:00:00 2001
From: Dhruva Chakrabarti <Dhruva.Chakrabarti at amd.com>
Date: Tue, 26 May 2026 18:33:57 -0500
Subject: [PATCH 06/15] Removed unneeded attributes.
---
.../AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-implicit-def-mir.mir | 2 +-
.../AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-implicit-def.ll | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-implicit-def-mir.mir b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-implicit-def-mir.mir
index c7b0b962e382f..b076ba91fbfb0 100644
--- a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-implicit-def-mir.mir
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-implicit-def-mir.mir
@@ -21,7 +21,7 @@
declare ptr addrspace(4) @llvm.amdgcn.kernarg.segment.ptr()
- attributes #0 = { "amdgpu-flat-work-group-size"="1, 256" "target-cpu"="gfx950" }
+ attributes #0 = { "amdgpu-flat-work-group-size"="1, 256" }
...
---
name: rewrite_vgpr_mfma_to_agpr_spill_implicit_def
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-implicit-def.ll b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-implicit-def.ll
index d25826121f100..df08aded2e080 100644
--- a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-implicit-def.ll
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-implicit-def.ll
@@ -156,4 +156,4 @@ bb49: ; preds = %bb2, %bb1
declare <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat>, <8 x bfloat>, <16 x float>, i32 immarg, i32 immarg, i32 immarg)
-attributes #0 = { "amdgpu-flat-work-group-size"="1, 256" "target-cpu"="gfx950" }
+attributes #0 = { "amdgpu-flat-work-group-size"="1, 256" }
>From 0319c2a0eb6db2d9e1064570abb845ee3d35af93 Mon Sep 17 00:00:00 2001
From: Dhruva Chakrabarti <Dhruva.Chakrabarti at amd.com>
Date: Tue, 26 May 2026 18:41:56 -0500
Subject: [PATCH 07/15] regex
---
.../AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-implicit-def-mir.mir | 2 +-
.../AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-implicit-def.ll | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-implicit-def-mir.mir b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-implicit-def-mir.mir
index b076ba91fbfb0..08f7edf9aa2b3 100644
--- a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-implicit-def-mir.mir
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-implicit-def-mir.mir
@@ -9,7 +9,7 @@
# When the AGPR rewrite pass unspills such a slot into a vreg, it must insert
# IMPLICIT_DEF so the vreg has defs on all paths.
-# CHECK: some reachable load not jointly dominated by stores
+# CHECK: Skipping ${{[a-zA-Z0-9_]+}}: some reachable load not jointly dominated by stores
--- |
define amdgpu_kernel void @rewrite_vgpr_mfma_to_agpr_spill_implicit_def(i1 %arg, <16 x float> %.sroa.366.2) #0 {
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-implicit-def.ll b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-implicit-def.ll
index df08aded2e080..feae7f4ecd730 100644
--- a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-implicit-def.ll
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-implicit-def.ll
@@ -10,7 +10,7 @@
; When the AGPR rewrite pass unspills such a slot into a vreg, it must insert
; IMPLICIT_DEF so the vreg has defs on all paths.
-; CHECK: some reachable load not jointly dominated by stores
+; CHECK: Skipping ${{[a-zA-Z0-9_]+}}: some reachable load not jointly dominated by stores
define amdgpu_kernel void @rewrite_vgpr_mfma_to_agpr_spill_implicit_def(i1 %arg, <16 x float> %.sroa.366.2) #0 {
.lr.ph.i:
>From 3fcd2121bef6f97248e7cbbbf7882a600e1ae55f Mon Sep 17 00:00:00 2001
From: Dhruva Chakrabarti <Dhruva.Chakrabarti at amd.com>
Date: Tue, 26 May 2026 18:56:20 -0500
Subject: [PATCH 08/15] Name cleanups.
---
... rewrite-vgpr-mfma-to-agpr-spill-joint-dom-mir.mir} | 10 +++++-----
...ll => rewrite-vgpr-mfma-to-agpr-spill-joint-dom.ll} | 8 ++++----
2 files changed, 9 insertions(+), 9 deletions(-)
rename llvm/test/CodeGen/AMDGPU/{rewrite-vgpr-mfma-to-agpr-spill-implicit-def-mir.mir => rewrite-vgpr-mfma-to-agpr-spill-joint-dom-mir.mir} (99%)
rename llvm/test/CodeGen/AMDGPU/{rewrite-vgpr-mfma-to-agpr-spill-implicit-def.ll => rewrite-vgpr-mfma-to-agpr-spill-joint-dom.ll} (97%)
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-implicit-def-mir.mir b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-joint-dom-mir.mir
similarity index 99%
rename from llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-implicit-def-mir.mir
rename to llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-joint-dom-mir.mir
index 08f7edf9aa2b3..87a3276b10891 100644
--- a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-implicit-def-mir.mir
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-joint-dom-mir.mir
@@ -5,14 +5,14 @@
# RUN: -debug-only=amdgpu-rewrite-agpr-copy-mfma -filetype=null %s 2>&1 \
# RUN: | FileCheck %s
-# It is legal for a spill reload to not have a dominating spill store.
-# When the AGPR rewrite pass unspills such a slot into a vreg, it must insert
-# IMPLICIT_DEF so the vreg has defs on all paths.
+# It is legal for a spill reload to not be jointly dominated by the slot's
+# spill stores. The AGPR rewrite pass must not unspill such a slot into a
+# vreg, otherwise the compiler will crash.
# CHECK: Skipping ${{[a-zA-Z0-9_]+}}: some reachable load not jointly dominated by stores
--- |
- define amdgpu_kernel void @rewrite_vgpr_mfma_to_agpr_spill_implicit_def(i1 %arg, <16 x float> %.sroa.366.2) #0 {
+ define amdgpu_kernel void @rewrite_vgpr_mfma_to_agpr_spill_joint_dom(i1 %arg, <16 x float> %.sroa.366.2) #0 {
%kernarg.segment = call ptr addrspace(4) @llvm.amdgcn.kernarg.segment.ptr()
%.kernarg.offset.align.down66 = bitcast ptr addrspace(4) %kernarg.segment to ptr addrspace(4)
%.sroa.366.2.kernarg.offset = getelementptr inbounds i8, ptr addrspace(4) %kernarg.segment, i64 64
@@ -24,7 +24,7 @@
attributes #0 = { "amdgpu-flat-work-group-size"="1, 256" }
...
---
-name: rewrite_vgpr_mfma_to_agpr_spill_implicit_def
+name: rewrite_vgpr_mfma_to_agpr_spill_joint_dom
tracksRegLiveness: true
liveins:
- { reg: '$sgpr4_sgpr5', virtual-reg: '%0' }
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-implicit-def.ll b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-joint-dom.ll
similarity index 97%
rename from llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-implicit-def.ll
rename to llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-joint-dom.ll
index feae7f4ecd730..13a230ea5d9a8 100644
--- a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-implicit-def.ll
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-joint-dom.ll
@@ -6,13 +6,13 @@
; Regression test from https://github.com/llvm/llvm-project/issues/196671
-; It is legal for a spill reload to not have a dominating spill store.
-; When the AGPR rewrite pass unspills such a slot into a vreg, it must insert
-; IMPLICIT_DEF so the vreg has defs on all paths.
+; It is legal for a spill reload to not be jointly dominated by the slot's
+; spill stores. The AGPR rewrite pass must not unspill such a slot into a
+; vreg, otherwise the compiler will crash.
; CHECK: Skipping ${{[a-zA-Z0-9_]+}}: some reachable load not jointly dominated by stores
-define amdgpu_kernel void @rewrite_vgpr_mfma_to_agpr_spill_implicit_def(i1 %arg, <16 x float> %.sroa.366.2) #0 {
+define amdgpu_kernel void @rewrite_vgpr_mfma_to_agpr_spill_joint_dom(i1 %arg, <16 x float> %.sroa.366.2) #0 {
.lr.ph.i:
br label %bb
>From 3e4713cb9960479c9a7a1eeb55c01502de87573b Mon Sep 17 00:00:00 2001
From: Dhruva Chakrabarti <Dhruva.Chakrabarti at amd.com>
Date: Wed, 10 Jun 2026 14:13:54 -0500
Subject: [PATCH 09/15] Addressed review comments: - Make
isLoadJointlyDominatedByStores a member instead of lambda - Remove Stores and
Loads, instead use SpillReferences directly.
---
.../AMDGPU/AMDGPURewriteAGPRCopyMFMA.cpp | 92 ++++++++++---------
1 file changed, 47 insertions(+), 45 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURewriteAGPRCopyMFMA.cpp b/llvm/lib/Target/AMDGPU/AMDGPURewriteAGPRCopyMFMA.cpp
index 1c236619987aa..193f14071155d 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURewriteAGPRCopyMFMA.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURewriteAGPRCopyMFMA.cpp
@@ -130,6 +130,15 @@ class AMDGPURewriteAGPRCopyMFMAImpl {
void collectSpillIndexUses(ArrayRef<LiveInterval *> StackIntervals,
SpillReferenceMap &Map) const;
+ /// Return true if the reload \p LoadMI of stack slot \p Slot is jointly
+ /// dominated by the slot's spill stores, i.e. every path from the entry
+ /// block to the load passes through a store to \p Slot before the load.
+ /// \p StoreFreeReachable is the set of blocks reachable from the entry block
+ /// without passing through any store block for \p Slot.
+ bool isLoadJointlyDominatedByStores(
+ const MachineInstr &LoadMI, int Slot,
+ const SmallPtrSetImpl<MachineBasicBlock *> &StoreFreeReachable) const;
+
/// Attempt to unspill VGPRs by finding a free register and replacing the
/// spill instructions with copies.
void eliminateSpillsOfReassignedVGPRs() const;
@@ -478,6 +487,34 @@ void AMDGPURewriteAGPRCopyMFMAImpl::collectSpillIndexUses(
}
}
+bool AMDGPURewriteAGPRCopyMFMAImpl::isLoadJointlyDominatedByStores(
+ const MachineInstr &LoadMI, int Slot,
+ const SmallPtrSetImpl<MachineBasicBlock *> &StoreFreeReachable) const {
+ const MachineBasicBlock *LoadMBB = LoadMI.getParent();
+ if (!MDT.isReachableFromEntry(LoadMBB))
+ return true;
+
+ // Check if every path passed through a store block.
+ if (!StoreFreeReachable.contains(LoadMBB))
+ return true;
+
+ // Otherwise, there exists a path to this block that has not seen any store
+ // yet. We must ensure that within this block there is a store to this slot
+ // before the load.
+ for (const MachineInstr &MI : *LoadMBB) {
+ if (&MI == &LoadMI)
+ break;
+ if (MI.mayStore()) {
+ for (const MachineOperand &MO : MI.operands()) {
+ if (MO.isFI() && MO.getIndex() == Slot)
+ return true;
+ }
+ }
+ }
+
+ return false;
+}
+
void AMDGPURewriteAGPRCopyMFMAImpl::eliminateSpillsOfReassignedVGPRs() const {
unsigned NumSlots = LSS.getNumIntervals();
if (NumSlots == 0)
@@ -533,20 +570,10 @@ void AMDGPURewriteAGPRCopyMFMAImpl::eliminateSpillsOfReassignedVGPRs() const {
// For each spill reload, every path from entry to the reload must pass
// through at least one spill store to the same stack slot.
- SmallVector<MachineInstr *, 4> Stores, Loads;
- Stores.reserve(SpillReferences->second.size());
- Loads.reserve(SpillReferences->second.size());
- for (MachineInstr *MI : SpillReferences->second) {
- if (MI->mayStore())
- Stores.push_back(MI);
- else if (MI->mayLoad())
- Loads.push_back(MI);
- }
-
SmallPtrSet<MachineBasicBlock *, 4> StoreBlocks;
- for (MachineInstr *S : Stores)
- if (MDT.isReachableFromEntry(S->getParent()))
- StoreBlocks.insert(S->getParent());
+ for (MachineInstr *MI : SpillReferences->second)
+ if (MI->mayStore() && MDT.isReachableFromEntry(MI->getParent()))
+ StoreBlocks.insert(MI->getParent());
if (StoreBlocks.empty()) {
LLVM_DEBUG(dbgs() << "Skipping " << printReg(Slot, &TRI)
@@ -556,12 +583,9 @@ void AMDGPURewriteAGPRCopyMFMAImpl::eliminateSpillsOfReassignedVGPRs() const {
// Compute blocks reachable from entry without passing through a store
// block.
- SmallPtrSet<MachineBasicBlock *, 16> StoreFreeReachable;
- SmallVector<MachineBasicBlock *, 16> Worklist;
-
MachineBasicBlock &EntryMBB = MF.front();
- Worklist.push_back(&EntryMBB);
- StoreFreeReachable.insert(&EntryMBB);
+ SmallPtrSet<MachineBasicBlock *, 16> StoreFreeReachable = {&EntryMBB};
+ SmallVector<MachineBasicBlock *, 16> Worklist = {&EntryMBB};
while (!Worklist.empty()) {
MachineBasicBlock *MBB = Worklist.pop_back_val();
@@ -574,33 +598,11 @@ void AMDGPURewriteAGPRCopyMFMAImpl::eliminateSpillsOfReassignedVGPRs() const {
}
}
- auto IsLoadJointlyDominatedByStores = [&](MachineInstr *LoadMI) -> bool {
- MachineBasicBlock *LoadMBB = LoadMI->getParent();
- if (!MDT.isReachableFromEntry(LoadMBB))
- return true;
-
- // Check if every path passed through a store block.
- if (!StoreFreeReachable.contains(LoadMBB))
- return true;
-
- // Otherwise, there exists a path to this block that has not seen any
- // store yet. We must ensure that within this block there is a store to
- // this slot before the load.
- for (MachineInstr &MI : *LoadMBB) {
- if (&MI == LoadMI)
- break;
- if (MI.mayStore()) {
- for (MachineOperand &MO : MI.operands()) {
- if (MO.isFI() && MO.getIndex() == Slot)
- return true;
- }
- }
- }
-
- return false;
- };
-
- if (!llvm::all_of(Loads, IsLoadJointlyDominatedByStores)) {
+ // Every reachable reload must be jointly dominated by the slot's stores.
+ if (!llvm::all_of(SpillReferences->second, [&](const MachineInstr *MI) {
+ return !MI->mayLoad() ||
+ isLoadJointlyDominatedByStores(*MI, Slot, StoreFreeReachable);
+ })) {
LLVM_DEBUG(
dbgs() << "Skipping " << printReg(Slot, &TRI)
<< ": some reachable load not jointly dominated by stores\n");
>From a2850e7511f09238047e9f4e6cfa978d94bcecd5 Mon Sep 17 00:00:00 2001
From: Dhruva Chakrabarti <Dhruva.Chakrabarti at amd.com>
Date: Wed, 10 Jun 2026 14:57:59 -0500
Subject: [PATCH 10/15] Removed extra space from flat-wg-size.
---
.../AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-joint-dom-mir.mir | 2 +-
.../CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-joint-dom.ll | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-joint-dom-mir.mir b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-joint-dom-mir.mir
index 87a3276b10891..a4fbce6cfdf48 100644
--- a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-joint-dom-mir.mir
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-joint-dom-mir.mir
@@ -21,7 +21,7 @@
declare ptr addrspace(4) @llvm.amdgcn.kernarg.segment.ptr()
- attributes #0 = { "amdgpu-flat-work-group-size"="1, 256" }
+ attributes #0 = { "amdgpu-flat-work-group-size"="1,256" }
...
---
name: rewrite_vgpr_mfma_to_agpr_spill_joint_dom
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-joint-dom.ll b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-joint-dom.ll
index 13a230ea5d9a8..e7e7ed3e01150 100644
--- a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-joint-dom.ll
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-joint-dom.ll
@@ -156,4 +156,4 @@ bb49: ; preds = %bb2, %bb1
declare <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat>, <8 x bfloat>, <16 x float>, i32 immarg, i32 immarg, i32 immarg)
-attributes #0 = { "amdgpu-flat-work-group-size"="1, 256" }
+attributes #0 = { "amdgpu-flat-work-group-size"="1,256" }
>From 024bd47f18a8801b9f7aff9d154ab9c09542c698 Mon Sep 17 00:00:00 2001
From: Dhruva Chakrabarti <Dhruva.Chakrabarti at amd.com>
Date: Wed, 10 Jun 2026 17:25:08 -0500
Subject: [PATCH 11/15] Dropped IR section instructions, removed IR references.
---
...ewrite-vgpr-mfma-to-agpr-spill-joint-dom-mir.mir | 13 ++++---------
1 file changed, 4 insertions(+), 9 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-joint-dom-mir.mir b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-joint-dom-mir.mir
index a4fbce6cfdf48..ef1d74b8eca54 100644
--- a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-joint-dom-mir.mir
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-joint-dom-mir.mir
@@ -12,15 +12,10 @@
# CHECK: Skipping ${{[a-zA-Z0-9_]+}}: some reachable load not jointly dominated by stores
--- |
- define amdgpu_kernel void @rewrite_vgpr_mfma_to_agpr_spill_joint_dom(i1 %arg, <16 x float> %.sroa.366.2) #0 {
- %kernarg.segment = call ptr addrspace(4) @llvm.amdgcn.kernarg.segment.ptr()
- %.kernarg.offset.align.down66 = bitcast ptr addrspace(4) %kernarg.segment to ptr addrspace(4)
- %.sroa.366.2.kernarg.offset = getelementptr inbounds i8, ptr addrspace(4) %kernarg.segment, i64 64
- unreachable
+ define amdgpu_kernel void @rewrite_vgpr_mfma_to_agpr_spill_joint_dom() #0 {
+ ret void
}
- declare ptr addrspace(4) @llvm.amdgcn.kernarg.segment.ptr()
-
attributes #0 = { "amdgpu-flat-work-group-size"="1,256" }
...
---
@@ -34,7 +29,7 @@ body: |
liveins: $sgpr4_sgpr5
%0:sgpr_64 = COPY killed $sgpr4_sgpr5
- %1:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM %0, 0, 0 :: (dereferenceable invariant load (s32) from %ir..kernarg.offset.align.down66, align 16, addrspace 4)
+ %1:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM %0, 0, 0 :: (dereferenceable invariant load (s32), align 16, addrspace 4)
S_BITCMP1_B32 killed %1, 0, implicit-def $scc
%2:sreg_64_xexec = S_CSELECT_B64 -1, 0, implicit killed $scc
%3:sreg_64 = S_MOV_B64 -1
@@ -57,7 +52,7 @@ body: |
%7.sub13:av_512_align2 = COPY %6
%7.sub14:av_512_align2 = COPY %6
%7.sub15:av_512_align2 = COPY %6
- early-clobber %8:sgpr_512 = S_LOAD_DWORDX16_IMM_ec killed %0, 64, 0 :: (dereferenceable invariant load (s512) from %ir..sroa.366.2.kernarg.offset, align 16, addrspace 4)
+ early-clobber %8:sgpr_512 = S_LOAD_DWORDX16_IMM_ec killed %0, 64, 0 :: (dereferenceable invariant load (s512), align 16, addrspace 4)
%9:sreg_64 = S_MOV_B64 0
%10:vreg_512_align2 = COPY %7, implicit $exec
%11:sreg_64 = S_AND_B64 $exec, killed %4, implicit-def dead $scc
>From 1aa093c51048c612b994f510d8ae1e80a5245ac9 Mon Sep 17 00:00:00 2001
From: Dhruva Chakrabarti <Dhruva.Chakrabarti at amd.com>
Date: Wed, 10 Jun 2026 17:35:07 -0500
Subject: [PATCH 12/15] Added braces to conform to AMDGPU coding style.
---
llvm/lib/Target/AMDGPU/AMDGPURewriteAGPRCopyMFMA.cpp | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURewriteAGPRCopyMFMA.cpp b/llvm/lib/Target/AMDGPU/AMDGPURewriteAGPRCopyMFMA.cpp
index 193f14071155d..67757b5fa449c 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURewriteAGPRCopyMFMA.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURewriteAGPRCopyMFMA.cpp
@@ -571,9 +571,10 @@ void AMDGPURewriteAGPRCopyMFMAImpl::eliminateSpillsOfReassignedVGPRs() const {
// For each spill reload, every path from entry to the reload must pass
// through at least one spill store to the same stack slot.
SmallPtrSet<MachineBasicBlock *, 4> StoreBlocks;
- for (MachineInstr *MI : SpillReferences->second)
+ for (MachineInstr *MI : SpillReferences->second) {
if (MI->mayStore() && MDT.isReachableFromEntry(MI->getParent()))
StoreBlocks.insert(MI->getParent());
+ }
if (StoreBlocks.empty()) {
LLVM_DEBUG(dbgs() << "Skipping " << printReg(Slot, &TRI)
>From 9496517d5d6db41b4307165676477a743a3ece60 Mon Sep 17 00:00:00 2001
From: Dhruva Chakrabarti <Dhruva.Chakrabarti at amd.com>
Date: Fri, 26 Jun 2026 20:14:23 -0500
Subject: [PATCH 13/15] Use LiveStacks instead of instruction scan within the
reload block.
---
.../AMDGPU/AMDGPURewriteAGPRCopyMFMA.cpp | 42 +++++++++----------
1 file changed, 21 insertions(+), 21 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURewriteAGPRCopyMFMA.cpp b/llvm/lib/Target/AMDGPU/AMDGPURewriteAGPRCopyMFMA.cpp
index 67757b5fa449c..f294a64a80a62 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURewriteAGPRCopyMFMA.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURewriteAGPRCopyMFMA.cpp
@@ -130,13 +130,13 @@ class AMDGPURewriteAGPRCopyMFMAImpl {
void collectSpillIndexUses(ArrayRef<LiveInterval *> StackIntervals,
SpillReferenceMap &Map) const;
- /// Return true if the reload \p LoadMI of stack slot \p Slot is jointly
- /// dominated by the slot's spill stores, i.e. every path from the entry
- /// block to the load passes through a store to \p Slot before the load.
- /// \p StoreFreeReachable is the set of blocks reachable from the entry block
- /// without passing through any store block for \p Slot.
+ /// Return true if the reload \p LoadMI of the stack slot with live interval
+ /// \p SlotLI is jointly dominated by the slot's spill stores, i.e. every path
+ /// from the entry block to the load passes through a store to the slot before
+ /// the load. \p StoreFreeReachable is the set of blocks reachable from the
+ /// entry block without passing through any store block for the slot.
bool isLoadJointlyDominatedByStores(
- const MachineInstr &LoadMI, int Slot,
+ const MachineInstr &LoadMI, const LiveInterval &SlotLI,
const SmallPtrSetImpl<MachineBasicBlock *> &StoreFreeReachable) const;
/// Attempt to unspill VGPRs by finding a free register and replacing the
@@ -488,7 +488,7 @@ void AMDGPURewriteAGPRCopyMFMAImpl::collectSpillIndexUses(
}
bool AMDGPURewriteAGPRCopyMFMAImpl::isLoadJointlyDominatedByStores(
- const MachineInstr &LoadMI, int Slot,
+ const MachineInstr &LoadMI, const LiveInterval &SlotLI,
const SmallPtrSetImpl<MachineBasicBlock *> &StoreFreeReachable) const {
const MachineBasicBlock *LoadMBB = LoadMI.getParent();
if (!MDT.isReachableFromEntry(LoadMBB))
@@ -500,19 +500,19 @@ bool AMDGPURewriteAGPRCopyMFMAImpl::isLoadJointlyDominatedByStores(
// Otherwise, there exists a path to this block that has not seen any store
// yet. We must ensure that within this block there is a store to this slot
- // before the load.
- for (const MachineInstr &MI : *LoadMBB) {
- if (&MI == &LoadMI)
- break;
- if (MI.mayStore()) {
- for (const MachineOperand &MO : MI.operands()) {
- if (MO.isFI() && MO.getIndex() == Slot)
- return true;
- }
- }
- }
-
- return false;
+ // before the load. Consult the slot's LiveStacks interval rather than
+ // scanning instructions: if a store to the slot precedes the load in this
+ // block, the slot's live range segment covering the load begins after the
+ // block start. If there is no such store, the segment begins at or before
+ // the block start. If a live-in segment has coalesced with an in-block
+ // segment, it would imply a reload before a store. That scenario would
+ // lead to a joint-dominance violation. If the load reads an undef value, the
+ // segment is null.
+ SlotIndex LoadIdx = LIS.getInstructionIndex(LoadMI);
+ SlotIndex BlockStart = LIS.getMBBStartIdx(LoadMBB);
+ const LiveRange::Segment *Seg =
+ SlotLI.getSegmentContaining(LoadIdx.getBaseIndex());
+ return Seg && Seg->start > BlockStart;
}
void AMDGPURewriteAGPRCopyMFMAImpl::eliminateSpillsOfReassignedVGPRs() const {
@@ -602,7 +602,7 @@ void AMDGPURewriteAGPRCopyMFMAImpl::eliminateSpillsOfReassignedVGPRs() const {
// Every reachable reload must be jointly dominated by the slot's stores.
if (!llvm::all_of(SpillReferences->second, [&](const MachineInstr *MI) {
return !MI->mayLoad() ||
- isLoadJointlyDominatedByStores(*MI, Slot, StoreFreeReachable);
+ isLoadJointlyDominatedByStores(*MI, *LI, StoreFreeReachable);
})) {
LLVM_DEBUG(
dbgs() << "Skipping " << printReg(Slot, &TRI)
>From aed8ea3fde9deaa4e08245b3447e585968d6abcd Mon Sep 17 00:00:00 2001
From: Dhruva Chakrabarti <Dhruva.Chakrabarti at amd.com>
Date: Tue, 30 Jun 2026 12:35:56 -0500
Subject: [PATCH 14/15] Addressed review comment: used isLiveInToMBB query.
---
.../Target/AMDGPU/AMDGPURewriteAGPRCopyMFMA.cpp | 17 +++++------------
1 file changed, 5 insertions(+), 12 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURewriteAGPRCopyMFMA.cpp b/llvm/lib/Target/AMDGPU/AMDGPURewriteAGPRCopyMFMA.cpp
index f294a64a80a62..5e27f39072ebb 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURewriteAGPRCopyMFMA.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURewriteAGPRCopyMFMA.cpp
@@ -500,19 +500,12 @@ bool AMDGPURewriteAGPRCopyMFMAImpl::isLoadJointlyDominatedByStores(
// Otherwise, there exists a path to this block that has not seen any store
// yet. We must ensure that within this block there is a store to this slot
- // before the load. Consult the slot's LiveStacks interval rather than
- // scanning instructions: if a store to the slot precedes the load in this
- // block, the slot's live range segment covering the load begins after the
- // block start. If there is no such store, the segment begins at or before
- // the block start. If a live-in segment has coalesced with an in-block
- // segment, it would imply a reload before a store. That scenario would
- // lead to a joint-dominance violation. If the load reads an undef value, the
- // segment is null.
+ // before the load. Consult the slot's LiveStacks interval: a store to the
+ // slot before the load means the slot is not live into this block but is
+ // live at the load. If the load reads an undef value, the slot is not live
+ // at the load, failing the joint-dominance check.
SlotIndex LoadIdx = LIS.getInstructionIndex(LoadMI);
- SlotIndex BlockStart = LIS.getMBBStartIdx(LoadMBB);
- const LiveRange::Segment *Seg =
- SlotLI.getSegmentContaining(LoadIdx.getBaseIndex());
- return Seg && Seg->start > BlockStart;
+ return SlotLI.liveAt(LoadIdx) && !LIS.isLiveInToMBB(SlotLI, LoadMBB);
}
void AMDGPURewriteAGPRCopyMFMAImpl::eliminateSpillsOfReassignedVGPRs() const {
>From 8850c674decc191d7f89f127e2ccc2ac43300ab4 Mon Sep 17 00:00:00 2001
From: Dhruva Chakrabarti <Dhruva.Chakrabarti at amd.com>
Date: Thu, 13 Aug 2026 18:46:21 -0500
Subject: [PATCH 15/15] Added review suggestion: moved RUN lines to new triple
format.
---
.../AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-joint-dom-mir.mir | 2 +-
.../CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-joint-dom.ll | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-joint-dom-mir.mir b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-joint-dom-mir.mir
index ef1d74b8eca54..c0190745f1e88 100644
--- a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-joint-dom-mir.mir
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-joint-dom-mir.mir
@@ -1,5 +1,5 @@
# REQUIRES: asserts
-# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 \
+# RUN: llc -mtriple=amdgpu9.50-amd-amdhsa \
# RUN: -start-before=register-coalescer \
# RUN: -stop-after=amdgpu-rewrite-agpr-copy-mfma \
# RUN: -debug-only=amdgpu-rewrite-agpr-copy-mfma -filetype=null %s 2>&1 \
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-joint-dom.ll b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-joint-dom.ll
index e7e7ed3e01150..c233012fd709b 100644
--- a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-joint-dom.ll
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-joint-dom.ll
@@ -1,5 +1,5 @@
; REQUIRES: asserts
-; RUN: llc -O3 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 \
+; RUN: llc -O3 -mtriple=amdgpu9.50-amd-amdhsa \
; RUN: -stop-after=amdgpu-rewrite-agpr-copy-mfma \
; RUN: -debug-only=amdgpu-rewrite-agpr-copy-mfma -filetype=null %s 2>&1 \
; RUN: | FileCheck %s
More information about the llvm-commits
mailing list