[llvm] [AMDGPU] RewriteMFMAFormStage: fix SpillCost early-return missing reset (PR #199708)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Jun 1 04:25:04 PDT 2026
https://github.com/xgxanq updated https://github.com/llvm/llvm-project/pull/199708
>From 03deede2b2175fbddb9020ededbcb04a356768f8 Mon Sep 17 00:00:00 2001
From: anqfu <anqfu at amd.com>
Date: Mon, 1 Jun 2026 06:29:18 +0000
Subject: [PATCH] [AMDGPU] RewriteMFMAFormStage: fix SpillCost early-return
missing reset
Extract resetRewriteCandsToVGPR() to consolidate the logic that restores
MFMA candidates from AGPR form back to VGPR form after cost analysis.
getRewriteCost() returned early when SpillCost > 0 without resetting
the AGPR-form register classes set by initHeuristics(), leaving MRI in
a corrupted state. Call resetRewriteCandsToVGPR() on both the
early-return path and the normal exit path to fix this.
Also change getRewriteCost() and rewrite() to take ArrayRef instead of
const std::vector& for cleaner API.
---
llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 60 +-
llvm/lib/Target/AMDGPU/GCNSchedStrategy.h | 9 +-
.../rewrite-mfma-form-spill-cost-reset.ll | 643 ++++++++++++++++++
3 files changed, 682 insertions(+), 30 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-spill-cost-reset.ll
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index 11b783a1024da..6f3fed4303dea 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -2263,6 +2263,29 @@ static bool isReachingDefAGPRForm(MachineInstr *RD,
return false;
}
+void RewriteMFMAFormStage::resetRewriteCandsToVGPR(
+ ArrayRef<std::pair<MachineInstr *, unsigned>> RewriteCands) {
+ for (auto [MI, OriginalOpcode] : RewriteCands) {
+ assert(TII->isMAI(*MI));
+ const TargetRegisterClass *ADefRC =
+ DAG.MRI.getRegClass(MI->getOperand(0).getReg());
+ const TargetRegisterClass *VDefRC = SRI->getEquivalentVGPRClass(ADefRC);
+ DAG.MRI.setRegClass(MI->getOperand(0).getReg(), VDefRC);
+ MI->setDesc(TII->get(OriginalOpcode));
+
+ MachineOperand *Src2 = TII->getNamedOperand(*MI, AMDGPU::OpName::src2);
+ if (!Src2->isReg())
+ continue;
+
+ // Have to get src types separately since subregs may cause C and D
+ // registers to be different types even though the actual operand is
+ // the same size.
+ const TargetRegisterClass *AUseRC = DAG.MRI.getRegClass(Src2->getReg());
+ const TargetRegisterClass *VUseRC = SRI->getEquivalentVGPRClass(AUseRC);
+ DAG.MRI.setRegClass(Src2->getReg(), VUseRC);
+ }
+}
+
bool RewriteMFMAFormStage::isRewriteCandidate(MachineInstr *MI) const {
if (!static_cast<const SIInstrInfo *>(DAG.TII)->isMAI(*MI))
return false;
@@ -2378,7 +2401,7 @@ bool RewriteMFMAFormStage::initHeuristics(
}
int64_t RewriteMFMAFormStage::getRewriteCost(
- const std::vector<std::pair<MachineInstr *, unsigned>> &RewriteCands,
+ ArrayRef<std::pair<MachineInstr *, unsigned>> RewriteCands,
const DenseMap<MachineBasicBlock *, std::set<Register>> &CopyForUse,
const SmallPtrSetImpl<MachineInstr *> &CopyForDef) {
MachineBlockFrequencyInfo *MBFI = DAG.MBFI;
@@ -2430,8 +2453,10 @@ int64_t RewriteMFMAFormStage::getRewriteCost(
SpillCost *= (int64_t)RelativeFreq;
// If we have increased spilling in any block, just bail.
- if (SpillCost > 0)
+ if (SpillCost > 0) {
+ resetRewriteCandsToVGPR(RewriteCands);
return SpillCost;
+ }
if (SpillCost < BestSpillCost)
BestSpillCost = SpillCost;
@@ -2468,36 +2493,17 @@ int64_t RewriteMFMAFormStage::getRewriteCost(
}
}
- // Reset the classes that were changed to AGPR for better RB analysis.
- // We must do rewriting after copy-insertion, as some defs of the register
- // may require VGPR. Additionally, if we bail out and don't perform the
- // rewrite then these need to be restored anyway.
- for (auto &[MI, OriginalOpcode] : RewriteCands) {
- assert(TII->isMAI(*MI));
- const TargetRegisterClass *ADefRC =
- DAG.MRI.getRegClass(MI->getOperand(0).getReg());
- const TargetRegisterClass *VDefRC = SRI->getEquivalentVGPRClass(ADefRC);
- DAG.MRI.setRegClass(MI->getOperand(0).getReg(), VDefRC);
- MI->setDesc(TII->get(OriginalOpcode));
-
- MachineOperand *Src2 = TII->getNamedOperand(*MI, AMDGPU::OpName::src2);
- assert(Src2);
- if (!Src2->isReg())
- continue;
-
- // Have to get src types separately since subregs may cause C and D
- // registers to be different types even though the actual operand is
- // the same size.
- const TargetRegisterClass *AUseRC = DAG.MRI.getRegClass(Src2->getReg());
- const TargetRegisterClass *VUseRC = SRI->getEquivalentVGPRClass(AUseRC);
- DAG.MRI.setRegClass(Src2->getReg(), VUseRC);
- }
+ // Reset the classes that were changed to AGPR for better register bank
+ // analysis. We must do rewriting after copy-insertion, as some defs of the
+ // register may require VGPR. Additionally, if we bail out and don't perform
+ // the rewrite then these need to be restored anyway.
+ resetRewriteCandsToVGPR(RewriteCands);
return Cost + CopyCost;
}
bool RewriteMFMAFormStage::rewrite(
- const std::vector<std::pair<MachineInstr *, unsigned>> &RewriteCands) {
+ ArrayRef<std::pair<MachineInstr *, unsigned>> RewriteCands) {
DenseMap<MachineInstr *, unsigned> FirstMIToRegion;
DenseMap<MachineInstr *, unsigned> LastMIToRegion;
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
index 3682f39c8259b..6b3e9a6e19a9a 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
@@ -458,17 +458,20 @@ class RewriteMFMAFormStage : public GCNSchedStage {
/// in initHeuristics. Uses \p CopyForUse and \p CopyForDef to calculate copy
/// costs, and \p RewriteCands to undo rewriting.
int64_t getRewriteCost(
- const std::vector<std::pair<MachineInstr *, unsigned>> &RewriteCands,
+ ArrayRef<std::pair<MachineInstr *, unsigned>> RewriteCands,
const DenseMap<MachineBasicBlock *, std::set<Register>> &CopyForUse,
const SmallPtrSetImpl<MachineInstr *> &CopyForDef);
/// Do the final rewrite on \p RewriteCands and insert any needed copies.
- bool
- rewrite(const std::vector<std::pair<MachineInstr *, unsigned>> &RewriteCands);
+ bool rewrite(ArrayRef<std::pair<MachineInstr *, unsigned>> RewriteCands);
/// \returns true if this MI is a rewrite candidate.
bool isRewriteCandidate(MachineInstr *MI) const;
+ /// Resets all candidates in \p RewriteCands back to VGPR form.
+ void resetRewriteCandsToVGPR(
+ ArrayRef<std::pair<MachineInstr *, unsigned>> RewriteCands);
+
/// Finds all the reaching defs of \p UseMO and stores the SlotIndexes into \p
/// DefIdxs
void findReachingDefs(MachineOperand &UseMO, LiveIntervals *LIS,
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-spill-cost-reset.ll b/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-spill-cost-reset.ll
new file mode 100644
index 0000000000000..5b4c6af801172
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-spill-cost-reset.ll
@@ -0,0 +1,643 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 \
+; RUN: -amdgpu-disable-rewrite-mfma-form-sched-stage=false \
+; RUN: -verify-machineinstrs \
+; RUN: -stop-after=machine-scheduler \
+; RUN: < %s | FileCheck %s
+;
+; Regression test for resetRewriteCandsToVGPR() called on the SpillCost > 0
+; early-return path inside getRewriteCost().
+;
+; Background
+; ----------
+; initHeuristics() speculatively rewrites MFMA candidates from VGPR form to
+; AGPR form (setDesc + setRegClass) so that getRealRegPressure() can evaluate
+; the post-rewrite spill cost. getRewriteCost() then loops over regions. If
+; SpillCost > 0 in any region (rewriting increases spilling), it bails early.
+;
+; Before the fix, the early-return path did not call resetRewriteCandsToVGPR(),
+; leaving MRI with AGPR register classes for registers that are still used as
+; VGPR operands. The corrupted MRI state causes downstream passes to emit AGPR
+; instructions even though the rewrite was supposed to be rejected.
+;
+; How SpillCost > 0 is triggered here
+; ------------------------------------
+; The "amdgpu-agpr-alloc"="0,0" attribute forces AGPRThreshold = 0 in
+; getMaxNumVectorRegs(). After initHeuristics() reclassifies MFMA accumulators
+; to AGPR, getRealRegPressure() reports all reclassified registers as AGPR
+; "spills" (every AGPR exceeds the 0 threshold). For region 1 (the hot loop):
+; SpillCostAfter = 76 (AGPR excess over AGPRThreshold=0)
+; SpillCostBefore = 72 (archVGPR excess before rewrite)
+; SpillCost = (76-72)*2*32 = 256 > 0 → early bail fired.
+;
+; Expected behavior WITH the fix
+; --------------------------------
+; resetRewriteCandsToVGPR() is called before returning, restoring all
+; candidate MFMA register classes and opcodes to VGPR form. rewrite() is
+; never called. MachineVerifier passes cleanly.
+;
+; Expected behavior WITHOUT the fix
+; ----------------------------------
+; The early-return skips resetRewriteCandsToVGPR(). MRI retains AGPR classes
+; and AGPR-form opcodes (V_MFMA_F32_16X16X32_F16_e64) for r1..r3, while r4
+; (not a rewrite candidate) stays VGPR-form and uses r3 as src2. The
+; MachineVerifier aborts: an areg_128_align2 register is used as src2 of a
+; VGPR-form MFMA, violating the operand register-class constraint.
+;
+; CFG: entry → loop (back-edge) → epilogue → ret
+;
+; Chain structure (12 independent chains, depth-4):
+; acc_X = phi [zeroinit, entry], [r1_X, loop]
+; r1_X = mfma(a0, b0, acc_X) ; level-1 loop-carried accumulator
+; r2_X = mfma(a1, b1, r1_X) ; level-2
+; r3_X = mfma(a0, b0, r2_X) ; level-3
+; r4_X = mfma(a1, b1, r3_X) ; level-4, dst escapes to epilogue (non-MAI)
+;
+; Pressure design (gfx950, 256 ArchVGPR limit):
+; 8 x <32 x float> loop-carried vector carriers = 256 VGPRs
+; 12 x r1_X loop-carried <4 x float> = 48 VGPRs (always live)
+; 12 x r4_X live-out to epilogue = 48 VGPRs (live at exit)
+; <8 x half> a0/b0 + a1/b1 = 16 VGPRs
+; Loop ArchVGPR peak ~= 328 > 256 → RegionsWithExcessArchVGPR set
+
+declare <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half>, <8 x half>, <4 x float>, i32 immarg, i32 immarg, i32 immarg)
+
+define amdgpu_kernel void @test_spill_cost_reset(
+ ; CHECK-LABEL: name: test_spill_cost_reset
+ ; CHECK: bb.0.entry:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: liveins: $sgpr4_sgpr5
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr4_sgpr5
+ ; CHECK-NEXT: early-clobber %1891:sgpr_256 = S_LOAD_DWORDX8_IMM_ec [[COPY]](p4), 16, 0 :: (dereferenceable invariant load (s256) from %ir.a0.kernarg.offset, align 16, addrspace 4)
+ ; CHECK-NEXT: early-clobber %1892:sgpr_256 = S_LOAD_DWORDX8_IMM_ec [[COPY]](p4), 48, 0 :: (dereferenceable invariant load (s256) from %ir.b0.kernarg.offset, align 16, addrspace 4)
+ ; CHECK-NEXT: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY]](p4), 80, 0 :: (dereferenceable invariant load (s32) from %ir.n.kernarg.offset, align 16, addrspace 4)
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_1:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_2:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_2:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_2:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_2:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_3:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_3:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_3:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_3:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_4:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_4:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_4:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_4:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_5:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_5:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_5:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_5:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_6:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_6:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_6:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_6:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_7:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_7:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_7:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_7:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_8:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_8:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_8:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_8:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_9:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_9:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_9:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_9:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_10:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_10:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_10:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_10:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_11:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_11:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_11:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_11:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_12:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_12:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_12:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_12:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:av_128_align2 = COPY %1891.sub0_sub1_sub2_sub3
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:av_128_align2 = COPY %1892.sub0_sub1_sub2_sub3
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:av_128_align2 = COPY %1891.sub4_sub5_sub6_sub7
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:av_128_align2 = COPY %1892.sub4_sub5_sub6_sub7
+ ; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY5:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY5:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY6:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY6:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY7:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY7:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY8:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY8:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY9:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY9:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY10:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY10:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY11:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY11:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY12:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY12:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY13:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY13:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY14:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY14:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY15:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY15:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY16:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY16:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY17:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY17:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY18:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY18:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY19:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY19:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY20:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY20:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY21:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY21:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY22:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY22:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY23:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY23:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY24:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY24:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY25:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY25:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY26:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY26:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY27:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY27:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY28:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY28:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY29:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY29:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY30:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY30:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY31:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY31:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY32:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY32:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY33:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY33:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY34:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY34:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY35:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY35:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY36:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY36:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY37:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY37:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY38:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY38:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY39:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY39:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY40:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY40:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY41:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY41:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY42:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY42:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY43:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY43:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY44:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY44:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY45:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY45:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY46:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY46:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY47:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY47:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY48:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY48:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY49:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY49:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY50:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY50:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY51:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY51:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY52:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY52:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY53:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY53:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY54:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY54:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY55:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY55:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY56:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY56:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY57:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY57:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY58:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY58:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY59:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY59:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY60:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY60:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY61:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY61:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY62:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY62:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY63:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY63:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY64:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY64:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY65:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY65:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY66:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY66:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY67:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY67:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY68:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY68:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY69:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY69:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY70:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY70:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY71:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY71:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY72:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY72:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY73:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY73:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY74:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY74:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY75:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY75:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY76:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY76:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY77:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY77:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY78:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY78:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY79:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY79:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY80:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY80:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY81:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY81:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY82:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY82:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY83:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY83:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY84:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY84:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY85:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY85:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY86:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY86:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY87:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY87:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY88:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY88:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY89:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY89:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY90:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY90:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY91:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY91:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY92:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY92:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY93:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY93:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY94:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY94:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY95:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY95:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY96:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY96:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY97:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY97:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY98:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY98:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY99:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY99:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY100:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY100:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY101:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY101:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY102:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY102:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY103:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY103:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY104:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY104:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY105:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY105:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY106:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY106:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY107:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY107:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY108:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY108:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY109:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY109:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY110:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY110:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY111:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY111:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY112:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY112:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY113:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY113:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY114:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY114:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY115:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY115:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY116:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY116:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY117:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY117:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY118:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY118:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY119:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY119:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY120:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY120:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY121:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY121:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY122:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY122:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY123:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY123:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY124:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY124:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY125:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY125:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY126:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY126:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY127:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY127:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY128:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY128:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY129:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY129:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY130:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY130:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY131:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY131:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1.loop:
+ ; CHECK-NEXT: successors: %bb.2(0x04000000), %bb.1(0x7c000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[V_MOV_B32_e32_12:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_12]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY131:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY131]], 8, [[COPY131]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY130:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY130]], 8, [[COPY130]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY129:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY129]], 8, [[COPY129]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY128:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY128]], 8, [[COPY128]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY127:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY127]], 8, [[COPY127]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_11:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_11]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY126:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY126]], 8, [[COPY126]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY125:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY125]], 8, [[COPY125]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY124:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY124]], 8, [[COPY124]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_10:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_10]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY123:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY123]], 8, [[COPY123]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY122:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY122]], 8, [[COPY122]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY121:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY121]], 8, [[COPY121]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_9:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_9]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY120:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY120]], 8, [[COPY120]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY119:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY119]], 8, [[COPY119]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY118:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY118]], 8, [[COPY118]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_8:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_8]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY117:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY117]], 8, [[COPY117]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY116:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY116]], 8, [[COPY116]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY115:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY115]], 8, [[COPY115]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_7:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_7]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY114:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY114]], 8, [[COPY114]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY113:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY113]], 8, [[COPY113]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY112:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY112]], 8, [[COPY112]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_6:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_6]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY111:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY111]], 8, [[COPY111]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY110:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY110]], 8, [[COPY110]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY109:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY109]], 8, [[COPY109]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_5:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_5]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY108:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY108]], 8, [[COPY108]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY107:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY107]], 8, [[COPY107]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY106:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY106]], 8, [[COPY106]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_4:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_4]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY105:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY105]], 8, [[COPY105]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY104:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY104]], 8, [[COPY104]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY103:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY103]], 8, [[COPY103]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_3:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_3]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY102:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY102]], 8, [[COPY102]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY101:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY101]], 8, [[COPY101]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY100:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY100]], 8, [[COPY100]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_2:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_2]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY99:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY99]], 8, [[COPY99]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY98:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY98]], 8, [[COPY98]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY97:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY97]], 8, [[COPY97]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_1]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY96:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY96]], 8, [[COPY96]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY95:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY95]], 8, [[COPY95]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY94:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY94]], 8, [[COPY94]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY93:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY93]], 8, [[COPY93]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY92:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY92]], 8, [[COPY92]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY91:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY91]], 8, [[COPY91]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_F16_vgprcd_e64_:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY3]], [[COPY4]], [[V_MOV_B32_e32_12]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_F16_vgprcd_e64_1:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MFMA_F32_16X16X32_F16_vgprcd_e64_]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY90:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY90]], 8, [[COPY90]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY89:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY89]], 8, [[COPY89]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY88:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY88]], 8, [[COPY88]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_F16_vgprcd_e64_2:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY3]], [[COPY4]], [[V_MFMA_F32_16X16X32_F16_vgprcd_e64_1]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY87:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY87]], 8, [[COPY87]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY86:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY86]], 8, [[COPY86]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY85:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY85]], 8, [[COPY85]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY84:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY84]], 8, [[COPY84]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY83:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY83]], 8, [[COPY83]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY82:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY82]], 8, [[COPY82]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY81:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY81]], 8, [[COPY81]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY80:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY80]], 8, [[COPY80]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY79:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY79]], 8, [[COPY79]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY78:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY78]], 8, [[COPY78]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY77:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY77]], 8, [[COPY77]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY76:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY76]], 8, [[COPY76]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY75:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY75]], 8, [[COPY75]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY74:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY74]], 8, [[COPY74]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY73:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY73]], 8, [[COPY73]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY72:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY72]], 8, [[COPY72]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY71:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY71]], 8, [[COPY71]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY70:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY70]], 8, [[COPY70]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY69:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY69]], 8, [[COPY69]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY68:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY68]], 8, [[COPY68]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY67:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY67]], 8, [[COPY67]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY66:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY66]], 8, [[COPY66]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY65:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY65]], 8, [[COPY65]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY64:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY64]], 8, [[COPY64]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY63:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY63]], 8, [[COPY63]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY62:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY62]], 8, [[COPY62]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY61:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY61]], 8, [[COPY61]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY60:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY60]], 8, [[COPY60]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY59:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY59]], 8, [[COPY59]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY58:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY58]], 8, [[COPY58]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY57:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY57]], 8, [[COPY57]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY56:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY56]], 8, [[COPY56]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY55:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY55]], 8, [[COPY55]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY54:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY54]], 8, [[COPY54]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY53:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY53]], 8, [[COPY53]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY52:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY52]], 8, [[COPY52]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY51:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY51]], 8, [[COPY51]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY50:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY50]], 8, [[COPY50]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY49:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY49]], 8, [[COPY49]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY48:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY48]], 8, [[COPY48]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY47:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY47]], 8, [[COPY47]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY46:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY46]], 8, [[COPY46]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY45:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY45]], 8, [[COPY45]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY44:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY44]], 8, [[COPY44]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY43:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY43]], 8, [[COPY43]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY42:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY42]], 8, [[COPY42]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY41:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY41]], 8, [[COPY41]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY40:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY40]], 8, [[COPY40]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY39:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY39]], 8, [[COPY39]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY38:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY38]], 8, [[COPY38]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY37:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY37]], 8, [[COPY37]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY36:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY36]], 8, [[COPY36]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY35:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY35]], 8, [[COPY35]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY34:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY34]], 8, [[COPY34]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY33:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY33]], 8, [[COPY33]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY32:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY32]], 8, [[COPY32]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY31:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY31]], 8, [[COPY31]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY30:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY30]], 8, [[COPY30]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY29:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY29]], 8, [[COPY29]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY28:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY28]], 8, [[COPY28]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY27:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY27]], 8, [[COPY27]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY26:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY26]], 8, [[COPY26]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY25:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY25]], 8, [[COPY25]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY24:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY24]], 8, [[COPY24]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY23:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY23]], 8, [[COPY23]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY22:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY22]], 8, [[COPY22]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY21:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY21]], 8, [[COPY21]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY20:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY20]], 8, [[COPY20]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY19:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY19]], 8, [[COPY19]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY18:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY18]], 8, [[COPY18]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY17:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY17]], 8, [[COPY17]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY16:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY16]], 8, [[COPY16]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY15:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY15]], 8, [[COPY15]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY14:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY14]], 8, [[COPY14]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY13:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY13]], 8, [[COPY13]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY12:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY12]], 8, [[COPY12]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY11:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY11]], 8, [[COPY11]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY10:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY10]], 8, [[COPY10]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY9:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY9]], 8, [[COPY9]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY8:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY8]], 8, [[COPY8]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY7:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY7]], 8, [[COPY7]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY6]], 8, [[COPY6]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY5:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY5]], 8, [[COPY5]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_ADD_I32 [[S_MOV_B32_]], 1, implicit-def dead $scc
+ ; CHECK-NEXT: S_CMP_LT_I32 [[S_MOV_B32_]], [[S_LOAD_DWORD_IMM]], implicit-def $scc
+ ; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_]], 8, [[V_MOV_B32_e32_]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: S_CBRANCH_SCC1 %bb.1, implicit killed $scc
+ ; CHECK-NEXT: S_BRANCH %bb.2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2.epilogue:
+ ; CHECK-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM [[COPY]](p4), 0, 0 :: (dereferenceable invariant load (s64) from %ir.out.kernarg.offset550, align 16, addrspace 4)
+ ; CHECK-NEXT: [[V_MOV_B32_e32_13:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: GLOBAL_STORE_DWORD_SADDR [[V_MOV_B32_e32_13]], [[V_MFMA_F32_16X16X32_F16_vgprcd_e64_2]].sub0, [[S_LOAD_DWORDX2_IMM]], 0, 0, implicit $exec :: (store (s32) into %ir.out.load, addrspace 1)
+ ; CHECK-NEXT: S_ENDPGM 0
+ ptr addrspace(1) %out,
+ <8 x half> %a0, <8 x half> %a1,
+ <8 x half> %b0, <8 x half> %b1,
+ i32 %n) #0 {
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ ; 8 x <32 x float> loop-carried vector carriers = 256 VGPRs.
+ %vc0 = phi <32 x float> [ zeroinitializer, %entry ], [ %vc0n, %loop ]
+ %vc1 = phi <32 x float> [ zeroinitializer, %entry ], [ %vc1n, %loop ]
+ %vc2 = phi <32 x float> [ zeroinitializer, %entry ], [ %vc2n, %loop ]
+ %vc3 = phi <32 x float> [ zeroinitializer, %entry ], [ %vc3n, %loop ]
+ %vc4 = phi <32 x float> [ zeroinitializer, %entry ], [ %vc4n, %loop ]
+ %vc5 = phi <32 x float> [ zeroinitializer, %entry ], [ %vc5n, %loop ]
+ %vc6 = phi <32 x float> [ zeroinitializer, %entry ], [ %vc6n, %loop ]
+ %vc7 = phi <32 x float> [ zeroinitializer, %entry ], [ %vc7n, %loop ]
+ ; 12 independent MFMA accumulators (loop-carried).
+ %acc0 = phi <4 x float> [ zeroinitializer, %entry ], [ %r1_0, %loop ]
+ %acc1 = phi <4 x float> [ zeroinitializer, %entry ], [ %r1_1, %loop ]
+ %acc2 = phi <4 x float> [ zeroinitializer, %entry ], [ %r1_2, %loop ]
+ %acc3 = phi <4 x float> [ zeroinitializer, %entry ], [ %r1_3, %loop ]
+ %acc4 = phi <4 x float> [ zeroinitializer, %entry ], [ %r1_4, %loop ]
+ %acc5 = phi <4 x float> [ zeroinitializer, %entry ], [ %r1_5, %loop ]
+ %acc6 = phi <4 x float> [ zeroinitializer, %entry ], [ %r1_6, %loop ]
+ %acc7 = phi <4 x float> [ zeroinitializer, %entry ], [ %r1_7, %loop ]
+ %acc8 = phi <4 x float> [ zeroinitializer, %entry ], [ %r1_8, %loop ]
+ %acc9 = phi <4 x float> [ zeroinitializer, %entry ], [ %r1_9, %loop ]
+ %acc10 = phi <4 x float> [ zeroinitializer, %entry ], [ %r1_10, %loop ]
+ %acc11 = phi <4 x float> [ zeroinitializer, %entry ], [ %r1_11, %loop ]
+ ; Level-1
+ %r1_0 = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %a0, <8 x half> %b0, <4 x float> %acc0, i32 0, i32 0, i32 0)
+ %r1_1 = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %a0, <8 x half> %b0, <4 x float> %acc1, i32 0, i32 0, i32 0)
+ %r1_2 = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %a0, <8 x half> %b0, <4 x float> %acc2, i32 0, i32 0, i32 0)
+ %r1_3 = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %a0, <8 x half> %b0, <4 x float> %acc3, i32 0, i32 0, i32 0)
+ %r1_4 = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %a0, <8 x half> %b0, <4 x float> %acc4, i32 0, i32 0, i32 0)
+ %r1_5 = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %a0, <8 x half> %b0, <4 x float> %acc5, i32 0, i32 0, i32 0)
+ %r1_6 = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %a0, <8 x half> %b0, <4 x float> %acc6, i32 0, i32 0, i32 0)
+ %r1_7 = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %a0, <8 x half> %b0, <4 x float> %acc7, i32 0, i32 0, i32 0)
+ %r1_8 = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %a0, <8 x half> %b0, <4 x float> %acc8, i32 0, i32 0, i32 0)
+ %r1_9 = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %a0, <8 x half> %b0, <4 x float> %acc9, i32 0, i32 0, i32 0)
+ %r1_10 = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %a0, <8 x half> %b0, <4 x float> %acc10, i32 0, i32 0, i32 0)
+ %r1_11 = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %a0, <8 x half> %b0, <4 x float> %acc11, i32 0, i32 0, i32 0)
+ ; Level-2
+ %r2_0 = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %a1, <8 x half> %b1, <4 x float> %r1_0, i32 0, i32 0, i32 0)
+ %r2_1 = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %a1, <8 x half> %b1, <4 x float> %r1_1, i32 0, i32 0, i32 0)
+ %r2_2 = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %a1, <8 x half> %b1, <4 x float> %r1_2, i32 0, i32 0, i32 0)
+ %r2_3 = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %a1, <8 x half> %b1, <4 x float> %r1_3, i32 0, i32 0, i32 0)
+ %r2_4 = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %a1, <8 x half> %b1, <4 x float> %r1_4, i32 0, i32 0, i32 0)
+ %r2_5 = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %a1, <8 x half> %b1, <4 x float> %r1_5, i32 0, i32 0, i32 0)
+ %r2_6 = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %a1, <8 x half> %b1, <4 x float> %r1_6, i32 0, i32 0, i32 0)
+ %r2_7 = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %a1, <8 x half> %b1, <4 x float> %r1_7, i32 0, i32 0, i32 0)
+ %r2_8 = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %a1, <8 x half> %b1, <4 x float> %r1_8, i32 0, i32 0, i32 0)
+ %r2_9 = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %a1, <8 x half> %b1, <4 x float> %r1_9, i32 0, i32 0, i32 0)
+ %r2_10 = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %a1, <8 x half> %b1, <4 x float> %r1_10, i32 0, i32 0, i32 0)
+ %r2_11 = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %a1, <8 x half> %b1, <4 x float> %r1_11, i32 0, i32 0, i32 0)
+ ; Level-3
+ %r3_0 = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %a0, <8 x half> %b0, <4 x float> %r2_0, i32 0, i32 0, i32 0)
+ %r3_1 = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %a0, <8 x half> %b0, <4 x float> %r2_1, i32 0, i32 0, i32 0)
+ %r3_2 = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %a0, <8 x half> %b0, <4 x float> %r2_2, i32 0, i32 0, i32 0)
+ %r3_3 = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %a0, <8 x half> %b0, <4 x float> %r2_3, i32 0, i32 0, i32 0)
+ %r3_4 = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %a0, <8 x half> %b0, <4 x float> %r2_4, i32 0, i32 0, i32 0)
+ %r3_5 = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %a0, <8 x half> %b0, <4 x float> %r2_5, i32 0, i32 0, i32 0)
+ %r3_6 = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %a0, <8 x half> %b0, <4 x float> %r2_6, i32 0, i32 0, i32 0)
+ %r3_7 = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %a0, <8 x half> %b0, <4 x float> %r2_7, i32 0, i32 0, i32 0)
+ %r3_8 = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %a0, <8 x half> %b0, <4 x float> %r2_8, i32 0, i32 0, i32 0)
+ %r3_9 = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %a0, <8 x half> %b0, <4 x float> %r2_9, i32 0, i32 0, i32 0)
+ %r3_10 = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %a0, <8 x half> %b0, <4 x float> %r2_10, i32 0, i32 0, i32 0)
+ %r3_11 = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %a0, <8 x half> %b0, <4 x float> %r2_11, i32 0, i32 0, i32 0)
+ ; Level-4 — dst escapes to epilogue via extractelement (non-MAI user).
+ %r4_0 = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %a1, <8 x half> %b1, <4 x float> %r3_0, i32 0, i32 0, i32 0)
+ %r4_1 = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %a1, <8 x half> %b1, <4 x float> %r3_1, i32 0, i32 0, i32 0)
+ %r4_2 = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %a1, <8 x half> %b1, <4 x float> %r3_2, i32 0, i32 0, i32 0)
+ %r4_3 = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %a1, <8 x half> %b1, <4 x float> %r3_3, i32 0, i32 0, i32 0)
+ %r4_4 = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %a1, <8 x half> %b1, <4 x float> %r3_4, i32 0, i32 0, i32 0)
+ %r4_5 = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %a1, <8 x half> %b1, <4 x float> %r3_5, i32 0, i32 0, i32 0)
+ %r4_6 = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %a1, <8 x half> %b1, <4 x float> %r3_6, i32 0, i32 0, i32 0)
+ %r4_7 = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %a1, <8 x half> %b1, <4 x float> %r3_7, i32 0, i32 0, i32 0)
+ %r4_8 = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %a1, <8 x half> %b1, <4 x float> %r3_8, i32 0, i32 0, i32 0)
+ %r4_9 = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %a1, <8 x half> %b1, <4 x float> %r3_9, i32 0, i32 0, i32 0)
+ %r4_10 = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %a1, <8 x half> %b1, <4 x float> %r3_10, i32 0, i32 0, i32 0)
+ %r4_11 = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %a1, <8 x half> %b1, <4 x float> %r3_11, i32 0, i32 0, i32 0)
+ ; Keep carriers live.
+ %vc0n = fadd <32 x float> %vc0, %vc0
+ %vc1n = fadd <32 x float> %vc1, %vc1
+ %vc2n = fadd <32 x float> %vc2, %vc2
+ %vc3n = fadd <32 x float> %vc3, %vc3
+ %vc4n = fadd <32 x float> %vc4, %vc4
+ %vc5n = fadd <32 x float> %vc5, %vc5
+ %vc6n = fadd <32 x float> %vc6, %vc6
+ %vc7n = fadd <32 x float> %vc7, %vc7
+ %i.next = add i32 %i, 1
+ %cond = icmp slt i32 %i.next, %n
+ br i1 %cond, label %loop, label %epilogue
+
+epilogue:
+ %e = extractelement <4 x float> %r4_0, i32 0
+ store float %e, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+attributes #0 = { "amdgpu-agpr-alloc"="0,0" "amdgpu-flat-work-group-size"="64,64" "amdgpu-waves-per-eu"="1,1" }
More information about the llvm-commits
mailing list