[llvm-branch-commits] [llvm] [AMDGPU] start using new pass manager for backend codegen (PR #210251)
Vikram Hegde via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Sun Sep 27 23:31:49 PDT 2026
https://github.com/vikramRH updated https://github.com/llvm/llvm-project/pull/210251
>From 40b9d6d4d0b7f6119eea9dc7be64d66bc0f0a946 Mon Sep 17 00:00:00 2001
From: vikhegde <vikram.hegde at amd.com>
Date: Thu, 16 Jul 2026 17:15:53 +0530
Subject: [PATCH] [AMDGPU] start using new pass manager for backend codegen
---
llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h | 2 +
.../AMDGPU/GlobalISel/amdgpu-irtranslator.ll | 2 +-
.../GlobalISel/dereferenceable-declaration.ll | 2 +-
.../AMDGPU/GlobalISel/function-returns.ll | 2 +-
.../GlobalISel/function-returns.v2i65.ll | 2 +-
.../AMDGPU/GlobalISel/irtranslate-bf16.ll | 2 +-
.../irtranslator-amdgcn-cs-chain.ll | 4 +-
.../GlobalISel/irtranslator-amdgcn-sendmsg.ll | 2 +-
...irtranslator-amdgpu_kernel-system-sgprs.ll | 2 +-
.../GlobalISel/irtranslator-amdgpu_kernel.ll | 4 +-
.../GlobalISel/irtranslator-amdgpu_ps.ll | 2 +-
.../GlobalISel/irtranslator-amdgpu_vs.ll | 2 +-
.../GlobalISel/irtranslator-assert-align.ll | 2 +-
.../GlobalISel/irtranslator-atomicrmw.ll | 2 +-
.../irtranslator-call-abi-attribute-hints.ll | 2 +-
.../irtranslator-call-implicit-args.ll | 4 +-
.../GlobalISel/irtranslator-call-non-fixed.ll | 2 +-
.../irtranslator-call-return-values.ll | 2 +-
.../GlobalISel/irtranslator-call-sret.ll | 2 +-
.../AMDGPU/GlobalISel/irtranslator-call.ll | 2 +-
.../irtranslator-constant-fold-vector-op.ll | 2 +-
.../GlobalISel/irtranslator-constantexpr.ll | 2 +-
.../GlobalISel/irtranslator-constrained-fp.ll | 2 +-
.../irtranslator-fast-math-flags.ll | 2 +-
.../AMDGPU/GlobalISel/irtranslator-fence.ll | 2 +-
...translator-fixed-function-abi-vgpr-args.ll | 2 +-
.../GlobalISel/irtranslator-function-args.ll | 2 +-
.../irtranslator-function-args.v2i65.ll | 2 +-
.../GlobalISel/irtranslator-getelementptr.ll | 2 +-
.../GlobalISel/irtranslator-indirect-call.ll | 2 +-
.../GlobalISel/irtranslator-inline-asm.ll | 2 +-
.../GlobalISel/irtranslator-invariant.ll | 2 +-
.../irtranslator-memory-intrinsics.ll | 2 +-
.../GlobalISel/irtranslator-metadata.ll | 2 +-
...tor-non-integral-address-spaces-vectors.ll | 2 +-
...rtranslator-non-integral-address-spaces.ll | 2 +-
.../GlobalISel/irtranslator-prefetch.ll | 2 +-
.../AMDGPU/GlobalISel/irtranslator-ptrmask.ll | 2 +-
...rtranslator-readnone-intrinsic-callsite.ll | 2 +-
.../AMDGPU/GlobalISel/irtranslator-sat.ll | 2 +-
.../GlobalISel/irtranslator-sibling-call.ll | 2 +-
.../irtranslator-struct-return-intrinsics.ll | 2 +-
.../GlobalISel/irtranslator-tail-call.ll | 2 +-
.../GlobalISel/irtranslator-zext-vec-index.ll | 2 +-
.../promote-dependency-on-invariant-result.ll | 2 +-
...ee-registers-assertion-after-ra-failure.ll | 2 +-
.../AMDGPU/alloc-aligned-tuples-gfx1250.mir | 2 +-
.../AMDGPU/alloc-aligned-tuples-gfx908.mir | 2 +-
.../AMDGPU/alloc-aligned-tuples-gfx90a.mir | 2 +-
...-amdgpu-flat-work-group-size-vgpr-limit.ll | 20 +-
.../attributor-flatscratchinit-globalisel.ll | 2 +-
.../AMDGPU/av_spill_cross_bb_usage.mir | 2 +-
.../bad-agpr-vgpr-regalloc-priority.mir | 2 +-
.../AMDGPU/bb-prolog-spill-during-regalloc.ll | 2 +-
.../block-should-not-be-in-alive-blocks.mir | 2 +-
.../AMDGPU/call-alias-register-usage-agpr.ll | 36 +-
.../AMDGPU/call-alias-register-usage0.ll | 18 +-
.../AMDGPU/call-alias-register-usage1.ll | 22 +-
.../AMDGPU/call-alias-register-usage2.ll | 22 +-
.../AMDGPU/call-alias-register-usage3.ll | 22 +-
llvm/test/CodeGen/AMDGPU/call-constexpr.ll | 57 +-
.../CodeGen/AMDGPU/convergence-laneops.ll | 4 +-
.../test/CodeGen/AMDGPU/convergence-tokens.ll | 2 +-
llvm/test/CodeGen/AMDGPU/dead_bundle.mir | 2 +-
.../dummy-regalloc-priority-advisor.mir | 4 +-
...gpr-vgpr-count-propagation-direct-chain.ll | 42 +-
.../dvgpr-vgpr-count-propagation-indirect.ll | 26 +-
.../AMDGPU/dvgpr-vgpr-count-propagation.ll | 50 +-
llvm/test/CodeGen/AMDGPU/emit-high-vgprs.mir | 2 +-
.../CodeGen/AMDGPU/expand-si-indirect.mir | 2 +-
.../AMDGPU/extend-wwm-virt-reg-liveness.mir | 2 +-
.../CodeGen/AMDGPU/fastregalloc-sgpr-only.mir | 2 +-
llvm/test/CodeGen/AMDGPU/fmamk_fmaak-t16.mir | 2 +-
.../CodeGen/AMDGPU/fold-reload-into-exec.mir | 2 +-
.../CodeGen/AMDGPU/fold-reload-into-m0.mir | 2 +-
.../CodeGen/AMDGPU/fold-restore-undef-use.mir | 2 +-
.../CodeGen/AMDGPU/function-resource-usage.ll | 92 +--
.../greedy-alloc-fail-sgpr1024-spill.mir | 2 +-
.../greedy-instruction-split-subrange.mir | 2 +-
.../greedy-remark-crash-unassigned-reg.mir | 2 +-
.../AMDGPU/illegal-eviction-assert.mir | 2 +-
.../AMDGPU/indirect-call-known-callees.ll | 73 +-
.../AMDGPU/indirect-reg-read-imm-idx.mir | 2 +-
.../CodeGen/AMDGPU/inflate-av-remat-imm.mir | 2 +-
...class-vgpr-mfma-to-agpr-negative-tests.mir | 2 +-
...class-vgpr-mfma-to-av-with-load-source.mir | 2 +-
.../inflate-reg-class-vgpr-mfma-to-av.mir | 2 +-
...-reg-class-snippet-copy-use-after-free.mir | 2 +-
...nfloop-subrange-spill-inspect-subrange.mir | 2 +-
.../CodeGen/AMDGPU/infloop-subrange-spill.mir | 2 +-
.../irtranslator-whole-wave-functions.ll | 2 +-
.../issue197467-virt-reg-rewrite-failure.mir | 2 +-
llvm/test/CodeGen/AMDGPU/issue48473.mir | 2 +-
...sue98474-assigned-physreg-interference.mir | 2 +-
...egrewriter-live-out-undef-subregisters.mir | 2 +-
.../AMDGPU/large-avgpr-assign-last.mir | 2 +-
llvm/test/CodeGen/AMDGPU/llc-pipeline.ll | 10 +-
.../llvm.amdgcn.ds.gws.barrier-bundle.ll | 8 +-
...llvm.amdgcn.ds.gws.barrier-fastregalloc.ll | 2 +-
.../AMDGPU/llvm.amdgcn.mfma.gfx950.bf16.ll | 144 ++--
.../CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll | 612 +++++++-------
....amdgcn.mfma.scale.f32.16x16x128.f8f6f4.ll | 460 +++++------
...m.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll | 456 +++++------
.../AMDGPU/llvm.amdgcn.smfmac.gfx950.ll | 764 +++++++++---------
.../CodeGen/AMDGPU/lo16-hi16-illegal-copy.mir | 8 +-
.../CodeGen/AMDGPU/lo16-hi16-physreg-copy.mir | 6 +-
.../AMDGPU/lo16-lo16-physreg-copy-agpr.mir | 2 +-
.../AMDGPU/lo16-lo16-physreg-copy-sgpr.mir | 4 +-
...wer-control-flow-live-variables-update.mir | 2 +-
...ntrol-flow-live-variables-update.xfail.mir | 2 +-
.../CodeGen/AMDGPU/lto-lower-module-lds.ll | 10 +-
.../CodeGen/AMDGPU/machine-sink-cycle.mir | 4 +-
.../CodeGen/AMDGPU/memcpy-crash-issue63986.ll | 2 +
.../CodeGen/AMDGPU/memcpy_const_compare.ll | 12 +-
llvm/test/CodeGen/AMDGPU/memmove-var-size.ll | 3 +-
.../AMDGPU/memset-param-combinations.ll | 156 ++--
.../AMDGPU/mfma-no-register-aliasing.ll | 2 +-
...-masking-pre-ra-update-liveness-wave32.mir | 2 +-
...pt-exec-masking-pre-ra-update-liveness.mir | 2 +-
...ptimize-exec-mask-pre-ra-alloc-failure.mir | 2 +-
...al-regcopy-and-spill-missed-at-regalloc.ll | 4 +-
llvm/test/CodeGen/AMDGPU/pr51516.mir | 4 +-
...ut-of-registers-error-all-regs-reserved.ll | 6 +-
.../AMDGPU/ran-out-of-registers-errors.ll | 10 +-
.../ran-out-of-sgprs-allocation-failure.mir | 2 +-
.../AMDGPU/recursive-resource-usage-mcexpr.ll | 131 +--
...lloc-failure-overlapping-insert-assert.mir | 2 +-
...fast-dont-drop-subreg-index-issue61134.mir | 2 +-
.../regalloc-introduces-copy-sgpr-to-agpr.mir | 2 +-
.../AMDGPU/regalloc-sgpr128-partial-def.mir | 2 +-
.../AMDGPU/regalloc-undef-copy-fold.mir | 2 +-
...ster-killed-error-after-alloc-failure0.mir | 4 +-
.../remaining-virtual-register-operands.mir | 2 +-
.../test/CodeGen/AMDGPU/remat-dead-subreg.mir | 2 +-
llvm/test/CodeGen/AMDGPU/remat-smrd.mir | 2 +-
llvm/test/CodeGen/AMDGPU/remat-sop.mir | 2 +-
llvm/test/CodeGen/AMDGPU/remat-vop.mir | 2 +-
.../AMDGPU/rewrite-partial-reg-uses-dbg.mir | 2 +-
.../AMDGPU/rewrite-partial-reg-uses-gen.mir | 2 +-
.../AMDGPU/rewrite-partial-reg-uses.mir | 2 +-
.../schedule-amdgpu-tracker-physreg-crash.ll | 3 +-
...schedule-regpressure-ilp-metric-spills.mir | 4 +-
...ule-regpressure-no-unclustered-regions.mir | 2 +-
.../CodeGen/AMDGPU/sgpr-regalloc-flags.ll | 16 +-
.../sgpr-spill-update-only-slot-indexes.ll | 2 +-
llvm/test/CodeGen/AMDGPU/shift-select.ll | 9 +-
.../si-lower-sgpr-spills-cycle-header.mir | 2 +-
...si-lower-sgpr-spills-multi-entry-cycle.mir | 2 +-
.../si-lower-sgpr-spills-vgpr-lanes-usage.mir | 2 +-
.../AMDGPU/snippet-copy-bundle-regression.mir | 2 +-
.../AMDGPU/spill-empty-live-interval.mir | 2 +-
.../CodeGen/AMDGPU/spill-scavenge-offset.ll | 2 +-
.../CodeGen/AMDGPU/spill-vector-superclass.ll | 2 +-
...-do-not-undo-subclass-split-with-remat.mir | 2 +-
.../stress-regalloc-limits-regalloc.mir | 6 +-
...-last-chance-recoloring-alloc-succeeds.mir | 2 +-
llvm/test/CodeGen/AMDGPU/threeaddr-wmma.mir | 2 +-
.../true16-ra-pre-gfx11-regression-test.mir | 2 +-
llvm/test/CodeGen/AMDGPU/twoaddr-constrain.ll | 2 +-
.../CodeGen/AMDGPU/vgpr-agpr-limit-gfx90a.ll | 2 +-
.../test/CodeGen/AMDGPU/vgpr-limit-gfx1250.ll | 2 +-
.../virtregrewrite-undef-identity-copy.mir | 2 +-
llvm/test/CodeGen/AMDGPU/vopc-remat.mir | 2 +-
llvm/test/CodeGen/AMDGPU/wave32.ll | 14 +-
.../AMDGPU/wwm-regalloc-partial-pool.mir | 4 +-
.../AMDGPU/wwm-spill-superclass-pseudo.mir | 4 +-
.../GlobalISel/irtranslator-byte-type.ll | 2 +-
llvm/test/DebugInfo/AMDGPU/debug-loc-copy.ll | 6 +-
168 files changed, 1814 insertions(+), 1810 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h
index 35ca216eee079..55771af5bc12a 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h
@@ -98,6 +98,8 @@ class GCNTargetMachine final : public AMDGPUTargetMachine {
bool useIPRA() const override { return true; }
+ bool shouldDefaultToNewPM() const override { return true; }
+
Error buildCodeGenPipeline(ModulePassManager &MPM, ModuleAnalysisManager &MAM,
raw_pwrite_stream &Out, raw_pwrite_stream *DwoOut,
CodeGenFileType FileType,
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/amdgpu-irtranslator.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/amdgpu-irtranslator.ll
index 672aec2551c03..f46ac5c0d93e8 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/amdgpu-irtranslator.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/amdgpu-irtranslator.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -mtriple=amdgpu8.03 -O0 -stop-after=irtranslator -global-isel %s -o - 2>&1 | FileCheck %s
+; RUN: llc -mtriple=amdgpu8.03 -O0 -stop-after=ir-translator -global-isel %s -o - 2>&1 | FileCheck %s
; This file checks that the translation from llvm IR to generic
; MachineInstr is correct.
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/dereferenceable-declaration.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/dereferenceable-declaration.ll
index 52ee17d8cc6aa..371460495b810 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/dereferenceable-declaration.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/dereferenceable-declaration.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -global-isel -mtriple=amdgpu8.03 -stop-after=irtranslator -o - %s | FileCheck %s
+; RUN: llc -global-isel -mtriple=amdgpu8.03 -stop-after=ir-translator -o - %s | FileCheck %s
declare align(8) dereferenceable(8) ptr @declared_with_ret_deref() #0
declare align(8) ptr @unknown_decl() #0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/function-returns.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/function-returns.ll
index 7ea054e6a7bb7..e765ca0caf9e9 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/function-returns.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/function-returns.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -global-isel -stop-after=irtranslator -mtriple=amdgpu8.03-mesa-mesa3d -o - %s | FileCheck -enable-var-scope %s
+; RUN: llc -global-isel -stop-after=ir-translator -mtriple=amdgpu8.03-mesa-mesa3d -o - %s | FileCheck -enable-var-scope %s
; FIXME: Also test with a pre-gfx8 target.
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/function-returns.v2i65.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/function-returns.v2i65.ll
index 98b6d6f8da429..85fd5ded9e4d8 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/function-returns.v2i65.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/function-returns.v2i65.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -global-isel -mtriple=amdgpu8.03-mesa-mesa3d -stop-after=irtranslator -o - %s | FileCheck %s
+; RUN: llc -global-isel -mtriple=amdgpu8.03-mesa-mesa3d -stop-after=ir-translator -o - %s | FileCheck %s
define <2 x i65> @v2i65_func_void() #0 {
; CHECK-LABEL: name: v2i65_func_void
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslate-bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslate-bf16.ll
index 5e4da6ba777b6..134af09936fbe 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslate-bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslate-bf16.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 4
-; RUN: llc < %s -global-isel -stop-after=irtranslator -mtriple=amdgpu9.00 | FileCheck %s -check-prefixes=GFX9
+; RUN: llc < %s -global-isel -stop-after=ir-translator -mtriple=amdgpu9.00 | FileCheck %s -check-prefixes=GFX9
; tests bf16 argument & return values lowering.
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgcn-cs-chain.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgcn-cs-chain.ll
index 442876257794c..008e1d0e21af7 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgcn-cs-chain.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgcn-cs-chain.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 2
-; RUN: llc --global-isel=1 -mtriple=amdgpu11.00--amdpal -stop-after=irtranslator %s -o - | FileCheck %s --check-prefix=GFX11
-; RUN: llc --global-isel=1 -mtriple=amdgpu10.30--amdpal -stop-after=irtranslator %s -o - | FileCheck %s --check-prefix=GFX10
+; RUN: llc --global-isel=1 -mtriple=amdgpu11.00--amdpal -stop-after=ir-translator %s -o - | FileCheck %s --check-prefix=GFX11
+; RUN: llc --global-isel=1 -mtriple=amdgpu10.30--amdpal -stop-after=ir-translator %s -o - | FileCheck %s --check-prefix=GFX10
declare amdgpu_cs_chain void @callee(<3 x i32> inreg, { i32, ptr addrspace(5), i32, i32 })
declare amdgpu_cs_chain_preserve void @callee_preserve(<3 x i32> inreg, { i32, ptr addrspace(5), i32, i32 })
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgcn-sendmsg.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgcn-sendmsg.ll
index 099c8e1bc0416..d402f605ad9d0 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgcn-sendmsg.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgcn-sendmsg.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -mtriple=amdgpu6.00 -O0 -stop-after=irtranslator -global-isel %s -o - | FileCheck %s
+; RUN: llc -mtriple=amdgpu6.00 -O0 -stop-after=ir-translator -global-isel %s -o - | FileCheck %s
declare void @llvm.amdgcn.s.sendmsg(i32 immarg, i32)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgpu_kernel-system-sgprs.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgpu_kernel-system-sgprs.ll
index 717f6a10cf691..293cb5b966d55 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgpu_kernel-system-sgprs.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgpu_kernel-system-sgprs.ll
@@ -1,4 +1,4 @@
-; RUN: opt -mtriple=amdgpu8.03-amd-amdhsa -passes=amdgpu-attributor < %s | llc -global-isel -mtriple=amdgpu8.03-amd-amdhsa -amdgpu-ir-lower-kernel-arguments=0 -stop-after=irtranslator -o - | FileCheck -check-prefix=HSA %s
+; RUN: opt -mtriple=amdgpu8.03-amd-amdhsa -passes=amdgpu-attributor < %s | llc -global-isel -mtriple=amdgpu8.03-amd-amdhsa -amdgpu-ir-lower-kernel-arguments=0 -stop-after=ir-translator -o - | FileCheck -check-prefix=HSA %s
; HSA-LABEL: name: default_kernel
; HSA: liveins:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgpu_kernel.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgpu_kernel.ll
index 75cb59e7d75f4..5e602576a3853 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgpu_kernel.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgpu_kernel.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -global-isel -mtriple=amdgpu8.03-amd-amdhsa -amdgpu-ir-lower-kernel-arguments=0 -stop-after=irtranslator %s -o - | FileCheck -check-prefix=HSA-VI %s
-; RUN: llc -global-isel -mtriple=amdgpu8.03-- -amdgpu-ir-lower-kernel-arguments=0 -stop-after=irtranslator %s -o - | FileCheck -check-prefix=LEGACY-MESA-VI %s
+; RUN: llc -global-isel -mtriple=amdgpu8.03-amd-amdhsa -amdgpu-ir-lower-kernel-arguments=0 -stop-after=ir-translator %s -o - | FileCheck -check-prefix=HSA-VI %s
+; RUN: llc -global-isel -mtriple=amdgpu8.03-- -amdgpu-ir-lower-kernel-arguments=0 -stop-after=ir-translator %s -o - | FileCheck -check-prefix=LEGACY-MESA-VI %s
define amdgpu_kernel void @i8_arg(ptr addrspace(1) nocapture %out, i8 %in) nounwind {
; HSA-VI-LABEL: name: i8_arg
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgpu_ps.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgpu_ps.ll
index aa44f28775ced..84df85cc0d4ed 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgpu_ps.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgpu_ps.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -global-isel -mtriple=amdgpu8.02-mesa-mesa3d -stop-after=irtranslator %s -o - | FileCheck %s
+; RUN: llc -global-isel -mtriple=amdgpu8.02-mesa-mesa3d -stop-after=ir-translator %s -o - | FileCheck %s
; Check that we correctly skip over disabled inputs
define amdgpu_ps void @disabled_input(float inreg %arg0, float %psinput0, float %psinput1) #1 {
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgpu_vs.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgpu_vs.ll
index f392820ba6c90..239fac44d7470 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgpu_vs.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgpu_vs.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -global-isel -mtriple=amdgpu8.03-mesa-mesa3d -stop-after=irtranslator %s -o - | FileCheck %s
+; RUN: llc -global-isel -mtriple=amdgpu8.03-mesa-mesa3d -stop-after=ir-translator %s -o - | FileCheck %s
define amdgpu_vs void @test_f32_inreg(float inreg %arg0) {
; CHECK-LABEL: name: test_f32_inreg
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-assert-align.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-assert-align.ll
index 5ed7d8d837ae7..a68b30fa33955 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-assert-align.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-assert-align.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -mtriple=amdgpu8.03 -O0 -stop-after=irtranslator -global-isel -o - %s | FileCheck %s
+; RUN: llc -mtriple=amdgpu8.03 -O0 -stop-after=ir-translator -global-isel -o - %s | FileCheck %s
; TODO: Could potentially insert it here
define void @arg_align_8(ptr addrspace(1) align 8 %arg0) {
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-atomicrmw.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-atomicrmw.ll
index d0be9b9c66f9f..010ee839253b9 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-atomicrmw.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-atomicrmw.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -global-isel -mtriple=amdgpu9.42 -O0 -stop-after=irtranslator -o - %s | FileCheck %s
+; RUN: llc -global-isel -mtriple=amdgpu9.42 -O0 -stop-after=ir-translator -o - %s | FileCheck %s
define float @test_atomicrmw_fadd(ptr addrspace(3) %addr) {
; CHECK-LABEL: name: test_atomicrmw_fadd
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call-abi-attribute-hints.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call-abi-attribute-hints.ll
index 2a996480365f6..029428504a058 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call-abi-attribute-hints.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call-abi-attribute-hints.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -global-isel -stop-after=irtranslator -mtriple=amdgpu9.00-amd-amdhsa -o - %s | FileCheck -enable-var-scope %s
+; RUN: llc -global-isel -stop-after=ir-translator -mtriple=amdgpu9.00-amd-amdhsa -o - %s | FileCheck -enable-var-scope %s
; Test that we don't insert code to pass implicit arguments we know
; the callee does not need.
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call-implicit-args.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call-implicit-args.ll
index 012e11e7babf0..30cda9dc3416b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call-implicit-args.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call-implicit-args.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -global-isel -stop-after=irtranslator -mtriple=amdgpu9.00-mesa-mesa3d -o - %s | FileCheck -enable-var-scope -check-prefix=GFX900 %s
-; RUN: llc -global-isel -stop-after=irtranslator -mtriple=amdgpu9.08-mesa-mesa3d -o - %s | FileCheck -enable-var-scope -check-prefix=GFX908 %s
+; RUN: llc -global-isel -stop-after=ir-translator -mtriple=amdgpu9.00-mesa-mesa3d -o - %s | FileCheck -enable-var-scope -check-prefix=GFX900 %s
+; RUN: llc -global-isel -stop-after=ir-translator -mtriple=amdgpu9.08-mesa-mesa3d -o - %s | FileCheck -enable-var-scope -check-prefix=GFX908 %s
; Workitem IDs are passed to the kernel differently for gfx908
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call-non-fixed.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call-non-fixed.ll
index 70d3009fa3263..56756f0ef3025 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call-non-fixed.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call-non-fixed.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -global-isel -stop-after=irtranslator -mtriple=amdgpu9.00-mesa-mesa3d -o - %s | FileCheck -enable-var-scope %s
+; RUN: llc -global-isel -stop-after=ir-translator -mtriple=amdgpu9.00-mesa-mesa3d -o - %s | FileCheck -enable-var-scope %s
; amdgpu_gfx calling convention
declare hidden amdgpu_gfx void @external_gfx_void_func_void() #0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call-return-values.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call-return-values.ll
index c4a86abf7ec78..e9baec38fe458 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call-return-values.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call-return-values.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -global-isel -mtriple=amdgpu9.00-amd-amdhsa -stop-after=irtranslator < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -global-isel -mtriple=amdgpu9.00-amd-amdhsa -stop-after=ir-translator < %s | FileCheck -check-prefix=GCN %s
declare i1 @external_i1_func_void() #0
declare zeroext i1 @external_i1_zeroext_func_void() #0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call-sret.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call-sret.ll
index b1909a9f0348a..bbffe60aa5d76 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call-sret.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call-sret.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -global-isel -mtriple=amdgpu9.00-amd-amdhsa -stop-after=irtranslator < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -global-isel -mtriple=amdgpu9.00-amd-amdhsa -stop-after=ir-translator < %s | FileCheck -check-prefix=GCN %s
declare hidden void @external_void_func_sret_struct_i8_i32_byval_struct_i8_i32(ptr addrspace(5) sret({ i8, i32 }), ptr addrspace(5) byval({ i8, i32 })) #0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call.ll
index 72ee1fa30b2b7..21a40411258fc 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -stop-after=irtranslator -mtriple=amdgpu9.00-mesa-mesa3d -o - %s | FileCheck -enable-var-scope %s
+; RUN: llc -global-isel -global-isel-abort=2 -stop-after=ir-translator -mtriple=amdgpu9.00-mesa-mesa3d -o - %s | FileCheck -enable-var-scope %s
declare hidden void @external_void_func_void() #0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-constant-fold-vector-op.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-constant-fold-vector-op.ll
index addbdde19fd98..afbc6575d624b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-constant-fold-vector-op.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-constant-fold-vector-op.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -O0 -global-isel -mtriple=amdgpu9.00-amd-amdhsa -stop-after=irtranslator -o - %s | FileCheck %s
+; RUN: llc -O0 -global-isel -mtriple=amdgpu9.00-amd-amdhsa -stop-after=ir-translator -o - %s | FileCheck %s
; The CSEMIRBuilder was mishandling the result operand when constant
; folding a vector operation, resulting in a missing def for the
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-constantexpr.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-constantexpr.ll
index 5614e7d422b00..2e28f8dd56021 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-constantexpr.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-constantexpr.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -global-isel -mtriple=amdgpu7.00-amd-amdhsa -stop-after=irtranslator -o - %s | FileCheck %s
+; RUN: llc -global-isel -mtriple=amdgpu7.00-amd-amdhsa -stop-after=ir-translator -o - %s | FileCheck %s
@var = global i32 poison
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-constrained-fp.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-constrained-fp.ll
index 6961a876a5ca3..02bd1976c67a2 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-constrained-fp.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-constrained-fp.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -global-isel -mtriple=amdgpu6.00 -stop-after=irtranslator %s -o - | FileCheck %s
+; RUN: llc -global-isel -mtriple=amdgpu6.00 -stop-after=ir-translator %s -o - | FileCheck %s
define float @v_constained_fadd_f32_fpexcept_strict(float %x, float %y) #0 {
; CHECK-LABEL: name: v_constained_fadd_f32_fpexcept_strict
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-fast-math-flags.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-fast-math-flags.ll
index 7163695b8606b..3ef761dad2a64 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-fast-math-flags.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-fast-math-flags.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=amdgpu8.03 -O0 -stop-after=irtranslator -global-isel %s -o - | FileCheck %s
+; RUN: llc -mtriple=amdgpu8.03 -O0 -stop-after=ir-translator -global-isel %s -o - | FileCheck %s
; Check flags are preserved for a regular instruction.
; CHECK-LABEL: name: fadd_nnan
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-fence.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-fence.ll
index 2d60999ff8773..bb863e30509de 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-fence.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-fence.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -global-isel -mtriple=amdgpu7.00-amd-amdhsa -stop-after=irtranslator < %s | FileCheck %s
+; RUN: llc -global-isel -mtriple=amdgpu7.00-amd-amdhsa -stop-after=ir-translator < %s | FileCheck %s
define amdgpu_kernel void @system_one_as_acquire() {
; CHECK-LABEL: name: system_one_as_acquire
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-fixed-function-abi-vgpr-args.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-fixed-function-abi-vgpr-args.ll
index 0feb2d76f0074..df6ea0920ab85 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-fixed-function-abi-vgpr-args.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-fixed-function-abi-vgpr-args.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -global-isel -mtriple=amdgpu7.00-amd-amdhsa -stop-after=irtranslator -o - %s | FileCheck --check-prefix=FIXED %s
+; RUN: llc -global-isel -mtriple=amdgpu7.00-amd-amdhsa -stop-after=ir-translator -o - %s | FileCheck --check-prefix=FIXED %s
; Make sure arg1 is not allocated in v31, which is reserved for
; workitem IDs with -amdgpu-fixed-function-abi.
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-function-args.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-function-args.ll
index 25f952df63f29..b89257cfeecb1 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-function-args.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-function-args.ll
@@ -2,7 +2,7 @@
; Note update_mir_test_checks does not support generating checks for
; the frame info, so some functions have manually added stack object
; checks.
-; RUN: llc -mtriple=amdgpu8.03 -O0 -stop-after=irtranslator -global-isel -o - %s | FileCheck %s
+; RUN: llc -mtriple=amdgpu8.03 -O0 -stop-after=ir-translator -global-isel -o - %s | FileCheck %s
; FIXME: pre-VI should have same ABI without legal i16 operations.
define void @void_func_empty_arg({} %arg0, i32 %arg1) #0 {
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-function-args.v2i65.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-function-args.v2i65.ll
index c03392d5423e6..1caafbc71e2a9 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-function-args.v2i65.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-function-args.v2i65.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -global-isel -mtriple=amdgpu8.03 -stop-after=irtranslator -o - %s | FileCheck %s
+; RUN: llc -global-isel -mtriple=amdgpu8.03 -stop-after=ir-translator -o - %s | FileCheck %s
define void @void_func_v2i65(<2 x i65> %arg0) #0 {
; CHECK-LABEL: name: void_func_v2i65
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-getelementptr.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-getelementptr.ll
index fd7b7056ad57d..b689c7eccaa1b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-getelementptr.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-getelementptr.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -global-isel -mtriple=amdgpu7.00-amd-amdhsa -stop-after=irtranslator -o - %s | FileCheck %s
+; RUN: llc -global-isel -mtriple=amdgpu7.00-amd-amdhsa -stop-after=ir-translator -o - %s | FileCheck %s
; Test 64-bit pointer with 64-bit index
define <2 x ptr addrspace(1)> @vector_gep_v2p1_index_v2i64(<2 x ptr addrspace(1)> %ptr, <2 x i64> %idx) {
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-indirect-call.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-indirect-call.ll
index c0d74ff42202e..8ea438099edfb 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-indirect-call.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-indirect-call.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -global-isel -stop-after=irtranslator -mtriple=amdgpu9.00-mesa-mesa3d -o - %s | FileCheck -enable-var-scope %s
+; RUN: llc -global-isel -stop-after=ir-translator -mtriple=amdgpu9.00-mesa-mesa3d -o - %s | FileCheck -enable-var-scope %s
define amdgpu_kernel void @test_indirect_call_sgpr_ptr(ptr %fptr) {
; CHECK-LABEL: name: test_indirect_call_sgpr_ptr
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-inline-asm.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-inline-asm.ll
index fda4647c73b4f..34105bf079609 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-inline-asm.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-inline-asm.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -mtriple=amdgpu9.08-amd-amdhsa -O0 -global-isel -stop-after=irtranslator -o - %s | FileCheck %s
+; RUN: llc -mtriple=amdgpu9.08-amd-amdhsa -O0 -global-isel -stop-after=ir-translator -o - %s | FileCheck %s
define amdgpu_kernel void @asm_convergent() convergent{
; CHECK-LABEL: name: asm_convergent
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-invariant.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-invariant.ll
index 418ca159196e0..ef89a93cbe10c 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-invariant.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-invariant.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -simplify-mir -global-isel -mtriple=amdgpu6.00 -stop-after=irtranslator %s -o - | FileCheck %s
+; RUN: llc -simplify-mir -global-isel -mtriple=amdgpu6.00 -stop-after=ir-translator %s -o - | FileCheck %s
; Check the flags set on the memory operands for loads determined to
; be constants by alias analysis.
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-memory-intrinsics.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-memory-intrinsics.ll
index 46498710bafc9..985c4853e3f91 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-memory-intrinsics.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-memory-intrinsics.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -global-isel -mtriple=amdgpu6.00 -O0 -stop-after=irtranslator %s -o - | FileCheck %s
+; RUN: llc -global-isel -mtriple=amdgpu6.00 -O0 -stop-after=ir-translator %s -o - | FileCheck %s
; Size operand should be the minimum of the two pointer sizes.
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-metadata.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-metadata.ll
index 1c6c52e0be081..2ac07a6907f63 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-metadata.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-metadata.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -global-isel -mtriple=amdgpu9.00 -stop-after=irtranslator -o - %s | FileCheck %s
+; RUN: llc -global-isel -mtriple=amdgpu9.00 -stop-after=ir-translator -o - %s | FileCheck %s
; Make sure intrinsics with metadata arguments are translated
define i32 @reloc_constant() {
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-non-integral-address-spaces-vectors.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-non-integral-address-spaces-vectors.ll
index fddf215dd272a..56b003f0db710 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-non-integral-address-spaces-vectors.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-non-integral-address-spaces-vectors.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 2
-; RUN: llc -global-isel -mtriple=amdgpu9.00-amd-amdpal -o - -stop-after=irtranslator < %s | FileCheck %s
+; RUN: llc -global-isel -mtriple=amdgpu9.00-amd-amdpal -o - -stop-after=ir-translator < %s | FileCheck %s
define <2 x ptr addrspace(7)> @no_auto_constfold_gep_vector() {
; CHECK-LABEL: name: no_auto_constfold_gep_vector
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-non-integral-address-spaces.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-non-integral-address-spaces.ll
index 944604bc8d727..bdbdb7b954f88 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-non-integral-address-spaces.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-non-integral-address-spaces.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -global-isel -mtriple=amdgpu9.00-amd-amdpal -o - -stop-after=irtranslator %s | FileCheck %s
+; RUN: llc -global-isel -mtriple=amdgpu9.00-amd-amdpal -o - -stop-after=ir-translator %s | FileCheck %s
; Check that the CSEMIRBuilder doesn't fold away the getelementptr during IRTranslator
define ptr addrspace(7) @no_auto_constfold_gep() {
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-prefetch.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-prefetch.ll
index 3629f6361ed03..a507d8c599305 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-prefetch.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-prefetch.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 4
-; RUN: llc -global-isel -mtriple=amdgpu6.00 -stop-after=irtranslator < %s | FileCheck %s
+; RUN: llc -global-isel -mtriple=amdgpu6.00 -stop-after=ir-translator < %s | FileCheck %s
define void @prefetch_read(ptr %ptr) {
; CHECK-LABEL: name: prefetch_read
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-ptrmask.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-ptrmask.ll
index 31da8e7c7179d..2474bb2d0ae7d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-ptrmask.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-ptrmask.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -global-isel -mtriple=amdgpu7.00-amd-amdhsa -stop-after=irtranslator < %s | FileCheck %s
+; RUN: llc -global-isel -mtriple=amdgpu7.00-amd-amdhsa -stop-after=ir-translator < %s | FileCheck %s
define ptr @ptrmask_flat_i64(ptr %ptr, i64 %mask) {
; CHECK-LABEL: name: ptrmask_flat_i64
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-readnone-intrinsic-callsite.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-readnone-intrinsic-callsite.ll
index 3e05ed51ea1bf..ae280dda8300e 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-readnone-intrinsic-callsite.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-readnone-intrinsic-callsite.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=amdgpu7.00-amd-amdhsa -global-isel -stop-after=irtranslator -o - %s | FileCheck %s
+; RUN: llc -mtriple=amdgpu7.00-amd-amdhsa -global-isel -stop-after=ir-translator -o - %s | FileCheck %s
; Make sure that an intrinsic declaration that has side effects, but
; called with a readnone call site is translated to
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-sat.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-sat.ll
index 55b4188ddbda0..1fb306fb2cf28 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-sat.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-sat.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -mtriple=amdgpu6.00 -global-isel -stop-after=irtranslator %s -o - | FileCheck %s
+; RUN: llc -mtriple=amdgpu6.00 -global-isel -stop-after=ir-translator %s -o - | FileCheck %s
define i16 @uaddsat_i16(i16 %lhs, i16 %rhs) {
; CHECK-LABEL: name: uaddsat_i16
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-sibling-call.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-sibling-call.ll
index d13da16dc57e2..b11e82c46244b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-sibling-call.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-sibling-call.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -global-isel -stop-after=irtranslator -mtriple=amdgpu9.00-amd-amdhsa < %s | FileCheck -enable-var-scope -check-prefix=GCN %s
+; RUN: llc -global-isel -stop-after=ir-translator -mtriple=amdgpu9.00-amd-amdhsa < %s | FileCheck -enable-var-scope -check-prefix=GCN %s
; This is a copy of sibling-call.ll, but stops after the IRTranslator.
define fastcc i32 @i32_fastcc_i32_i32(i32 %arg0, i32 %arg1) #1 {
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-struct-return-intrinsics.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-struct-return-intrinsics.ll
index ad3424e82475f..861d1b1d71cab 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-struct-return-intrinsics.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-struct-return-intrinsics.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -mtriple=amdgpu6.00-mesa-mesa3d -global-isel -stop-after=irtranslator -o - %s | FileCheck %s
+; RUN: llc -mtriple=amdgpu6.00-mesa-mesa3d -global-isel -stop-after=ir-translator -o - %s | FileCheck %s
declare { float, i1 } @llvm.amdgcn.div.scale.f32(float, float, i1)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-tail-call.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-tail-call.ll
index 27de5521fc859..f7270ca094d67 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-tail-call.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-tail-call.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -global-isel -stop-after=irtranslator -mtriple=amdgpu9.00-mesa-mesa3d -o - %s | FileCheck -enable-var-scope %s
+; RUN: llc -global-isel -stop-after=ir-translator -mtriple=amdgpu9.00-mesa-mesa3d -o - %s | FileCheck -enable-var-scope %s
declare hidden void @external_void_func_void()
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-zext-vec-index.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-zext-vec-index.ll
index 27dd893b68386..1a3be34e03bd7 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-zext-vec-index.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-zext-vec-index.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -mtriple=amdgpu6.00 -O0 -stop-after=irtranslator -global-isel %s -o - | FileCheck %s
+; RUN: llc -mtriple=amdgpu6.00 -O0 -stop-after=ir-translator -global-isel %s -o - | FileCheck %s
define i8 @f_i1_1() {
; CHECK-LABEL: name: f_i1_1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/promote-dependency-on-invariant-result.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/promote-dependency-on-invariant-result.ll
index a3caff4c1e04e..f7c0e7d9e6230 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/promote-dependency-on-invariant-result.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/promote-dependency-on-invariant-result.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -mtriple=amdgpu9.0a -O0 -global-isel=true --stop-after=irtranslator -o - %s | FileCheck %s
+; RUN: llc -mtriple=amdgpu9.0a -O0 -global-isel=true --stop-after=ir-translator -o - %s | FileCheck %s
declare ptr @llvm.invariant.start.p5(i64 immarg, ptr addrspace(5) nocapture)
declare void @llvm.invariant.end.p5(ptr, i64 immarg, ptr addrspace(5) nocapture)
diff --git a/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers-assertion-after-ra-failure.ll b/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers-assertion-after-ra-failure.ll
index d35f04c64ac89..f3b0a315d6166 100644
--- a/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers-assertion-after-ra-failure.ll
+++ b/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers-assertion-after-ra-failure.ll
@@ -1,5 +1,5 @@
; RUN: not llc -mtriple=amdgpu9.0a-amd-amdhsa -filetype=null %s 2>&1 | FileCheck -check-prefix=ERR %s
-; RUN: not llc -mtriple=amdgpu9.0a-amd-amdhsa -vgpr-regalloc=basic -filetype=null %s 2>&1 | FileCheck -check-prefix=ERR %s
+; RUN: not llc -enable-new-pm=0 -mtriple=amdgpu9.0a-amd-amdhsa -vgpr-regalloc=basic -filetype=null %s 2>&1 | FileCheck -check-prefix=ERR %s
; ERR: error: <unknown>:0:0: no registers from class available to allocate in function 'no_free_vgprs_at_agpr_to_agpr_copy'
diff --git a/llvm/test/CodeGen/AMDGPU/alloc-aligned-tuples-gfx1250.mir b/llvm/test/CodeGen/AMDGPU/alloc-aligned-tuples-gfx1250.mir
index 8c5645ad049e6..cfb03e87d86c0 100644
--- a/llvm/test/CodeGen/AMDGPU/alloc-aligned-tuples-gfx1250.mir
+++ b/llvm/test/CodeGen/AMDGPU/alloc-aligned-tuples-gfx1250.mir
@@ -1,4 +1,4 @@
-# RUN: llc -mtriple=amdgpu12.50 -start-before=greedy,0 -stop-after=virtregrewriter,2 -verify-machineinstrs -o - %s | FileCheck --check-prefixes=GCN,GFX1250 %s
+# RUN: llc -mtriple=amdgpu12.50 -start-before=greedy,1 -stop-after=virt-reg-rewriter,3 -verify-machineinstrs -o - %s | FileCheck --check-prefixes=GCN,GFX1250 %s
# Using the unaligned vector tuples are OK as long as they aren't used
# in a real instruction.
diff --git a/llvm/test/CodeGen/AMDGPU/alloc-aligned-tuples-gfx908.mir b/llvm/test/CodeGen/AMDGPU/alloc-aligned-tuples-gfx908.mir
index cb042fa6bc261..76c807aaeb0da 100644
--- a/llvm/test/CodeGen/AMDGPU/alloc-aligned-tuples-gfx908.mir
+++ b/llvm/test/CodeGen/AMDGPU/alloc-aligned-tuples-gfx908.mir
@@ -1,4 +1,4 @@
-# RUN: llc -mtriple=amdgpu9.08 -start-before=greedy,0 -stop-after=virtregrewriter,2 -verify-machineinstrs -o - %s | FileCheck --check-prefixes=GCN,GFX908 %s
+# RUN: llc -mtriple=amdgpu9.08 -start-before=greedy,1 -stop-after=virt-reg-rewriter,3 -verify-machineinstrs -o - %s | FileCheck --check-prefixes=GCN,GFX908 %s
---
# GCN-LABEL: name: alloc_vgpr_64
diff --git a/llvm/test/CodeGen/AMDGPU/alloc-aligned-tuples-gfx90a.mir b/llvm/test/CodeGen/AMDGPU/alloc-aligned-tuples-gfx90a.mir
index c6593c0dbeb45..6bebdb59a1548 100644
--- a/llvm/test/CodeGen/AMDGPU/alloc-aligned-tuples-gfx90a.mir
+++ b/llvm/test/CodeGen/AMDGPU/alloc-aligned-tuples-gfx90a.mir
@@ -1,4 +1,4 @@
-# RUN: llc -mtriple=amdgpu9.0a -start-before=greedy,0 -stop-after=virtregrewriter,2 -verify-machineinstrs -o - %s | FileCheck --check-prefixes=GCN,GFX90A %s
+# RUN: llc -mtriple=amdgpu9.0a -start-before=greedy,1 -stop-after=virt-reg-rewriter,3 -verify-machineinstrs -o - %s | FileCheck --check-prefixes=GCN,GFX90A %s
# RUN: llc -enable-new-pm -mtriple=amdgpu9.0a -passes='greedy<sgpr>,virt-reg-rewriter<no-clear-vregs>,stack-slot-coloring,si-lower-sgpr-spills,si-pre-allocate-wwm-regs,greedy<wwm>,si-lower-wwm-copies,virt-reg-rewriter<no-clear-vregs>,amdgpu-reserve-wwm-regs,greedy<vgpr>,amdgpu-nsa-reassign,virt-reg-rewriter' -o - %s | FileCheck --check-prefixes=GCN,GFX90A %s
# Using the unaligned vector tuples are OK as long as they aren't used
diff --git a/llvm/test/CodeGen/AMDGPU/attr-amdgpu-flat-work-group-size-vgpr-limit.ll b/llvm/test/CodeGen/AMDGPU/attr-amdgpu-flat-work-group-size-vgpr-limit.ll
index 412933f78660b..332f237d8d43e 100644
--- a/llvm/test/CodeGen/AMDGPU/attr-amdgpu-flat-work-group-size-vgpr-limit.ll
+++ b/llvm/test/CodeGen/AMDGPU/attr-amdgpu-flat-work-group-size-vgpr-limit.ll
@@ -1,15 +1,15 @@
; -enable-misched=false makes the register usage more predictable
; -regalloc=fast just makes the test run faster
-; RUN: llc -mtriple=amdgpu9.00 -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc=fast -vgpr-regalloc=fast -wwm-regalloc=fast < %s | FileCheck %s --check-prefixes=GCN,GFX9
-; RUN: llc -mtriple=amdgpu9.0a -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc=fast -vgpr-regalloc=fast -wwm-regalloc=fast < %s | FileCheck %s --check-prefixes=GCN,GFX90A
-; RUN: llc -mtriple=amdgpu10.10 -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc=fast -vgpr-regalloc=fast -wwm-regalloc=fast < %s | FileCheck %s --check-prefixes=GCN,GFX10WGP-WAVE32
-; RUN: llc -mtriple=amdgpu10.10 -mattr=+wavefrontsize64 -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc=fast -vgpr-regalloc=fast -wwm-regalloc=fast < %s | FileCheck %s --check-prefixes=GCN,GFX10WGP-WAVE64
-; RUN: llc -mtriple=amdgpu10.10 -mattr=+cumode -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc=fast -vgpr-regalloc=fast -wwm-regalloc=fast < %s | FileCheck %s --check-prefixes=GCN,GFX10CU-WAVE32
-; RUN: llc -mtriple=amdgpu10.10 -mattr=+cumode,+wavefrontsize64 -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc=fast -vgpr-regalloc=fast -wwm-regalloc=fast < %s | FileCheck %s --check-prefixes=GCN,GFX10CU-WAVE64
-; RUN: llc -mtriple=amdgpu11.00 -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc=fast -vgpr-regalloc=fast -wwm-regalloc=fast < %s | FileCheck %s --check-prefixes=GCN,GFX11WGP-WAVE32
-; RUN: llc -mtriple=amdgpu11.00 -mattr=+wavefrontsize64 -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc=fast -vgpr-regalloc=fast -wwm-regalloc=fast < %s | FileCheck %s --check-prefixes=GCN,GFX11WGP-WAVE64
-; RUN: llc -mtriple=amdgpu11.00 -mattr=+cumode -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc=fast -vgpr-regalloc=fast -wwm-regalloc=fast < %s | FileCheck %s --check-prefixes=GCN,GFX11CU-WAVE32
-; RUN: llc -mtriple=amdgpu11.00 -mattr=+cumode,+wavefrontsize64 -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc=fast -vgpr-regalloc=fast -wwm-regalloc=fast < %s | FileCheck %s --check-prefixes=GCN,GFX11CU-WAVE64
+; RUN: llc -mtriple=amdgpu9.00 -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc-npm=fast -vgpr-regalloc-npm=fast -wwm-regalloc-npm=fast < %s | FileCheck %s --check-prefixes=GCN,GFX9
+; RUN: llc -mtriple=amdgpu9.0a -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc-npm=fast -vgpr-regalloc-npm=fast -wwm-regalloc-npm=fast < %s | FileCheck %s --check-prefixes=GCN,GFX90A
+; RUN: llc -mtriple=amdgpu10.10 -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc-npm=fast -vgpr-regalloc-npm=fast -wwm-regalloc-npm=fast < %s | FileCheck %s --check-prefixes=GCN,GFX10WGP-WAVE32
+; RUN: llc -mtriple=amdgpu10.10 -mattr=+wavefrontsize64 -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc-npm=fast -vgpr-regalloc-npm=fast -wwm-regalloc-npm=fast < %s | FileCheck %s --check-prefixes=GCN,GFX10WGP-WAVE64
+; RUN: llc -mtriple=amdgpu10.10 -mattr=+cumode -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc-npm=fast -vgpr-regalloc-npm=fast -wwm-regalloc-npm=fast < %s | FileCheck %s --check-prefixes=GCN,GFX10CU-WAVE32
+; RUN: llc -mtriple=amdgpu10.10 -mattr=+cumode,+wavefrontsize64 -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc-npm=fast -vgpr-regalloc-npm=fast -wwm-regalloc-npm=fast < %s | FileCheck %s --check-prefixes=GCN,GFX10CU-WAVE64
+; RUN: llc -mtriple=amdgpu11.00 -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc-npm=fast -vgpr-regalloc-npm=fast -wwm-regalloc-npm=fast < %s | FileCheck %s --check-prefixes=GCN,GFX11WGP-WAVE32
+; RUN: llc -mtriple=amdgpu11.00 -mattr=+wavefrontsize64 -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc-npm=fast -vgpr-regalloc-npm=fast -wwm-regalloc-npm=fast < %s | FileCheck %s --check-prefixes=GCN,GFX11WGP-WAVE64
+; RUN: llc -mtriple=amdgpu11.00 -mattr=+cumode -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc-npm=fast -vgpr-regalloc-npm=fast -wwm-regalloc-npm=fast < %s | FileCheck %s --check-prefixes=GCN,GFX11CU-WAVE32
+; RUN: llc -mtriple=amdgpu11.00 -mattr=+cumode,+wavefrontsize64 -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc-npm=fast -vgpr-regalloc-npm=fast -wwm-regalloc-npm=fast < %s | FileCheck %s --check-prefixes=GCN,GFX11CU-WAVE64
define internal void @use256vgprs() {
%v0 = call i32 asm sideeffect "; def $0", "=v"()
diff --git a/llvm/test/CodeGen/AMDGPU/attributor-flatscratchinit-globalisel.ll b/llvm/test/CodeGen/AMDGPU/attributor-flatscratchinit-globalisel.ll
index aab2968ae7e20..a701557105551 100644
--- a/llvm/test/CodeGen/AMDGPU/attributor-flatscratchinit-globalisel.ll
+++ b/llvm/test/CodeGen/AMDGPU/attributor-flatscratchinit-globalisel.ll
@@ -1,5 +1,5 @@
; Test the generation of the attribute amdgpu-no-flat-scratch-init
-; RUN: opt -S -O2 -mtriple=amdgpu10.10-amd-amdhsa %s | llc -global-isel -stop-after=irtranslator | FileCheck -check-prefixes=GFX10 %s
+; RUN: opt -S -O2 -mtriple=amdgpu10.10-amd-amdhsa %s | llc -global-isel -stop-after=ir-translator | FileCheck -check-prefixes=GFX10 %s
;; tests of addrspacecast
diff --git a/llvm/test/CodeGen/AMDGPU/av_spill_cross_bb_usage.mir b/llvm/test/CodeGen/AMDGPU/av_spill_cross_bb_usage.mir
index 94165dbfb8849..7246ec05fdc38 100644
--- a/llvm/test/CodeGen/AMDGPU/av_spill_cross_bb_usage.mir
+++ b/llvm/test/CodeGen/AMDGPU/av_spill_cross_bb_usage.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu9.08-amd-amdhsa -start-before=greedy,1 -stop-after=prolog-epilog -verify-machineinstrs -verify-regalloc -o - %s | FileCheck --check-prefixes=GCN %s
+# RUN: llc -mtriple=amdgpu9.08-amd-amdhsa -start-before=greedy,2 -stop-after=prolog-epilog -verify-machineinstrs -verify-regalloc -o - %s | FileCheck --check-prefixes=GCN %s
# The VGPR pair spilled and restored around the callsite is used in the next basic block.
#
diff --git a/llvm/test/CodeGen/AMDGPU/bad-agpr-vgpr-regalloc-priority.mir b/llvm/test/CodeGen/AMDGPU/bad-agpr-vgpr-regalloc-priority.mir
index a2763a1a1ccfd..7d0ced376f1cb 100644
--- a/llvm/test/CodeGen/AMDGPU/bad-agpr-vgpr-regalloc-priority.mir
+++ b/llvm/test/CodeGen/AMDGPU/bad-agpr-vgpr-regalloc-priority.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
-# RUN: llc -mtriple=amdgpu9.08-amd-amdhsa -start-before=greedy,2 -stop-after=virtregrewriter,2 -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.08-amd-amdhsa -start-before=greedy,3 -stop-after=virt-reg-rewriter,3 -o - %s | FileCheck %s
---
name: bad_ra
diff --git a/llvm/test/CodeGen/AMDGPU/bb-prolog-spill-during-regalloc.ll b/llvm/test/CodeGen/AMDGPU/bb-prolog-spill-during-regalloc.ll
index 7694e30a612af..71134ed6b67fb 100644
--- a/llvm/test/CodeGen/AMDGPU/bb-prolog-spill-during-regalloc.ll
+++ b/llvm/test/CodeGen/AMDGPU/bb-prolog-spill-during-regalloc.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 3
-; RUN: llc -mtriple=amdgpu9.06-amd-amdhsa -O0 --stop-after=regallocfast,2 -o - %s | FileCheck -check-prefix=REGALLOC %s
+; RUN: llc -mtriple=amdgpu9.06-amd-amdhsa -O0 --stop-after=regallocfast,3 -o - %s | FileCheck -check-prefix=REGALLOC %s
; Test to check if the bb prolog spills are inserted correctly during regalloc.
define i32 @prolog_spill(i32 %arg0, i32 %arg1, i32 %arg2) {
diff --git a/llvm/test/CodeGen/AMDGPU/block-should-not-be-in-alive-blocks.mir b/llvm/test/CodeGen/AMDGPU/block-should-not-be-in-alive-blocks.mir
index 875e2fcc671e6..90b301ec0b3b4 100644
--- a/llvm/test/CodeGen/AMDGPU/block-should-not-be-in-alive-blocks.mir
+++ b/llvm/test/CodeGen/AMDGPU/block-should-not-be-in-alive-blocks.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu10.31-amd-amdhsa -verify-machineinstrs -start-after=unreachable-mbb-elimination -stop-after=phi-node-elimination -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu10.31-amd-amdhsa -verify-machineinstrs -start-after=unreachable-mbb-elimination -stop-after=si-lower-control-flow -o - %s | FileCheck %s
# FIXME: Should be able to just use run-pass, but need to keep
# LiveVariables live after for the verifier. Also -start-before
diff --git a/llvm/test/CodeGen/AMDGPU/call-alias-register-usage-agpr.ll b/llvm/test/CodeGen/AMDGPU/call-alias-register-usage-agpr.ll
index 4ee6585c0a84a..b39b6cc4712de 100644
--- a/llvm/test/CodeGen/AMDGPU/call-alias-register-usage-agpr.ll
+++ b/llvm/test/CodeGen/AMDGPU/call-alias-register-usage-agpr.ll
@@ -1,16 +1,25 @@
-; RUN: llc -O0 -mtriple=amdgpu9.08-amd-amdhsa < %s | FileCheck -check-prefix=ALL %s
+; RUN: llc -O0 -mtriple=amdgpu9.08-amd-amdhsa < %s | FileCheck -check-prefix=ALL,GFX908 %s
; RUN: llc -O0 -mtriple=amdgpu9.0a-amd-amdhsa < %s | FileCheck -check-prefixes=ALL,GFX90A %s
-; CallGraphAnalysis, which CodeGenSCC order depends on, does not look
-; through aliases. If GlobalOpt is never run, we do not see direct
-; calls,
-
@alias = hidden alias void (), ptr @aliasee_default
-; ALL-LABEL: {{^}}kernel:
-; ALL: .amdhsa_next_free_vgpr max(totalnumvgprs(.Lkernel.num_agpr, .Lkernel.num_vgpr), 1, 0)
-; ALL-NEXT: .amdhsa_next_free_sgpr max(.Lkernel.numbered_sgpr+extrasgprs(.Lkernel.uses_vcc, .Lkernel.uses_flat_scratch, 1), 1, 0)-extrasgprs(.Lkernel.uses_vcc, .Lkernel.uses_flat_scratch, 1)
-; GFX90A-NEXT: .amdhsa_accum_offset (((((alignto(max(1, .Lkernel.num_vgpr), 4)/4)-1)&~65536)&63)+1)*4
+define internal void @aliasee_default() #1 {
+bb:
+ call void asm sideeffect "; clobber a26 ", "~{a26}"()
+ ret void
+}
+; ALL: .set .Laliasee_default.num_vgpr, 0
+; ALL-NEXT: .set .Laliasee_default.num_agpr, 27
+; ALL-NEXT: .set .Laliasee_default.numbered_sgpr, 32
+
+; GFX908-LABEL: {{^}}kernel:
+; GFX908: .amdhsa_next_free_vgpr 41
+; GFX908-NEXT: .amdhsa_next_free_sgpr 33
+
+; GFX90A-LABEL: {{^}}kernel:
+; GFX90A: .amdhsa_next_free_vgpr 71
+; GFX90A-NEXT: .amdhsa_next_free_sgpr 33
+; GFX90A-NEXT: .amdhsa_accum_offset 44
; ALL: .set .Lkernel.num_vgpr, max(41, .Laliasee_default.num_vgpr)
; ALL-NEXT: .set .Lkernel.num_agpr, max(0, .Laliasee_default.num_agpr)
@@ -21,15 +30,6 @@ bb:
ret void
}
-define internal void @aliasee_default() #1 {
-bb:
- call void asm sideeffect "; clobber a26 ", "~{a26}"()
- ret void
-}
-; ALL: .set .Laliasee_default.num_vgpr, 0
-; ALL-NEXT: .set .Laliasee_default.num_agpr, 27
-; ALL-NEXT: .set .Laliasee_default.numbered_sgpr, 32
-
attributes #0 = { noinline norecurse nounwind optnone }
attributes #1 = { noinline norecurse nounwind readnone willreturn }
attributes #2 = { nounwind readnone willreturn }
diff --git a/llvm/test/CodeGen/AMDGPU/call-alias-register-usage0.ll b/llvm/test/CodeGen/AMDGPU/call-alias-register-usage0.ll
index 8e74dcdd9490f..296507e2c2bbf 100644
--- a/llvm/test/CodeGen/AMDGPU/call-alias-register-usage0.ll
+++ b/llvm/test/CodeGen/AMDGPU/call-alias-register-usage0.ll
@@ -6,6 +6,15 @@
@alias0 = hidden alias void (), ptr @aliasee_default_vgpr64_sgpr102
+; CHECK: .set .Laliasee_default_vgpr64_sgpr102.num_vgpr, 53
+; CHECK-NEXT: .set .Laliasee_default_vgpr64_sgpr102.num_agpr, 0
+; CHECK-NEXT: .set .Laliasee_default_vgpr64_sgpr102.numbered_sgpr, 32
+define internal void @aliasee_default_vgpr64_sgpr102() #1 {
+bb:
+ call void asm sideeffect "; clobber v52 ", "~{v52}"()
+ ret void
+}
+
; CHECK-LABEL: {{^}}kernel0:
; CHECK: .set .Lkernel0.num_vgpr, max(41, .Laliasee_default_vgpr64_sgpr102.num_vgpr)
; CHECK-NEXT: .set .Lkernel0.num_agpr, max(0, .Laliasee_default_vgpr64_sgpr102.num_agpr)
@@ -16,15 +25,6 @@ bb:
ret void
}
-; CHECK: .set .Laliasee_default_vgpr64_sgpr102.num_vgpr, 53
-; CHECK-NEXT: .set .Laliasee_default_vgpr64_sgpr102.num_agpr, 0
-; CHECK-NEXT: .set .Laliasee_default_vgpr64_sgpr102.numbered_sgpr, 32
-define internal void @aliasee_default_vgpr64_sgpr102() #1 {
-bb:
- call void asm sideeffect "; clobber v52 ", "~{v52}"()
- ret void
-}
-
attributes #0 = { noinline norecurse nounwind optnone }
attributes #1 = { noinline norecurse nounwind readnone willreturn }
attributes #2 = { nounwind readnone willreturn }
diff --git a/llvm/test/CodeGen/AMDGPU/call-alias-register-usage1.ll b/llvm/test/CodeGen/AMDGPU/call-alias-register-usage1.ll
index 05412b343ecb7..81ee5b96a297f 100644
--- a/llvm/test/CodeGen/AMDGPU/call-alias-register-usage1.ll
+++ b/llvm/test/CodeGen/AMDGPU/call-alias-register-usage1.ll
@@ -8,9 +8,18 @@
; The parent kernel has a higher VGPR usage than the possible callees.
+; CHECK: .set .Laliasee_vgpr32_sgpr76.num_vgpr, 27
+; CHECK-NEXT: .set .Laliasee_vgpr32_sgpr76.num_agpr, 0
+; CHECK-NEXT: .set .Laliasee_vgpr32_sgpr76.numbered_sgpr, 32
+define internal void @aliasee_vgpr32_sgpr76() #1 {
+bb:
+ call void asm sideeffect "; clobber v26 ", "~{v26}"()
+ ret void
+}
+
; CHECK-LABEL: {{^}}kernel1:
-; CHECK: .amdhsa_next_free_vgpr max(totalnumvgprs(.Lkernel1.num_agpr, .Lkernel1.num_vgpr), 1, 0)
-; CHECK-NEXT: .amdhsa_next_free_sgpr max(.Lkernel1.numbered_sgpr+extrasgprs(.Lkernel1.uses_vcc, .Lkernel1.uses_flat_scratch, 1), 1, 0)-extrasgprs(.Lkernel1.uses_vcc, .Lkernel1.uses_flat_scratch, 1)
+; CHECK: .amdhsa_next_free_vgpr 42
+; CHECK-NEXT: .amdhsa_next_free_sgpr 33
; CHECK: .set .Lkernel1.num_vgpr, max(42, .Laliasee_vgpr32_sgpr76.num_vgpr)
; CHECK-NEXT: .set .Lkernel1.num_agpr, max(0, .Laliasee_vgpr32_sgpr76.num_agpr)
@@ -22,15 +31,6 @@ bb:
ret void
}
-; CHECK: .set .Laliasee_vgpr32_sgpr76.num_vgpr, 27
-; CHECK-NEXT: .set .Laliasee_vgpr32_sgpr76.num_agpr, 0
-; CHECK-NEXT: .set .Laliasee_vgpr32_sgpr76.numbered_sgpr, 32
-define internal void @aliasee_vgpr32_sgpr76() #1 {
-bb:
- call void asm sideeffect "; clobber v26 ", "~{v26}"()
- ret void
-}
-
attributes #0 = { noinline norecurse nounwind optnone }
attributes #1 = { noinline norecurse nounwind readnone willreturn "amdgpu-waves-per-eu"="8,10" }
attributes #2 = { nounwind readnone willreturn }
diff --git a/llvm/test/CodeGen/AMDGPU/call-alias-register-usage2.ll b/llvm/test/CodeGen/AMDGPU/call-alias-register-usage2.ll
index 5735ef21b084d..afcac9cb4327d 100644
--- a/llvm/test/CodeGen/AMDGPU/call-alias-register-usage2.ll
+++ b/llvm/test/CodeGen/AMDGPU/call-alias-register-usage2.ll
@@ -6,9 +6,18 @@
@alias2 = hidden alias void (), ptr @aliasee_vgpr64_sgpr102
+; CHECK: .set .Laliasee_vgpr64_sgpr102.num_vgpr, 53
+; CHECK-NEXT: .set .Laliasee_vgpr64_sgpr102.num_agpr, 0
+; CHECK-NEXT: .set .Laliasee_vgpr64_sgpr102.numbered_sgpr, 32
+define internal void @aliasee_vgpr64_sgpr102() #1 {
+bb:
+ call void asm sideeffect "; clobber v52 ", "~{v52}"()
+ ret void
+}
+
; CHECK-LABEL: {{^}}kernel2:
-; CHECK: .amdhsa_next_free_vgpr max(totalnumvgprs(.Lkernel2.num_agpr, .Lkernel2.num_vgpr), 1, 0)
-; CHECK-NEXT: .amdhsa_next_free_sgpr max(.Lkernel2.numbered_sgpr+extrasgprs(.Lkernel2.uses_vcc, .Lkernel2.uses_flat_scratch, 1), 1, 0)-extrasgprs(.Lkernel2.uses_vcc, .Lkernel2.uses_flat_scratch, 1)
+; CHECK: .amdhsa_next_free_vgpr 53
+; CHECK-NEXT: .amdhsa_next_free_sgpr 33
; CHECK: .set .Lkernel2.num_vgpr, max(41, .Laliasee_vgpr64_sgpr102.num_vgpr)
; CHECK-NEXT: .set .Lkernel2.num_agpr, max(0, .Laliasee_vgpr64_sgpr102.num_agpr)
@@ -19,15 +28,6 @@ bb:
ret void
}
-; CHECK: .set .Laliasee_vgpr64_sgpr102.num_vgpr, 53
-; CHECK-NEXT: .set .Laliasee_vgpr64_sgpr102.num_agpr, 0
-; CHECK-NEXT: .set .Laliasee_vgpr64_sgpr102.numbered_sgpr, 32
-define internal void @aliasee_vgpr64_sgpr102() #1 {
-bb:
- call void asm sideeffect "; clobber v52 ", "~{v52}"()
- ret void
-}
-
attributes #0 = { noinline norecurse nounwind optnone }
attributes #1 = { noinline norecurse nounwind readnone willreturn "amdgpu-waves-per-eu"="4,10" }
attributes #2 = { nounwind readnone willreturn }
diff --git a/llvm/test/CodeGen/AMDGPU/call-alias-register-usage3.ll b/llvm/test/CodeGen/AMDGPU/call-alias-register-usage3.ll
index 7e6be8593e3c5..4c094d75ecd70 100644
--- a/llvm/test/CodeGen/AMDGPU/call-alias-register-usage3.ll
+++ b/llvm/test/CodeGen/AMDGPU/call-alias-register-usage3.ll
@@ -6,9 +6,18 @@
@alias3 = hidden alias void (), ptr @aliasee_vgpr256_sgpr102
+; CHECK: .set .Laliasee_vgpr256_sgpr102.num_vgpr, 253
+; CHECK-NEXT: .set .Laliasee_vgpr256_sgpr102.num_agpr, 0
+; CHECK-NEXT: .set .Laliasee_vgpr256_sgpr102.numbered_sgpr, 33
+define internal void @aliasee_vgpr256_sgpr102() #1 {
+bb:
+ call void asm sideeffect "; clobber v252 ", "~{v252}"()
+ ret void
+}
+
; CHECK-LABEL: {{^}}kernel3:
-; CHECK: .amdhsa_next_free_vgpr max(totalnumvgprs(.Lkernel3.num_agpr, .Lkernel3.num_vgpr), 1, 0)
-; CHECK-NEXT: .amdhsa_next_free_sgpr max(.Lkernel3.numbered_sgpr+extrasgprs(.Lkernel3.uses_vcc, .Lkernel3.uses_flat_scratch, 1), 1, 0)-extrasgprs(.Lkernel3.uses_vcc, .Lkernel3.uses_flat_scratch, 1)
+; CHECK: .amdhsa_next_free_vgpr 253
+; CHECK-NEXT: .amdhsa_next_free_sgpr 33
; CHECK: .set .Lkernel3.num_vgpr, max(41, .Laliasee_vgpr256_sgpr102.num_vgpr)
; CHECK-NEXT: .set .Lkernel3.num_agpr, max(0, .Laliasee_vgpr256_sgpr102.num_agpr)
@@ -19,15 +28,6 @@ bb:
ret void
}
-; CHECK: .set .Laliasee_vgpr256_sgpr102.num_vgpr, 253
-; CHECK-NEXT: .set .Laliasee_vgpr256_sgpr102.num_agpr, 0
-; CHECK-NEXT: .set .Laliasee_vgpr256_sgpr102.numbered_sgpr, 33
-define internal void @aliasee_vgpr256_sgpr102() #1 {
-bb:
- call void asm sideeffect "; clobber v252 ", "~{v252}"()
- ret void
-}
-
attributes #0 = { noinline norecurse nounwind optnone }
attributes #1 = { noinline norecurse nounwind readnone willreturn "amdgpu-flat-work-group-size"="1,256" "amdgpu-waves-per-eu"="1,1" }
attributes #2 = { nounwind readnone willreturn }
diff --git a/llvm/test/CodeGen/AMDGPU/call-constexpr.ll b/llvm/test/CodeGen/AMDGPU/call-constexpr.ll
index 53867b9031f6b..e39883c6602ce 100644
--- a/llvm/test/CodeGen/AMDGPU/call-constexpr.ll
+++ b/llvm/test/CodeGen/AMDGPU/call-constexpr.ll
@@ -1,7 +1,33 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -global-isel=0 -mtriple=amdgpu7.00-amd-amdhsa < %s | FileCheck -check-prefixes=GCN,SDAG %s
; RUN: llc -global-isel=1 -mtriple=amdgpu7.00-amd-amdhsa < %s | FileCheck -check-prefixes=GCN,GISEL %s
+define hidden i32 @ret_i32_noinline() #0 {
+; GCN-LABEL: ret_i32_noinline:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mov_b32_e32 v0, 4
+; GCN-NEXT: s_setpc_b64 s[30:31]
+ ret i32 4
+}
+
+define hidden i32 @ret_i32_alwaysinline() #1 {
+; GCN-LABEL: ret_i32_alwaysinline:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mov_b32_e32 v0, 4
+; GCN-NEXT: s_setpc_b64 s[30:31]
+ ret i32 4
+}
+
+define hidden i32 @ident_i32(i32 %i) #0 {
+; GCN-LABEL: ident_i32:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: s_setpc_b64 s[30:31]
+ ret i32 %i
+}
+
define amdgpu_kernel void @test_bitcast_return_type_noinline() #0 {
; SDAG-LABEL: test_bitcast_return_type_noinline:
; SDAG: ; %bb.0:
@@ -351,35 +377,6 @@ continue:
ret void
}
-; Callees appears last in source file to test that we still lower their
-; arguments before we lower any calls to them.
-
-define hidden i32 @ret_i32_noinline() #0 {
-; GCN-LABEL: ret_i32_noinline:
-; GCN: ; %bb.0:
-; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v0, 4
-; GCN-NEXT: s_setpc_b64 s[30:31]
- ret i32 4
-}
-
-define hidden i32 @ret_i32_alwaysinline() #1 {
-; GCN-LABEL: ret_i32_alwaysinline:
-; GCN: ; %bb.0:
-; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v0, 4
-; GCN-NEXT: s_setpc_b64 s[30:31]
- ret i32 4
-}
-
-define hidden i32 @ident_i32(i32 %i) #0 {
-; GCN-LABEL: ident_i32:
-; GCN: ; %bb.0:
-; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: s_setpc_b64 s[30:31]
- ret i32 %i
-}
-
declare i32 @llvm.amdgcn.workitem.id.x() #2
attributes #0 = { nounwind noinline }
diff --git a/llvm/test/CodeGen/AMDGPU/convergence-laneops.ll b/llvm/test/CodeGen/AMDGPU/convergence-laneops.ll
index 64a6edcc72c93..3353f200629dc 100644
--- a/llvm/test/CodeGen/AMDGPU/convergence-laneops.ll
+++ b/llvm/test/CodeGen/AMDGPU/convergence-laneops.ll
@@ -1,5 +1,5 @@
-; RUN: llc -stop-after=amdgpu-isel -mtriple=amdgpu11.00-- -verify-machineinstrs -o - %s | FileCheck --check-prefixes=CHECK,ISEL %s
-; RUN: not --crash llc -mtriple=amdgpu11.00--amdhsa -verify-machineinstrs < %s 2>&1 | FileCheck --check-prefix=CHECK-ERROR %s
+; RUN: llc -stop-after=amdgpu-isel -mtriple=amdgpu11.00-- -verify-each -o - %s | FileCheck --check-prefixes=CHECK,ISEL %s
+; RUN: not --crash llc -mtriple=amdgpu11.00--amdhsa -verify-each < %s 2>&1 | FileCheck --check-prefix=CHECK-ERROR %s
; FIXME: Merge these tests with existing lane op tests (llvm.amdgcn.readlane.ll, llvm.amdgcn.writelane.ll ...) once the crash is fixed.
diff --git a/llvm/test/CodeGen/AMDGPU/convergence-tokens.ll b/llvm/test/CodeGen/AMDGPU/convergence-tokens.ll
index 78909952aeb70..d0ed15514b24e 100644
--- a/llvm/test/CodeGen/AMDGPU/convergence-tokens.ll
+++ b/llvm/test/CodeGen/AMDGPU/convergence-tokens.ll
@@ -1,6 +1,6 @@
; RUN: llc -stop-after=amdgpu-isel -mtriple=amdgpu9.00-- -o - %s | FileCheck --check-prefixes=CHECK,ISEL %s
; RUN: llc -stop-after=dead-mi-elimination -mtriple=amdgpu9.00-- -o - %s | FileCheck --check-prefixes=CHECK,DEADMI %s
-; RUN: llc -global-isel -stop-after=irtranslator -mtriple=amdgpu9.00-- -o - %s | FileCheck %s --check-prefixes=CHECK,GISEL
+; RUN: llc -global-isel -stop-after=ir-translator -mtriple=amdgpu9.00-- -o - %s | FileCheck %s --check-prefixes=CHECK,GISEL
; CHECK-LABEL: name: basic_call
; CHECK: [[TOKEN:%[0-9]+]]{{[^ ]*}} = CONVERGENCECTRL_ENTRY
diff --git a/llvm/test/CodeGen/AMDGPU/dead_bundle.mir b/llvm/test/CodeGen/AMDGPU/dead_bundle.mir
index 77d4a437d167d..b761deb2f4cf7 100644
--- a/llvm/test/CodeGen/AMDGPU/dead_bundle.mir
+++ b/llvm/test/CodeGen/AMDGPU/dead_bundle.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu11.00--amdpal -verify-machineinstrs=1 -start-before=greedy,0 -stop-after=virtregrewriter,0 -stress-regalloc=5 %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgpu11.00--amdpal -verify-machineinstrs=1 -start-before=greedy,1 -stop-after=virt-reg-rewriter,1 -stress-regalloc=5 %s -o - | FileCheck %s
# This test checks that dead bundles are handled correctly.
---
diff --git a/llvm/test/CodeGen/AMDGPU/dummy-regalloc-priority-advisor.mir b/llvm/test/CodeGen/AMDGPU/dummy-regalloc-priority-advisor.mir
index 3f034d709b102..f32ebfaeb4a78 100644
--- a/llvm/test/CodeGen/AMDGPU/dummy-regalloc-priority-advisor.mir
+++ b/llvm/test/CodeGen/AMDGPU/dummy-regalloc-priority-advisor.mir
@@ -1,6 +1,6 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
-# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -start-before=greedy,2 -stress-regalloc=4 -stop-after=virtregrewriter,2 -regalloc-enable-priority-advisor=default -o - %s | FileCheck -check-prefixes=CHECK,DEFAULT %s
-# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -start-before=greedy,2 -stress-regalloc=4 -stop-after=virtregrewriter,2 -regalloc-enable-priority-advisor=dummy -o - %s | FileCheck -check-prefixes=CHECK,DUMMY %s
+# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -start-before=greedy,3 -stress-regalloc=4 -stop-after=virt-reg-rewriter,3 -regalloc-enable-priority-advisor=default -o - %s | FileCheck -check-prefixes=CHECK,DEFAULT %s
+# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -start-before=greedy,3 -stress-regalloc=4 -stop-after=virt-reg-rewriter,3 -regalloc-enable-priority-advisor=dummy -o - %s | FileCheck -check-prefixes=CHECK,DUMMY %s
# Check that the regalloc-enable-priority-advisor=dummy option works
# and the result is different from the default. Ordinarily %1 would be
diff --git a/llvm/test/CodeGen/AMDGPU/dvgpr-vgpr-count-propagation-direct-chain.ll b/llvm/test/CodeGen/AMDGPU/dvgpr-vgpr-count-propagation-direct-chain.ll
index 88683131078cf..08e2cb9332ede 100644
--- a/llvm/test/CodeGen/AMDGPU/dvgpr-vgpr-count-propagation-direct-chain.ll
+++ b/llvm/test/CodeGen/AMDGPU/dvgpr-vgpr-count-propagation-direct-chain.ll
@@ -10,27 +10,27 @@
; func.2 and func.4 have DVGPRs disabled
-; DVGPR: .set .Lgfx_func_a.num_vgpr, 40
-; DVGPR: .set .Lamdgpu_cs_main.num_vgpr, max(42, .Lgfx_func_a.num_vgpr)
-; DVGPR: .set .Lfunc.0.num_vgpr, max(11, .Lgfx_func_a.num_vgpr)
-; DVGPR: .set .Lfunc.1.num_vgpr, 82
-; DVGPR: .set .Lfunc.2.num_vgpr, max(11, amdgpu.max_num_vgpr)
-; DVGPR: .set .Lfunc.3.num_vgpr, 11
-; DVGPR: .set .Lfunc.4.num_vgpr, max(82, amdgpu.max_num_vgpr)
-; DVGPR: .set .Lretry_vgpr_alloc.num_vgpr, 11
-; DVGPR: .set .Lfirst_retry_wrapper.num_vgpr, 11
-; DVGPR: .set amdgpu.max_num_vgpr, 82
-
-; NODVGPR: .set .Lgfx_func_a.num_vgpr, 40
-; NODVGPR: .set .Lamdgpu_cs_main.num_vgpr, max(42, amdgpu.max_num_vgpr)
-; NODVGPR: .set .Lfunc.0.num_vgpr, max(11, amdgpu.max_num_vgpr)
-; NODVGPR: .set .Lfunc.1.num_vgpr, max(82, amdgpu.max_num_vgpr)
-; NODVGPR: .set .Lfunc.2.num_vgpr, max(11, amdgpu.max_num_vgpr)
-; NODVGPR: .set .Lfunc.3.num_vgpr, max(11, amdgpu.max_num_vgpr)
-; NODVGPR: .set .Lfunc.4.num_vgpr, max(82, amdgpu.max_num_vgpr)
-; NODVGPR: .set .Lretry_vgpr_alloc.num_vgpr, max(11, amdgpu.max_num_vgpr)
-; NODVGPR: .set .Lfirst_retry_wrapper.num_vgpr, max(11, amdgpu.max_num_vgpr)
-; NODVGPR: .set amdgpu.max_num_vgpr, 82
+; DVGPR-DAG: .set .Lgfx_func_a.num_vgpr, 40
+; DVGPR-DAG: .set .Lamdgpu_cs_main.num_vgpr, max(42, .Lgfx_func_a.num_vgpr)
+; DVGPR-DAG: .set .Lfunc.0.num_vgpr, max(11, .Lgfx_func_a.num_vgpr)
+; DVGPR-DAG: .set .Lfunc.1.num_vgpr, 82
+; DVGPR-DAG: .set .Lfunc.2.num_vgpr, max(11, amdgpu.max_num_vgpr)
+; DVGPR-DAG: .set .Lfunc.3.num_vgpr, 11
+; DVGPR-DAG: .set .Lfunc.4.num_vgpr, max(82, amdgpu.max_num_vgpr)
+; DVGPR-DAG: .set .Lretry_vgpr_alloc.num_vgpr, 11
+; DVGPR-DAG: .set .Lfirst_retry_wrapper.num_vgpr, 11
+; DVGPR-DAG: .set amdgpu.max_num_vgpr, 82
+
+; NODVGPR-DAG: .set .Lgfx_func_a.num_vgpr, 40
+; NODVGPR-DAG: .set .Lamdgpu_cs_main.num_vgpr, max(42, amdgpu.max_num_vgpr)
+; NODVGPR-DAG: .set .Lfunc.0.num_vgpr, max(11, amdgpu.max_num_vgpr)
+; NODVGPR-DAG: .set .Lfunc.1.num_vgpr, max(82, amdgpu.max_num_vgpr)
+; NODVGPR-DAG: .set .Lfunc.2.num_vgpr, max(11, amdgpu.max_num_vgpr)
+; NODVGPR-DAG: .set .Lfunc.3.num_vgpr, max(11, amdgpu.max_num_vgpr)
+; NODVGPR-DAG: .set .Lfunc.4.num_vgpr, max(82, amdgpu.max_num_vgpr)
+; NODVGPR-DAG: .set .Lretry_vgpr_alloc.num_vgpr, max(11, amdgpu.max_num_vgpr)
+; NODVGPR-DAG: .set .Lfirst_retry_wrapper.num_vgpr, max(11, amdgpu.max_num_vgpr)
+; NODVGPR-DAG: .set amdgpu.max_num_vgpr, 82
; DVGPR: - .hardware_stages:
; DVGPR: .vgpr_count: 0x2a
diff --git a/llvm/test/CodeGen/AMDGPU/dvgpr-vgpr-count-propagation-indirect.ll b/llvm/test/CodeGen/AMDGPU/dvgpr-vgpr-count-propagation-indirect.ll
index 21028afc34003..b9f27d01205eb 100644
--- a/llvm/test/CodeGen/AMDGPU/dvgpr-vgpr-count-propagation-indirect.ll
+++ b/llvm/test/CodeGen/AMDGPU/dvgpr-vgpr-count-propagation-indirect.ll
@@ -6,19 +6,19 @@
; their `num_vgpr` count set to the max of their local count and the module-wide
; max VGPR count of all non-chain functions.
-; DVGPR: .set .Lgfx_func_a.num_vgpr, 40
-; DVGPR: .set .Lgfx_func_b.num_vgpr, 80
-; DVGPR: .set .Lfunc_with_indirect_call.num_vgpr, max(11, amdgpu.max_num_vgpr)
-; DVGPR: .set .Lfunc_direct_only.num_vgpr, max(11, .Lgfx_func_a.num_vgpr)
-; DVGPR: .set .Lfunc_chain_only.num_vgpr, 11
-; DVGPR: .set amdgpu.max_num_vgpr, 80
-
-; NODVGPR: .set .Lgfx_func_a.num_vgpr, 40
-; NODVGPR: .set .Lgfx_func_b.num_vgpr, 80
-; NODVGPR: .set .Lfunc_with_indirect_call.num_vgpr, max(11, amdgpu.max_num_vgpr)
-; NODVGPR: .set .Lfunc_direct_only.num_vgpr, max(11, amdgpu.max_num_vgpr)
-; NODVGPR: .set .Lfunc_chain_only.num_vgpr, max(11, amdgpu.max_num_vgpr)
-; NODVGPR: .set amdgpu.max_num_vgpr, 80
+; DVGPR-DAG: .set .Lgfx_func_a.num_vgpr, 40
+; DVGPR-DAG: .set .Lgfx_func_b.num_vgpr, 80
+; DVGPR-DAG: .set .Lfunc_with_indirect_call.num_vgpr, max(11, amdgpu.max_num_vgpr)
+; DVGPR-DAG: .set .Lfunc_direct_only.num_vgpr, max(11, .Lgfx_func_a.num_vgpr)
+; DVGPR-DAG: .set .Lfunc_chain_only.num_vgpr, 11
+; DVGPR-DAG: .set amdgpu.max_num_vgpr, 80
+
+; NODVGPR-DAG: .set .Lgfx_func_a.num_vgpr, 40
+; NODVGPR-DAG: .set .Lgfx_func_b.num_vgpr, 80
+; NODVGPR-DAG: .set .Lfunc_with_indirect_call.num_vgpr, max(11, amdgpu.max_num_vgpr)
+; NODVGPR-DAG: .set .Lfunc_direct_only.num_vgpr, max(11, amdgpu.max_num_vgpr)
+; NODVGPR-DAG: .set .Lfunc_chain_only.num_vgpr, max(11, amdgpu.max_num_vgpr)
+; NODVGPR-DAG: .set amdgpu.max_num_vgpr, 80
define amdgpu_gfx void @gfx_func_a() #0 {
call void asm sideeffect "", "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7},~{v8},~{v9},~{v10},~{v11},~{v12},~{v13},~{v14},~{v15},~{v16},~{v17},~{v18},~{v19},~{v20},~{v21},~{v22},~{v23},~{v24},~{v25},~{v26},~{v27},~{v28},~{v29},~{v30},~{v31},~{v32},~{v33},~{v34},~{v35},~{v36},~{v37},~{v38},~{v39}"()
diff --git a/llvm/test/CodeGen/AMDGPU/dvgpr-vgpr-count-propagation.ll b/llvm/test/CodeGen/AMDGPU/dvgpr-vgpr-count-propagation.ll
index 22ea185019cf9..43f107611f4b8 100644
--- a/llvm/test/CodeGen/AMDGPU/dvgpr-vgpr-count-propagation.ll
+++ b/llvm/test/CodeGen/AMDGPU/dvgpr-vgpr-count-propagation.ll
@@ -11,31 +11,31 @@
; non-chain functions. With DVGPRs disabled, it represents module-wide max VGPR
; count.
-; DVGPR: .set .Lgfx_func_a.num_vgpr, 40
-; DVGPR: .set .Lgfx_func_b2.num_vgpr, 80
-; DVGPR: .set .Lgfx_func_b.num_vgpr, max(61, .Lgfx_func_b2.num_vgpr)
-; DVGPR: .set .Lamdgpu_cs_main.num_vgpr, max(42, .Lgfx_func_a.num_vgpr)
-; DVGPR: .set .Lfunc.0.num_vgpr, 13
-; DVGPR: .set .Lfunc.1.num_vgpr, max(14, .Lgfx_func_a.num_vgpr, .Lgfx_func_b.num_vgpr)
-; DVGPR: .set .Lfunc.2.num_vgpr, max(16, .Lgfx_func_a.num_vgpr)
-; DVGPR: .set .Lfunc.3.num_vgpr, max(15, .Lgfx_func_b.num_vgpr)
-; DVGPR: .set .Lfunc.4.num_vgpr, max(100, .Lgfx_func_b.num_vgpr)
-; DVGPR: .set .Lretry_vgpr_alloc.num_vgpr, 11
-; DVGPR: .set .Lfirst_retry_wrapper.num_vgpr, 11
-; DVGPR: .set amdgpu.max_num_vgpr, 80
-
-; NODVGPR: .set .Lgfx_func_a.num_vgpr, 40
-; NODVGPR: .set .Lgfx_func_b2.num_vgpr, 80
-; NODVGPR: .set .Lgfx_func_b.num_vgpr, max(61, .Lgfx_func_b2.num_vgpr)
-; NODVGPR: .set .Lamdgpu_cs_main.num_vgpr, max(42, amdgpu.max_num_vgpr)
-; NODVGPR: .set .Lfunc.0.num_vgpr, max(13, amdgpu.max_num_vgpr)
-; NODVGPR: .set .Lfunc.1.num_vgpr, max(14, amdgpu.max_num_vgpr)
-; NODVGPR: .set .Lfunc.2.num_vgpr, max(16, amdgpu.max_num_vgpr)
-; NODVGPR: .set .Lfunc.3.num_vgpr, max(15, amdgpu.max_num_vgpr)
-; NODVGPR: .set .Lfunc.4.num_vgpr, max(100, amdgpu.max_num_vgpr)
-; NODVGPR: .set .Lretry_vgpr_alloc.num_vgpr, max(11, amdgpu.max_num_vgpr)
-; NODVGPR: .set .Lfirst_retry_wrapper.num_vgpr, max(11, amdgpu.max_num_vgpr)
-; NODVGPR: .set amdgpu.max_num_vgpr, 100
+; DVGPR-DAG: .set .Lgfx_func_a.num_vgpr, 40
+; DVGPR-DAG: .set .Lgfx_func_b2.num_vgpr, 80
+; DVGPR-DAG: .set .Lgfx_func_b.num_vgpr, max(61, .Lgfx_func_b2.num_vgpr)
+; DVGPR-DAG: .set .Lamdgpu_cs_main.num_vgpr, max(42, .Lgfx_func_a.num_vgpr)
+; DVGPR-DAG: .set .Lfunc.0.num_vgpr, 13
+; DVGPR-DAG: .set .Lfunc.1.num_vgpr, max(14, .Lgfx_func_a.num_vgpr, .Lgfx_func_b.num_vgpr)
+; DVGPR-DAG: .set .Lfunc.2.num_vgpr, max(16, .Lgfx_func_a.num_vgpr)
+; DVGPR-DAG: .set .Lfunc.3.num_vgpr, max(15, .Lgfx_func_b.num_vgpr)
+; DVGPR-DAG: .set .Lfunc.4.num_vgpr, max(100, .Lgfx_func_b.num_vgpr)
+; DVGPR-DAG: .set .Lretry_vgpr_alloc.num_vgpr, 11
+; DVGPR-DAG: .set .Lfirst_retry_wrapper.num_vgpr, 11
+; DVGPR-DAG: .set amdgpu.max_num_vgpr, 80
+
+; NODVGPR-DAG: .set .Lgfx_func_a.num_vgpr, 40
+; NODVGPR-DAG: .set .Lgfx_func_b2.num_vgpr, 80
+; NODVGPR-DAG: .set .Lgfx_func_b.num_vgpr, max(61, .Lgfx_func_b2.num_vgpr)
+; NODVGPR-DAG: .set .Lamdgpu_cs_main.num_vgpr, max(42, amdgpu.max_num_vgpr)
+; NODVGPR-DAG: .set .Lfunc.0.num_vgpr, max(13, amdgpu.max_num_vgpr)
+; NODVGPR-DAG: .set .Lfunc.1.num_vgpr, max(14, amdgpu.max_num_vgpr)
+; NODVGPR-DAG: .set .Lfunc.2.num_vgpr, max(16, amdgpu.max_num_vgpr)
+; NODVGPR-DAG: .set .Lfunc.3.num_vgpr, max(15, amdgpu.max_num_vgpr)
+; NODVGPR-DAG: .set .Lfunc.4.num_vgpr, max(100, amdgpu.max_num_vgpr)
+; NODVGPR-DAG: .set .Lretry_vgpr_alloc.num_vgpr, max(11, amdgpu.max_num_vgpr)
+; NODVGPR-DAG: .set .Lfirst_retry_wrapper.num_vgpr, max(11, amdgpu.max_num_vgpr)
+; NODVGPR-DAG: .set amdgpu.max_num_vgpr, 100
; DVGPR: - .hardware_stages:
; DVGPR: .vgpr_count: 0x2a
diff --git a/llvm/test/CodeGen/AMDGPU/emit-high-vgprs.mir b/llvm/test/CodeGen/AMDGPU/emit-high-vgprs.mir
index df569571c6045..2fbd5c2a7b644 100644
--- a/llvm/test/CodeGen/AMDGPU/emit-high-vgprs.mir
+++ b/llvm/test/CodeGen/AMDGPU/emit-high-vgprs.mir
@@ -1,4 +1,4 @@
-# RUN: llc -mtriple=amdgpu12.50 -show-mc-encoding -start-before=amdgpu-resource-usage -o - %s | FileCheck -check-prefix=GCN %s
+# RUN: llc -mtriple=amdgpu12.50 -show-mc-encoding -start-before=amdgpu-asm-printer -o - %s | FileCheck -check-prefix=GCN %s
# GCN-LABEL: {{^}}high_vgprs:
---
diff --git a/llvm/test/CodeGen/AMDGPU/expand-si-indirect.mir b/llvm/test/CodeGen/AMDGPU/expand-si-indirect.mir
index 8b4bf8cf8492b..247a71ee0e905 100644
--- a/llvm/test/CodeGen/AMDGPU/expand-si-indirect.mir
+++ b/llvm/test/CodeGen/AMDGPU/expand-si-indirect.mir
@@ -1,4 +1,4 @@
-# RUN: llc -mtriple=amdgpu9.00 -start-before=twoaddressinstruction %s -o - | FileCheck %s -check-prefix=GCN
+# RUN: llc -mtriple=amdgpu9.00 -start-before=two-address-instruction %s -o - | FileCheck %s -check-prefix=GCN
# Wait to sxpand SI_INDIRECT sequences that use VGPR indexing until after
# register allocation. We don’t want to reschedule the mode switching or to
diff --git a/llvm/test/CodeGen/AMDGPU/extend-wwm-virt-reg-liveness.mir b/llvm/test/CodeGen/AMDGPU/extend-wwm-virt-reg-liveness.mir
index 3433534698dd4..0b7ad9256c71d 100644
--- a/llvm/test/CodeGen/AMDGPU/extend-wwm-virt-reg-liveness.mir
+++ b/llvm/test/CodeGen/AMDGPU/extend-wwm-virt-reg-liveness.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu9.06-amd-amdhsa -start-before=si-lower-sgpr-spills -stop-after=virtregrewriter,1 -verify-machineinstrs %s -o - | FileCheck -check-prefix=GCN %s
+# RUN: llc -mtriple=amdgpu9.06-amd-amdhsa -start-before=si-lower-sgpr-spills -stop-after=virt-reg-rewriter,2 -verify-machineinstrs %s -o - | FileCheck -check-prefix=GCN %s
# Tests to check the conservative lieness extension for the wwm registers during SGPR spill lowering.
diff --git a/llvm/test/CodeGen/AMDGPU/fastregalloc-sgpr-only.mir b/llvm/test/CodeGen/AMDGPU/fastregalloc-sgpr-only.mir
index 22908c4834b63..305ec1ff8708c 100644
--- a/llvm/test/CodeGen/AMDGPU/fastregalloc-sgpr-only.mir
+++ b/llvm/test/CodeGen/AMDGPU/fastregalloc-sgpr-only.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -O0 -mtriple=amdgpu9.00-amd-amdhsa -start-before=regallocfast,0 --stop-after=regallocfast,0 -o - %s | FileCheck -check-prefix=GCN %s
+# RUN: llc -O0 -mtriple=amdgpu9.00-amd-amdhsa -start-before=regallocfast,1 --stop-after=regallocfast,1 -o - %s | FileCheck -check-prefix=GCN %s
---
name: copy_vgpr_allocation
diff --git a/llvm/test/CodeGen/AMDGPU/fmamk_fmaak-t16.mir b/llvm/test/CodeGen/AMDGPU/fmamk_fmaak-t16.mir
index 24bfc46380e6a..d6f0e129782f6 100644
--- a/llvm/test/CodeGen/AMDGPU/fmamk_fmaak-t16.mir
+++ b/llvm/test/CodeGen/AMDGPU/fmamk_fmaak-t16.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
-# RUN: llc -mtriple=amdgpu11.00 -mattr=+real-true16 -start-before=greedy,0 -stop-after=virtregrewriter,2 -o - %s | FileCheck -check-prefix=GFX11 %s
+# RUN: llc -mtriple=amdgpu11.00 -mattr=+real-true16 -start-before=greedy,1 -stop-after=virt-reg-rewriter,3 -o - %s | FileCheck -check-prefix=GFX11 %s
---
name: v_fmamk_f16
diff --git a/llvm/test/CodeGen/AMDGPU/fold-reload-into-exec.mir b/llvm/test/CodeGen/AMDGPU/fold-reload-into-exec.mir
index aa8b2e5b3f604..8d07d9ff5d90b 100644
--- a/llvm/test/CodeGen/AMDGPU/fold-reload-into-exec.mir
+++ b/llvm/test/CodeGen/AMDGPU/fold-reload-into-exec.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -verify-machineinstrs -stress-regalloc=2 -start-before=greedy -stop-after=virtregmap -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -verify-machineinstrs -stress-regalloc=2 -start-before=greedy -stop-after=amdgpu-preload-kern-arg-prolog -o - %s | FileCheck %s
# Test that a spill of a copy of exec is not folded to be a spill of exec directly.
diff --git a/llvm/test/CodeGen/AMDGPU/fold-reload-into-m0.mir b/llvm/test/CodeGen/AMDGPU/fold-reload-into-m0.mir
index ab1bc8641842f..604e7e4cc1e97 100644
--- a/llvm/test/CodeGen/AMDGPU/fold-reload-into-m0.mir
+++ b/llvm/test/CodeGen/AMDGPU/fold-reload-into-m0.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -verify-machineinstrs -stress-regalloc=2 -start-before=greedy -stop-after=virtregmap -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -verify-machineinstrs -stress-regalloc=2 -start-before=greedy -stop-after=amdgpu-preload-kern-arg-prolog -o - %s | FileCheck %s
# Test that a spill of a copy of m0 is not folded to be a spill of m0 directly.
diff --git a/llvm/test/CodeGen/AMDGPU/fold-restore-undef-use.mir b/llvm/test/CodeGen/AMDGPU/fold-restore-undef-use.mir
index 9297b44695c68..06725af984963 100644
--- a/llvm/test/CodeGen/AMDGPU/fold-restore-undef-use.mir
+++ b/llvm/test/CodeGen/AMDGPU/fold-restore-undef-use.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu9.0a-amd-amdhsa -stress-regalloc=4 -verify-regalloc -start-before=greedy,0 -stop-after=virtregrewriter,0 %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.0a-amd-amdhsa -stress-regalloc=4 -verify-regalloc -start-before=greedy,1 -stop-after=virt-reg-rewriter,1 %s -o - | FileCheck %s
# RUN: llc -enable-new-pm -mtriple=amdgpu9.0a-amd-amdhsa -stress-regalloc=4 -verify-regalloc -passes="greedy<sgpr>,virt-reg-rewriter<no-clear-vregs>" %s -o - | FileCheck %s
diff --git a/llvm/test/CodeGen/AMDGPU/function-resource-usage.ll b/llvm/test/CodeGen/AMDGPU/function-resource-usage.ll
index 8da7a82d41f19..bd9f2bfa24596 100644
--- a/llvm/test/CodeGen/AMDGPU/function-resource-usage.ll
+++ b/llvm/test/CodeGen/AMDGPU/function-resource-usage.ll
@@ -481,29 +481,29 @@ define amdgpu_kernel void @usage_direct_recursion(i32 %n) #0 {
ret void
}
-; GCN-LABEL: {{^}}multi_stage_recurse2:
-; GCN: .set .Lmulti_stage_recurse2.num_vgpr, max(43, .Lmulti_stage_recurse1.num_vgpr)
-; GCN: .set .Lmulti_stage_recurse2.num_agpr, max(0, .Lmulti_stage_recurse1.num_agpr)
-; GCN: .set .Lmulti_stage_recurse2.numbered_sgpr, max(34, .Lmulti_stage_recurse1.numbered_sgpr)
-; GCN: .set .Lmulti_stage_recurse2.private_seg_size, 16+max(.Lmulti_stage_recurse1.private_seg_size)
-; GCN: .set .Lmulti_stage_recurse2.uses_vcc, or(1, .Lmulti_stage_recurse1.uses_vcc)
-; GCN: .set .Lmulti_stage_recurse2.uses_flat_scratch, or(0, .Lmulti_stage_recurse1.uses_flat_scratch)
-; GCN: .set .Lmulti_stage_recurse2.has_dyn_sized_stack, or(0, .Lmulti_stage_recurse1.has_dyn_sized_stack)
-; GCN: .set .Lmulti_stage_recurse2.has_recursion, or(1, .Lmulti_stage_recurse1.has_recursion)
-; GCN: .set .Lmulti_stage_recurse2.has_indirect_call, or(0, .Lmulti_stage_recurse1.has_indirect_call)
-; GCN: TotalNumSgprs: .Lmulti_stage_recurse2.numbered_sgpr+extrasgprs(.Lmulti_stage_recurse2.uses_vcc, .Lmulti_stage_recurse2.uses_flat_scratch, 1)
-; GCN: NumVgprs: max(43, .Lmulti_stage_recurse1.num_vgpr)
-; GCN: ScratchSize: 16+max(.Lmulti_stage_recurse1.private_seg_size)
; GCN-LABEL: {{^}}multi_stage_recurse1:
-; GCN: .set .Lmulti_stage_recurse1.num_vgpr, max(48, 43)
-; GCN: .set .Lmulti_stage_recurse1.num_agpr, max(0, 0)
-; GCN: .set .Lmulti_stage_recurse1.numbered_sgpr, max(34, 34)
-; GCN: .set .Lmulti_stage_recurse1.private_seg_size, 16
-; GCN: .set .Lmulti_stage_recurse1.uses_vcc, 1
-; GCN: .set .Lmulti_stage_recurse1.uses_flat_scratch, 0
-; GCN: .set .Lmulti_stage_recurse1.has_dyn_sized_stack, 0
-; GCN: .set .Lmulti_stage_recurse1.has_recursion, 1
-; GCN: .set .Lmulti_stage_recurse1.has_indirect_call, 0
+; GCN: .set .Lmulti_stage_recurse1.num_vgpr, max(48, .Lmulti_stage_recurse2.num_vgpr)
+; GCN: .set .Lmulti_stage_recurse1.num_agpr, max(0, .Lmulti_stage_recurse2.num_agpr)
+; GCN: .set .Lmulti_stage_recurse1.numbered_sgpr, max(34, .Lmulti_stage_recurse2.numbered_sgpr)
+; GCN: .set .Lmulti_stage_recurse1.private_seg_size, 16+max(.Lmulti_stage_recurse2.private_seg_size)
+; GCN: .set .Lmulti_stage_recurse1.uses_vcc, or(1, .Lmulti_stage_recurse2.uses_vcc)
+; GCN: .set .Lmulti_stage_recurse1.uses_flat_scratch, or(0, .Lmulti_stage_recurse2.uses_flat_scratch)
+; GCN: .set .Lmulti_stage_recurse1.has_dyn_sized_stack, or(0, .Lmulti_stage_recurse2.has_dyn_sized_stack)
+; GCN: .set .Lmulti_stage_recurse1.has_recursion, or(1, .Lmulti_stage_recurse2.has_recursion)
+; GCN: .set .Lmulti_stage_recurse1.has_indirect_call, or(0, .Lmulti_stage_recurse2.has_indirect_call)
+; GCN: TotalNumSgprs: .Lmulti_stage_recurse1.numbered_sgpr+extrasgprs(.Lmulti_stage_recurse1.uses_vcc, .Lmulti_stage_recurse1.uses_flat_scratch, 1)
+; GCN: NumVgprs: max(48, .Lmulti_stage_recurse2.num_vgpr)
+; GCN: ScratchSize: 16+max(.Lmulti_stage_recurse2.private_seg_size)
+; GCN-LABEL: {{^}}multi_stage_recurse2:
+; GCN: .set .Lmulti_stage_recurse2.num_vgpr, max(43, 48)
+; GCN: .set .Lmulti_stage_recurse2.num_agpr, max(0, 0)
+; GCN: .set .Lmulti_stage_recurse2.numbered_sgpr, max(34, 34)
+; GCN: .set .Lmulti_stage_recurse2.private_seg_size, 16
+; GCN: .set .Lmulti_stage_recurse2.uses_vcc, 1
+; GCN: .set .Lmulti_stage_recurse2.uses_flat_scratch, 0
+; GCN: .set .Lmulti_stage_recurse2.has_dyn_sized_stack, 0
+; GCN: .set .Lmulti_stage_recurse2.has_recursion, 1
+; GCN: .set .Lmulti_stage_recurse2.has_indirect_call, 0
; GCN: TotalNumSgprs: 38
; GCN: NumVgprs: 48
; GCN: ScratchSize: 16
@@ -530,35 +530,35 @@ define void @multi_stage_recurse2(i32 %val) #2 {
; GCN: .set .Lusage_multi_stage_recurse.has_indirect_call, or(0, .Lmulti_stage_recurse1.has_indirect_call)
; GCN: TotalNumSgprs: 40
; GCN: NumVgprs: 48
-; GCN: ScratchSize: 16
+; GCN: ScratchSize: 32
define amdgpu_kernel void @usage_multi_stage_recurse(i32 %n) #0 {
call void @multi_stage_recurse1(i32 %n)
ret void
}
-; GCN-LABEL: {{^}}multi_stage_recurse_noattr2:
-; GCN: .set .Lmulti_stage_recurse_noattr2.num_vgpr, max(41, .Lmulti_stage_recurse_noattr1.num_vgpr)
-; GCN: .set .Lmulti_stage_recurse_noattr2.num_agpr, max(0, .Lmulti_stage_recurse_noattr1.num_agpr)
-; GCN: .set .Lmulti_stage_recurse_noattr2.numbered_sgpr, max(54, .Lmulti_stage_recurse_noattr1.numbered_sgpr)
-; GCN: .set .Lmulti_stage_recurse_noattr2.private_seg_size, 16+max(.Lmulti_stage_recurse_noattr1.private_seg_size)
-; GCN: .set .Lmulti_stage_recurse_noattr2.uses_vcc, or(1, .Lmulti_stage_recurse_noattr1.uses_vcc)
-; GCN: .set .Lmulti_stage_recurse_noattr2.uses_flat_scratch, or(0, .Lmulti_stage_recurse_noattr1.uses_flat_scratch)
-; GCN: .set .Lmulti_stage_recurse_noattr2.has_dyn_sized_stack, or(0, .Lmulti_stage_recurse_noattr1.has_dyn_sized_stack)
-; GCN: .set .Lmulti_stage_recurse_noattr2.has_recursion, or(0, .Lmulti_stage_recurse_noattr1.has_recursion)
-; GCN: .set .Lmulti_stage_recurse_noattr2.has_indirect_call, or(0, .Lmulti_stage_recurse_noattr1.has_indirect_call)
-; GCN: TotalNumSgprs: .Lmulti_stage_recurse_noattr2.numbered_sgpr+extrasgprs(.Lmulti_stage_recurse_noattr2.uses_vcc, .Lmulti_stage_recurse_noattr2.uses_flat_scratch, 1)
-; GCN: NumVgprs: max(41, .Lmulti_stage_recurse_noattr1.num_vgpr)
-; GCN: ScratchSize: 16+max(.Lmulti_stage_recurse_noattr1.private_seg_size)
; GCN-LABEL: {{^}}multi_stage_recurse_noattr1:
-; GCN: .set .Lmulti_stage_recurse_noattr1.num_vgpr, max(41, 41)
-; GCN: .set .Lmulti_stage_recurse_noattr1.num_agpr, max(0, 0)
-; GCN: .set .Lmulti_stage_recurse_noattr1.numbered_sgpr, max(57, 54)
-; GCN: .set .Lmulti_stage_recurse_noattr1.private_seg_size, 16
-; GCN: .set .Lmulti_stage_recurse_noattr1.uses_vcc, 1
-; GCN: .set .Lmulti_stage_recurse_noattr1.uses_flat_scratch, 0
-; GCN: .set .Lmulti_stage_recurse_noattr1.has_dyn_sized_stack, 0
-; GCN: .set .Lmulti_stage_recurse_noattr1.has_recursion, 0
-; GCN: .set .Lmulti_stage_recurse_noattr1.has_indirect_call, 0
+; GCN: .set .Lmulti_stage_recurse_noattr1.num_vgpr, max(41, .Lmulti_stage_recurse_noattr2.num_vgpr)
+; GCN: .set .Lmulti_stage_recurse_noattr1.num_agpr, max(0, .Lmulti_stage_recurse_noattr2.num_agpr)
+; GCN: .set .Lmulti_stage_recurse_noattr1.numbered_sgpr, max(57, .Lmulti_stage_recurse_noattr2.numbered_sgpr)
+; GCN: .set .Lmulti_stage_recurse_noattr1.private_seg_size, 16+max(.Lmulti_stage_recurse_noattr2.private_seg_size)
+; GCN: .set .Lmulti_stage_recurse_noattr1.uses_vcc, or(1, .Lmulti_stage_recurse_noattr2.uses_vcc)
+; GCN: .set .Lmulti_stage_recurse_noattr1.uses_flat_scratch, or(0, .Lmulti_stage_recurse_noattr2.uses_flat_scratch)
+; GCN: .set .Lmulti_stage_recurse_noattr1.has_dyn_sized_stack, or(0, .Lmulti_stage_recurse_noattr2.has_dyn_sized_stack)
+; GCN: .set .Lmulti_stage_recurse_noattr1.has_recursion, or(0, .Lmulti_stage_recurse_noattr2.has_recursion)
+; GCN: .set .Lmulti_stage_recurse_noattr1.has_indirect_call, or(0, .Lmulti_stage_recurse_noattr2.has_indirect_call)
+; GCN: TotalNumSgprs: .Lmulti_stage_recurse_noattr1.numbered_sgpr+extrasgprs(.Lmulti_stage_recurse_noattr1.uses_vcc, .Lmulti_stage_recurse_noattr1.uses_flat_scratch, 1)
+; GCN: NumVgprs: max(41, .Lmulti_stage_recurse_noattr2.num_vgpr)
+; GCN: ScratchSize: 16+max(.Lmulti_stage_recurse_noattr2.private_seg_size)
+; GCN-LABEL: {{^}}multi_stage_recurse_noattr2:
+; GCN: .set .Lmulti_stage_recurse_noattr2.num_vgpr, max(41, 41)
+; GCN: .set .Lmulti_stage_recurse_noattr2.num_agpr, max(0, 0)
+; GCN: .set .Lmulti_stage_recurse_noattr2.numbered_sgpr, max(54, 57)
+; GCN: .set .Lmulti_stage_recurse_noattr2.private_seg_size, 16
+; GCN: .set .Lmulti_stage_recurse_noattr2.uses_vcc, 1
+; GCN: .set .Lmulti_stage_recurse_noattr2.uses_flat_scratch, 0
+; GCN: .set .Lmulti_stage_recurse_noattr2.has_dyn_sized_stack, 0
+; GCN: .set .Lmulti_stage_recurse_noattr2.has_recursion, 0
+; GCN: .set .Lmulti_stage_recurse_noattr2.has_indirect_call, 0
; GCN: TotalNumSgprs: 61
; GCN: NumVgprs: 41
; GCN: ScratchSize: 16
@@ -585,7 +585,7 @@ define void @multi_stage_recurse_noattr2(i32 %val) #0 {
; GCN: .set .Lusage_multi_stage_recurse_noattrs.has_indirect_call, or(0, .Lmulti_stage_recurse_noattr1.has_indirect_call)
; GCN: TotalNumSgprs: 63
; GCN: NumVgprs: 41
-; GCN: ScratchSize: 16
+; GCN: ScratchSize: 32
define amdgpu_kernel void @usage_multi_stage_recurse_noattrs(i32 %n) #0 {
call void @multi_stage_recurse_noattr1(i32 %n)
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/greedy-alloc-fail-sgpr1024-spill.mir b/llvm/test/CodeGen/AMDGPU/greedy-alloc-fail-sgpr1024-spill.mir
index 94c22b1aa8664..20ec57429619b 100644
--- a/llvm/test/CodeGen/AMDGPU/greedy-alloc-fail-sgpr1024-spill.mir
+++ b/llvm/test/CodeGen/AMDGPU/greedy-alloc-fail-sgpr1024-spill.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu9.08-amd-amdhsa -start-before=greedy,0 -stop-after=virtregrewriter,0 -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.08-amd-amdhsa -start-before=greedy,1 -stop-after=virt-reg-rewriter,1 -o - %s | FileCheck %s
# This testcase used to fail due to introducing a spill of an SGPR
# 1024 for every subregister use inside the loop. With overlapping
diff --git a/llvm/test/CodeGen/AMDGPU/greedy-instruction-split-subrange.mir b/llvm/test/CodeGen/AMDGPU/greedy-instruction-split-subrange.mir
index a3523221c7637..dceb62e0907c9 100644
--- a/llvm/test/CodeGen/AMDGPU/greedy-instruction-split-subrange.mir
+++ b/llvm/test/CodeGen/AMDGPU/greedy-instruction-split-subrange.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -verify-regalloc -stress-regalloc=3 -start-before=greedy,1 -stop-before=virtregrewriter,1 -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -verify-regalloc -stress-regalloc=3 -start-before=greedy,2 -stop-before=virt-reg-rewriter,2 -o - %s | FileCheck %s
---
name: split_instruction_subranges
alignment: 1
diff --git a/llvm/test/CodeGen/AMDGPU/greedy-remark-crash-unassigned-reg.mir b/llvm/test/CodeGen/AMDGPU/greedy-remark-crash-unassigned-reg.mir
index 71e99a6589882..1f3aa4030d623 100644
--- a/llvm/test/CodeGen/AMDGPU/greedy-remark-crash-unassigned-reg.mir
+++ b/llvm/test/CodeGen/AMDGPU/greedy-remark-crash-unassigned-reg.mir
@@ -1,5 +1,5 @@
# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa \
-# RUN: -start-before=greedy,0 -stop-after=virtregrewriter,0 -pass-remarks='.*' -pass-remarks-output=%t.yaml -filetype=null %s
+# RUN: -start-before=greedy,1 -stop-after=virt-reg-rewriter,1 -pass-remarks='.*' -pass-remarks-output=%t.yaml -filetype=null %s
# RUN: FileCheck %s < %t.yaml
# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa \
diff --git a/llvm/test/CodeGen/AMDGPU/illegal-eviction-assert.mir b/llvm/test/CodeGen/AMDGPU/illegal-eviction-assert.mir
index 7666aa35a4214..cdad480b3aaaa 100644
--- a/llvm/test/CodeGen/AMDGPU/illegal-eviction-assert.mir
+++ b/llvm/test/CodeGen/AMDGPU/illegal-eviction-assert.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: not llc -mtriple=amdgpu9.00 -start-before=greedy,0 -stop-after=virtregrewriter,2 -o - 2>%t.err %s | FileCheck %s
+# RUN: not llc -mtriple=amdgpu9.00 -start-before=greedy,1 -stop-after=virt-reg-rewriter,3 -o - 2>%t.err %s | FileCheck %s
# RUN: FileCheck -check-prefix=ERR %s < %t.err
# This testcase cannot be compiled. An attempted eviction legality
diff --git a/llvm/test/CodeGen/AMDGPU/indirect-call-known-callees.ll b/llvm/test/CodeGen/AMDGPU/indirect-call-known-callees.ll
index 9f67a48cf3f53..6dd134d71c827 100644
--- a/llvm/test/CodeGen/AMDGPU/indirect-call-known-callees.ll
+++ b/llvm/test/CodeGen/AMDGPU/indirect-call-known-callees.ll
@@ -8,6 +8,42 @@
; FIXME: Passing real values for workitem ID, and 0s that can be undef
+define void @snork() {
+; GFX9-LABEL: snork:
+; GFX9: ; %bb.0: ; %bb
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: snork:
+; GFX12: ; %bb.0: ; %bb
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+bb:
+ ret void
+}
+
+define void @wobble() {
+; GFX9-LABEL: wobble:
+; GFX9: ; %bb.0: ; %bb
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: wobble:
+; GFX12: ; %bb.0: ; %bb
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+bb:
+ ret void
+}
+
define amdgpu_kernel void @indirect_call_known_no_special_inputs() {
; GFX9-LABEL: indirect_call_known_no_special_inputs:
; GFX9: ; %bb.0: ; %bb
@@ -74,46 +110,9 @@ define amdgpu_kernel void @indirect_call_known_no_special_inputs() {
; GFX12-NEXT: s_mov_b64 s[6:7], s[2:3]
; GFX12-NEXT: s_swappc_b64 s[30:31], s[12:13]
; GFX12-NEXT: s_endpgm
-
bb:
%cond = load i1, ptr addrspace(4) null
%tmp = select i1 %cond, ptr @wobble, ptr @snork
call void %tmp(ptr poison, i32 poison, ptr poison)
ret void
}
-
-define void @wobble() {
-; GFX9-LABEL: wobble:
-; GFX9: ; %bb.0: ; %bb
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-LABEL: wobble:
-; GFX12: ; %bb.0: ; %bb
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_setpc_b64 s[30:31]
-bb:
- ret void
-}
-
-define void @snork() {
-; GFX9-LABEL: snork:
-; GFX9: ; %bb.0: ; %bb
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-LABEL: snork:
-; GFX12: ; %bb.0: ; %bb
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_setpc_b64 s[30:31]
-bb:
- ret void
-}
diff --git a/llvm/test/CodeGen/AMDGPU/indirect-reg-read-imm-idx.mir b/llvm/test/CodeGen/AMDGPU/indirect-reg-read-imm-idx.mir
index 5a7572fa4a1a6..f693f286375d5 100644
--- a/llvm/test/CodeGen/AMDGPU/indirect-reg-read-imm-idx.mir
+++ b/llvm/test/CodeGen/AMDGPU/indirect-reg-read-imm-idx.mir
@@ -1,4 +1,4 @@
-# RUN: llc -mtriple=amdgpu9.0a -start-before=twoaddressinstruction %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.0a -start-before=two-address-instruction %s -o - | FileCheck %s
# Test that V_INDIRECT_REG_READ_GPR_IDX expansion handles immediate index operands.
diff --git a/llvm/test/CodeGen/AMDGPU/inflate-av-remat-imm.mir b/llvm/test/CodeGen/AMDGPU/inflate-av-remat-imm.mir
index 498a5cea9c3f2..6bca79c818280 100644
--- a/llvm/test/CodeGen/AMDGPU/inflate-av-remat-imm.mir
+++ b/llvm/test/CodeGen/AMDGPU/inflate-av-remat-imm.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
-# RUN: llc -mtriple=amdgpu9.0a-amd-amdhsa -simplify-mir -start-before=greedy,2 -stress-regalloc=3 -stop-after=postrapseudos -o - -verify-regalloc %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.0a-amd-amdhsa -simplify-mir -start-before=greedy,3 -stress-regalloc=3 -stop-after=post-ra-pseudos -o - -verify-regalloc %s | FileCheck %s
# Compare results of using V_MOV_B32 vs. AV_MOV_B32_IMM_PSEUDO during
# allocation.
diff --git a/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-agpr-negative-tests.mir b/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-agpr-negative-tests.mir
index 462714a57e30e..5e0e62b367261 100644
--- a/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-agpr-negative-tests.mir
+++ b/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-agpr-negative-tests.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
-# RUN: llc -mtriple=amdgpu9.42-amd-amdhsa -start-before=greedy,2 -stop-after=virtregrewriter,2 -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.42-amd-amdhsa -start-before=greedy,3 -stop-after=virt-reg-rewriter,3 -o - %s | FileCheck %s
--- |
diff --git a/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-av-with-load-source.mir b/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-av-with-load-source.mir
index 5bc25d5994ff6..c35ccca162e2b 100644
--- a/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-av-with-load-source.mir
+++ b/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-av-with-load-source.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
-# RUN: llc -mtriple=amdgpu9.42-amd-amdhsa -start-before=greedy,2 -stop-after=virtregrewriter,2 -verify-regalloc -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.42-amd-amdhsa -start-before=greedy,3 -stop-after=virt-reg-rewriter,3 -verify-regalloc -verify-machineinstrs -o - %s | FileCheck %s
# There aren't enough VGPRs for %0 to stay in a VGPR. %0 should be
# split and inflated to AV_512. The VGPR version of the instruction
diff --git a/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-av.mir b/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-av.mir
index 3cb539d3a423a..7cf0ee7fcb0ed 100644
--- a/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-av.mir
+++ b/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-av.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
-# RUN: llc -mtriple=amdgpu9.42-amd-amdhsa -start-before=greedy,2 -stop-after=virtregrewriter,2 -verify-regalloc -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.42-amd-amdhsa -start-before=greedy,3 -stop-after=virt-reg-rewriter,3 -verify-regalloc -verify-machineinstrs -o - %s | FileCheck %s
# There aren't enough VGPRs for %0 to stay in a VGPR. %0 should be
# split and inflated to AV_512. The VGPR version of the instruction
diff --git a/llvm/test/CodeGen/AMDGPU/inflated-reg-class-snippet-copy-use-after-free.mir b/llvm/test/CodeGen/AMDGPU/inflated-reg-class-snippet-copy-use-after-free.mir
index 3ed98899d6019..97cef18bd6254 100644
--- a/llvm/test/CodeGen/AMDGPU/inflated-reg-class-snippet-copy-use-after-free.mir
+++ b/llvm/test/CodeGen/AMDGPU/inflated-reg-class-snippet-copy-use-after-free.mir
@@ -1,4 +1,4 @@
-# RUN: not llc -mtriple=amdgpu9.0a-amd-amdhsa -simplify-mir -start-before=greedy,2 -stress-regalloc=4 -stop-before=virtregrewriter,2 -o - -verify-regalloc %s 2> %t.err | FileCheck %s
+# RUN: not llc -mtriple=amdgpu9.0a-amd-amdhsa -simplify-mir -start-before=greedy,3 -stress-regalloc=4 -stop-before=virt-reg-rewriter,3 -o - -verify-regalloc %s 2> %t.err | FileCheck %s
# RUN: FileCheck -check-prefix=ERR %s < %t.err
# To allocate the vreg_512_align2, the allocation will attempt to
diff --git a/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill-inspect-subrange.mir b/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill-inspect-subrange.mir
index 63ee4473a56e5..befc33e18aa94 100644
--- a/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill-inspect-subrange.mir
+++ b/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill-inspect-subrange.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 4
-# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -verify-regalloc -start-before=greedy,0 -stop-after=virtregrewriter,0 -simplify-mir -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -verify-regalloc -start-before=greedy,1 -stop-after=virt-reg-rewriter,1 -simplify-mir -o - %s | FileCheck %s
--- |
diff --git a/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill.mir b/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill.mir
index a8419a4bfc749..e6f29a8c11afc 100644
--- a/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill.mir
+++ b/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 3
-# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -start-before=greedy,0 -stop-after=virtregrewriter,0 -simplify-mir -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -start-before=greedy,1 -stop-after=virt-reg-rewriter,1 -simplify-mir -o - %s | FileCheck %s
--- |
define void @main() #0 {
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/irtranslator-whole-wave-functions.ll b/llvm/test/CodeGen/AMDGPU/irtranslator-whole-wave-functions.ll
index 4b6d904cb6e74..3a66b0fb600f5 100644
--- a/llvm/test/CodeGen/AMDGPU/irtranslator-whole-wave-functions.ll
+++ b/llvm/test/CodeGen/AMDGPU/irtranslator-whole-wave-functions.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -global-isel=1 -mtriple=amdgpu12.00--amdpal -stop-after=irtranslator < %s | FileCheck %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00--amdpal -stop-after=ir-translator < %s | FileCheck %s
define amdgpu_gfx_whole_wave i32 @basic_test(i1 %active, i32 %a, i32 %b) {
; CHECK-LABEL: name: basic_test
diff --git a/llvm/test/CodeGen/AMDGPU/issue197467-virt-reg-rewrite-failure.mir b/llvm/test/CodeGen/AMDGPU/issue197467-virt-reg-rewrite-failure.mir
index f28b600ca5fb8..9fba250470697 100644
--- a/llvm/test/CodeGen/AMDGPU/issue197467-virt-reg-rewrite-failure.mir
+++ b/llvm/test/CodeGen/AMDGPU/issue197467-virt-reg-rewrite-failure.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
-# RUN: llc -mtriple=amdgpu9.0a-amd-amdhsa -start-before=greedy -stop-after=virtregrewriter %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.0a-amd-amdhsa -start-before=greedy -stop-after=virt-reg-rewriter %s -o - | FileCheck %s
# Check VirtRegRewriter does not crashed with "cannot represent required
# subregister defs". Regression caused by #197467.
diff --git a/llvm/test/CodeGen/AMDGPU/issue48473.mir b/llvm/test/CodeGen/AMDGPU/issue48473.mir
index 7a8fde7bef6b5..097a11ee88493 100644
--- a/llvm/test/CodeGen/AMDGPU/issue48473.mir
+++ b/llvm/test/CodeGen/AMDGPU/issue48473.mir
@@ -1,4 +1,4 @@
-# RUN: not llc -mtriple=amdgpu9.08-amd-amdhsa -start-before=greedy,0 -stop-after=virtregrewriter,1 -verify-machineinstrs -o - 2> %t.err %s | FileCheck %s
+# RUN: not llc -mtriple=amdgpu9.08-amd-amdhsa -start-before=greedy,1 -stop-after=virt-reg-rewriter,2 -verify-machineinstrs -o - 2> %t.err %s | FileCheck %s
# RUN: FileCheck -check-prefix=ERR %s < %t.err
# ERR: error: register allocation failed: maximum depth for recoloring reached. Use -fexhaustive-register-search to skip cutoffs
diff --git a/llvm/test/CodeGen/AMDGPU/issue98474-assigned-physreg-interference.mir b/llvm/test/CodeGen/AMDGPU/issue98474-assigned-physreg-interference.mir
index f8d7635d8a6c1..a078979e8b33b 100644
--- a/llvm/test/CodeGen/AMDGPU/issue98474-assigned-physreg-interference.mir
+++ b/llvm/test/CodeGen/AMDGPU/issue98474-assigned-physreg-interference.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
-# RUN: llc -mtriple=amdgpu9.00-amd-amdpal -start-before=greedy,2 -stop-after=tailduplication -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.00-amd-amdpal -start-before=greedy,3 -stop-after=tailduplication -verify-machineinstrs -o - %s | FileCheck %s
---
name: undef_subreg_def_live_out_tailduplicate_vreg96_undef_sub1_sub2_assigned_physreg_interference
diff --git a/llvm/test/CodeGen/AMDGPU/issue98474-virtregrewriter-live-out-undef-subregisters.mir b/llvm/test/CodeGen/AMDGPU/issue98474-virtregrewriter-live-out-undef-subregisters.mir
index 886c8e6494be9..5e5aa5782ef82 100644
--- a/llvm/test/CodeGen/AMDGPU/issue98474-virtregrewriter-live-out-undef-subregisters.mir
+++ b/llvm/test/CodeGen/AMDGPU/issue98474-virtregrewriter-live-out-undef-subregisters.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
-# RUN: llc -mtriple=amdgpu9.00-amd-amdpal -start-before=greedy,2 -stop-after=tailduplication -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.00-amd-amdpal -start-before=greedy,3 -stop-after=tailduplication -verify-machineinstrs -o - %s | FileCheck %s
# The partial def of %0 introduces a live out undef def of %0.sub1
# into bb.3. We need to maintain this liveness with an explicit def of
diff --git a/llvm/test/CodeGen/AMDGPU/large-avgpr-assign-last.mir b/llvm/test/CodeGen/AMDGPU/large-avgpr-assign-last.mir
index fcb8461963dbb..854677b970e61 100644
--- a/llvm/test/CodeGen/AMDGPU/large-avgpr-assign-last.mir
+++ b/llvm/test/CodeGen/AMDGPU/large-avgpr-assign-last.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
-# RUN: llc -mtriple=amdgpu9.50-amd-amdhsa -verify-regalloc -greedy-regclass-priority-trumps-globalness=1 -start-after=machine-scheduler -stop-after=virtregrewriter,2 -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.50-amd-amdhsa -verify-regalloc -greedy-regclass-priority-trumps-globalness=1 -start-after=machine-scheduler -stop-after=virt-reg-rewriter,3 -o - %s | FileCheck %s
--- |
define void @temp_vgpr_to_agpr_should_not_undo_split_with_remat() #0 {
diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
index 41002382b042f..e51993398e7cb 100644
--- a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
+++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
@@ -1,14 +1,14 @@
; UNSUPPORTED: expensive_checks
-; RUN: llc -O0 -mtriple=amdgpu7.00--amdhsa -disable-verify -debug-pass=Structure < %s 2>&1 \
+; RUN: llc -O0 -enable-new-pm=0 -mtriple=amdgpu7.00--amdhsa -disable-verify -debug-pass=Structure < %s 2>&1 \
; RUN: | FileCheck -match-full-lines -strict-whitespace -check-prefix=GCN-O0 %s
-; RUN: llc -O1 -mtriple=amdgpu7.00--amdhsa -disable-verify -debug-pass=Structure < %s 2>&1 \
+; RUN: llc -O1 -enable-new-pm=0 -mtriple=amdgpu7.00--amdhsa -disable-verify -debug-pass=Structure < %s 2>&1 \
; RUN: | FileCheck -match-full-lines -strict-whitespace -check-prefix=GCN-O1 %s
-; RUN: llc -O1 -mtriple=amdgpu7.00--amdhsa -disable-verify -amdgpu-scalar-ir-passes -amdgpu-sdwa-peephole \
+; RUN: llc -O1 -enable-new-pm=0 -mtriple=amdgpu7.00--amdhsa -disable-verify -amdgpu-scalar-ir-passes -amdgpu-sdwa-peephole \
; RUN: -amdgpu-load-store-vectorizer -amdgpu-enable-pre-ra-optimizations -amdgpu-loop-prefetch -debug-pass=Structure < %s 2>&1 \
; RUN: | FileCheck -match-full-lines -strict-whitespace -check-prefix=GCN-O1-OPTS %s
-; RUN: llc -O2 -mtriple=amdgpu7.00--amdhsa -disable-verify -debug-pass=Structure < %s 2>&1 \
+; RUN: llc -O2 -enable-new-pm=0 -mtriple=amdgpu7.00--amdhsa -disable-verify -debug-pass=Structure < %s 2>&1 \
; RUN: | FileCheck -match-full-lines -strict-whitespace -check-prefix=GCN-O2 %s
-; RUN: llc -O3 -mtriple=amdgpu7.00--amdhsa -disable-verify -debug-pass=Structure < %s 2>&1 \
+; RUN: llc -O3 -enable-new-pm=0 -mtriple=amdgpu7.00--amdhsa -disable-verify -debug-pass=Structure < %s 2>&1 \
; RUN: | FileCheck -match-full-lines -strict-whitespace -check-prefix=GCN-O3 %s
; REQUIRES: asserts
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.barrier-bundle.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.barrier-bundle.ll
index d5fca840bd4e6..960ed7f1a1441 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.barrier-bundle.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.barrier-bundle.ll
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -global-isel=0 -mtriple=amdgpu6.00-mesa-mesa3d -stop-after=postrapseudos < %s | FileCheck -check-prefix=GFX6-SDAG %s
-; RUN: llc -global-isel=1 -mtriple=amdgpu6.00-mesa-mesa3d -stop-after=postrapseudos < %s | FileCheck -check-prefix=GFX6-GISEL %s
-; RUN: llc -global-isel=0 -mtriple=amdgpu9.00-mesa-mesa3d -stop-after=postrapseudos < %s | FileCheck -check-prefix=GFX9-SDAG %s
-; RUN: llc -global-isel=1 -mtriple=amdgpu9.00-mesa-mesa3d -stop-after=postrapseudos < %s | FileCheck -check-prefix=GFX9-GISEL %s
+; RUN: llc -global-isel=0 -mtriple=amdgpu6.00-mesa-mesa3d -stop-after=post-ra-pseudos < %s | FileCheck -check-prefix=GFX6-SDAG %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu6.00-mesa-mesa3d -stop-after=post-ra-pseudos < %s | FileCheck -check-prefix=GFX6-GISEL %s
+; RUN: llc -global-isel=0 -mtriple=amdgpu9.00-mesa-mesa3d -stop-after=post-ra-pseudos < %s | FileCheck -check-prefix=GFX9-SDAG %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.00-mesa-mesa3d -stop-after=post-ra-pseudos < %s | FileCheck -check-prefix=GFX9-GISEL %s
; Make sure the op is emitted bundled with a waitcnt with and without the retry loop, and the bundle is not removed by ExpandPostRAPseudos.
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.barrier-fastregalloc.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.barrier-fastregalloc.ll
index 62aceaf290e94..bf56e181cda51 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.barrier-fastregalloc.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.barrier-fastregalloc.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -mtriple=amdgpu9.00-mesa-mesa3d -O0 -stop-after=postrapseudos -o - < %s | FileCheck -enable-var-scope -check-prefix=MIR %s
+; RUN: llc -mtriple=amdgpu9.00-mesa-mesa3d -O0 -stop-after=post-ra-pseudos -o - < %s | FileCheck -enable-var-scope -check-prefix=MIR %s
define amdgpu_kernel void @gws_barrier_offset0(i32 %val) #0 {
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.bf16.ll
index ce660cd4bb6c3..5f01aaa18fa7d 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.bf16.ll
@@ -933,12 +933,12 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16__vgprcd(<8 x bfloat> %arg
; SDAG-AGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; SDAG-AGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; SDAG-AGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v32, 0
+; SDAG-AGPR-NEXT: v_mov_b32_e32 v36, 0
; SDAG-AGPR-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-AGPR-NEXT: v_mov_b64_e32 v[36:37], s[26:27]
-; SDAG-AGPR-NEXT: v_mov_b64_e32 v[34:35], s[24:25]
-; SDAG-AGPR-NEXT: v_mov_b64_e32 v[40:41], s[30:31]
-; SDAG-AGPR-NEXT: v_mov_b64_e32 v[38:39], s[28:29]
+; SDAG-AGPR-NEXT: v_mov_b64_e32 v[40:41], s[26:27]
+; SDAG-AGPR-NEXT: v_mov_b64_e32 v[38:39], s[24:25]
+; SDAG-AGPR-NEXT: v_mov_b64_e32 v[44:45], s[30:31]
+; SDAG-AGPR-NEXT: v_mov_b64_e32 v[42:43], s[28:29]
; SDAG-AGPR-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; SDAG-AGPR-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; SDAG-AGPR-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
@@ -947,41 +947,41 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16__vgprcd(<8 x bfloat> %arg
; SDAG-AGPR-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
; SDAG-AGPR-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; SDAG-AGPR-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v42, s20
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v43, s21
-; SDAG-AGPR-NEXT: v_mfma_f32_32x32x16_bf16 v[0:15], v[34:37], v[38:41], v[16:31]
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v44, s22
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v45, s23
-; SDAG-AGPR-NEXT: global_store_dwordx4 v32, v[42:45], s[0:1] offset:48 sc0 sc1
+; SDAG-AGPR-NEXT: v_mov_b32_e32 v32, s20
+; SDAG-AGPR-NEXT: v_mov_b32_e32 v33, s21
+; SDAG-AGPR-NEXT: v_mfma_f32_32x32x16_bf16 v[0:15], v[38:41], v[42:45], v[16:31]
+; SDAG-AGPR-NEXT: v_mov_b32_e32 v34, s22
+; SDAG-AGPR-NEXT: v_mov_b32_e32 v35, s23
+; SDAG-AGPR-NEXT: global_store_dwordx4 v36, v[32:35], s[0:1] offset:48 sc0 sc1
; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-AGPR-NEXT: s_nop 2
; SDAG-AGPR-NEXT: v_mov_b32_e32 v16, s16
; SDAG-AGPR-NEXT: v_mov_b32_e32 v17, s17
; SDAG-AGPR-NEXT: v_mov_b32_e32 v18, s18
; SDAG-AGPR-NEXT: v_mov_b32_e32 v19, s19
-; SDAG-AGPR-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1
+; SDAG-AGPR-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] offset:32 sc0 sc1
; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-AGPR-NEXT: s_nop 0
; SDAG-AGPR-NEXT: v_mov_b32_e32 v16, s12
; SDAG-AGPR-NEXT: v_mov_b32_e32 v17, s13
; SDAG-AGPR-NEXT: v_mov_b32_e32 v18, s14
; SDAG-AGPR-NEXT: v_mov_b32_e32 v19, s15
-; SDAG-AGPR-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1
+; SDAG-AGPR-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] offset:16 sc0 sc1
; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-AGPR-NEXT: s_nop 0
; SDAG-AGPR-NEXT: v_mov_b32_e32 v16, s8
; SDAG-AGPR-NEXT: v_mov_b32_e32 v17, s9
; SDAG-AGPR-NEXT: v_mov_b32_e32 v18, s10
; SDAG-AGPR-NEXT: v_mov_b32_e32 v19, s11
-; SDAG-AGPR-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1
+; SDAG-AGPR-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] sc0 sc1
; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-AGPR-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32 sc0 sc1
+; SDAG-AGPR-NEXT: global_store_dwordx4 v36, v[8:11], s[0:1] offset:32 sc0 sc1
; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-AGPR-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48 sc0 sc1
+; SDAG-AGPR-NEXT: global_store_dwordx4 v36, v[12:15], s[0:1] offset:48 sc0 sc1
; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-AGPR-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] sc0 sc1
+; SDAG-AGPR-NEXT: global_store_dwordx4 v36, v[0:3], s[0:1] sc0 sc1
; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-AGPR-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16 sc0 sc1
+; SDAG-AGPR-NEXT: global_store_dwordx4 v36, v[4:7], s[0:1] offset:16 sc0 sc1
; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-AGPR-NEXT: s_endpgm
;
@@ -1036,12 +1036,12 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16__vgprcd(<8 x bfloat> %arg
; SDAG-VGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; SDAG-VGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; SDAG-VGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v32, 0
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v36, 0
; SDAG-VGPR-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-VGPR-NEXT: v_mov_b64_e32 v[36:37], s[26:27]
-; SDAG-VGPR-NEXT: v_mov_b64_e32 v[34:35], s[24:25]
-; SDAG-VGPR-NEXT: v_mov_b64_e32 v[40:41], s[30:31]
-; SDAG-VGPR-NEXT: v_mov_b64_e32 v[38:39], s[28:29]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[40:41], s[26:27]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[38:39], s[24:25]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[44:45], s[30:31]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[42:43], s[28:29]
; SDAG-VGPR-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; SDAG-VGPR-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; SDAG-VGPR-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
@@ -1050,41 +1050,41 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16__vgprcd(<8 x bfloat> %arg
; SDAG-VGPR-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
; SDAG-VGPR-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; SDAG-VGPR-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v42, s20
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v43, s21
-; SDAG-VGPR-NEXT: v_mfma_f32_32x32x16_bf16 v[0:15], v[34:37], v[38:41], v[16:31]
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v44, s22
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v45, s23
-; SDAG-VGPR-NEXT: global_store_dwordx4 v32, v[42:45], s[0:1] offset:48 sc0 sc1
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v32, s20
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v33, s21
+; SDAG-VGPR-NEXT: v_mfma_f32_32x32x16_bf16 v[0:15], v[38:41], v[42:45], v[16:31]
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v34, s22
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v35, s23
+; SDAG-VGPR-NEXT: global_store_dwordx4 v36, v[32:35], s[0:1] offset:48 sc0 sc1
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-VGPR-NEXT: s_nop 2
; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, s16
; SDAG-VGPR-NEXT: v_mov_b32_e32 v17, s17
; SDAG-VGPR-NEXT: v_mov_b32_e32 v18, s18
; SDAG-VGPR-NEXT: v_mov_b32_e32 v19, s19
-; SDAG-VGPR-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1
+; SDAG-VGPR-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] offset:32 sc0 sc1
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-VGPR-NEXT: s_nop 0
; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, s12
; SDAG-VGPR-NEXT: v_mov_b32_e32 v17, s13
; SDAG-VGPR-NEXT: v_mov_b32_e32 v18, s14
; SDAG-VGPR-NEXT: v_mov_b32_e32 v19, s15
-; SDAG-VGPR-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1
+; SDAG-VGPR-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] offset:16 sc0 sc1
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-VGPR-NEXT: s_nop 0
; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, s8
; SDAG-VGPR-NEXT: v_mov_b32_e32 v17, s9
; SDAG-VGPR-NEXT: v_mov_b32_e32 v18, s10
; SDAG-VGPR-NEXT: v_mov_b32_e32 v19, s11
-; SDAG-VGPR-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1
+; SDAG-VGPR-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] sc0 sc1
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-VGPR-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32 sc0 sc1
+; SDAG-VGPR-NEXT: global_store_dwordx4 v36, v[8:11], s[0:1] offset:32 sc0 sc1
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-VGPR-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48 sc0 sc1
+; SDAG-VGPR-NEXT: global_store_dwordx4 v36, v[12:15], s[0:1] offset:48 sc0 sc1
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-VGPR-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] sc0 sc1
+; SDAG-VGPR-NEXT: global_store_dwordx4 v36, v[0:3], s[0:1] sc0 sc1
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-VGPR-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16 sc0 sc1
+; SDAG-VGPR-NEXT: global_store_dwordx4 v36, v[4:7], s[0:1] offset:16 sc0 sc1
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-VGPR-NEXT: s_endpgm
;
@@ -1202,12 +1202,12 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16__vgprcd__flags(<8 x bfloa
; SDAG-AGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; SDAG-AGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; SDAG-AGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v32, 0
+; SDAG-AGPR-NEXT: v_mov_b32_e32 v36, 0
; SDAG-AGPR-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-AGPR-NEXT: v_mov_b64_e32 v[36:37], s[26:27]
-; SDAG-AGPR-NEXT: v_mov_b64_e32 v[34:35], s[24:25]
-; SDAG-AGPR-NEXT: v_mov_b64_e32 v[40:41], s[30:31]
-; SDAG-AGPR-NEXT: v_mov_b64_e32 v[38:39], s[28:29]
+; SDAG-AGPR-NEXT: v_mov_b64_e32 v[40:41], s[26:27]
+; SDAG-AGPR-NEXT: v_mov_b64_e32 v[38:39], s[24:25]
+; SDAG-AGPR-NEXT: v_mov_b64_e32 v[44:45], s[30:31]
+; SDAG-AGPR-NEXT: v_mov_b64_e32 v[42:43], s[28:29]
; SDAG-AGPR-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; SDAG-AGPR-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; SDAG-AGPR-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
@@ -1216,41 +1216,41 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16__vgprcd__flags(<8 x bfloa
; SDAG-AGPR-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
; SDAG-AGPR-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; SDAG-AGPR-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v42, s20
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v43, s21
-; SDAG-AGPR-NEXT: v_mfma_f32_32x32x16_bf16 v[0:15], v[34:37], v[38:41], v[16:31] cbsz:1 abid:2 blgp:3
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v44, s22
-; SDAG-AGPR-NEXT: v_mov_b32_e32 v45, s23
-; SDAG-AGPR-NEXT: global_store_dwordx4 v32, v[42:45], s[0:1] offset:48 sc0 sc1
+; SDAG-AGPR-NEXT: v_mov_b32_e32 v32, s20
+; SDAG-AGPR-NEXT: v_mov_b32_e32 v33, s21
+; SDAG-AGPR-NEXT: v_mfma_f32_32x32x16_bf16 v[0:15], v[38:41], v[42:45], v[16:31] cbsz:1 abid:2 blgp:3
+; SDAG-AGPR-NEXT: v_mov_b32_e32 v34, s22
+; SDAG-AGPR-NEXT: v_mov_b32_e32 v35, s23
+; SDAG-AGPR-NEXT: global_store_dwordx4 v36, v[32:35], s[0:1] offset:48 sc0 sc1
; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-AGPR-NEXT: s_nop 2
; SDAG-AGPR-NEXT: v_mov_b32_e32 v16, s16
; SDAG-AGPR-NEXT: v_mov_b32_e32 v17, s17
; SDAG-AGPR-NEXT: v_mov_b32_e32 v18, s18
; SDAG-AGPR-NEXT: v_mov_b32_e32 v19, s19
-; SDAG-AGPR-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1
+; SDAG-AGPR-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] offset:32 sc0 sc1
; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-AGPR-NEXT: s_nop 0
; SDAG-AGPR-NEXT: v_mov_b32_e32 v16, s12
; SDAG-AGPR-NEXT: v_mov_b32_e32 v17, s13
; SDAG-AGPR-NEXT: v_mov_b32_e32 v18, s14
; SDAG-AGPR-NEXT: v_mov_b32_e32 v19, s15
-; SDAG-AGPR-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1
+; SDAG-AGPR-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] offset:16 sc0 sc1
; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-AGPR-NEXT: s_nop 0
; SDAG-AGPR-NEXT: v_mov_b32_e32 v16, s8
; SDAG-AGPR-NEXT: v_mov_b32_e32 v17, s9
; SDAG-AGPR-NEXT: v_mov_b32_e32 v18, s10
; SDAG-AGPR-NEXT: v_mov_b32_e32 v19, s11
-; SDAG-AGPR-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1
+; SDAG-AGPR-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] sc0 sc1
; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-AGPR-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32 sc0 sc1
+; SDAG-AGPR-NEXT: global_store_dwordx4 v36, v[8:11], s[0:1] offset:32 sc0 sc1
; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-AGPR-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48 sc0 sc1
+; SDAG-AGPR-NEXT: global_store_dwordx4 v36, v[12:15], s[0:1] offset:48 sc0 sc1
; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-AGPR-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] sc0 sc1
+; SDAG-AGPR-NEXT: global_store_dwordx4 v36, v[0:3], s[0:1] sc0 sc1
; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-AGPR-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16 sc0 sc1
+; SDAG-AGPR-NEXT: global_store_dwordx4 v36, v[4:7], s[0:1] offset:16 sc0 sc1
; SDAG-AGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-AGPR-NEXT: s_endpgm
;
@@ -1305,12 +1305,12 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16__vgprcd__flags(<8 x bfloa
; SDAG-VGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; SDAG-VGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; SDAG-VGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v32, 0
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v36, 0
; SDAG-VGPR-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-VGPR-NEXT: v_mov_b64_e32 v[36:37], s[26:27]
-; SDAG-VGPR-NEXT: v_mov_b64_e32 v[34:35], s[24:25]
-; SDAG-VGPR-NEXT: v_mov_b64_e32 v[40:41], s[30:31]
-; SDAG-VGPR-NEXT: v_mov_b64_e32 v[38:39], s[28:29]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[40:41], s[26:27]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[38:39], s[24:25]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[44:45], s[30:31]
+; SDAG-VGPR-NEXT: v_mov_b64_e32 v[42:43], s[28:29]
; SDAG-VGPR-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; SDAG-VGPR-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; SDAG-VGPR-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
@@ -1319,41 +1319,41 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16__vgprcd__flags(<8 x bfloa
; SDAG-VGPR-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
; SDAG-VGPR-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; SDAG-VGPR-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v42, s20
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v43, s21
-; SDAG-VGPR-NEXT: v_mfma_f32_32x32x16_bf16 v[0:15], v[34:37], v[38:41], v[16:31] cbsz:1 abid:2 blgp:3
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v44, s22
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v45, s23
-; SDAG-VGPR-NEXT: global_store_dwordx4 v32, v[42:45], s[0:1] offset:48 sc0 sc1
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v32, s20
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v33, s21
+; SDAG-VGPR-NEXT: v_mfma_f32_32x32x16_bf16 v[0:15], v[38:41], v[42:45], v[16:31] cbsz:1 abid:2 blgp:3
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v34, s22
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v35, s23
+; SDAG-VGPR-NEXT: global_store_dwordx4 v36, v[32:35], s[0:1] offset:48 sc0 sc1
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-VGPR-NEXT: s_nop 2
; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, s16
; SDAG-VGPR-NEXT: v_mov_b32_e32 v17, s17
; SDAG-VGPR-NEXT: v_mov_b32_e32 v18, s18
; SDAG-VGPR-NEXT: v_mov_b32_e32 v19, s19
-; SDAG-VGPR-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1
+; SDAG-VGPR-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] offset:32 sc0 sc1
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-VGPR-NEXT: s_nop 0
; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, s12
; SDAG-VGPR-NEXT: v_mov_b32_e32 v17, s13
; SDAG-VGPR-NEXT: v_mov_b32_e32 v18, s14
; SDAG-VGPR-NEXT: v_mov_b32_e32 v19, s15
-; SDAG-VGPR-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1
+; SDAG-VGPR-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] offset:16 sc0 sc1
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-VGPR-NEXT: s_nop 0
; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, s8
; SDAG-VGPR-NEXT: v_mov_b32_e32 v17, s9
; SDAG-VGPR-NEXT: v_mov_b32_e32 v18, s10
; SDAG-VGPR-NEXT: v_mov_b32_e32 v19, s11
-; SDAG-VGPR-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1
+; SDAG-VGPR-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] sc0 sc1
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-VGPR-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32 sc0 sc1
+; SDAG-VGPR-NEXT: global_store_dwordx4 v36, v[8:11], s[0:1] offset:32 sc0 sc1
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-VGPR-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48 sc0 sc1
+; SDAG-VGPR-NEXT: global_store_dwordx4 v36, v[12:15], s[0:1] offset:48 sc0 sc1
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-VGPR-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] sc0 sc1
+; SDAG-VGPR-NEXT: global_store_dwordx4 v36, v[0:3], s[0:1] sc0 sc1
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
-; SDAG-VGPR-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16 sc0 sc1
+; SDAG-VGPR-NEXT: global_store_dwordx4 v36, v[4:7], s[0:1] offset:16 sc0 sc1
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-VGPR-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll
index 42cd877c42261..0e30ce829b315 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll
@@ -1401,12 +1401,12 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd(<8 x half> %arg0,
; SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
-; SDAG-NEXT: v_mov_b32_e32 v32, 0
+; SDAG-NEXT: v_mov_b32_e32 v36, 0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b64_e32 v[36:37], s[26:27]
-; SDAG-NEXT: v_mov_b64_e32 v[34:35], s[24:25]
-; SDAG-NEXT: v_mov_b64_e32 v[40:41], s[30:31]
-; SDAG-NEXT: v_mov_b64_e32 v[38:39], s[28:29]
+; SDAG-NEXT: v_mov_b64_e32 v[40:41], s[26:27]
+; SDAG-NEXT: v_mov_b64_e32 v[38:39], s[24:25]
+; SDAG-NEXT: v_mov_b64_e32 v[44:45], s[30:31]
+; SDAG-NEXT: v_mov_b64_e32 v[42:43], s[28:29]
; SDAG-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; SDAG-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; SDAG-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
@@ -1415,41 +1415,41 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd(<8 x half> %arg0,
; SDAG-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
-; SDAG-NEXT: v_mov_b32_e32 v42, s20
-; SDAG-NEXT: v_mov_b32_e32 v43, s21
-; SDAG-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[34:37], v[38:41], v[16:31]
-; SDAG-NEXT: v_mov_b32_e32 v44, s22
-; SDAG-NEXT: v_mov_b32_e32 v45, s23
-; SDAG-NEXT: global_store_dwordx4 v32, v[42:45], s[0:1] offset:48 sc0 sc1
+; SDAG-NEXT: v_mov_b32_e32 v32, s20
+; SDAG-NEXT: v_mov_b32_e32 v33, s21
+; SDAG-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[38:41], v[42:45], v[16:31]
+; SDAG-NEXT: v_mov_b32_e32 v34, s22
+; SDAG-NEXT: v_mov_b32_e32 v35, s23
+; SDAG-NEXT: global_store_dwordx4 v36, v[32:35], s[0:1] offset:48 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 2
; SDAG-NEXT: v_mov_b32_e32 v16, s16
; SDAG-NEXT: v_mov_b32_e32 v17, s17
; SDAG-NEXT: v_mov_b32_e32 v18, s18
; SDAG-NEXT: v_mov_b32_e32 v19, s19
-; SDAG-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] offset:32 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_mov_b32_e32 v16, s12
; SDAG-NEXT: v_mov_b32_e32 v17, s13
; SDAG-NEXT: v_mov_b32_e32 v18, s14
; SDAG-NEXT: v_mov_b32_e32 v19, s15
-; SDAG-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] offset:16 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_mov_b32_e32 v16, s8
; SDAG-NEXT: v_mov_b32_e32 v17, s9
; SDAG-NEXT: v_mov_b32_e32 v18, s10
; SDAG-NEXT: v_mov_b32_e32 v19, s11
-; SDAG-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32 sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v36, v[8:11], s[0:1] offset:32 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48 sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v36, v[12:15], s[0:1] offset:48 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v36, v[0:3], s[0:1] sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16 sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v36, v[4:7], s[0:1] offset:16 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_endpgm
;
@@ -1504,12 +1504,12 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd(<8 x half> %arg0,
; HEURRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; HEURRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
-; HEURRC-NEXT: v_mov_b32_e32 v32, 0
+; HEURRC-NEXT: v_mov_b32_e32 v36, 0
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
-; HEURRC-NEXT: v_mov_b64_e32 v[36:37], s[26:27]
-; HEURRC-NEXT: v_mov_b64_e32 v[34:35], s[24:25]
-; HEURRC-NEXT: v_mov_b64_e32 v[40:41], s[30:31]
-; HEURRC-NEXT: v_mov_b64_e32 v[38:39], s[28:29]
+; HEURRC-NEXT: v_mov_b64_e32 v[40:41], s[26:27]
+; HEURRC-NEXT: v_mov_b64_e32 v[38:39], s[24:25]
+; HEURRC-NEXT: v_mov_b64_e32 v[44:45], s[30:31]
+; HEURRC-NEXT: v_mov_b64_e32 v[42:43], s[28:29]
; HEURRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; HEURRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; HEURRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
@@ -1518,41 +1518,41 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd(<8 x half> %arg0,
; HEURRC-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
; HEURRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; HEURRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
-; HEURRC-NEXT: v_mov_b32_e32 v42, s20
-; HEURRC-NEXT: v_mov_b32_e32 v43, s21
-; HEURRC-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[34:37], v[38:41], v[16:31]
-; HEURRC-NEXT: v_mov_b32_e32 v44, s22
-; HEURRC-NEXT: v_mov_b32_e32 v45, s23
-; HEURRC-NEXT: global_store_dwordx4 v32, v[42:45], s[0:1] offset:48 sc0 sc1
+; HEURRC-NEXT: v_mov_b32_e32 v32, s20
+; HEURRC-NEXT: v_mov_b32_e32 v33, s21
+; HEURRC-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[38:41], v[42:45], v[16:31]
+; HEURRC-NEXT: v_mov_b32_e32 v34, s22
+; HEURRC-NEXT: v_mov_b32_e32 v35, s23
+; HEURRC-NEXT: global_store_dwordx4 v36, v[32:35], s[0:1] offset:48 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_nop 2
; HEURRC-NEXT: v_mov_b32_e32 v16, s16
; HEURRC-NEXT: v_mov_b32_e32 v17, s17
; HEURRC-NEXT: v_mov_b32_e32 v18, s18
; HEURRC-NEXT: v_mov_b32_e32 v19, s19
-; HEURRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] offset:32 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_nop 0
; HEURRC-NEXT: v_mov_b32_e32 v16, s12
; HEURRC-NEXT: v_mov_b32_e32 v17, s13
; HEURRC-NEXT: v_mov_b32_e32 v18, s14
; HEURRC-NEXT: v_mov_b32_e32 v19, s15
-; HEURRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] offset:16 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_nop 0
; HEURRC-NEXT: v_mov_b32_e32 v16, s8
; HEURRC-NEXT: v_mov_b32_e32 v17, s9
; HEURRC-NEXT: v_mov_b32_e32 v18, s10
; HEURRC-NEXT: v_mov_b32_e32 v19, s11
-; HEURRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32 sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v36, v[8:11], s[0:1] offset:32 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48 sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v36, v[12:15], s[0:1] offset:48 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v36, v[0:3], s[0:1] sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16 sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v36, v[4:7], s[0:1] offset:16 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_endpgm
;
@@ -1561,12 +1561,12 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd(<8 x half> %arg0,
; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPRRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
-; VGPRRC-NEXT: v_mov_b32_e32 v32, 0
+; VGPRRC-NEXT: v_mov_b32_e32 v36, 0
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
-; VGPRRC-NEXT: v_mov_b64_e32 v[36:37], s[26:27]
-; VGPRRC-NEXT: v_mov_b64_e32 v[34:35], s[24:25]
-; VGPRRC-NEXT: v_mov_b64_e32 v[40:41], s[30:31]
-; VGPRRC-NEXT: v_mov_b64_e32 v[38:39], s[28:29]
+; VGPRRC-NEXT: v_mov_b64_e32 v[40:41], s[26:27]
+; VGPRRC-NEXT: v_mov_b64_e32 v[38:39], s[24:25]
+; VGPRRC-NEXT: v_mov_b64_e32 v[44:45], s[30:31]
+; VGPRRC-NEXT: v_mov_b64_e32 v[42:43], s[28:29]
; VGPRRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; VGPRRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; VGPRRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
@@ -1575,41 +1575,41 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd(<8 x half> %arg0,
; VGPRRC-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
-; VGPRRC-NEXT: v_mov_b32_e32 v42, s20
-; VGPRRC-NEXT: v_mov_b32_e32 v43, s21
-; VGPRRC-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[34:37], v[38:41], v[16:31]
-; VGPRRC-NEXT: v_mov_b32_e32 v44, s22
-; VGPRRC-NEXT: v_mov_b32_e32 v45, s23
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[42:45], s[0:1] offset:48 sc0 sc1
+; VGPRRC-NEXT: v_mov_b32_e32 v32, s20
+; VGPRRC-NEXT: v_mov_b32_e32 v33, s21
+; VGPRRC-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[38:41], v[42:45], v[16:31]
+; VGPRRC-NEXT: v_mov_b32_e32 v34, s22
+; VGPRRC-NEXT: v_mov_b32_e32 v35, s23
+; VGPRRC-NEXT: global_store_dwordx4 v36, v[32:35], s[0:1] offset:48 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 2
; VGPRRC-NEXT: v_mov_b32_e32 v16, s16
; VGPRRC-NEXT: v_mov_b32_e32 v17, s17
; VGPRRC-NEXT: v_mov_b32_e32 v18, s18
; VGPRRC-NEXT: v_mov_b32_e32 v19, s19
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] offset:32 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b32_e32 v16, s12
; VGPRRC-NEXT: v_mov_b32_e32 v17, s13
; VGPRRC-NEXT: v_mov_b32_e32 v18, s14
; VGPRRC-NEXT: v_mov_b32_e32 v19, s15
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] offset:16 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b32_e32 v16, s8
; VGPRRC-NEXT: v_mov_b32_e32 v17, s9
; VGPRRC-NEXT: v_mov_b32_e32 v18, s10
; VGPRRC-NEXT: v_mov_b32_e32 v19, s11
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32 sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v36, v[8:11], s[0:1] offset:32 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48 sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v36, v[12:15], s[0:1] offset:48 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v36, v[0:3], s[0:1] sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16 sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v36, v[4:7], s[0:1] offset:16 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_endpgm
; AGPR-LABEL: test_mfma_f32_32x32x16_f16__vgprcd:
@@ -1743,12 +1743,12 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd__flags(<8 x half>
; SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
-; SDAG-NEXT: v_mov_b32_e32 v32, 0
+; SDAG-NEXT: v_mov_b32_e32 v36, 0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b64_e32 v[36:37], s[26:27]
-; SDAG-NEXT: v_mov_b64_e32 v[34:35], s[24:25]
-; SDAG-NEXT: v_mov_b64_e32 v[40:41], s[30:31]
-; SDAG-NEXT: v_mov_b64_e32 v[38:39], s[28:29]
+; SDAG-NEXT: v_mov_b64_e32 v[40:41], s[26:27]
+; SDAG-NEXT: v_mov_b64_e32 v[38:39], s[24:25]
+; SDAG-NEXT: v_mov_b64_e32 v[44:45], s[30:31]
+; SDAG-NEXT: v_mov_b64_e32 v[42:43], s[28:29]
; SDAG-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; SDAG-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; SDAG-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
@@ -1757,41 +1757,41 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd__flags(<8 x half>
; SDAG-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
-; SDAG-NEXT: v_mov_b32_e32 v42, s20
-; SDAG-NEXT: v_mov_b32_e32 v43, s21
-; SDAG-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[34:37], v[38:41], v[16:31] cbsz:1 abid:2 blgp:3
-; SDAG-NEXT: v_mov_b32_e32 v44, s22
-; SDAG-NEXT: v_mov_b32_e32 v45, s23
-; SDAG-NEXT: global_store_dwordx4 v32, v[42:45], s[0:1] offset:48 sc0 sc1
+; SDAG-NEXT: v_mov_b32_e32 v32, s20
+; SDAG-NEXT: v_mov_b32_e32 v33, s21
+; SDAG-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[38:41], v[42:45], v[16:31] cbsz:1 abid:2 blgp:3
+; SDAG-NEXT: v_mov_b32_e32 v34, s22
+; SDAG-NEXT: v_mov_b32_e32 v35, s23
+; SDAG-NEXT: global_store_dwordx4 v36, v[32:35], s[0:1] offset:48 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 2
; SDAG-NEXT: v_mov_b32_e32 v16, s16
; SDAG-NEXT: v_mov_b32_e32 v17, s17
; SDAG-NEXT: v_mov_b32_e32 v18, s18
; SDAG-NEXT: v_mov_b32_e32 v19, s19
-; SDAG-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] offset:32 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_mov_b32_e32 v16, s12
; SDAG-NEXT: v_mov_b32_e32 v17, s13
; SDAG-NEXT: v_mov_b32_e32 v18, s14
; SDAG-NEXT: v_mov_b32_e32 v19, s15
-; SDAG-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] offset:16 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_mov_b32_e32 v16, s8
; SDAG-NEXT: v_mov_b32_e32 v17, s9
; SDAG-NEXT: v_mov_b32_e32 v18, s10
; SDAG-NEXT: v_mov_b32_e32 v19, s11
-; SDAG-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32 sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v36, v[8:11], s[0:1] offset:32 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48 sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v36, v[12:15], s[0:1] offset:48 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v36, v[0:3], s[0:1] sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16 sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v36, v[4:7], s[0:1] offset:16 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_endpgm
;
@@ -1846,12 +1846,12 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd__flags(<8 x half>
; HEURRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; HEURRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
-; HEURRC-NEXT: v_mov_b32_e32 v32, 0
+; HEURRC-NEXT: v_mov_b32_e32 v36, 0
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
-; HEURRC-NEXT: v_mov_b64_e32 v[36:37], s[26:27]
-; HEURRC-NEXT: v_mov_b64_e32 v[34:35], s[24:25]
-; HEURRC-NEXT: v_mov_b64_e32 v[40:41], s[30:31]
-; HEURRC-NEXT: v_mov_b64_e32 v[38:39], s[28:29]
+; HEURRC-NEXT: v_mov_b64_e32 v[40:41], s[26:27]
+; HEURRC-NEXT: v_mov_b64_e32 v[38:39], s[24:25]
+; HEURRC-NEXT: v_mov_b64_e32 v[44:45], s[30:31]
+; HEURRC-NEXT: v_mov_b64_e32 v[42:43], s[28:29]
; HEURRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; HEURRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; HEURRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
@@ -1860,41 +1860,41 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd__flags(<8 x half>
; HEURRC-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
; HEURRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; HEURRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
-; HEURRC-NEXT: v_mov_b32_e32 v42, s20
-; HEURRC-NEXT: v_mov_b32_e32 v43, s21
-; HEURRC-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[34:37], v[38:41], v[16:31] cbsz:1 abid:2 blgp:3
-; HEURRC-NEXT: v_mov_b32_e32 v44, s22
-; HEURRC-NEXT: v_mov_b32_e32 v45, s23
-; HEURRC-NEXT: global_store_dwordx4 v32, v[42:45], s[0:1] offset:48 sc0 sc1
+; HEURRC-NEXT: v_mov_b32_e32 v32, s20
+; HEURRC-NEXT: v_mov_b32_e32 v33, s21
+; HEURRC-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[38:41], v[42:45], v[16:31] cbsz:1 abid:2 blgp:3
+; HEURRC-NEXT: v_mov_b32_e32 v34, s22
+; HEURRC-NEXT: v_mov_b32_e32 v35, s23
+; HEURRC-NEXT: global_store_dwordx4 v36, v[32:35], s[0:1] offset:48 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_nop 2
; HEURRC-NEXT: v_mov_b32_e32 v16, s16
; HEURRC-NEXT: v_mov_b32_e32 v17, s17
; HEURRC-NEXT: v_mov_b32_e32 v18, s18
; HEURRC-NEXT: v_mov_b32_e32 v19, s19
-; HEURRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] offset:32 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_nop 0
; HEURRC-NEXT: v_mov_b32_e32 v16, s12
; HEURRC-NEXT: v_mov_b32_e32 v17, s13
; HEURRC-NEXT: v_mov_b32_e32 v18, s14
; HEURRC-NEXT: v_mov_b32_e32 v19, s15
-; HEURRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] offset:16 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_nop 0
; HEURRC-NEXT: v_mov_b32_e32 v16, s8
; HEURRC-NEXT: v_mov_b32_e32 v17, s9
; HEURRC-NEXT: v_mov_b32_e32 v18, s10
; HEURRC-NEXT: v_mov_b32_e32 v19, s11
-; HEURRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32 sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v36, v[8:11], s[0:1] offset:32 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48 sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v36, v[12:15], s[0:1] offset:48 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v36, v[0:3], s[0:1] sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16 sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v36, v[4:7], s[0:1] offset:16 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_endpgm
;
@@ -1903,12 +1903,12 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd__flags(<8 x half>
; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPRRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
-; VGPRRC-NEXT: v_mov_b32_e32 v32, 0
+; VGPRRC-NEXT: v_mov_b32_e32 v36, 0
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
-; VGPRRC-NEXT: v_mov_b64_e32 v[36:37], s[26:27]
-; VGPRRC-NEXT: v_mov_b64_e32 v[34:35], s[24:25]
-; VGPRRC-NEXT: v_mov_b64_e32 v[40:41], s[30:31]
-; VGPRRC-NEXT: v_mov_b64_e32 v[38:39], s[28:29]
+; VGPRRC-NEXT: v_mov_b64_e32 v[40:41], s[26:27]
+; VGPRRC-NEXT: v_mov_b64_e32 v[38:39], s[24:25]
+; VGPRRC-NEXT: v_mov_b64_e32 v[44:45], s[30:31]
+; VGPRRC-NEXT: v_mov_b64_e32 v[42:43], s[28:29]
; VGPRRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; VGPRRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; VGPRRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
@@ -1917,41 +1917,41 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd__flags(<8 x half>
; VGPRRC-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
-; VGPRRC-NEXT: v_mov_b32_e32 v42, s20
-; VGPRRC-NEXT: v_mov_b32_e32 v43, s21
-; VGPRRC-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[34:37], v[38:41], v[16:31] cbsz:1 abid:2 blgp:3
-; VGPRRC-NEXT: v_mov_b32_e32 v44, s22
-; VGPRRC-NEXT: v_mov_b32_e32 v45, s23
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[42:45], s[0:1] offset:48 sc0 sc1
+; VGPRRC-NEXT: v_mov_b32_e32 v32, s20
+; VGPRRC-NEXT: v_mov_b32_e32 v33, s21
+; VGPRRC-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[38:41], v[42:45], v[16:31] cbsz:1 abid:2 blgp:3
+; VGPRRC-NEXT: v_mov_b32_e32 v34, s22
+; VGPRRC-NEXT: v_mov_b32_e32 v35, s23
+; VGPRRC-NEXT: global_store_dwordx4 v36, v[32:35], s[0:1] offset:48 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 2
; VGPRRC-NEXT: v_mov_b32_e32 v16, s16
; VGPRRC-NEXT: v_mov_b32_e32 v17, s17
; VGPRRC-NEXT: v_mov_b32_e32 v18, s18
; VGPRRC-NEXT: v_mov_b32_e32 v19, s19
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] offset:32 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b32_e32 v16, s12
; VGPRRC-NEXT: v_mov_b32_e32 v17, s13
; VGPRRC-NEXT: v_mov_b32_e32 v18, s14
; VGPRRC-NEXT: v_mov_b32_e32 v19, s15
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] offset:16 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b32_e32 v16, s8
; VGPRRC-NEXT: v_mov_b32_e32 v17, s9
; VGPRRC-NEXT: v_mov_b32_e32 v18, s10
; VGPRRC-NEXT: v_mov_b32_e32 v19, s11
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32 sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v36, v[8:11], s[0:1] offset:32 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48 sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v36, v[12:15], s[0:1] offset:48 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v36, v[0:3], s[0:1] sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16 sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v36, v[4:7], s[0:1] offset:16 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_endpgm
; AGPR-LABEL: test_mfma_f32_32x32x16_f16__vgprcd__flags:
@@ -2557,24 +2557,24 @@ define amdgpu_kernel void @test_mfma_i32_16x16x64_i8_no_agpr__vgprcd(ptr addrspa
; SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
; SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
-; SDAG-NEXT: v_mov_b32_e32 v4, 0
+; SDAG-NEXT: v_mov_b32_e32 v12, 0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v6, s8
-; SDAG-NEXT: v_mov_b32_e32 v7, s9
-; SDAG-NEXT: v_mov_b32_e32 v8, s10
-; SDAG-NEXT: v_mov_b32_e32 v9, s11
-; SDAG-NEXT: v_mov_b32_e32 v10, s12
-; SDAG-NEXT: v_mov_b32_e32 v11, s13
-; SDAG-NEXT: v_mov_b32_e32 v12, s14
-; SDAG-NEXT: v_mov_b32_e32 v13, s15
-; SDAG-NEXT: v_mov_b32_e32 v0, s0
-; SDAG-NEXT: v_mov_b32_e32 v1, s1
-; SDAG-NEXT: v_mov_b32_e32 v2, s2
-; SDAG-NEXT: v_mov_b32_e32 v3, s3
+; SDAG-NEXT: v_mov_b32_e32 v0, s8
+; SDAG-NEXT: v_mov_b32_e32 v1, s9
+; SDAG-NEXT: v_mov_b32_e32 v2, s10
+; SDAG-NEXT: v_mov_b32_e32 v3, s11
+; SDAG-NEXT: v_mov_b32_e32 v4, s12
+; SDAG-NEXT: v_mov_b32_e32 v5, s13
+; SDAG-NEXT: v_mov_b32_e32 v6, s14
+; SDAG-NEXT: v_mov_b32_e32 v7, s15
+; SDAG-NEXT: v_mov_b32_e32 v8, s0
+; SDAG-NEXT: v_mov_b32_e32 v9, s1
+; SDAG-NEXT: v_mov_b32_e32 v10, s2
+; SDAG-NEXT: v_mov_b32_e32 v11, s3
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[6:9], v[10:13], v[0:3]
+; SDAG-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[0:3], v[4:7], v[8:11]
; SDAG-NEXT: s_nop 7
-; SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; SDAG-NEXT: global_store_dwordx4 v12, v[0:3], s[6:7]
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: test_mfma_i32_16x16x64_i8_no_agpr__vgprcd:
@@ -2601,24 +2601,24 @@ define amdgpu_kernel void @test_mfma_i32_16x16x64_i8_no_agpr__vgprcd(ptr addrspa
; HEURRC-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
; HEURRC-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; HEURRC-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
-; HEURRC-NEXT: v_mov_b32_e32 v4, 0
+; HEURRC-NEXT: v_mov_b32_e32 v12, 0
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
-; HEURRC-NEXT: v_mov_b32_e32 v6, s8
-; HEURRC-NEXT: v_mov_b32_e32 v7, s9
-; HEURRC-NEXT: v_mov_b32_e32 v8, s10
-; HEURRC-NEXT: v_mov_b32_e32 v9, s11
-; HEURRC-NEXT: v_mov_b32_e32 v10, s12
-; HEURRC-NEXT: v_mov_b32_e32 v11, s13
-; HEURRC-NEXT: v_mov_b32_e32 v12, s14
-; HEURRC-NEXT: v_mov_b32_e32 v13, s15
-; HEURRC-NEXT: v_mov_b32_e32 v0, s0
-; HEURRC-NEXT: v_mov_b32_e32 v1, s1
-; HEURRC-NEXT: v_mov_b32_e32 v2, s2
-; HEURRC-NEXT: v_mov_b32_e32 v3, s3
+; HEURRC-NEXT: v_mov_b32_e32 v0, s8
+; HEURRC-NEXT: v_mov_b32_e32 v1, s9
+; HEURRC-NEXT: v_mov_b32_e32 v2, s10
+; HEURRC-NEXT: v_mov_b32_e32 v3, s11
+; HEURRC-NEXT: v_mov_b32_e32 v4, s12
+; HEURRC-NEXT: v_mov_b32_e32 v5, s13
+; HEURRC-NEXT: v_mov_b32_e32 v6, s14
+; HEURRC-NEXT: v_mov_b32_e32 v7, s15
+; HEURRC-NEXT: v_mov_b32_e32 v8, s0
+; HEURRC-NEXT: v_mov_b32_e32 v9, s1
+; HEURRC-NEXT: v_mov_b32_e32 v10, s2
+; HEURRC-NEXT: v_mov_b32_e32 v11, s3
; HEURRC-NEXT: s_nop 1
-; HEURRC-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[6:9], v[10:13], v[0:3]
+; HEURRC-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[0:3], v[4:7], v[8:11]
; HEURRC-NEXT: s_nop 7
-; HEURRC-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; HEURRC-NEXT: global_store_dwordx4 v12, v[0:3], s[6:7]
; HEURRC-NEXT: s_endpgm
;
; VGPRRC-LABEL: test_mfma_i32_16x16x64_i8_no_agpr__vgprcd:
@@ -2626,24 +2626,24 @@ define amdgpu_kernel void @test_mfma_i32_16x16x64_i8_no_agpr__vgprcd(ptr addrspa
; VGPRRC-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
; VGPRRC-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; VGPRRC-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
-; VGPRRC-NEXT: v_mov_b32_e32 v4, 0
+; VGPRRC-NEXT: v_mov_b32_e32 v12, 0
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
-; VGPRRC-NEXT: v_mov_b32_e32 v6, s8
-; VGPRRC-NEXT: v_mov_b32_e32 v7, s9
-; VGPRRC-NEXT: v_mov_b32_e32 v8, s10
-; VGPRRC-NEXT: v_mov_b32_e32 v9, s11
-; VGPRRC-NEXT: v_mov_b32_e32 v10, s12
-; VGPRRC-NEXT: v_mov_b32_e32 v11, s13
-; VGPRRC-NEXT: v_mov_b32_e32 v12, s14
-; VGPRRC-NEXT: v_mov_b32_e32 v13, s15
-; VGPRRC-NEXT: v_mov_b32_e32 v0, s0
-; VGPRRC-NEXT: v_mov_b32_e32 v1, s1
-; VGPRRC-NEXT: v_mov_b32_e32 v2, s2
-; VGPRRC-NEXT: v_mov_b32_e32 v3, s3
+; VGPRRC-NEXT: v_mov_b32_e32 v0, s8
+; VGPRRC-NEXT: v_mov_b32_e32 v1, s9
+; VGPRRC-NEXT: v_mov_b32_e32 v2, s10
+; VGPRRC-NEXT: v_mov_b32_e32 v3, s11
+; VGPRRC-NEXT: v_mov_b32_e32 v4, s12
+; VGPRRC-NEXT: v_mov_b32_e32 v5, s13
+; VGPRRC-NEXT: v_mov_b32_e32 v6, s14
+; VGPRRC-NEXT: v_mov_b32_e32 v7, s15
+; VGPRRC-NEXT: v_mov_b32_e32 v8, s0
+; VGPRRC-NEXT: v_mov_b32_e32 v9, s1
+; VGPRRC-NEXT: v_mov_b32_e32 v10, s2
+; VGPRRC-NEXT: v_mov_b32_e32 v11, s3
; VGPRRC-NEXT: s_nop 1
-; VGPRRC-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[6:9], v[10:13], v[0:3]
+; VGPRRC-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[0:3], v[4:7], v[8:11]
; VGPRRC-NEXT: s_nop 7
-; VGPRRC-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; VGPRRC-NEXT: global_store_dwordx4 v12, v[0:3], s[6:7]
; VGPRRC-NEXT: s_endpgm
; AGPR-LABEL: test_mfma_i32_16x16x64_i8_no_agpr__vgprcd:
; AGPR: ; %bb.0:
@@ -2704,24 +2704,24 @@ define amdgpu_kernel void @test_mfma_i32_16x16x64_i8_no_agpr__vgprcd__flags(ptr
; SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
; SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
-; SDAG-NEXT: v_mov_b32_e32 v4, 0
+; SDAG-NEXT: v_mov_b32_e32 v12, 0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v6, s8
-; SDAG-NEXT: v_mov_b32_e32 v7, s9
-; SDAG-NEXT: v_mov_b32_e32 v8, s10
-; SDAG-NEXT: v_mov_b32_e32 v9, s11
-; SDAG-NEXT: v_mov_b32_e32 v10, s12
-; SDAG-NEXT: v_mov_b32_e32 v11, s13
-; SDAG-NEXT: v_mov_b32_e32 v12, s14
-; SDAG-NEXT: v_mov_b32_e32 v13, s15
-; SDAG-NEXT: v_mov_b32_e32 v0, s0
-; SDAG-NEXT: v_mov_b32_e32 v1, s1
-; SDAG-NEXT: v_mov_b32_e32 v2, s2
-; SDAG-NEXT: v_mov_b32_e32 v3, s3
+; SDAG-NEXT: v_mov_b32_e32 v0, s8
+; SDAG-NEXT: v_mov_b32_e32 v1, s9
+; SDAG-NEXT: v_mov_b32_e32 v2, s10
+; SDAG-NEXT: v_mov_b32_e32 v3, s11
+; SDAG-NEXT: v_mov_b32_e32 v4, s12
+; SDAG-NEXT: v_mov_b32_e32 v5, s13
+; SDAG-NEXT: v_mov_b32_e32 v6, s14
+; SDAG-NEXT: v_mov_b32_e32 v7, s15
+; SDAG-NEXT: v_mov_b32_e32 v8, s0
+; SDAG-NEXT: v_mov_b32_e32 v9, s1
+; SDAG-NEXT: v_mov_b32_e32 v10, s2
+; SDAG-NEXT: v_mov_b32_e32 v11, s3
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[6:9], v[10:13], v[0:3] cbsz:3 abid:2 blgp:1
+; SDAG-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[0:3], v[4:7], v[8:11] cbsz:3 abid:2 blgp:1
; SDAG-NEXT: s_nop 7
-; SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; SDAG-NEXT: global_store_dwordx4 v12, v[0:3], s[6:7]
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: test_mfma_i32_16x16x64_i8_no_agpr__vgprcd__flags:
@@ -2748,24 +2748,24 @@ define amdgpu_kernel void @test_mfma_i32_16x16x64_i8_no_agpr__vgprcd__flags(ptr
; HEURRC-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
; HEURRC-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; HEURRC-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
-; HEURRC-NEXT: v_mov_b32_e32 v4, 0
+; HEURRC-NEXT: v_mov_b32_e32 v12, 0
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
-; HEURRC-NEXT: v_mov_b32_e32 v6, s8
-; HEURRC-NEXT: v_mov_b32_e32 v7, s9
-; HEURRC-NEXT: v_mov_b32_e32 v8, s10
-; HEURRC-NEXT: v_mov_b32_e32 v9, s11
-; HEURRC-NEXT: v_mov_b32_e32 v10, s12
-; HEURRC-NEXT: v_mov_b32_e32 v11, s13
-; HEURRC-NEXT: v_mov_b32_e32 v12, s14
-; HEURRC-NEXT: v_mov_b32_e32 v13, s15
-; HEURRC-NEXT: v_mov_b32_e32 v0, s0
-; HEURRC-NEXT: v_mov_b32_e32 v1, s1
-; HEURRC-NEXT: v_mov_b32_e32 v2, s2
-; HEURRC-NEXT: v_mov_b32_e32 v3, s3
+; HEURRC-NEXT: v_mov_b32_e32 v0, s8
+; HEURRC-NEXT: v_mov_b32_e32 v1, s9
+; HEURRC-NEXT: v_mov_b32_e32 v2, s10
+; HEURRC-NEXT: v_mov_b32_e32 v3, s11
+; HEURRC-NEXT: v_mov_b32_e32 v4, s12
+; HEURRC-NEXT: v_mov_b32_e32 v5, s13
+; HEURRC-NEXT: v_mov_b32_e32 v6, s14
+; HEURRC-NEXT: v_mov_b32_e32 v7, s15
+; HEURRC-NEXT: v_mov_b32_e32 v8, s0
+; HEURRC-NEXT: v_mov_b32_e32 v9, s1
+; HEURRC-NEXT: v_mov_b32_e32 v10, s2
+; HEURRC-NEXT: v_mov_b32_e32 v11, s3
; HEURRC-NEXT: s_nop 1
-; HEURRC-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[6:9], v[10:13], v[0:3] cbsz:3 abid:2 blgp:1
+; HEURRC-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[0:3], v[4:7], v[8:11] cbsz:3 abid:2 blgp:1
; HEURRC-NEXT: s_nop 7
-; HEURRC-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; HEURRC-NEXT: global_store_dwordx4 v12, v[0:3], s[6:7]
; HEURRC-NEXT: s_endpgm
;
; VGPRRC-LABEL: test_mfma_i32_16x16x64_i8_no_agpr__vgprcd__flags:
@@ -2773,24 +2773,24 @@ define amdgpu_kernel void @test_mfma_i32_16x16x64_i8_no_agpr__vgprcd__flags(ptr
; VGPRRC-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
; VGPRRC-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; VGPRRC-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
-; VGPRRC-NEXT: v_mov_b32_e32 v4, 0
+; VGPRRC-NEXT: v_mov_b32_e32 v12, 0
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
-; VGPRRC-NEXT: v_mov_b32_e32 v6, s8
-; VGPRRC-NEXT: v_mov_b32_e32 v7, s9
-; VGPRRC-NEXT: v_mov_b32_e32 v8, s10
-; VGPRRC-NEXT: v_mov_b32_e32 v9, s11
-; VGPRRC-NEXT: v_mov_b32_e32 v10, s12
-; VGPRRC-NEXT: v_mov_b32_e32 v11, s13
-; VGPRRC-NEXT: v_mov_b32_e32 v12, s14
-; VGPRRC-NEXT: v_mov_b32_e32 v13, s15
-; VGPRRC-NEXT: v_mov_b32_e32 v0, s0
-; VGPRRC-NEXT: v_mov_b32_e32 v1, s1
-; VGPRRC-NEXT: v_mov_b32_e32 v2, s2
-; VGPRRC-NEXT: v_mov_b32_e32 v3, s3
+; VGPRRC-NEXT: v_mov_b32_e32 v0, s8
+; VGPRRC-NEXT: v_mov_b32_e32 v1, s9
+; VGPRRC-NEXT: v_mov_b32_e32 v2, s10
+; VGPRRC-NEXT: v_mov_b32_e32 v3, s11
+; VGPRRC-NEXT: v_mov_b32_e32 v4, s12
+; VGPRRC-NEXT: v_mov_b32_e32 v5, s13
+; VGPRRC-NEXT: v_mov_b32_e32 v6, s14
+; VGPRRC-NEXT: v_mov_b32_e32 v7, s15
+; VGPRRC-NEXT: v_mov_b32_e32 v8, s0
+; VGPRRC-NEXT: v_mov_b32_e32 v9, s1
+; VGPRRC-NEXT: v_mov_b32_e32 v10, s2
+; VGPRRC-NEXT: v_mov_b32_e32 v11, s3
; VGPRRC-NEXT: s_nop 1
-; VGPRRC-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[6:9], v[10:13], v[0:3] cbsz:3 abid:2 blgp:1
+; VGPRRC-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[0:3], v[4:7], v[8:11] cbsz:3 abid:2 blgp:1
; VGPRRC-NEXT: s_nop 7
-; VGPRRC-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; VGPRRC-NEXT: global_store_dwordx4 v12, v[0:3], s[6:7]
; VGPRRC-NEXT: s_endpgm
; AGPR-LABEL: test_mfma_i32_16x16x64_i8_no_agpr__vgprcd__flags:
; AGPR: ; %bb.0:
@@ -3952,17 +3952,17 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd(<4 x i32> %arg0, <4
; SDAG: ; %bb.0:
; SDAG-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
-; SDAG-NEXT: v_mov_b32_e32 v32, 0
+; SDAG-NEXT: v_mov_b32_e32 v40, 0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v34, s20
-; SDAG-NEXT: v_mov_b32_e32 v35, s21
-; SDAG-NEXT: v_mov_b32_e32 v36, s22
-; SDAG-NEXT: v_mov_b32_e32 v37, s23
+; SDAG-NEXT: v_mov_b32_e32 v32, s20
+; SDAG-NEXT: v_mov_b32_e32 v33, s21
+; SDAG-NEXT: v_mov_b32_e32 v34, s22
+; SDAG-NEXT: v_mov_b32_e32 v35, s23
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
-; SDAG-NEXT: v_mov_b32_e32 v38, s24
-; SDAG-NEXT: v_mov_b32_e32 v39, s25
-; SDAG-NEXT: v_mov_b32_e32 v40, s26
-; SDAG-NEXT: v_mov_b32_e32 v41, s27
+; SDAG-NEXT: v_mov_b32_e32 v36, s24
+; SDAG-NEXT: v_mov_b32_e32 v37, s25
+; SDAG-NEXT: v_mov_b32_e32 v38, s26
+; SDAG-NEXT: v_mov_b32_e32 v39, s27
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; SDAG-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
@@ -3973,42 +3973,42 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd(<4 x i32> %arg0, <4
; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[34:37], v[38:41], v[16:31]
+; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31]
; SDAG-NEXT: s_nop 6
; SDAG-NEXT: v_mov_b32_e32 v16, s20
; SDAG-NEXT: v_mov_b32_e32 v17, s21
; SDAG-NEXT: v_mov_b32_e32 v18, s22
; SDAG-NEXT: v_mov_b32_e32 v19, s23
-; SDAG-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:48 sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:48 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_mov_b32_e32 v16, s16
; SDAG-NEXT: v_mov_b32_e32 v17, s17
; SDAG-NEXT: v_mov_b32_e32 v18, s18
; SDAG-NEXT: v_mov_b32_e32 v19, s19
-; SDAG-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:32 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_mov_b32_e32 v16, s12
; SDAG-NEXT: v_mov_b32_e32 v17, s13
; SDAG-NEXT: v_mov_b32_e32 v18, s14
; SDAG-NEXT: v_mov_b32_e32 v19, s15
-; SDAG-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:16 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_mov_b32_e32 v16, s8
; SDAG-NEXT: v_mov_b32_e32 v17, s9
; SDAG-NEXT: v_mov_b32_e32 v18, s10
; SDAG-NEXT: v_mov_b32_e32 v19, s11
-; SDAG-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32 sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v40, v[8:11], s[0:1] offset:32 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48 sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v40, v[12:15], s[0:1] offset:48 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v40, v[0:3], s[0:1] sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16 sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v40, v[4:7], s[0:1] offset:16 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_endpgm
;
@@ -4062,17 +4062,17 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd(<4 x i32> %arg0, <4
; HEURRC: ; %bb.0:
; HEURRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
; HEURRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
-; HEURRC-NEXT: v_mov_b32_e32 v32, 0
+; HEURRC-NEXT: v_mov_b32_e32 v40, 0
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
-; HEURRC-NEXT: v_mov_b32_e32 v34, s20
-; HEURRC-NEXT: v_mov_b32_e32 v35, s21
-; HEURRC-NEXT: v_mov_b32_e32 v36, s22
-; HEURRC-NEXT: v_mov_b32_e32 v37, s23
+; HEURRC-NEXT: v_mov_b32_e32 v32, s20
+; HEURRC-NEXT: v_mov_b32_e32 v33, s21
+; HEURRC-NEXT: v_mov_b32_e32 v34, s22
+; HEURRC-NEXT: v_mov_b32_e32 v35, s23
; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
-; HEURRC-NEXT: v_mov_b32_e32 v38, s24
-; HEURRC-NEXT: v_mov_b32_e32 v39, s25
-; HEURRC-NEXT: v_mov_b32_e32 v40, s26
-; HEURRC-NEXT: v_mov_b32_e32 v41, s27
+; HEURRC-NEXT: v_mov_b32_e32 v36, s24
+; HEURRC-NEXT: v_mov_b32_e32 v37, s25
+; HEURRC-NEXT: v_mov_b32_e32 v38, s26
+; HEURRC-NEXT: v_mov_b32_e32 v39, s27
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
; HEURRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; HEURRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
@@ -4083,42 +4083,42 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd(<4 x i32> %arg0, <4
; HEURRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; HEURRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; HEURRC-NEXT: s_nop 1
-; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[34:37], v[38:41], v[16:31]
+; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31]
; HEURRC-NEXT: s_nop 6
; HEURRC-NEXT: v_mov_b32_e32 v16, s20
; HEURRC-NEXT: v_mov_b32_e32 v17, s21
; HEURRC-NEXT: v_mov_b32_e32 v18, s22
; HEURRC-NEXT: v_mov_b32_e32 v19, s23
-; HEURRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:48 sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:48 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_nop 0
; HEURRC-NEXT: v_mov_b32_e32 v16, s16
; HEURRC-NEXT: v_mov_b32_e32 v17, s17
; HEURRC-NEXT: v_mov_b32_e32 v18, s18
; HEURRC-NEXT: v_mov_b32_e32 v19, s19
-; HEURRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:32 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_nop 0
; HEURRC-NEXT: v_mov_b32_e32 v16, s12
; HEURRC-NEXT: v_mov_b32_e32 v17, s13
; HEURRC-NEXT: v_mov_b32_e32 v18, s14
; HEURRC-NEXT: v_mov_b32_e32 v19, s15
-; HEURRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:16 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_nop 0
; HEURRC-NEXT: v_mov_b32_e32 v16, s8
; HEURRC-NEXT: v_mov_b32_e32 v17, s9
; HEURRC-NEXT: v_mov_b32_e32 v18, s10
; HEURRC-NEXT: v_mov_b32_e32 v19, s11
-; HEURRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32 sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v40, v[8:11], s[0:1] offset:32 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48 sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v40, v[12:15], s[0:1] offset:48 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v40, v[0:3], s[0:1] sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16 sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v40, v[4:7], s[0:1] offset:16 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_endpgm
;
@@ -4126,17 +4126,17 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd(<4 x i32> %arg0, <4
; VGPRRC: ; %bb.0:
; VGPRRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
; VGPRRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
-; VGPRRC-NEXT: v_mov_b32_e32 v32, 0
+; VGPRRC-NEXT: v_mov_b32_e32 v40, 0
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
-; VGPRRC-NEXT: v_mov_b32_e32 v34, s20
-; VGPRRC-NEXT: v_mov_b32_e32 v35, s21
-; VGPRRC-NEXT: v_mov_b32_e32 v36, s22
-; VGPRRC-NEXT: v_mov_b32_e32 v37, s23
+; VGPRRC-NEXT: v_mov_b32_e32 v32, s20
+; VGPRRC-NEXT: v_mov_b32_e32 v33, s21
+; VGPRRC-NEXT: v_mov_b32_e32 v34, s22
+; VGPRRC-NEXT: v_mov_b32_e32 v35, s23
; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
-; VGPRRC-NEXT: v_mov_b32_e32 v38, s24
-; VGPRRC-NEXT: v_mov_b32_e32 v39, s25
-; VGPRRC-NEXT: v_mov_b32_e32 v40, s26
-; VGPRRC-NEXT: v_mov_b32_e32 v41, s27
+; VGPRRC-NEXT: v_mov_b32_e32 v36, s24
+; VGPRRC-NEXT: v_mov_b32_e32 v37, s25
+; VGPRRC-NEXT: v_mov_b32_e32 v38, s26
+; VGPRRC-NEXT: v_mov_b32_e32 v39, s27
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
; VGPRRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; VGPRRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
@@ -4147,42 +4147,42 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd(<4 x i32> %arg0, <4
; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; VGPRRC-NEXT: s_nop 1
-; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[34:37], v[38:41], v[16:31]
+; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31]
; VGPRRC-NEXT: s_nop 6
; VGPRRC-NEXT: v_mov_b32_e32 v16, s20
; VGPRRC-NEXT: v_mov_b32_e32 v17, s21
; VGPRRC-NEXT: v_mov_b32_e32 v18, s22
; VGPRRC-NEXT: v_mov_b32_e32 v19, s23
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:48 sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:48 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b32_e32 v16, s16
; VGPRRC-NEXT: v_mov_b32_e32 v17, s17
; VGPRRC-NEXT: v_mov_b32_e32 v18, s18
; VGPRRC-NEXT: v_mov_b32_e32 v19, s19
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:32 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b32_e32 v16, s12
; VGPRRC-NEXT: v_mov_b32_e32 v17, s13
; VGPRRC-NEXT: v_mov_b32_e32 v18, s14
; VGPRRC-NEXT: v_mov_b32_e32 v19, s15
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:16 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b32_e32 v16, s8
; VGPRRC-NEXT: v_mov_b32_e32 v17, s9
; VGPRRC-NEXT: v_mov_b32_e32 v18, s10
; VGPRRC-NEXT: v_mov_b32_e32 v19, s11
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32 sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v40, v[8:11], s[0:1] offset:32 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48 sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v40, v[12:15], s[0:1] offset:48 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v40, v[0:3], s[0:1] sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16 sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v40, v[4:7], s[0:1] offset:16 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_endpgm
; AGPR-LABEL: test_mfma_i32_32x32x32_i8__vgprcd:
@@ -4329,17 +4329,17 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd__flags(<4 x i32> %a
; SDAG: ; %bb.0:
; SDAG-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
-; SDAG-NEXT: v_mov_b32_e32 v32, 0
+; SDAG-NEXT: v_mov_b32_e32 v40, 0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v34, s20
-; SDAG-NEXT: v_mov_b32_e32 v35, s21
-; SDAG-NEXT: v_mov_b32_e32 v36, s22
-; SDAG-NEXT: v_mov_b32_e32 v37, s23
+; SDAG-NEXT: v_mov_b32_e32 v32, s20
+; SDAG-NEXT: v_mov_b32_e32 v33, s21
+; SDAG-NEXT: v_mov_b32_e32 v34, s22
+; SDAG-NEXT: v_mov_b32_e32 v35, s23
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
-; SDAG-NEXT: v_mov_b32_e32 v38, s24
-; SDAG-NEXT: v_mov_b32_e32 v39, s25
-; SDAG-NEXT: v_mov_b32_e32 v40, s26
-; SDAG-NEXT: v_mov_b32_e32 v41, s27
+; SDAG-NEXT: v_mov_b32_e32 v36, s24
+; SDAG-NEXT: v_mov_b32_e32 v37, s25
+; SDAG-NEXT: v_mov_b32_e32 v38, s26
+; SDAG-NEXT: v_mov_b32_e32 v39, s27
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; SDAG-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
@@ -4350,42 +4350,42 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd__flags(<4 x i32> %a
; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[34:37], v[38:41], v[16:31] cbsz:1 abid:2 blgp:3
+; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31] cbsz:1 abid:2 blgp:3
; SDAG-NEXT: s_nop 6
; SDAG-NEXT: v_mov_b32_e32 v16, s20
; SDAG-NEXT: v_mov_b32_e32 v17, s21
; SDAG-NEXT: v_mov_b32_e32 v18, s22
; SDAG-NEXT: v_mov_b32_e32 v19, s23
-; SDAG-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:48 sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:48 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_mov_b32_e32 v16, s16
; SDAG-NEXT: v_mov_b32_e32 v17, s17
; SDAG-NEXT: v_mov_b32_e32 v18, s18
; SDAG-NEXT: v_mov_b32_e32 v19, s19
-; SDAG-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:32 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_mov_b32_e32 v16, s12
; SDAG-NEXT: v_mov_b32_e32 v17, s13
; SDAG-NEXT: v_mov_b32_e32 v18, s14
; SDAG-NEXT: v_mov_b32_e32 v19, s15
-; SDAG-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:16 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_mov_b32_e32 v16, s8
; SDAG-NEXT: v_mov_b32_e32 v17, s9
; SDAG-NEXT: v_mov_b32_e32 v18, s10
; SDAG-NEXT: v_mov_b32_e32 v19, s11
-; SDAG-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32 sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v40, v[8:11], s[0:1] offset:32 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48 sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v40, v[12:15], s[0:1] offset:48 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v40, v[0:3], s[0:1] sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16 sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v40, v[4:7], s[0:1] offset:16 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_endpgm
;
@@ -4439,17 +4439,17 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd__flags(<4 x i32> %a
; HEURRC: ; %bb.0:
; HEURRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
; HEURRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
-; HEURRC-NEXT: v_mov_b32_e32 v32, 0
+; HEURRC-NEXT: v_mov_b32_e32 v40, 0
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
-; HEURRC-NEXT: v_mov_b32_e32 v34, s20
-; HEURRC-NEXT: v_mov_b32_e32 v35, s21
-; HEURRC-NEXT: v_mov_b32_e32 v36, s22
-; HEURRC-NEXT: v_mov_b32_e32 v37, s23
+; HEURRC-NEXT: v_mov_b32_e32 v32, s20
+; HEURRC-NEXT: v_mov_b32_e32 v33, s21
+; HEURRC-NEXT: v_mov_b32_e32 v34, s22
+; HEURRC-NEXT: v_mov_b32_e32 v35, s23
; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
-; HEURRC-NEXT: v_mov_b32_e32 v38, s24
-; HEURRC-NEXT: v_mov_b32_e32 v39, s25
-; HEURRC-NEXT: v_mov_b32_e32 v40, s26
-; HEURRC-NEXT: v_mov_b32_e32 v41, s27
+; HEURRC-NEXT: v_mov_b32_e32 v36, s24
+; HEURRC-NEXT: v_mov_b32_e32 v37, s25
+; HEURRC-NEXT: v_mov_b32_e32 v38, s26
+; HEURRC-NEXT: v_mov_b32_e32 v39, s27
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
; HEURRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; HEURRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
@@ -4460,42 +4460,42 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd__flags(<4 x i32> %a
; HEURRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; HEURRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; HEURRC-NEXT: s_nop 1
-; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[34:37], v[38:41], v[16:31] cbsz:1 abid:2 blgp:3
+; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31] cbsz:1 abid:2 blgp:3
; HEURRC-NEXT: s_nop 6
; HEURRC-NEXT: v_mov_b32_e32 v16, s20
; HEURRC-NEXT: v_mov_b32_e32 v17, s21
; HEURRC-NEXT: v_mov_b32_e32 v18, s22
; HEURRC-NEXT: v_mov_b32_e32 v19, s23
-; HEURRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:48 sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:48 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_nop 0
; HEURRC-NEXT: v_mov_b32_e32 v16, s16
; HEURRC-NEXT: v_mov_b32_e32 v17, s17
; HEURRC-NEXT: v_mov_b32_e32 v18, s18
; HEURRC-NEXT: v_mov_b32_e32 v19, s19
-; HEURRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:32 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_nop 0
; HEURRC-NEXT: v_mov_b32_e32 v16, s12
; HEURRC-NEXT: v_mov_b32_e32 v17, s13
; HEURRC-NEXT: v_mov_b32_e32 v18, s14
; HEURRC-NEXT: v_mov_b32_e32 v19, s15
-; HEURRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:16 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_nop 0
; HEURRC-NEXT: v_mov_b32_e32 v16, s8
; HEURRC-NEXT: v_mov_b32_e32 v17, s9
; HEURRC-NEXT: v_mov_b32_e32 v18, s10
; HEURRC-NEXT: v_mov_b32_e32 v19, s11
-; HEURRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32 sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v40, v[8:11], s[0:1] offset:32 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48 sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v40, v[12:15], s[0:1] offset:48 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v40, v[0:3], s[0:1] sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16 sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v40, v[4:7], s[0:1] offset:16 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_endpgm
;
@@ -4503,17 +4503,17 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd__flags(<4 x i32> %a
; VGPRRC: ; %bb.0:
; VGPRRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
; VGPRRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
-; VGPRRC-NEXT: v_mov_b32_e32 v32, 0
+; VGPRRC-NEXT: v_mov_b32_e32 v40, 0
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
-; VGPRRC-NEXT: v_mov_b32_e32 v34, s20
-; VGPRRC-NEXT: v_mov_b32_e32 v35, s21
-; VGPRRC-NEXT: v_mov_b32_e32 v36, s22
-; VGPRRC-NEXT: v_mov_b32_e32 v37, s23
+; VGPRRC-NEXT: v_mov_b32_e32 v32, s20
+; VGPRRC-NEXT: v_mov_b32_e32 v33, s21
+; VGPRRC-NEXT: v_mov_b32_e32 v34, s22
+; VGPRRC-NEXT: v_mov_b32_e32 v35, s23
; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
-; VGPRRC-NEXT: v_mov_b32_e32 v38, s24
-; VGPRRC-NEXT: v_mov_b32_e32 v39, s25
-; VGPRRC-NEXT: v_mov_b32_e32 v40, s26
-; VGPRRC-NEXT: v_mov_b32_e32 v41, s27
+; VGPRRC-NEXT: v_mov_b32_e32 v36, s24
+; VGPRRC-NEXT: v_mov_b32_e32 v37, s25
+; VGPRRC-NEXT: v_mov_b32_e32 v38, s26
+; VGPRRC-NEXT: v_mov_b32_e32 v39, s27
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
; VGPRRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; VGPRRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
@@ -4524,42 +4524,42 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd__flags(<4 x i32> %a
; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; VGPRRC-NEXT: s_nop 1
-; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[34:37], v[38:41], v[16:31] cbsz:1 abid:2 blgp:3
+; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31] cbsz:1 abid:2 blgp:3
; VGPRRC-NEXT: s_nop 6
; VGPRRC-NEXT: v_mov_b32_e32 v16, s20
; VGPRRC-NEXT: v_mov_b32_e32 v17, s21
; VGPRRC-NEXT: v_mov_b32_e32 v18, s22
; VGPRRC-NEXT: v_mov_b32_e32 v19, s23
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:48 sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:48 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b32_e32 v16, s16
; VGPRRC-NEXT: v_mov_b32_e32 v17, s17
; VGPRRC-NEXT: v_mov_b32_e32 v18, s18
; VGPRRC-NEXT: v_mov_b32_e32 v19, s19
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:32 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b32_e32 v16, s12
; VGPRRC-NEXT: v_mov_b32_e32 v17, s13
; VGPRRC-NEXT: v_mov_b32_e32 v18, s14
; VGPRRC-NEXT: v_mov_b32_e32 v19, s15
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:16 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b32_e32 v16, s8
; VGPRRC-NEXT: v_mov_b32_e32 v17, s9
; VGPRRC-NEXT: v_mov_b32_e32 v18, s10
; VGPRRC-NEXT: v_mov_b32_e32 v19, s11
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32 sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v40, v[8:11], s[0:1] offset:32 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48 sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v40, v[12:15], s[0:1] offset:48 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v40, v[0:3], s[0:1] sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16 sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v40, v[4:7], s[0:1] offset:16 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_endpgm
; AGPR-LABEL: test_mfma_i32_32x32x32_i8__vgprcd__flags:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.ll
index aeeb5958e9ca3..0977caa3d677a 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.ll
@@ -2513,36 +2513,36 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd(<8 x i32
; SDAG-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd:
; SDAG: ; %bb.0:
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x0
-; SDAG-NEXT: v_mov_b32_e32 v4, 0
-; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v8, s8
-; SDAG-NEXT: v_mov_b32_e32 v9, s9
-; SDAG-NEXT: v_mov_b32_e32 v10, s10
-; SDAG-NEXT: v_mov_b32_e32 v11, s11
-; SDAG-NEXT: v_mov_b32_e32 v12, s12
-; SDAG-NEXT: v_mov_b32_e32 v13, s13
-; SDAG-NEXT: v_mov_b32_e32 v14, s14
-; SDAG-NEXT: v_mov_b32_e32 v15, s15
-; SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x40
-; SDAG-NEXT: v_mov_b32_e32 v16, s16
-; SDAG-NEXT: v_mov_b32_e32 v17, s17
-; SDAG-NEXT: v_mov_b32_e32 v18, s18
-; SDAG-NEXT: v_mov_b32_e32 v19, s19
-; SDAG-NEXT: v_mov_b32_e32 v20, s20
-; SDAG-NEXT: v_mov_b32_e32 v21, s21
-; SDAG-NEXT: v_mov_b32_e32 v22, s22
-; SDAG-NEXT: v_mov_b32_e32 v23, s23
+; SDAG-NEXT: v_mov_b32_e32 v20, 0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: v_mov_b32_e32 v0, s8
; SDAG-NEXT: v_mov_b32_e32 v1, s9
; SDAG-NEXT: v_mov_b32_e32 v2, s10
; SDAG-NEXT: v_mov_b32_e32 v3, s11
-; SDAG-NEXT: v_mov_b32_e32 v5, s12
-; SDAG-NEXT: v_mov_b32_e32 v6, s13
+; SDAG-NEXT: v_mov_b32_e32 v4, s12
+; SDAG-NEXT: v_mov_b32_e32 v5, s13
+; SDAG-NEXT: v_mov_b32_e32 v6, s14
+; SDAG-NEXT: v_mov_b32_e32 v7, s15
+; SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x40
+; SDAG-NEXT: v_mov_b32_e32 v8, s16
+; SDAG-NEXT: v_mov_b32_e32 v9, s17
+; SDAG-NEXT: v_mov_b32_e32 v10, s18
+; SDAG-NEXT: v_mov_b32_e32 v11, s19
+; SDAG-NEXT: v_mov_b32_e32 v12, s20
+; SDAG-NEXT: v_mov_b32_e32 v13, s21
+; SDAG-NEXT: v_mov_b32_e32 v14, s22
+; SDAG-NEXT: v_mov_b32_e32 v15, s23
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: v_mov_b32_e32 v16, s8
+; SDAG-NEXT: v_mov_b32_e32 v17, s9
+; SDAG-NEXT: v_mov_b32_e32 v18, s10
+; SDAG-NEXT: v_mov_b32_e32 v19, s11
+; SDAG-NEXT: v_mov_b32_e32 v21, s12
+; SDAG-NEXT: v_mov_b32_e32 v22, s13
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[8:15], v[16:23], v[0:3], v5, v6 op_sel:[1,1,0] op_sel_hi:[1,0,0] blgp:2
+; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v21, v22 op_sel:[1,1,0] op_sel_hi:[1,0,0] blgp:2
; SDAG-NEXT: s_nop 11
-; SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[14:15]
+; SDAG-NEXT: global_store_dwordx4 v20, v[0:3], s[14:15]
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd:
@@ -2572,36 +2572,36 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd(<8 x i32
; AGPR-SDAG-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd:
; AGPR-SDAG: ; %bb.0:
; AGPR-SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x0
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v4, 0
-; AGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v8, s8
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v9, s9
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v10, s10
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v11, s11
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v12, s12
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v13, s13
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v14, s14
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v15, s15
-; AGPR-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x40
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s16
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s17
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s18
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s19
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s20
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s21
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, s22
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v23, s23
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, 0
; AGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; AGPR-SDAG-NEXT: v_mov_b32_e32 v0, s8
; AGPR-SDAG-NEXT: v_mov_b32_e32 v1, s9
; AGPR-SDAG-NEXT: v_mov_b32_e32 v2, s10
; AGPR-SDAG-NEXT: v_mov_b32_e32 v3, s11
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v5, s12
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v6, s13
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v4, s12
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v5, s13
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v6, s14
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v7, s15
+; AGPR-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x40
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v8, s16
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v9, s17
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v10, s18
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v11, s19
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v12, s20
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v13, s21
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v14, s22
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v15, s23
+; AGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s8
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s9
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s10
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s11
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s12
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, s13
; AGPR-SDAG-NEXT: s_nop 1
-; AGPR-SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[8:15], v[16:23], v[0:3], v5, v6 op_sel:[1,1,0] op_sel_hi:[1,0,0] blgp:2
+; AGPR-SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v21, v22 op_sel:[1,1,0] op_sel_hi:[1,0,0] blgp:2
; AGPR-SDAG-NEXT: s_nop 11
-; AGPR-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[14:15]
+; AGPR-SDAG-NEXT: global_store_dwordx4 v20, v[0:3], s[14:15]
; AGPR-SDAG-NEXT: s_endpgm
;
; AGPR-GISEL-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd:
@@ -2638,32 +2638,32 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x0
; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x40
; SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x50
-; SDAG-NEXT: v_mov_b32_e32 v5, -2
-; SDAG-NEXT: v_mov_b32_e32 v6, 0x41
-; SDAG-NEXT: v_mov_b32_e32 v4, 0
+; SDAG-NEXT: v_mov_b32_e32 v21, -2
+; SDAG-NEXT: v_mov_b32_e32 v22, 0x41
+; SDAG-NEXT: v_mov_b32_e32 v20, 0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v8, s8
-; SDAG-NEXT: v_mov_b32_e32 v9, s9
-; SDAG-NEXT: v_mov_b32_e32 v10, s10
-; SDAG-NEXT: v_mov_b32_e32 v11, s11
-; SDAG-NEXT: v_mov_b32_e32 v12, s12
-; SDAG-NEXT: v_mov_b32_e32 v13, s13
-; SDAG-NEXT: v_mov_b32_e32 v14, s14
-; SDAG-NEXT: v_mov_b32_e32 v15, s15
-; SDAG-NEXT: v_mov_b32_e32 v16, s16
-; SDAG-NEXT: v_mov_b32_e32 v17, s17
-; SDAG-NEXT: v_mov_b32_e32 v18, s18
-; SDAG-NEXT: v_mov_b32_e32 v19, s19
-; SDAG-NEXT: v_mov_b32_e32 v20, s20
-; SDAG-NEXT: v_mov_b32_e32 v21, s21
-; SDAG-NEXT: v_mov_b32_e32 v22, s22
-; SDAG-NEXT: v_mov_b32_e32 v23, s23
-; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; SDAG-NEXT: v_mov_b32_e32 v0, s8
+; SDAG-NEXT: v_mov_b32_e32 v1, s9
+; SDAG-NEXT: v_mov_b32_e32 v2, s10
+; SDAG-NEXT: v_mov_b32_e32 v3, s11
+; SDAG-NEXT: v_mov_b32_e32 v4, s12
+; SDAG-NEXT: v_mov_b32_e32 v5, s13
+; SDAG-NEXT: v_mov_b32_e32 v6, s14
+; SDAG-NEXT: v_mov_b32_e32 v7, s15
+; SDAG-NEXT: v_mov_b32_e32 v8, s16
+; SDAG-NEXT: v_mov_b32_e32 v9, s17
+; SDAG-NEXT: v_mov_b32_e32 v10, s18
+; SDAG-NEXT: v_mov_b32_e32 v11, s19
+; SDAG-NEXT: v_mov_b32_e32 v12, s20
+; SDAG-NEXT: v_mov_b32_e32 v13, s21
+; SDAG-NEXT: v_mov_b32_e32 v14, s22
+; SDAG-NEXT: v_mov_b32_e32 v15, s23
+; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[2:3]
+; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[0:1]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[8:15], v[16:23], v[0:3], v6, v5 op_sel:[1,1,0] op_sel_hi:[1,0,0]
+; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v22, v21 op_sel:[1,1,0] op_sel_hi:[1,0,0]
; SDAG-NEXT: s_nop 11
-; SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; SDAG-NEXT: global_store_dwordx4 v20, v[0:3], s[6:7]
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA_kimm__scaleB__inlineimm:
@@ -2696,32 +2696,32 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; AGPR-SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x0
; AGPR-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x40
; AGPR-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x50
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v5, -2
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v6, 0x41
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v4, 0
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, -2
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, 0x41
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, 0
; AGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v8, s8
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v9, s9
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v10, s10
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v11, s11
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v12, s12
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v13, s13
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v14, s14
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v15, s15
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s16
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s17
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s18
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s19
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s20
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s21
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, s22
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v23, s23
-; AGPR-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; AGPR-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v0, s8
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v1, s9
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v2, s10
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v3, s11
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v4, s12
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v5, s13
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v6, s14
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v7, s15
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v8, s16
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v9, s17
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v10, s18
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v11, s19
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v12, s20
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v13, s21
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v14, s22
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v15, s23
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[18:19], s[2:3]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[16:17], s[0:1]
; AGPR-SDAG-NEXT: s_nop 1
-; AGPR-SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[8:15], v[16:23], v[0:3], v6, v5 op_sel:[1,1,0] op_sel_hi:[1,0,0]
+; AGPR-SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v22, v21 op_sel:[1,1,0] op_sel_hi:[1,0,0]
; AGPR-SDAG-NEXT: s_nop 11
-; AGPR-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; AGPR-SDAG-NEXT: global_store_dwordx4 v20, v[0:3], s[6:7]
; AGPR-SDAG-NEXT: s_endpgm
;
; AGPR-GISEL-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA_kimm__scaleB__inlineimm:
@@ -2759,32 +2759,32 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x0
; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x40
; SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x50
-; SDAG-NEXT: v_mov_b32_e32 v5, 1.0
-; SDAG-NEXT: v_mov_b32_e32 v6, 0x41
-; SDAG-NEXT: v_mov_b32_e32 v4, 0
+; SDAG-NEXT: v_mov_b32_e32 v21, 1.0
+; SDAG-NEXT: v_mov_b32_e32 v22, 0x41
+; SDAG-NEXT: v_mov_b32_e32 v20, 0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v8, s8
-; SDAG-NEXT: v_mov_b32_e32 v9, s9
-; SDAG-NEXT: v_mov_b32_e32 v10, s10
-; SDAG-NEXT: v_mov_b32_e32 v11, s11
-; SDAG-NEXT: v_mov_b32_e32 v12, s12
-; SDAG-NEXT: v_mov_b32_e32 v13, s13
-; SDAG-NEXT: v_mov_b32_e32 v14, s14
-; SDAG-NEXT: v_mov_b32_e32 v15, s15
-; SDAG-NEXT: v_mov_b32_e32 v16, s16
-; SDAG-NEXT: v_mov_b32_e32 v17, s17
-; SDAG-NEXT: v_mov_b32_e32 v18, s18
-; SDAG-NEXT: v_mov_b32_e32 v19, s19
-; SDAG-NEXT: v_mov_b32_e32 v20, s20
-; SDAG-NEXT: v_mov_b32_e32 v21, s21
-; SDAG-NEXT: v_mov_b32_e32 v22, s22
-; SDAG-NEXT: v_mov_b32_e32 v23, s23
-; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; SDAG-NEXT: v_mov_b32_e32 v0, s8
+; SDAG-NEXT: v_mov_b32_e32 v1, s9
+; SDAG-NEXT: v_mov_b32_e32 v2, s10
+; SDAG-NEXT: v_mov_b32_e32 v3, s11
+; SDAG-NEXT: v_mov_b32_e32 v4, s12
+; SDAG-NEXT: v_mov_b32_e32 v5, s13
+; SDAG-NEXT: v_mov_b32_e32 v6, s14
+; SDAG-NEXT: v_mov_b32_e32 v7, s15
+; SDAG-NEXT: v_mov_b32_e32 v8, s16
+; SDAG-NEXT: v_mov_b32_e32 v9, s17
+; SDAG-NEXT: v_mov_b32_e32 v10, s18
+; SDAG-NEXT: v_mov_b32_e32 v11, s19
+; SDAG-NEXT: v_mov_b32_e32 v12, s20
+; SDAG-NEXT: v_mov_b32_e32 v13, s21
+; SDAG-NEXT: v_mov_b32_e32 v14, s22
+; SDAG-NEXT: v_mov_b32_e32 v15, s23
+; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[2:3]
+; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[0:1]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[8:15], v[16:23], v[0:3], v6, v5 op_sel:[1,1,0] op_sel_hi:[1,0,0]
+; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v22, v21 op_sel:[1,1,0] op_sel_hi:[1,0,0]
; SDAG-NEXT: s_nop 11
-; SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; SDAG-NEXT: global_store_dwordx4 v20, v[0:3], s[6:7]
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA_kimm__scaleB__FP_literal:
@@ -2817,32 +2817,32 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; AGPR-SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x0
; AGPR-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x40
; AGPR-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x50
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v5, 1.0
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v6, 0x41
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v4, 0
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, 1.0
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, 0x41
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, 0
; AGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v8, s8
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v9, s9
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v10, s10
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v11, s11
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v12, s12
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v13, s13
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v14, s14
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v15, s15
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s16
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s17
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s18
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s19
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s20
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s21
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, s22
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v23, s23
-; AGPR-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; AGPR-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v0, s8
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v1, s9
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v2, s10
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v3, s11
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v4, s12
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v5, s13
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v6, s14
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v7, s15
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v8, s16
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v9, s17
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v10, s18
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v11, s19
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v12, s20
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v13, s21
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v14, s22
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v15, s23
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[18:19], s[2:3]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[16:17], s[0:1]
; AGPR-SDAG-NEXT: s_nop 1
-; AGPR-SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[8:15], v[16:23], v[0:3], v6, v5 op_sel:[1,1,0] op_sel_hi:[1,0,0]
+; AGPR-SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v22, v21 op_sel:[1,1,0] op_sel_hi:[1,0,0]
; AGPR-SDAG-NEXT: s_nop 11
-; AGPR-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; AGPR-SDAG-NEXT: global_store_dwordx4 v20, v[0:3], s[6:7]
; AGPR-SDAG-NEXT: s_endpgm
;
; AGPR-GISEL-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA_kimm__scaleB__FP_literal:
@@ -2880,32 +2880,32 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x0
; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x40
; SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x50
-; SDAG-NEXT: v_mov_b32_e32 v5, -2
-; SDAG-NEXT: v_mov_b32_e32 v6, 1.0
-; SDAG-NEXT: v_mov_b32_e32 v4, 0
+; SDAG-NEXT: v_mov_b32_e32 v21, -2
+; SDAG-NEXT: v_mov_b32_e32 v22, 1.0
+; SDAG-NEXT: v_mov_b32_e32 v20, 0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v8, s8
-; SDAG-NEXT: v_mov_b32_e32 v9, s9
-; SDAG-NEXT: v_mov_b32_e32 v10, s10
-; SDAG-NEXT: v_mov_b32_e32 v11, s11
-; SDAG-NEXT: v_mov_b32_e32 v12, s12
-; SDAG-NEXT: v_mov_b32_e32 v13, s13
-; SDAG-NEXT: v_mov_b32_e32 v14, s14
-; SDAG-NEXT: v_mov_b32_e32 v15, s15
-; SDAG-NEXT: v_mov_b32_e32 v16, s16
-; SDAG-NEXT: v_mov_b32_e32 v17, s17
-; SDAG-NEXT: v_mov_b32_e32 v18, s18
-; SDAG-NEXT: v_mov_b32_e32 v19, s19
-; SDAG-NEXT: v_mov_b32_e32 v20, s20
-; SDAG-NEXT: v_mov_b32_e32 v21, s21
-; SDAG-NEXT: v_mov_b32_e32 v22, s22
-; SDAG-NEXT: v_mov_b32_e32 v23, s23
-; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; SDAG-NEXT: v_mov_b32_e32 v0, s8
+; SDAG-NEXT: v_mov_b32_e32 v1, s9
+; SDAG-NEXT: v_mov_b32_e32 v2, s10
+; SDAG-NEXT: v_mov_b32_e32 v3, s11
+; SDAG-NEXT: v_mov_b32_e32 v4, s12
+; SDAG-NEXT: v_mov_b32_e32 v5, s13
+; SDAG-NEXT: v_mov_b32_e32 v6, s14
+; SDAG-NEXT: v_mov_b32_e32 v7, s15
+; SDAG-NEXT: v_mov_b32_e32 v8, s16
+; SDAG-NEXT: v_mov_b32_e32 v9, s17
+; SDAG-NEXT: v_mov_b32_e32 v10, s18
+; SDAG-NEXT: v_mov_b32_e32 v11, s19
+; SDAG-NEXT: v_mov_b32_e32 v12, s20
+; SDAG-NEXT: v_mov_b32_e32 v13, s21
+; SDAG-NEXT: v_mov_b32_e32 v14, s22
+; SDAG-NEXT: v_mov_b32_e32 v15, s23
+; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[2:3]
+; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[0:1]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[8:15], v[16:23], v[0:3], v6, v5 op_sel:[1,1,0] op_sel_hi:[1,0,0]
+; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v22, v21 op_sel:[1,1,0] op_sel_hi:[1,0,0]
; SDAG-NEXT: s_nop 11
-; SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; SDAG-NEXT: global_store_dwordx4 v20, v[0:3], s[6:7]
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA_FP_literal__scaleB__inline_imm:
@@ -2938,32 +2938,32 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; AGPR-SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x0
; AGPR-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x40
; AGPR-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x50
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v5, -2
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v6, 1.0
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v4, 0
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, -2
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, 1.0
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, 0
; AGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v8, s8
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v9, s9
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v10, s10
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v11, s11
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v12, s12
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v13, s13
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v14, s14
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v15, s15
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s16
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s17
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s18
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s19
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s20
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s21
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, s22
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v23, s23
-; AGPR-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; AGPR-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v0, s8
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v1, s9
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v2, s10
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v3, s11
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v4, s12
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v5, s13
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v6, s14
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v7, s15
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v8, s16
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v9, s17
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v10, s18
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v11, s19
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v12, s20
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v13, s21
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v14, s22
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v15, s23
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[18:19], s[2:3]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[16:17], s[0:1]
; AGPR-SDAG-NEXT: s_nop 1
-; AGPR-SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[8:15], v[16:23], v[0:3], v6, v5 op_sel:[1,1,0] op_sel_hi:[1,0,0]
+; AGPR-SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v22, v21 op_sel:[1,1,0] op_sel_hi:[1,0,0]
; AGPR-SDAG-NEXT: s_nop 11
-; AGPR-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; AGPR-SDAG-NEXT: global_store_dwordx4 v20, v[0:3], s[6:7]
; AGPR-SDAG-NEXT: s_endpgm
;
; AGPR-GISEL-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA_FP_literal__scaleB__inline_imm:
@@ -3001,32 +3001,32 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x0
; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x40
; SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x50
-; SDAG-NEXT: v_mov_b32_e32 v5, 0.15915494
-; SDAG-NEXT: v_mov_b32_e32 v6, 1.0
-; SDAG-NEXT: v_mov_b32_e32 v4, 0
+; SDAG-NEXT: v_mov_b32_e32 v21, 0.15915494
+; SDAG-NEXT: v_mov_b32_e32 v22, 1.0
+; SDAG-NEXT: v_mov_b32_e32 v20, 0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v8, s8
-; SDAG-NEXT: v_mov_b32_e32 v9, s9
-; SDAG-NEXT: v_mov_b32_e32 v10, s10
-; SDAG-NEXT: v_mov_b32_e32 v11, s11
-; SDAG-NEXT: v_mov_b32_e32 v12, s12
-; SDAG-NEXT: v_mov_b32_e32 v13, s13
-; SDAG-NEXT: v_mov_b32_e32 v14, s14
-; SDAG-NEXT: v_mov_b32_e32 v15, s15
-; SDAG-NEXT: v_mov_b32_e32 v16, s16
-; SDAG-NEXT: v_mov_b32_e32 v17, s17
-; SDAG-NEXT: v_mov_b32_e32 v18, s18
-; SDAG-NEXT: v_mov_b32_e32 v19, s19
-; SDAG-NEXT: v_mov_b32_e32 v20, s20
-; SDAG-NEXT: v_mov_b32_e32 v21, s21
-; SDAG-NEXT: v_mov_b32_e32 v22, s22
-; SDAG-NEXT: v_mov_b32_e32 v23, s23
-; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; SDAG-NEXT: v_mov_b32_e32 v0, s8
+; SDAG-NEXT: v_mov_b32_e32 v1, s9
+; SDAG-NEXT: v_mov_b32_e32 v2, s10
+; SDAG-NEXT: v_mov_b32_e32 v3, s11
+; SDAG-NEXT: v_mov_b32_e32 v4, s12
+; SDAG-NEXT: v_mov_b32_e32 v5, s13
+; SDAG-NEXT: v_mov_b32_e32 v6, s14
+; SDAG-NEXT: v_mov_b32_e32 v7, s15
+; SDAG-NEXT: v_mov_b32_e32 v8, s16
+; SDAG-NEXT: v_mov_b32_e32 v9, s17
+; SDAG-NEXT: v_mov_b32_e32 v10, s18
+; SDAG-NEXT: v_mov_b32_e32 v11, s19
+; SDAG-NEXT: v_mov_b32_e32 v12, s20
+; SDAG-NEXT: v_mov_b32_e32 v13, s21
+; SDAG-NEXT: v_mov_b32_e32 v14, s22
+; SDAG-NEXT: v_mov_b32_e32 v15, s23
+; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[2:3]
+; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[0:1]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[8:15], v[16:23], v[0:3], v6, v5 op_sel:[1,1,0] op_sel_hi:[1,0,0]
+; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v22, v21 op_sel:[1,1,0] op_sel_hi:[1,0,0]
; SDAG-NEXT: s_nop 11
-; SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; SDAG-NEXT: global_store_dwordx4 v20, v[0:3], s[6:7]
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA_FP_literal__scaleB__FP_literal:
@@ -3059,32 +3059,32 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; AGPR-SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x0
; AGPR-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x40
; AGPR-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x50
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v5, 0.15915494
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v6, 1.0
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v4, 0
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, 0.15915494
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, 1.0
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, 0
; AGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v8, s8
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v9, s9
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v10, s10
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v11, s11
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v12, s12
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v13, s13
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v14, s14
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v15, s15
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s16
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s17
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s18
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s19
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s20
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s21
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, s22
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v23, s23
-; AGPR-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; AGPR-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v0, s8
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v1, s9
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v2, s10
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v3, s11
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v4, s12
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v5, s13
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v6, s14
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v7, s15
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v8, s16
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v9, s17
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v10, s18
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v11, s19
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v12, s20
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v13, s21
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v14, s22
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v15, s23
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[18:19], s[2:3]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[16:17], s[0:1]
; AGPR-SDAG-NEXT: s_nop 1
-; AGPR-SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[8:15], v[16:23], v[0:3], v6, v5 op_sel:[1,1,0] op_sel_hi:[1,0,0]
+; AGPR-SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v22, v21 op_sel:[1,1,0] op_sel_hi:[1,0,0]
; AGPR-SDAG-NEXT: s_nop 11
-; AGPR-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; AGPR-SDAG-NEXT: global_store_dwordx4 v20, v[0:3], s[6:7]
; AGPR-SDAG-NEXT: s_endpgm
;
; AGPR-GISEL-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA_FP_literal__scaleB__FP_literal:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll
index 749f9e5ae5ef2..202f9fc81f290 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll
@@ -7307,22 +7307,22 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4__vgprcd(<8 x i32>
; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x80
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[36:37]
-; SDAG-NEXT: v_mov_b32_e32 v18, s8
-; SDAG-NEXT: v_mov_b32_e32 v19, s9
-; SDAG-NEXT: v_mov_b32_e32 v20, s10
-; SDAG-NEXT: v_mov_b32_e32 v21, s11
-; SDAG-NEXT: v_mov_b32_e32 v22, s12
-; SDAG-NEXT: v_mov_b32_e32 v23, s13
-; SDAG-NEXT: v_mov_b32_e32 v24, s14
-; SDAG-NEXT: v_mov_b32_e32 v25, s15
-; SDAG-NEXT: v_mov_b32_e32 v26, s16
-; SDAG-NEXT: v_mov_b32_e32 v27, s17
-; SDAG-NEXT: v_mov_b32_e32 v28, s18
-; SDAG-NEXT: v_mov_b32_e32 v29, s19
-; SDAG-NEXT: v_mov_b32_e32 v30, s20
-; SDAG-NEXT: v_mov_b32_e32 v31, s21
-; SDAG-NEXT: v_mov_b32_e32 v32, s22
-; SDAG-NEXT: v_mov_b32_e32 v33, s23
+; SDAG-NEXT: v_mov_b32_e32 v16, s8
+; SDAG-NEXT: v_mov_b32_e32 v17, s9
+; SDAG-NEXT: v_mov_b32_e32 v18, s10
+; SDAG-NEXT: v_mov_b32_e32 v19, s11
+; SDAG-NEXT: v_mov_b32_e32 v20, s12
+; SDAG-NEXT: v_mov_b32_e32 v21, s13
+; SDAG-NEXT: v_mov_b32_e32 v22, s14
+; SDAG-NEXT: v_mov_b32_e32 v23, s15
+; SDAG-NEXT: v_mov_b32_e32 v24, s16
+; SDAG-NEXT: v_mov_b32_e32 v25, s17
+; SDAG-NEXT: v_mov_b32_e32 v26, s18
+; SDAG-NEXT: v_mov_b32_e32 v27, s19
+; SDAG-NEXT: v_mov_b32_e32 v28, s20
+; SDAG-NEXT: v_mov_b32_e32 v29, s21
+; SDAG-NEXT: v_mov_b32_e32 v30, s22
+; SDAG-NEXT: v_mov_b32_e32 v31, s23
; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[38:39]
; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[40:41]
; SDAG-NEXT: v_mov_b64_e32 v[6:7], s[42:43]
@@ -7330,10 +7330,10 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4__vgprcd(<8 x i32>
; SDAG-NEXT: v_mov_b64_e32 v[10:11], s[46:47]
; SDAG-NEXT: v_mov_b64_e32 v[12:13], s[48:49]
; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[50:51]
-; SDAG-NEXT: v_mov_b32_e32 v16, s0
-; SDAG-NEXT: v_mov_b32_e32 v17, s1
+; SDAG-NEXT: v_mov_b32_e32 v32, s0
+; SDAG-NEXT: v_mov_b32_e32 v33, s1
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[18:25], v[26:33], v[0:15], v16, v17 op_sel:[1,1,0] op_sel_hi:[1,0,0] blgp:2
+; SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[16:23], v[24:31], v[0:15], v32, v33 op_sel:[1,1,0] op_sel_hi:[1,0,0] blgp:2
; SDAG-NEXT: v_mov_b32_e32 v16, 0
; SDAG-NEXT: s_nop 15
; SDAG-NEXT: s_nop 2
@@ -7385,22 +7385,22 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4__vgprcd(<8 x i32>
; AGPR-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x80
; AGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[36:37]
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s8
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s9
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s10
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s11
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, s12
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v23, s13
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v24, s14
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v25, s15
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v26, s16
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v27, s17
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v28, s18
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v29, s19
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v30, s20
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v31, s21
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v32, s22
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v33, s23
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s8
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s9
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s10
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s11
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s12
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s13
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, s14
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v23, s15
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v24, s16
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v25, s17
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v26, s18
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v27, s19
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v28, s20
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v29, s21
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v30, s22
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v31, s23
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[38:39]
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[40:41]
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[42:43]
@@ -7408,10 +7408,10 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4__vgprcd(<8 x i32>
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[46:47]
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[12:13], s[48:49]
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[14:15], s[50:51]
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s0
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s1
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v32, s0
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v33, s1
; AGPR-SDAG-NEXT: s_nop 1
-; AGPR-SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[18:25], v[26:33], v[0:15], v16, v17 op_sel:[1,1,0] op_sel_hi:[1,0,0] blgp:2
+; AGPR-SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[16:23], v[24:31], v[0:15], v32, v33 op_sel:[1,1,0] op_sel_hi:[1,0,0] blgp:2
; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, 0
; AGPR-SDAG-NEXT: s_nop 15
; AGPR-SDAG-NEXT: s_nop 2
@@ -7465,26 +7465,26 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4__vgprcd___scaleA_
; SDAG: ; %bb.0:
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x0
; SDAG-NEXT: s_load_dwordx16 s[36:51], s[4:5], 0x40
-; SDAG-NEXT: v_mov_b32_e32 v16, -2
-; SDAG-NEXT: v_mov_b32_e32 v17, 0x41
+; SDAG-NEXT: v_mov_b32_e32 v32, -2
+; SDAG-NEXT: v_mov_b32_e32 v33, 0x41
; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x80
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v18, s8
-; SDAG-NEXT: v_mov_b32_e32 v19, s9
-; SDAG-NEXT: v_mov_b32_e32 v20, s10
-; SDAG-NEXT: v_mov_b32_e32 v21, s11
-; SDAG-NEXT: v_mov_b32_e32 v22, s12
-; SDAG-NEXT: v_mov_b32_e32 v23, s13
-; SDAG-NEXT: v_mov_b32_e32 v24, s14
-; SDAG-NEXT: v_mov_b32_e32 v25, s15
-; SDAG-NEXT: v_mov_b32_e32 v26, s16
-; SDAG-NEXT: v_mov_b32_e32 v27, s17
-; SDAG-NEXT: v_mov_b32_e32 v28, s18
-; SDAG-NEXT: v_mov_b32_e32 v29, s19
-; SDAG-NEXT: v_mov_b32_e32 v30, s20
-; SDAG-NEXT: v_mov_b32_e32 v31, s21
-; SDAG-NEXT: v_mov_b32_e32 v32, s22
-; SDAG-NEXT: v_mov_b32_e32 v33, s23
+; SDAG-NEXT: v_mov_b32_e32 v16, s8
+; SDAG-NEXT: v_mov_b32_e32 v17, s9
+; SDAG-NEXT: v_mov_b32_e32 v18, s10
+; SDAG-NEXT: v_mov_b32_e32 v19, s11
+; SDAG-NEXT: v_mov_b32_e32 v20, s12
+; SDAG-NEXT: v_mov_b32_e32 v21, s13
+; SDAG-NEXT: v_mov_b32_e32 v22, s14
+; SDAG-NEXT: v_mov_b32_e32 v23, s15
+; SDAG-NEXT: v_mov_b32_e32 v24, s16
+; SDAG-NEXT: v_mov_b32_e32 v25, s17
+; SDAG-NEXT: v_mov_b32_e32 v26, s18
+; SDAG-NEXT: v_mov_b32_e32 v27, s19
+; SDAG-NEXT: v_mov_b32_e32 v28, s20
+; SDAG-NEXT: v_mov_b32_e32 v29, s21
+; SDAG-NEXT: v_mov_b32_e32 v30, s22
+; SDAG-NEXT: v_mov_b32_e32 v31, s23
; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[36:37]
; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[38:39]
; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[40:41]
@@ -7494,7 +7494,7 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4__vgprcd___scaleA_
; SDAG-NEXT: v_mov_b64_e32 v[12:13], s[48:49]
; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[50:51]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[18:25], v[26:33], v[0:15], v17, v16 op_sel:[1,1,0] op_sel_hi:[1,0,0] blgp:2
+; SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[16:23], v[24:31], v[0:15], v33, v32 op_sel:[1,1,0] op_sel_hi:[1,0,0] blgp:2
; SDAG-NEXT: v_mov_b32_e32 v16, 0
; SDAG-NEXT: s_nop 15
; SDAG-NEXT: s_nop 2
@@ -7543,26 +7543,26 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4__vgprcd___scaleA_
; AGPR-SDAG: ; %bb.0:
; AGPR-SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x0
; AGPR-SDAG-NEXT: s_load_dwordx16 s[36:51], s[4:5], 0x40
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, -2
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, 0x41
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v32, -2
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v33, 0x41
; AGPR-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x80
; AGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s8
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s9
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s10
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s11
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, s12
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v23, s13
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v24, s14
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v25, s15
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v26, s16
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v27, s17
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v28, s18
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v29, s19
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v30, s20
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v31, s21
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v32, s22
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v33, s23
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s8
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s9
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s10
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s11
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s12
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s13
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, s14
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v23, s15
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v24, s16
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v25, s17
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v26, s18
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v27, s19
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v28, s20
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v29, s21
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v30, s22
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v31, s23
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[36:37]
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[38:39]
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[40:41]
@@ -7572,7 +7572,7 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4__vgprcd___scaleA_
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[12:13], s[48:49]
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[14:15], s[50:51]
; AGPR-SDAG-NEXT: s_nop 1
-; AGPR-SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[18:25], v[26:33], v[0:15], v17, v16 op_sel:[1,1,0] op_sel_hi:[1,0,0] blgp:2
+; AGPR-SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[16:23], v[24:31], v[0:15], v33, v32 op_sel:[1,1,0] op_sel_hi:[1,0,0] blgp:2
; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, 0
; AGPR-SDAG-NEXT: s_nop 15
; AGPR-SDAG-NEXT: s_nop 2
@@ -8245,41 +8245,42 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__vgprcd_nonma
; SDAG-NEXT: s_nop 1
; SDAG-NEXT: v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[32:39], v[40:47], v[16:31] blgp:2
; SDAG-NEXT: s_nop 14
-; SDAG-NEXT: v_mov_b32_e32 v18, s20
-; SDAG-NEXT: v_mov_b32_e32 v19, s21
-; SDAG-NEXT: v_mov_b32_e32 v20, s22
-; SDAG-NEXT: v_mov_b32_e32 v21, s23
-; SDAG-NEXT: v_mov_b64_e32 v[16:17], 48
-; SDAG-NEXT: global_store_dwordx4 v[16:17], v[18:21], off sc0 sc1
+; SDAG-NEXT: v_mov_b32_e32 v16, s20
+; SDAG-NEXT: v_mov_b32_e32 v17, s21
+; SDAG-NEXT: v_mov_b32_e32 v18, s22
+; SDAG-NEXT: v_mov_b32_e32 v19, s23
+; SDAG-NEXT: v_mov_b64_e32 v[20:21], 48
+; SDAG-NEXT: global_store_dwordx4 v[20:21], v[16:19], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v22, s18
-; SDAG-NEXT: v_mov_b32_e32 v23, s19
-; SDAG-NEXT: v_mov_b32_e32 v20, s16
-; SDAG-NEXT: v_mov_b32_e32 v21, s17
-; SDAG-NEXT: v_mov_b64_e32 v[18:19], 32
-; SDAG-NEXT: global_store_dwordx4 v[18:19], v[20:23], off sc0 sc1
+; SDAG-NEXT: v_mov_b64_e32 v[22:23], 32
+; SDAG-NEXT: v_mov_b64_e32 v[24:25], 16
+; SDAG-NEXT: v_mov_b32_e32 v16, s16
+; SDAG-NEXT: v_mov_b32_e32 v17, s17
+; SDAG-NEXT: v_mov_b32_e32 v18, s18
+; SDAG-NEXT: v_mov_b32_e32 v19, s19
+; SDAG-NEXT: global_store_dwordx4 v[22:23], v[16:19], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v24, s14
-; SDAG-NEXT: v_mov_b32_e32 v25, s15
-; SDAG-NEXT: v_mov_b32_e32 v22, s12
-; SDAG-NEXT: v_mov_b32_e32 v23, s13
-; SDAG-NEXT: v_mov_b64_e32 v[20:21], 16
-; SDAG-NEXT: global_store_dwordx4 v[20:21], v[22:25], off sc0 sc1
+; SDAG-NEXT: v_mov_b64_e32 v[26:27], 0
+; SDAG-NEXT: v_mov_b32_e32 v16, s12
+; SDAG-NEXT: v_mov_b32_e32 v17, s13
+; SDAG-NEXT: v_mov_b32_e32 v18, s14
+; SDAG-NEXT: v_mov_b32_e32 v19, s15
+; SDAG-NEXT: global_store_dwordx4 v[24:25], v[16:19], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v26, s10
-; SDAG-NEXT: v_mov_b32_e32 v27, s11
-; SDAG-NEXT: v_mov_b32_e32 v24, s8
-; SDAG-NEXT: v_mov_b32_e32 v25, s9
-; SDAG-NEXT: v_mov_b64_e32 v[22:23], 0
-; SDAG-NEXT: global_store_dwordx4 v[22:23], v[24:27], off sc0 sc1
+; SDAG-NEXT: s_nop 0
+; SDAG-NEXT: v_mov_b32_e32 v16, s8
+; SDAG-NEXT: v_mov_b32_e32 v17, s9
+; SDAG-NEXT: v_mov_b32_e32 v18, s10
+; SDAG-NEXT: v_mov_b32_e32 v19, s11
+; SDAG-NEXT: global_store_dwordx4 v[26:27], v[16:19], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v[18:19], v[8:11], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[22:23], v[8:11], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v[16:17], v[12:15], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[20:21], v[12:15], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v[22:23], v[0:3], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[26:27], v[0:3], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v[20:21], v[4:7], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[24:25], v[4:7], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_endpgm
;
@@ -8362,41 +8363,42 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__vgprcd_nonma
; AGPR-SDAG-NEXT: s_nop 1
; AGPR-SDAG-NEXT: v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[32:39], v[40:47], v[16:31] blgp:2
; AGPR-SDAG-NEXT: s_nop 14
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s20
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s21
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s22
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s23
-; AGPR-SDAG-NEXT: v_mov_b64_e32 v[16:17], 48
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[16:17], v[18:21], off sc0 sc1
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s20
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s21
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s22
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s23
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[20:21], 48
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[20:21], v[16:19], off sc0 sc1
; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, s18
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v23, s19
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s16
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s17
-; AGPR-SDAG-NEXT: v_mov_b64_e32 v[18:19], 32
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[18:19], v[20:23], off sc0 sc1
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[22:23], 32
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[24:25], 16
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s16
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s17
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s18
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s19
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[22:23], v[16:19], off sc0 sc1
; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v24, s14
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v25, s15
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, s12
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v23, s13
-; AGPR-SDAG-NEXT: v_mov_b64_e32 v[20:21], 16
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[20:21], v[22:25], off sc0 sc1
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[26:27], 0
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s12
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s13
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s14
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s15
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[24:25], v[16:19], off sc0 sc1
; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v26, s10
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v27, s11
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v24, s8
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v25, s9
-; AGPR-SDAG-NEXT: v_mov_b64_e32 v[22:23], 0
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[22:23], v[24:27], off sc0 sc1
+; AGPR-SDAG-NEXT: s_nop 0
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s8
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s9
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s10
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s11
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[26:27], v[16:19], off sc0 sc1
; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[18:19], v[8:11], off sc0 sc1
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[22:23], v[8:11], off sc0 sc1
; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[16:17], v[12:15], off sc0 sc1
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[20:21], v[12:15], off sc0 sc1
; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[22:23], v[0:3], off sc0 sc1
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[26:27], v[0:3], off sc0 sc1
; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[20:21], v[4:7], off sc0 sc1
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[24:25], v[4:7], off sc0 sc1
; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
; AGPR-SDAG-NEXT: s_endpgm
;
@@ -8455,26 +8457,26 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_25_42__vgprcd_non
; SDAG-LABEL: test_mfma_scale_f32_32x32x64_f8f6f4_25_42__vgprcd_nonmac:
; SDAG: ; %bb.0:
; SDAG-NEXT: s_load_dwordx16 s[12:27], s[4:5], 0x0
-; SDAG-NEXT: v_mov_b32_e32 v16, 42
-; SDAG-NEXT: v_mov_b32_e32 v17, 25
+; SDAG-NEXT: v_mov_b32_e32 v32, 42
+; SDAG-NEXT: v_mov_b32_e32 v33, 25
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v18, s12
-; SDAG-NEXT: v_mov_b32_e32 v19, s13
-; SDAG-NEXT: v_mov_b32_e32 v20, s14
-; SDAG-NEXT: v_mov_b32_e32 v21, s15
-; SDAG-NEXT: v_mov_b32_e32 v22, s16
-; SDAG-NEXT: v_mov_b32_e32 v23, s17
-; SDAG-NEXT: v_mov_b32_e32 v24, s18
-; SDAG-NEXT: v_mov_b32_e32 v25, s19
-; SDAG-NEXT: v_mov_b32_e32 v26, s20
-; SDAG-NEXT: v_mov_b32_e32 v27, s21
-; SDAG-NEXT: v_mov_b32_e32 v28, s22
-; SDAG-NEXT: v_mov_b32_e32 v29, s23
+; SDAG-NEXT: v_mov_b32_e32 v16, s12
+; SDAG-NEXT: v_mov_b32_e32 v17, s13
+; SDAG-NEXT: v_mov_b32_e32 v18, s14
+; SDAG-NEXT: v_mov_b32_e32 v19, s15
+; SDAG-NEXT: v_mov_b32_e32 v20, s16
+; SDAG-NEXT: v_mov_b32_e32 v21, s17
+; SDAG-NEXT: v_mov_b32_e32 v22, s18
+; SDAG-NEXT: v_mov_b32_e32 v23, s19
+; SDAG-NEXT: v_mov_b32_e32 v24, s20
+; SDAG-NEXT: v_mov_b32_e32 v25, s21
+; SDAG-NEXT: v_mov_b32_e32 v26, s22
+; SDAG-NEXT: v_mov_b32_e32 v27, s23
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x40
-; SDAG-NEXT: v_mov_b32_e32 v30, s24
-; SDAG-NEXT: v_mov_b32_e32 v31, s25
-; SDAG-NEXT: v_mov_b32_e32 v32, s26
-; SDAG-NEXT: v_mov_b32_e32 v33, s27
+; SDAG-NEXT: v_mov_b32_e32 v28, s24
+; SDAG-NEXT: v_mov_b32_e32 v29, s25
+; SDAG-NEXT: v_mov_b32_e32 v30, s26
+; SDAG-NEXT: v_mov_b32_e32 v31, s27
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
@@ -8485,42 +8487,43 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_25_42__vgprcd_non
; SDAG-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[18:25], v[26:33], v[0:15], v17, v16 op_sel_hi:[0,0,0] blgp:2
-; SDAG-NEXT: v_mov_b32_e32 v18, s20
-; SDAG-NEXT: v_mov_b32_e32 v19, s21
-; SDAG-NEXT: v_mov_b32_e32 v20, s22
-; SDAG-NEXT: v_mov_b32_e32 v21, s23
-; SDAG-NEXT: v_mov_b64_e32 v[16:17], 48
-; SDAG-NEXT: global_store_dwordx4 v[16:17], v[18:21], off sc0 sc1
+; SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[16:23], v[24:31], v[0:15], v33, v32 op_sel_hi:[0,0,0] blgp:2
+; SDAG-NEXT: v_mov_b32_e32 v16, s20
+; SDAG-NEXT: v_mov_b32_e32 v17, s21
+; SDAG-NEXT: v_mov_b32_e32 v18, s22
+; SDAG-NEXT: v_mov_b32_e32 v19, s23
+; SDAG-NEXT: v_mov_b64_e32 v[20:21], 48
+; SDAG-NEXT: global_store_dwordx4 v[20:21], v[16:19], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v22, s18
-; SDAG-NEXT: v_mov_b32_e32 v23, s19
-; SDAG-NEXT: v_mov_b32_e32 v20, s16
-; SDAG-NEXT: v_mov_b32_e32 v21, s17
-; SDAG-NEXT: v_mov_b64_e32 v[18:19], 32
-; SDAG-NEXT: global_store_dwordx4 v[18:19], v[20:23], off sc0 sc1
+; SDAG-NEXT: v_mov_b64_e32 v[22:23], 32
+; SDAG-NEXT: v_mov_b64_e32 v[24:25], 16
+; SDAG-NEXT: v_mov_b32_e32 v16, s16
+; SDAG-NEXT: v_mov_b32_e32 v17, s17
+; SDAG-NEXT: v_mov_b32_e32 v18, s18
+; SDAG-NEXT: v_mov_b32_e32 v19, s19
+; SDAG-NEXT: global_store_dwordx4 v[22:23], v[16:19], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v24, s14
-; SDAG-NEXT: v_mov_b32_e32 v25, s15
-; SDAG-NEXT: v_mov_b32_e32 v22, s12
-; SDAG-NEXT: v_mov_b32_e32 v23, s13
-; SDAG-NEXT: v_mov_b64_e32 v[20:21], 16
-; SDAG-NEXT: global_store_dwordx4 v[20:21], v[22:25], off sc0 sc1
+; SDAG-NEXT: v_mov_b64_e32 v[26:27], 0
+; SDAG-NEXT: v_mov_b32_e32 v16, s12
+; SDAG-NEXT: v_mov_b32_e32 v17, s13
+; SDAG-NEXT: v_mov_b32_e32 v18, s14
+; SDAG-NEXT: v_mov_b32_e32 v19, s15
+; SDAG-NEXT: global_store_dwordx4 v[24:25], v[16:19], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v26, s10
-; SDAG-NEXT: v_mov_b32_e32 v27, s11
-; SDAG-NEXT: v_mov_b32_e32 v24, s8
-; SDAG-NEXT: v_mov_b32_e32 v25, s9
-; SDAG-NEXT: v_mov_b64_e32 v[22:23], 0
-; SDAG-NEXT: global_store_dwordx4 v[22:23], v[24:27], off sc0 sc1
+; SDAG-NEXT: s_nop 0
+; SDAG-NEXT: v_mov_b32_e32 v16, s8
+; SDAG-NEXT: v_mov_b32_e32 v17, s9
+; SDAG-NEXT: v_mov_b32_e32 v18, s10
+; SDAG-NEXT: v_mov_b32_e32 v19, s11
+; SDAG-NEXT: global_store_dwordx4 v[26:27], v[16:19], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v[18:19], v[8:11], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[22:23], v[8:11], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v[16:17], v[12:15], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[20:21], v[12:15], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v[22:23], v[0:3], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[26:27], v[0:3], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v[20:21], v[4:7], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[24:25], v[4:7], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_endpgm
;
@@ -8582,26 +8585,26 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_25_42__vgprcd_non
; AGPR-SDAG-LABEL: test_mfma_scale_f32_32x32x64_f8f6f4_25_42__vgprcd_nonmac:
; AGPR-SDAG: ; %bb.0:
; AGPR-SDAG-NEXT: s_load_dwordx16 s[12:27], s[4:5], 0x0
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, 42
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, 25
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v32, 42
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v33, 25
; AGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s12
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s13
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s14
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s15
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, s16
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v23, s17
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v24, s18
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v25, s19
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v26, s20
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v27, s21
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v28, s22
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v29, s23
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s12
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s13
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s14
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s15
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s16
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s17
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, s18
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v23, s19
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v24, s20
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v25, s21
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v26, s22
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v27, s23
; AGPR-SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x40
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v30, s24
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v31, s25
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v32, s26
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v33, s27
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v28, s24
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v29, s25
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v30, s26
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v31, s27
; AGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
@@ -8612,42 +8615,43 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_25_42__vgprcd_non
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; AGPR-SDAG-NEXT: s_nop 1
-; AGPR-SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[18:25], v[26:33], v[0:15], v17, v16 op_sel_hi:[0,0,0] blgp:2
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s20
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s21
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s22
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s23
-; AGPR-SDAG-NEXT: v_mov_b64_e32 v[16:17], 48
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[16:17], v[18:21], off sc0 sc1
+; AGPR-SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[16:23], v[24:31], v[0:15], v33, v32 op_sel_hi:[0,0,0] blgp:2
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s20
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s21
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s22
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s23
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[20:21], 48
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[20:21], v[16:19], off sc0 sc1
; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, s18
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v23, s19
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s16
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s17
-; AGPR-SDAG-NEXT: v_mov_b64_e32 v[18:19], 32
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[18:19], v[20:23], off sc0 sc1
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[22:23], 32
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[24:25], 16
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s16
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s17
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s18
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s19
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[22:23], v[16:19], off sc0 sc1
; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v24, s14
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v25, s15
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, s12
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v23, s13
-; AGPR-SDAG-NEXT: v_mov_b64_e32 v[20:21], 16
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[20:21], v[22:25], off sc0 sc1
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[26:27], 0
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s12
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s13
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s14
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s15
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[24:25], v[16:19], off sc0 sc1
; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v26, s10
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v27, s11
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v24, s8
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v25, s9
-; AGPR-SDAG-NEXT: v_mov_b64_e32 v[22:23], 0
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[22:23], v[24:27], off sc0 sc1
+; AGPR-SDAG-NEXT: s_nop 0
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s8
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s9
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s10
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s11
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[26:27], v[16:19], off sc0 sc1
; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[18:19], v[8:11], off sc0 sc1
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[22:23], v[8:11], off sc0 sc1
; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[16:17], v[12:15], off sc0 sc1
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[20:21], v[12:15], off sc0 sc1
; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[22:23], v[0:3], off sc0 sc1
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[26:27], v[0:3], off sc0 sc1
; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[20:21], v[4:7], off sc0 sc1
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[24:25], v[4:7], off sc0 sc1
; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
; AGPR-SDAG-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.smfmac.gfx950.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.smfmac.gfx950.ll
index 0f8c5ca34c501..61f90d4a6100c 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.smfmac.gfx950.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.smfmac.gfx950.ll
@@ -890,20 +890,20 @@ define amdgpu_kernel void @test_smfmac_f32_16x16x64_bf16__vgpr(ptr addrspace(1)
; GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x44
; GISEL-NEXT: s_load_dword s16, s[4:5], 0x64
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GISEL-NEXT: global_load_dwordx4 v[14:17], v0, s[6:7]
-; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[2:3]
-; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[0:1]
+; GISEL-NEXT: global_load_dwordx4 v[8:11], v0, s[6:7]
+; GISEL-NEXT: v_mov_b64_e32 v[14:15], s[2:3]
+; GISEL-NEXT: v_mov_b64_e32 v[12:13], s[0:1]
; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
-; GISEL-NEXT: v_mov_b32_e32 v12, s16
+; GISEL-NEXT: v_mov_b32_e32 v16, s16
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_nop 0
-; GISEL-NEXT: v_smfmac_f32_16x16x64_bf16 v[14:17], v[8:11], v[0:7], v12 cbsz:1 abid:2
+; GISEL-NEXT: v_smfmac_f32_16x16x64_bf16 v[8:11], v[12:15], v[0:7], v16 cbsz:1 abid:2
; GISEL-NEXT: v_mov_b32_e32 v0, 0
; GISEL-NEXT: s_nop 6
-; GISEL-NEXT: global_store_dwordx4 v0, v[14:17], s[6:7]
+; GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[6:7]
; GISEL-NEXT: s_endpgm
;
; SDAG-VGPR-LABEL: test_smfmac_f32_16x16x64_bf16__vgpr:
@@ -939,20 +939,20 @@ define amdgpu_kernel void @test_smfmac_f32_16x16x64_bf16__vgpr(ptr addrspace(1)
; GISEL-VGPR-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x44
; GISEL-VGPR-NEXT: s_load_dword s16, s[4:5], 0x64
; GISEL-VGPR-NEXT: s_waitcnt lgkmcnt(0)
-; GISEL-VGPR-NEXT: global_load_dwordx4 v[14:17], v0, s[6:7]
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[10:11], s[2:3]
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[8:9], s[0:1]
+; GISEL-VGPR-NEXT: global_load_dwordx4 v[8:11], v0, s[6:7]
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[14:15], s[2:3]
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[12:13], s[0:1]
; GISEL-VGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; GISEL-VGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; GISEL-VGPR-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; GISEL-VGPR-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
-; GISEL-VGPR-NEXT: v_mov_b32_e32 v12, s16
+; GISEL-VGPR-NEXT: v_mov_b32_e32 v16, s16
; GISEL-VGPR-NEXT: s_waitcnt vmcnt(0)
; GISEL-VGPR-NEXT: s_nop 0
-; GISEL-VGPR-NEXT: v_smfmac_f32_16x16x64_bf16 v[14:17], v[8:11], v[0:7], v12 cbsz:1 abid:2
+; GISEL-VGPR-NEXT: v_smfmac_f32_16x16x64_bf16 v[8:11], v[12:15], v[0:7], v16 cbsz:1 abid:2
; GISEL-VGPR-NEXT: v_mov_b32_e32 v0, 0
; GISEL-VGPR-NEXT: s_nop 6
-; GISEL-VGPR-NEXT: global_store_dwordx4 v0, v[14:17], s[6:7]
+; GISEL-VGPR-NEXT: global_store_dwordx4 v0, v[8:11], s[6:7]
; GISEL-VGPR-NEXT: s_endpgm
bb:
%id = call i32 @llvm.amdgcn.workitem.id.x()
@@ -1709,26 +1709,26 @@ define amdgpu_kernel void @test_smfmac_i32_16x16x128_i8__vgpr(ptr addrspace(1) %
; SDAG-NEXT: s_load_dword s16, s[4:5], 0x64
; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: global_load_dwordx4 v[10:13], v0, s[6:7]
-; SDAG-NEXT: v_mov_b32_e32 v0, 0
-; SDAG-NEXT: v_mov_b32_e32 v14, s8
-; SDAG-NEXT: v_mov_b32_e32 v15, s9
-; SDAG-NEXT: v_mov_b32_e32 v16, s10
-; SDAG-NEXT: v_mov_b32_e32 v17, s11
-; SDAG-NEXT: v_mov_b32_e32 v2, s12
-; SDAG-NEXT: v_mov_b32_e32 v3, s13
-; SDAG-NEXT: v_mov_b32_e32 v4, s14
-; SDAG-NEXT: v_mov_b32_e32 v5, s15
-; SDAG-NEXT: v_mov_b32_e32 v6, s0
-; SDAG-NEXT: v_mov_b32_e32 v7, s1
-; SDAG-NEXT: v_mov_b32_e32 v8, s2
-; SDAG-NEXT: v_mov_b32_e32 v9, s3
-; SDAG-NEXT: v_mov_b32_e32 v1, s16
+; SDAG-NEXT: global_load_dwordx4 v[14:17], v0, s[6:7]
+; SDAG-NEXT: v_mov_b32_e32 v12, 0
+; SDAG-NEXT: v_mov_b32_e32 v8, s8
+; SDAG-NEXT: v_mov_b32_e32 v9, s9
+; SDAG-NEXT: v_mov_b32_e32 v10, s10
+; SDAG-NEXT: v_mov_b32_e32 v11, s11
+; SDAG-NEXT: v_mov_b32_e32 v0, s12
+; SDAG-NEXT: v_mov_b32_e32 v1, s13
+; SDAG-NEXT: v_mov_b32_e32 v2, s14
+; SDAG-NEXT: v_mov_b32_e32 v3, s15
+; SDAG-NEXT: v_mov_b32_e32 v4, s0
+; SDAG-NEXT: v_mov_b32_e32 v5, s1
+; SDAG-NEXT: v_mov_b32_e32 v6, s2
+; SDAG-NEXT: v_mov_b32_e32 v7, s3
+; SDAG-NEXT: v_mov_b32_e32 v13, s16
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
-; SDAG-NEXT: v_smfmac_i32_16x16x128_i8 v[10:13], v[14:17], v[2:9], v1 cbsz:1 abid:2
+; SDAG-NEXT: v_smfmac_i32_16x16x128_i8 v[14:17], v[8:11], v[0:7], v13 cbsz:1 abid:2
; SDAG-NEXT: s_nop 7
-; SDAG-NEXT: global_store_dwordx4 v0, v[10:13], s[6:7]
+; SDAG-NEXT: global_store_dwordx4 v12, v[14:17], s[6:7]
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: test_smfmac_i32_16x16x128_i8__vgpr:
@@ -1740,20 +1740,20 @@ define amdgpu_kernel void @test_smfmac_i32_16x16x128_i8__vgpr(ptr addrspace(1) %
; GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x54
; GISEL-NEXT: s_load_dword s2, s[4:5], 0x64
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GISEL-NEXT: global_load_dwordx4 v[14:17], v0, s[0:1]
-; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
-; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GISEL-NEXT: global_load_dwordx4 v[8:11], v0, s[0:1]
+; GISEL-NEXT: v_mov_b64_e32 v[14:15], s[10:11]
+; GISEL-NEXT: v_mov_b64_e32 v[12:13], s[8:9]
; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[12:13]
; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[14:15]
; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[16:17]
; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[18:19]
-; GISEL-NEXT: v_mov_b32_e32 v12, s2
+; GISEL-NEXT: v_mov_b32_e32 v16, s2
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_nop 0
-; GISEL-NEXT: v_smfmac_i32_16x16x128_i8 v[14:17], v[8:11], v[0:7], v12 cbsz:1 abid:2
+; GISEL-NEXT: v_smfmac_i32_16x16x128_i8 v[8:11], v[12:15], v[0:7], v16 cbsz:1 abid:2
; GISEL-NEXT: v_mov_b32_e32 v0, 0
; GISEL-NEXT: s_nop 6
-; GISEL-NEXT: global_store_dwordx4 v0, v[14:17], s[0:1]
+; GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[0:1]
; GISEL-NEXT: s_endpgm
;
; SDAG-VGPR-LABEL: test_smfmac_i32_16x16x128_i8__vgpr:
@@ -1765,26 +1765,26 @@ define amdgpu_kernel void @test_smfmac_i32_16x16x128_i8__vgpr(ptr addrspace(1) %
; SDAG-VGPR-NEXT: s_load_dword s16, s[4:5], 0x64
; SDAG-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
; SDAG-VGPR-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-VGPR-NEXT: global_load_dwordx4 v[10:13], v0, s[6:7]
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v0, 0
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v14, s8
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v15, s9
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, s10
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v17, s11
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v2, s12
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v3, s13
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v4, s14
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v5, s15
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v6, s0
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v7, s1
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v8, s2
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v9, s3
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v1, s16
+; SDAG-VGPR-NEXT: global_load_dwordx4 v[14:17], v0, s[6:7]
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v12, 0
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v8, s8
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v9, s9
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v10, s10
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v11, s11
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v0, s12
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v1, s13
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v2, s14
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v3, s15
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v4, s0
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v5, s1
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v6, s2
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v7, s3
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v13, s16
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-VGPR-NEXT: s_nop 0
-; SDAG-VGPR-NEXT: v_smfmac_i32_16x16x128_i8 v[10:13], v[14:17], v[2:9], v1 cbsz:1 abid:2
+; SDAG-VGPR-NEXT: v_smfmac_i32_16x16x128_i8 v[14:17], v[8:11], v[0:7], v13 cbsz:1 abid:2
; SDAG-VGPR-NEXT: s_nop 7
-; SDAG-VGPR-NEXT: global_store_dwordx4 v0, v[10:13], s[6:7]
+; SDAG-VGPR-NEXT: global_store_dwordx4 v12, v[14:17], s[6:7]
; SDAG-VGPR-NEXT: s_endpgm
;
; GISEL-VGPR-LABEL: test_smfmac_i32_16x16x128_i8__vgpr:
@@ -1796,20 +1796,20 @@ define amdgpu_kernel void @test_smfmac_i32_16x16x128_i8__vgpr(ptr addrspace(1) %
; GISEL-VGPR-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x54
; GISEL-VGPR-NEXT: s_load_dword s2, s[4:5], 0x64
; GISEL-VGPR-NEXT: s_waitcnt lgkmcnt(0)
-; GISEL-VGPR-NEXT: global_load_dwordx4 v[14:17], v0, s[0:1]
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GISEL-VGPR-NEXT: global_load_dwordx4 v[8:11], v0, s[0:1]
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[14:15], s[10:11]
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[12:13], s[8:9]
; GISEL-VGPR-NEXT: v_mov_b64_e32 v[0:1], s[12:13]
; GISEL-VGPR-NEXT: v_mov_b64_e32 v[2:3], s[14:15]
; GISEL-VGPR-NEXT: v_mov_b64_e32 v[4:5], s[16:17]
; GISEL-VGPR-NEXT: v_mov_b64_e32 v[6:7], s[18:19]
-; GISEL-VGPR-NEXT: v_mov_b32_e32 v12, s2
+; GISEL-VGPR-NEXT: v_mov_b32_e32 v16, s2
; GISEL-VGPR-NEXT: s_waitcnt vmcnt(0)
; GISEL-VGPR-NEXT: s_nop 0
-; GISEL-VGPR-NEXT: v_smfmac_i32_16x16x128_i8 v[14:17], v[8:11], v[0:7], v12 cbsz:1 abid:2
+; GISEL-VGPR-NEXT: v_smfmac_i32_16x16x128_i8 v[8:11], v[12:15], v[0:7], v16 cbsz:1 abid:2
; GISEL-VGPR-NEXT: v_mov_b32_e32 v0, 0
; GISEL-VGPR-NEXT: s_nop 6
-; GISEL-VGPR-NEXT: global_store_dwordx4 v0, v[14:17], s[0:1]
+; GISEL-VGPR-NEXT: global_store_dwordx4 v0, v[8:11], s[0:1]
; GISEL-VGPR-NEXT: s_endpgm
bb:
%id = call i32 @llvm.amdgcn.workitem.id.x()
@@ -2034,22 +2034,22 @@ define amdgpu_kernel void @test_smfmac_i32_32x32x64_i8__vgpr(ptr addrspace(1) %a
; SDAG-NEXT: global_load_dwordx4 v[8:11], v16, s[6:7] offset:32
; SDAG-NEXT: global_load_dwordx4 v[4:7], v16, s[6:7] offset:16
; SDAG-NEXT: global_load_dwordx4 v[0:3], v16, s[6:7]
-; SDAG-NEXT: v_mov_b32_e32 v26, s8
-; SDAG-NEXT: v_mov_b32_e32 v27, s9
-; SDAG-NEXT: v_mov_b32_e32 v28, s10
-; SDAG-NEXT: v_mov_b32_e32 v29, s11
-; SDAG-NEXT: v_mov_b32_e32 v18, s12
-; SDAG-NEXT: v_mov_b32_e32 v19, s13
-; SDAG-NEXT: v_mov_b32_e32 v20, s14
-; SDAG-NEXT: v_mov_b32_e32 v21, s15
-; SDAG-NEXT: v_mov_b32_e32 v22, s0
-; SDAG-NEXT: v_mov_b32_e32 v23, s1
-; SDAG-NEXT: v_mov_b32_e32 v24, s2
-; SDAG-NEXT: v_mov_b32_e32 v25, s3
-; SDAG-NEXT: v_mov_b32_e32 v16, s16
+; SDAG-NEXT: v_mov_b32_e32 v24, s8
+; SDAG-NEXT: v_mov_b32_e32 v25, s9
+; SDAG-NEXT: v_mov_b32_e32 v26, s10
+; SDAG-NEXT: v_mov_b32_e32 v27, s11
+; SDAG-NEXT: v_mov_b32_e32 v16, s12
+; SDAG-NEXT: v_mov_b32_e32 v17, s13
+; SDAG-NEXT: v_mov_b32_e32 v18, s14
+; SDAG-NEXT: v_mov_b32_e32 v19, s15
+; SDAG-NEXT: v_mov_b32_e32 v20, s0
+; SDAG-NEXT: v_mov_b32_e32 v21, s1
+; SDAG-NEXT: v_mov_b32_e32 v22, s2
+; SDAG-NEXT: v_mov_b32_e32 v23, s3
+; SDAG-NEXT: v_mov_b32_e32 v28, s16
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
-; SDAG-NEXT: v_smfmac_i32_32x32x64_i8 v[0:15], v[26:29], v[18:25], v16 cbsz:1 abid:2
+; SDAG-NEXT: v_smfmac_i32_32x32x64_i8 v[0:15], v[24:27], v[16:23], v28 cbsz:1 abid:2
; SDAG-NEXT: v_mov_b32_e32 v16, 0
; SDAG-NEXT: s_nop 10
; SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[6:7] offset:32
@@ -2102,22 +2102,22 @@ define amdgpu_kernel void @test_smfmac_i32_32x32x64_i8__vgpr(ptr addrspace(1) %a
; SDAG-VGPR-NEXT: global_load_dwordx4 v[8:11], v16, s[6:7] offset:32
; SDAG-VGPR-NEXT: global_load_dwordx4 v[4:7], v16, s[6:7] offset:16
; SDAG-VGPR-NEXT: global_load_dwordx4 v[0:3], v16, s[6:7]
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v26, s8
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v27, s9
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v28, s10
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v29, s11
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v18, s12
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v19, s13
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v20, s14
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v21, s15
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v22, s0
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v23, s1
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v24, s2
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v25, s3
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, s16
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v24, s8
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v25, s9
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v26, s10
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v27, s11
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, s12
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v17, s13
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v18, s14
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v19, s15
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v20, s0
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v21, s1
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v22, s2
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v23, s3
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v28, s16
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-VGPR-NEXT: s_nop 0
-; SDAG-VGPR-NEXT: v_smfmac_i32_32x32x64_i8 v[0:15], v[26:29], v[18:25], v16 cbsz:1 abid:2
+; SDAG-VGPR-NEXT: v_smfmac_i32_32x32x64_i8 v[0:15], v[24:27], v[16:23], v28 cbsz:1 abid:2
; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, 0
; SDAG-VGPR-NEXT: s_nop 10
; SDAG-VGPR-NEXT: global_store_dwordx4 v16, v[8:11], s[6:7] offset:32
@@ -2578,26 +2578,26 @@ define amdgpu_kernel void @test_smfmac_f32_16x16x128_bf8_bf8__vgpr(ptr addrspace
; SDAG-NEXT: s_load_dword s16, s[4:5], 0x64
; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: global_load_dwordx4 v[10:13], v0, s[6:7]
-; SDAG-NEXT: v_mov_b32_e32 v0, 0
-; SDAG-NEXT: v_mov_b32_e32 v14, s8
-; SDAG-NEXT: v_mov_b32_e32 v15, s9
-; SDAG-NEXT: v_mov_b32_e32 v16, s10
-; SDAG-NEXT: v_mov_b32_e32 v17, s11
-; SDAG-NEXT: v_mov_b32_e32 v2, s12
-; SDAG-NEXT: v_mov_b32_e32 v3, s13
-; SDAG-NEXT: v_mov_b32_e32 v4, s14
-; SDAG-NEXT: v_mov_b32_e32 v5, s15
-; SDAG-NEXT: v_mov_b32_e32 v6, s0
-; SDAG-NEXT: v_mov_b32_e32 v7, s1
-; SDAG-NEXT: v_mov_b32_e32 v8, s2
-; SDAG-NEXT: v_mov_b32_e32 v9, s3
-; SDAG-NEXT: v_mov_b32_e32 v1, s16
+; SDAG-NEXT: global_load_dwordx4 v[14:17], v0, s[6:7]
+; SDAG-NEXT: v_mov_b32_e32 v12, 0
+; SDAG-NEXT: v_mov_b32_e32 v8, s8
+; SDAG-NEXT: v_mov_b32_e32 v9, s9
+; SDAG-NEXT: v_mov_b32_e32 v10, s10
+; SDAG-NEXT: v_mov_b32_e32 v11, s11
+; SDAG-NEXT: v_mov_b32_e32 v0, s12
+; SDAG-NEXT: v_mov_b32_e32 v1, s13
+; SDAG-NEXT: v_mov_b32_e32 v2, s14
+; SDAG-NEXT: v_mov_b32_e32 v3, s15
+; SDAG-NEXT: v_mov_b32_e32 v4, s0
+; SDAG-NEXT: v_mov_b32_e32 v5, s1
+; SDAG-NEXT: v_mov_b32_e32 v6, s2
+; SDAG-NEXT: v_mov_b32_e32 v7, s3
+; SDAG-NEXT: v_mov_b32_e32 v13, s16
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
-; SDAG-NEXT: v_smfmac_f32_16x16x128_bf8_bf8 v[10:13], v[14:17], v[2:9], v1 cbsz:1 abid:2
+; SDAG-NEXT: v_smfmac_f32_16x16x128_bf8_bf8 v[14:17], v[8:11], v[0:7], v13 cbsz:1 abid:2
; SDAG-NEXT: s_nop 7
-; SDAG-NEXT: global_store_dwordx4 v0, v[10:13], s[6:7]
+; SDAG-NEXT: global_store_dwordx4 v12, v[14:17], s[6:7]
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: test_smfmac_f32_16x16x128_bf8_bf8__vgpr:
@@ -2609,20 +2609,20 @@ define amdgpu_kernel void @test_smfmac_f32_16x16x128_bf8_bf8__vgpr(ptr addrspace
; GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x54
; GISEL-NEXT: s_load_dword s2, s[4:5], 0x64
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GISEL-NEXT: global_load_dwordx4 v[14:17], v0, s[0:1]
-; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
-; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GISEL-NEXT: global_load_dwordx4 v[8:11], v0, s[0:1]
+; GISEL-NEXT: v_mov_b64_e32 v[14:15], s[10:11]
+; GISEL-NEXT: v_mov_b64_e32 v[12:13], s[8:9]
; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[12:13]
; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[14:15]
; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[16:17]
; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[18:19]
-; GISEL-NEXT: v_mov_b32_e32 v12, s2
+; GISEL-NEXT: v_mov_b32_e32 v16, s2
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_nop 0
-; GISEL-NEXT: v_smfmac_f32_16x16x128_bf8_bf8 v[14:17], v[8:11], v[0:7], v12 cbsz:1 abid:2
+; GISEL-NEXT: v_smfmac_f32_16x16x128_bf8_bf8 v[8:11], v[12:15], v[0:7], v16 cbsz:1 abid:2
; GISEL-NEXT: v_mov_b32_e32 v0, 0
; GISEL-NEXT: s_nop 6
-; GISEL-NEXT: global_store_dwordx4 v0, v[14:17], s[0:1]
+; GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[0:1]
; GISEL-NEXT: s_endpgm
;
; SDAG-VGPR-LABEL: test_smfmac_f32_16x16x128_bf8_bf8__vgpr:
@@ -2634,26 +2634,26 @@ define amdgpu_kernel void @test_smfmac_f32_16x16x128_bf8_bf8__vgpr(ptr addrspace
; SDAG-VGPR-NEXT: s_load_dword s16, s[4:5], 0x64
; SDAG-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
; SDAG-VGPR-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-VGPR-NEXT: global_load_dwordx4 v[10:13], v0, s[6:7]
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v0, 0
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v14, s8
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v15, s9
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, s10
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v17, s11
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v2, s12
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v3, s13
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v4, s14
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v5, s15
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v6, s0
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v7, s1
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v8, s2
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v9, s3
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v1, s16
+; SDAG-VGPR-NEXT: global_load_dwordx4 v[14:17], v0, s[6:7]
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v12, 0
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v8, s8
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v9, s9
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v10, s10
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v11, s11
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v0, s12
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v1, s13
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v2, s14
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v3, s15
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v4, s0
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v5, s1
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v6, s2
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v7, s3
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v13, s16
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-VGPR-NEXT: s_nop 0
-; SDAG-VGPR-NEXT: v_smfmac_f32_16x16x128_bf8_bf8 v[10:13], v[14:17], v[2:9], v1 cbsz:1 abid:2
+; SDAG-VGPR-NEXT: v_smfmac_f32_16x16x128_bf8_bf8 v[14:17], v[8:11], v[0:7], v13 cbsz:1 abid:2
; SDAG-VGPR-NEXT: s_nop 7
-; SDAG-VGPR-NEXT: global_store_dwordx4 v0, v[10:13], s[6:7]
+; SDAG-VGPR-NEXT: global_store_dwordx4 v12, v[14:17], s[6:7]
; SDAG-VGPR-NEXT: s_endpgm
;
; GISEL-VGPR-LABEL: test_smfmac_f32_16x16x128_bf8_bf8__vgpr:
@@ -2665,20 +2665,20 @@ define amdgpu_kernel void @test_smfmac_f32_16x16x128_bf8_bf8__vgpr(ptr addrspace
; GISEL-VGPR-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x54
; GISEL-VGPR-NEXT: s_load_dword s2, s[4:5], 0x64
; GISEL-VGPR-NEXT: s_waitcnt lgkmcnt(0)
-; GISEL-VGPR-NEXT: global_load_dwordx4 v[14:17], v0, s[0:1]
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GISEL-VGPR-NEXT: global_load_dwordx4 v[8:11], v0, s[0:1]
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[14:15], s[10:11]
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[12:13], s[8:9]
; GISEL-VGPR-NEXT: v_mov_b64_e32 v[0:1], s[12:13]
; GISEL-VGPR-NEXT: v_mov_b64_e32 v[2:3], s[14:15]
; GISEL-VGPR-NEXT: v_mov_b64_e32 v[4:5], s[16:17]
; GISEL-VGPR-NEXT: v_mov_b64_e32 v[6:7], s[18:19]
-; GISEL-VGPR-NEXT: v_mov_b32_e32 v12, s2
+; GISEL-VGPR-NEXT: v_mov_b32_e32 v16, s2
; GISEL-VGPR-NEXT: s_waitcnt vmcnt(0)
; GISEL-VGPR-NEXT: s_nop 0
-; GISEL-VGPR-NEXT: v_smfmac_f32_16x16x128_bf8_bf8 v[14:17], v[8:11], v[0:7], v12 cbsz:1 abid:2
+; GISEL-VGPR-NEXT: v_smfmac_f32_16x16x128_bf8_bf8 v[8:11], v[12:15], v[0:7], v16 cbsz:1 abid:2
; GISEL-VGPR-NEXT: v_mov_b32_e32 v0, 0
; GISEL-VGPR-NEXT: s_nop 6
-; GISEL-VGPR-NEXT: global_store_dwordx4 v0, v[14:17], s[0:1]
+; GISEL-VGPR-NEXT: global_store_dwordx4 v0, v[8:11], s[0:1]
; GISEL-VGPR-NEXT: s_endpgm
bb:
%id = call i32 @llvm.amdgcn.workitem.id.x()
@@ -2899,26 +2899,26 @@ define amdgpu_kernel void @test_smfmac_f32_16x16x128_bf8_fp8__vgpr(ptr addrspace
; SDAG-NEXT: s_load_dword s16, s[4:5], 0x64
; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: global_load_dwordx4 v[10:13], v0, s[6:7]
-; SDAG-NEXT: v_mov_b32_e32 v0, 0
-; SDAG-NEXT: v_mov_b32_e32 v14, s8
-; SDAG-NEXT: v_mov_b32_e32 v15, s9
-; SDAG-NEXT: v_mov_b32_e32 v16, s10
-; SDAG-NEXT: v_mov_b32_e32 v17, s11
-; SDAG-NEXT: v_mov_b32_e32 v2, s12
-; SDAG-NEXT: v_mov_b32_e32 v3, s13
-; SDAG-NEXT: v_mov_b32_e32 v4, s14
-; SDAG-NEXT: v_mov_b32_e32 v5, s15
-; SDAG-NEXT: v_mov_b32_e32 v6, s0
-; SDAG-NEXT: v_mov_b32_e32 v7, s1
-; SDAG-NEXT: v_mov_b32_e32 v8, s2
-; SDAG-NEXT: v_mov_b32_e32 v9, s3
-; SDAG-NEXT: v_mov_b32_e32 v1, s16
+; SDAG-NEXT: global_load_dwordx4 v[14:17], v0, s[6:7]
+; SDAG-NEXT: v_mov_b32_e32 v12, 0
+; SDAG-NEXT: v_mov_b32_e32 v8, s8
+; SDAG-NEXT: v_mov_b32_e32 v9, s9
+; SDAG-NEXT: v_mov_b32_e32 v10, s10
+; SDAG-NEXT: v_mov_b32_e32 v11, s11
+; SDAG-NEXT: v_mov_b32_e32 v0, s12
+; SDAG-NEXT: v_mov_b32_e32 v1, s13
+; SDAG-NEXT: v_mov_b32_e32 v2, s14
+; SDAG-NEXT: v_mov_b32_e32 v3, s15
+; SDAG-NEXT: v_mov_b32_e32 v4, s0
+; SDAG-NEXT: v_mov_b32_e32 v5, s1
+; SDAG-NEXT: v_mov_b32_e32 v6, s2
+; SDAG-NEXT: v_mov_b32_e32 v7, s3
+; SDAG-NEXT: v_mov_b32_e32 v13, s16
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
-; SDAG-NEXT: v_smfmac_f32_16x16x128_bf8_fp8 v[10:13], v[14:17], v[2:9], v1 cbsz:1 abid:2
+; SDAG-NEXT: v_smfmac_f32_16x16x128_bf8_fp8 v[14:17], v[8:11], v[0:7], v13 cbsz:1 abid:2
; SDAG-NEXT: s_nop 7
-; SDAG-NEXT: global_store_dwordx4 v0, v[10:13], s[6:7]
+; SDAG-NEXT: global_store_dwordx4 v12, v[14:17], s[6:7]
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: test_smfmac_f32_16x16x128_bf8_fp8__vgpr:
@@ -2930,20 +2930,20 @@ define amdgpu_kernel void @test_smfmac_f32_16x16x128_bf8_fp8__vgpr(ptr addrspace
; GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x54
; GISEL-NEXT: s_load_dword s2, s[4:5], 0x64
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GISEL-NEXT: global_load_dwordx4 v[14:17], v0, s[0:1]
-; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
-; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GISEL-NEXT: global_load_dwordx4 v[8:11], v0, s[0:1]
+; GISEL-NEXT: v_mov_b64_e32 v[14:15], s[10:11]
+; GISEL-NEXT: v_mov_b64_e32 v[12:13], s[8:9]
; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[12:13]
; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[14:15]
; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[16:17]
; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[18:19]
-; GISEL-NEXT: v_mov_b32_e32 v12, s2
+; GISEL-NEXT: v_mov_b32_e32 v16, s2
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_nop 0
-; GISEL-NEXT: v_smfmac_f32_16x16x128_bf8_fp8 v[14:17], v[8:11], v[0:7], v12 cbsz:1 abid:2
+; GISEL-NEXT: v_smfmac_f32_16x16x128_bf8_fp8 v[8:11], v[12:15], v[0:7], v16 cbsz:1 abid:2
; GISEL-NEXT: v_mov_b32_e32 v0, 0
; GISEL-NEXT: s_nop 6
-; GISEL-NEXT: global_store_dwordx4 v0, v[14:17], s[0:1]
+; GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[0:1]
; GISEL-NEXT: s_endpgm
;
; SDAG-VGPR-LABEL: test_smfmac_f32_16x16x128_bf8_fp8__vgpr:
@@ -2955,26 +2955,26 @@ define amdgpu_kernel void @test_smfmac_f32_16x16x128_bf8_fp8__vgpr(ptr addrspace
; SDAG-VGPR-NEXT: s_load_dword s16, s[4:5], 0x64
; SDAG-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
; SDAG-VGPR-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-VGPR-NEXT: global_load_dwordx4 v[10:13], v0, s[6:7]
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v0, 0
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v14, s8
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v15, s9
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, s10
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v17, s11
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v2, s12
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v3, s13
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v4, s14
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v5, s15
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v6, s0
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v7, s1
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v8, s2
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v9, s3
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v1, s16
+; SDAG-VGPR-NEXT: global_load_dwordx4 v[14:17], v0, s[6:7]
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v12, 0
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v8, s8
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v9, s9
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v10, s10
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v11, s11
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v0, s12
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v1, s13
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v2, s14
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v3, s15
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v4, s0
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v5, s1
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v6, s2
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v7, s3
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v13, s16
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-VGPR-NEXT: s_nop 0
-; SDAG-VGPR-NEXT: v_smfmac_f32_16x16x128_bf8_fp8 v[10:13], v[14:17], v[2:9], v1 cbsz:1 abid:2
+; SDAG-VGPR-NEXT: v_smfmac_f32_16x16x128_bf8_fp8 v[14:17], v[8:11], v[0:7], v13 cbsz:1 abid:2
; SDAG-VGPR-NEXT: s_nop 7
-; SDAG-VGPR-NEXT: global_store_dwordx4 v0, v[10:13], s[6:7]
+; SDAG-VGPR-NEXT: global_store_dwordx4 v12, v[14:17], s[6:7]
; SDAG-VGPR-NEXT: s_endpgm
;
; GISEL-VGPR-LABEL: test_smfmac_f32_16x16x128_bf8_fp8__vgpr:
@@ -2986,20 +2986,20 @@ define amdgpu_kernel void @test_smfmac_f32_16x16x128_bf8_fp8__vgpr(ptr addrspace
; GISEL-VGPR-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x54
; GISEL-VGPR-NEXT: s_load_dword s2, s[4:5], 0x64
; GISEL-VGPR-NEXT: s_waitcnt lgkmcnt(0)
-; GISEL-VGPR-NEXT: global_load_dwordx4 v[14:17], v0, s[0:1]
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GISEL-VGPR-NEXT: global_load_dwordx4 v[8:11], v0, s[0:1]
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[14:15], s[10:11]
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[12:13], s[8:9]
; GISEL-VGPR-NEXT: v_mov_b64_e32 v[0:1], s[12:13]
; GISEL-VGPR-NEXT: v_mov_b64_e32 v[2:3], s[14:15]
; GISEL-VGPR-NEXT: v_mov_b64_e32 v[4:5], s[16:17]
; GISEL-VGPR-NEXT: v_mov_b64_e32 v[6:7], s[18:19]
-; GISEL-VGPR-NEXT: v_mov_b32_e32 v12, s2
+; GISEL-VGPR-NEXT: v_mov_b32_e32 v16, s2
; GISEL-VGPR-NEXT: s_waitcnt vmcnt(0)
; GISEL-VGPR-NEXT: s_nop 0
-; GISEL-VGPR-NEXT: v_smfmac_f32_16x16x128_bf8_fp8 v[14:17], v[8:11], v[0:7], v12 cbsz:1 abid:2
+; GISEL-VGPR-NEXT: v_smfmac_f32_16x16x128_bf8_fp8 v[8:11], v[12:15], v[0:7], v16 cbsz:1 abid:2
; GISEL-VGPR-NEXT: v_mov_b32_e32 v0, 0
; GISEL-VGPR-NEXT: s_nop 6
-; GISEL-VGPR-NEXT: global_store_dwordx4 v0, v[14:17], s[0:1]
+; GISEL-VGPR-NEXT: global_store_dwordx4 v0, v[8:11], s[0:1]
; GISEL-VGPR-NEXT: s_endpgm
bb:
%id = call i32 @llvm.amdgcn.workitem.id.x()
@@ -3220,26 +3220,26 @@ define amdgpu_kernel void @test_smfmac_f32_16x16x128_fp8_bf8__vgpr(ptr addrspace
; SDAG-NEXT: s_load_dword s16, s[4:5], 0x64
; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: global_load_dwordx4 v[10:13], v0, s[6:7]
-; SDAG-NEXT: v_mov_b32_e32 v0, 0
-; SDAG-NEXT: v_mov_b32_e32 v14, s8
-; SDAG-NEXT: v_mov_b32_e32 v15, s9
-; SDAG-NEXT: v_mov_b32_e32 v16, s10
-; SDAG-NEXT: v_mov_b32_e32 v17, s11
-; SDAG-NEXT: v_mov_b32_e32 v2, s12
-; SDAG-NEXT: v_mov_b32_e32 v3, s13
-; SDAG-NEXT: v_mov_b32_e32 v4, s14
-; SDAG-NEXT: v_mov_b32_e32 v5, s15
-; SDAG-NEXT: v_mov_b32_e32 v6, s0
-; SDAG-NEXT: v_mov_b32_e32 v7, s1
-; SDAG-NEXT: v_mov_b32_e32 v8, s2
-; SDAG-NEXT: v_mov_b32_e32 v9, s3
-; SDAG-NEXT: v_mov_b32_e32 v1, s16
+; SDAG-NEXT: global_load_dwordx4 v[14:17], v0, s[6:7]
+; SDAG-NEXT: v_mov_b32_e32 v12, 0
+; SDAG-NEXT: v_mov_b32_e32 v8, s8
+; SDAG-NEXT: v_mov_b32_e32 v9, s9
+; SDAG-NEXT: v_mov_b32_e32 v10, s10
+; SDAG-NEXT: v_mov_b32_e32 v11, s11
+; SDAG-NEXT: v_mov_b32_e32 v0, s12
+; SDAG-NEXT: v_mov_b32_e32 v1, s13
+; SDAG-NEXT: v_mov_b32_e32 v2, s14
+; SDAG-NEXT: v_mov_b32_e32 v3, s15
+; SDAG-NEXT: v_mov_b32_e32 v4, s0
+; SDAG-NEXT: v_mov_b32_e32 v5, s1
+; SDAG-NEXT: v_mov_b32_e32 v6, s2
+; SDAG-NEXT: v_mov_b32_e32 v7, s3
+; SDAG-NEXT: v_mov_b32_e32 v13, s16
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
-; SDAG-NEXT: v_smfmac_f32_16x16x128_fp8_bf8 v[10:13], v[14:17], v[2:9], v1 cbsz:1 abid:2
+; SDAG-NEXT: v_smfmac_f32_16x16x128_fp8_bf8 v[14:17], v[8:11], v[0:7], v13 cbsz:1 abid:2
; SDAG-NEXT: s_nop 7
-; SDAG-NEXT: global_store_dwordx4 v0, v[10:13], s[6:7]
+; SDAG-NEXT: global_store_dwordx4 v12, v[14:17], s[6:7]
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: test_smfmac_f32_16x16x128_fp8_bf8__vgpr:
@@ -3251,20 +3251,20 @@ define amdgpu_kernel void @test_smfmac_f32_16x16x128_fp8_bf8__vgpr(ptr addrspace
; GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x54
; GISEL-NEXT: s_load_dword s2, s[4:5], 0x64
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GISEL-NEXT: global_load_dwordx4 v[14:17], v0, s[0:1]
-; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
-; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GISEL-NEXT: global_load_dwordx4 v[8:11], v0, s[0:1]
+; GISEL-NEXT: v_mov_b64_e32 v[14:15], s[10:11]
+; GISEL-NEXT: v_mov_b64_e32 v[12:13], s[8:9]
; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[12:13]
; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[14:15]
; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[16:17]
; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[18:19]
-; GISEL-NEXT: v_mov_b32_e32 v12, s2
+; GISEL-NEXT: v_mov_b32_e32 v16, s2
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_nop 0
-; GISEL-NEXT: v_smfmac_f32_16x16x128_fp8_bf8 v[14:17], v[8:11], v[0:7], v12 cbsz:1 abid:2
+; GISEL-NEXT: v_smfmac_f32_16x16x128_fp8_bf8 v[8:11], v[12:15], v[0:7], v16 cbsz:1 abid:2
; GISEL-NEXT: v_mov_b32_e32 v0, 0
; GISEL-NEXT: s_nop 6
-; GISEL-NEXT: global_store_dwordx4 v0, v[14:17], s[0:1]
+; GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[0:1]
; GISEL-NEXT: s_endpgm
;
; SDAG-VGPR-LABEL: test_smfmac_f32_16x16x128_fp8_bf8__vgpr:
@@ -3276,26 +3276,26 @@ define amdgpu_kernel void @test_smfmac_f32_16x16x128_fp8_bf8__vgpr(ptr addrspace
; SDAG-VGPR-NEXT: s_load_dword s16, s[4:5], 0x64
; SDAG-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
; SDAG-VGPR-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-VGPR-NEXT: global_load_dwordx4 v[10:13], v0, s[6:7]
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v0, 0
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v14, s8
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v15, s9
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, s10
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v17, s11
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v2, s12
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v3, s13
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v4, s14
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v5, s15
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v6, s0
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v7, s1
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v8, s2
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v9, s3
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v1, s16
+; SDAG-VGPR-NEXT: global_load_dwordx4 v[14:17], v0, s[6:7]
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v12, 0
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v8, s8
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v9, s9
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v10, s10
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v11, s11
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v0, s12
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v1, s13
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v2, s14
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v3, s15
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v4, s0
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v5, s1
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v6, s2
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v7, s3
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v13, s16
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-VGPR-NEXT: s_nop 0
-; SDAG-VGPR-NEXT: v_smfmac_f32_16x16x128_fp8_bf8 v[10:13], v[14:17], v[2:9], v1 cbsz:1 abid:2
+; SDAG-VGPR-NEXT: v_smfmac_f32_16x16x128_fp8_bf8 v[14:17], v[8:11], v[0:7], v13 cbsz:1 abid:2
; SDAG-VGPR-NEXT: s_nop 7
-; SDAG-VGPR-NEXT: global_store_dwordx4 v0, v[10:13], s[6:7]
+; SDAG-VGPR-NEXT: global_store_dwordx4 v12, v[14:17], s[6:7]
; SDAG-VGPR-NEXT: s_endpgm
;
; GISEL-VGPR-LABEL: test_smfmac_f32_16x16x128_fp8_bf8__vgpr:
@@ -3307,20 +3307,20 @@ define amdgpu_kernel void @test_smfmac_f32_16x16x128_fp8_bf8__vgpr(ptr addrspace
; GISEL-VGPR-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x54
; GISEL-VGPR-NEXT: s_load_dword s2, s[4:5], 0x64
; GISEL-VGPR-NEXT: s_waitcnt lgkmcnt(0)
-; GISEL-VGPR-NEXT: global_load_dwordx4 v[14:17], v0, s[0:1]
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GISEL-VGPR-NEXT: global_load_dwordx4 v[8:11], v0, s[0:1]
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[14:15], s[10:11]
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[12:13], s[8:9]
; GISEL-VGPR-NEXT: v_mov_b64_e32 v[0:1], s[12:13]
; GISEL-VGPR-NEXT: v_mov_b64_e32 v[2:3], s[14:15]
; GISEL-VGPR-NEXT: v_mov_b64_e32 v[4:5], s[16:17]
; GISEL-VGPR-NEXT: v_mov_b64_e32 v[6:7], s[18:19]
-; GISEL-VGPR-NEXT: v_mov_b32_e32 v12, s2
+; GISEL-VGPR-NEXT: v_mov_b32_e32 v16, s2
; GISEL-VGPR-NEXT: s_waitcnt vmcnt(0)
; GISEL-VGPR-NEXT: s_nop 0
-; GISEL-VGPR-NEXT: v_smfmac_f32_16x16x128_fp8_bf8 v[14:17], v[8:11], v[0:7], v12 cbsz:1 abid:2
+; GISEL-VGPR-NEXT: v_smfmac_f32_16x16x128_fp8_bf8 v[8:11], v[12:15], v[0:7], v16 cbsz:1 abid:2
; GISEL-VGPR-NEXT: v_mov_b32_e32 v0, 0
; GISEL-VGPR-NEXT: s_nop 6
-; GISEL-VGPR-NEXT: global_store_dwordx4 v0, v[14:17], s[0:1]
+; GISEL-VGPR-NEXT: global_store_dwordx4 v0, v[8:11], s[0:1]
; GISEL-VGPR-NEXT: s_endpgm
bb:
%id = call i32 @llvm.amdgcn.workitem.id.x()
@@ -3541,26 +3541,26 @@ define amdgpu_kernel void @test_smfmac_f32_16x16x128_fp8_fp8__vgpr(ptr addrspace
; SDAG-NEXT: s_load_dword s16, s[4:5], 0x64
; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: global_load_dwordx4 v[10:13], v0, s[6:7]
-; SDAG-NEXT: v_mov_b32_e32 v0, 0
-; SDAG-NEXT: v_mov_b32_e32 v14, s8
-; SDAG-NEXT: v_mov_b32_e32 v15, s9
-; SDAG-NEXT: v_mov_b32_e32 v16, s10
-; SDAG-NEXT: v_mov_b32_e32 v17, s11
-; SDAG-NEXT: v_mov_b32_e32 v2, s12
-; SDAG-NEXT: v_mov_b32_e32 v3, s13
-; SDAG-NEXT: v_mov_b32_e32 v4, s14
-; SDAG-NEXT: v_mov_b32_e32 v5, s15
-; SDAG-NEXT: v_mov_b32_e32 v6, s0
-; SDAG-NEXT: v_mov_b32_e32 v7, s1
-; SDAG-NEXT: v_mov_b32_e32 v8, s2
-; SDAG-NEXT: v_mov_b32_e32 v9, s3
-; SDAG-NEXT: v_mov_b32_e32 v1, s16
+; SDAG-NEXT: global_load_dwordx4 v[14:17], v0, s[6:7]
+; SDAG-NEXT: v_mov_b32_e32 v12, 0
+; SDAG-NEXT: v_mov_b32_e32 v8, s8
+; SDAG-NEXT: v_mov_b32_e32 v9, s9
+; SDAG-NEXT: v_mov_b32_e32 v10, s10
+; SDAG-NEXT: v_mov_b32_e32 v11, s11
+; SDAG-NEXT: v_mov_b32_e32 v0, s12
+; SDAG-NEXT: v_mov_b32_e32 v1, s13
+; SDAG-NEXT: v_mov_b32_e32 v2, s14
+; SDAG-NEXT: v_mov_b32_e32 v3, s15
+; SDAG-NEXT: v_mov_b32_e32 v4, s0
+; SDAG-NEXT: v_mov_b32_e32 v5, s1
+; SDAG-NEXT: v_mov_b32_e32 v6, s2
+; SDAG-NEXT: v_mov_b32_e32 v7, s3
+; SDAG-NEXT: v_mov_b32_e32 v13, s16
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
-; SDAG-NEXT: v_smfmac_f32_16x16x128_fp8_fp8 v[10:13], v[14:17], v[2:9], v1 cbsz:1 abid:2
+; SDAG-NEXT: v_smfmac_f32_16x16x128_fp8_fp8 v[14:17], v[8:11], v[0:7], v13 cbsz:1 abid:2
; SDAG-NEXT: s_nop 7
-; SDAG-NEXT: global_store_dwordx4 v0, v[10:13], s[6:7]
+; SDAG-NEXT: global_store_dwordx4 v12, v[14:17], s[6:7]
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: test_smfmac_f32_16x16x128_fp8_fp8__vgpr:
@@ -3572,20 +3572,20 @@ define amdgpu_kernel void @test_smfmac_f32_16x16x128_fp8_fp8__vgpr(ptr addrspace
; GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x54
; GISEL-NEXT: s_load_dword s2, s[4:5], 0x64
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GISEL-NEXT: global_load_dwordx4 v[14:17], v0, s[0:1]
-; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
-; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GISEL-NEXT: global_load_dwordx4 v[8:11], v0, s[0:1]
+; GISEL-NEXT: v_mov_b64_e32 v[14:15], s[10:11]
+; GISEL-NEXT: v_mov_b64_e32 v[12:13], s[8:9]
; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[12:13]
; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[14:15]
; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[16:17]
; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[18:19]
-; GISEL-NEXT: v_mov_b32_e32 v12, s2
+; GISEL-NEXT: v_mov_b32_e32 v16, s2
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_nop 0
-; GISEL-NEXT: v_smfmac_f32_16x16x128_fp8_fp8 v[14:17], v[8:11], v[0:7], v12 cbsz:1 abid:2
+; GISEL-NEXT: v_smfmac_f32_16x16x128_fp8_fp8 v[8:11], v[12:15], v[0:7], v16 cbsz:1 abid:2
; GISEL-NEXT: v_mov_b32_e32 v0, 0
; GISEL-NEXT: s_nop 6
-; GISEL-NEXT: global_store_dwordx4 v0, v[14:17], s[0:1]
+; GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[0:1]
; GISEL-NEXT: s_endpgm
;
; SDAG-VGPR-LABEL: test_smfmac_f32_16x16x128_fp8_fp8__vgpr:
@@ -3597,26 +3597,26 @@ define amdgpu_kernel void @test_smfmac_f32_16x16x128_fp8_fp8__vgpr(ptr addrspace
; SDAG-VGPR-NEXT: s_load_dword s16, s[4:5], 0x64
; SDAG-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
; SDAG-VGPR-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-VGPR-NEXT: global_load_dwordx4 v[10:13], v0, s[6:7]
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v0, 0
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v14, s8
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v15, s9
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, s10
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v17, s11
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v2, s12
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v3, s13
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v4, s14
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v5, s15
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v6, s0
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v7, s1
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v8, s2
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v9, s3
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v1, s16
+; SDAG-VGPR-NEXT: global_load_dwordx4 v[14:17], v0, s[6:7]
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v12, 0
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v8, s8
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v9, s9
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v10, s10
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v11, s11
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v0, s12
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v1, s13
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v2, s14
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v3, s15
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v4, s0
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v5, s1
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v6, s2
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v7, s3
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v13, s16
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-VGPR-NEXT: s_nop 0
-; SDAG-VGPR-NEXT: v_smfmac_f32_16x16x128_fp8_fp8 v[10:13], v[14:17], v[2:9], v1 cbsz:1 abid:2
+; SDAG-VGPR-NEXT: v_smfmac_f32_16x16x128_fp8_fp8 v[14:17], v[8:11], v[0:7], v13 cbsz:1 abid:2
; SDAG-VGPR-NEXT: s_nop 7
-; SDAG-VGPR-NEXT: global_store_dwordx4 v0, v[10:13], s[6:7]
+; SDAG-VGPR-NEXT: global_store_dwordx4 v12, v[14:17], s[6:7]
; SDAG-VGPR-NEXT: s_endpgm
;
; GISEL-VGPR-LABEL: test_smfmac_f32_16x16x128_fp8_fp8__vgpr:
@@ -3628,20 +3628,20 @@ define amdgpu_kernel void @test_smfmac_f32_16x16x128_fp8_fp8__vgpr(ptr addrspace
; GISEL-VGPR-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x54
; GISEL-VGPR-NEXT: s_load_dword s2, s[4:5], 0x64
; GISEL-VGPR-NEXT: s_waitcnt lgkmcnt(0)
-; GISEL-VGPR-NEXT: global_load_dwordx4 v[14:17], v0, s[0:1]
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
-; GISEL-VGPR-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GISEL-VGPR-NEXT: global_load_dwordx4 v[8:11], v0, s[0:1]
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[14:15], s[10:11]
+; GISEL-VGPR-NEXT: v_mov_b64_e32 v[12:13], s[8:9]
; GISEL-VGPR-NEXT: v_mov_b64_e32 v[0:1], s[12:13]
; GISEL-VGPR-NEXT: v_mov_b64_e32 v[2:3], s[14:15]
; GISEL-VGPR-NEXT: v_mov_b64_e32 v[4:5], s[16:17]
; GISEL-VGPR-NEXT: v_mov_b64_e32 v[6:7], s[18:19]
-; GISEL-VGPR-NEXT: v_mov_b32_e32 v12, s2
+; GISEL-VGPR-NEXT: v_mov_b32_e32 v16, s2
; GISEL-VGPR-NEXT: s_waitcnt vmcnt(0)
; GISEL-VGPR-NEXT: s_nop 0
-; GISEL-VGPR-NEXT: v_smfmac_f32_16x16x128_fp8_fp8 v[14:17], v[8:11], v[0:7], v12 cbsz:1 abid:2
+; GISEL-VGPR-NEXT: v_smfmac_f32_16x16x128_fp8_fp8 v[8:11], v[12:15], v[0:7], v16 cbsz:1 abid:2
; GISEL-VGPR-NEXT: v_mov_b32_e32 v0, 0
; GISEL-VGPR-NEXT: s_nop 6
-; GISEL-VGPR-NEXT: global_store_dwordx4 v0, v[14:17], s[0:1]
+; GISEL-VGPR-NEXT: global_store_dwordx4 v0, v[8:11], s[0:1]
; GISEL-VGPR-NEXT: s_endpgm
bb:
%id = call i32 @llvm.amdgcn.workitem.id.x()
@@ -3866,22 +3866,22 @@ define amdgpu_kernel void @test_smfmac_f32_32x32x64_bf8_bf8__vgpr(ptr addrspace(
; SDAG-NEXT: global_load_dwordx4 v[8:11], v16, s[6:7] offset:32
; SDAG-NEXT: global_load_dwordx4 v[4:7], v16, s[6:7] offset:16
; SDAG-NEXT: global_load_dwordx4 v[0:3], v16, s[6:7]
-; SDAG-NEXT: v_mov_b32_e32 v26, s8
-; SDAG-NEXT: v_mov_b32_e32 v27, s9
-; SDAG-NEXT: v_mov_b32_e32 v28, s10
-; SDAG-NEXT: v_mov_b32_e32 v29, s11
-; SDAG-NEXT: v_mov_b32_e32 v18, s12
-; SDAG-NEXT: v_mov_b32_e32 v19, s13
-; SDAG-NEXT: v_mov_b32_e32 v20, s14
-; SDAG-NEXT: v_mov_b32_e32 v21, s15
-; SDAG-NEXT: v_mov_b32_e32 v22, s0
-; SDAG-NEXT: v_mov_b32_e32 v23, s1
-; SDAG-NEXT: v_mov_b32_e32 v24, s2
-; SDAG-NEXT: v_mov_b32_e32 v25, s3
-; SDAG-NEXT: v_mov_b32_e32 v16, s16
+; SDAG-NEXT: v_mov_b32_e32 v24, s8
+; SDAG-NEXT: v_mov_b32_e32 v25, s9
+; SDAG-NEXT: v_mov_b32_e32 v26, s10
+; SDAG-NEXT: v_mov_b32_e32 v27, s11
+; SDAG-NEXT: v_mov_b32_e32 v16, s12
+; SDAG-NEXT: v_mov_b32_e32 v17, s13
+; SDAG-NEXT: v_mov_b32_e32 v18, s14
+; SDAG-NEXT: v_mov_b32_e32 v19, s15
+; SDAG-NEXT: v_mov_b32_e32 v20, s0
+; SDAG-NEXT: v_mov_b32_e32 v21, s1
+; SDAG-NEXT: v_mov_b32_e32 v22, s2
+; SDAG-NEXT: v_mov_b32_e32 v23, s3
+; SDAG-NEXT: v_mov_b32_e32 v28, s16
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
-; SDAG-NEXT: v_smfmac_f32_32x32x64_bf8_bf8 v[0:15], v[26:29], v[18:25], v16 cbsz:1 abid:2
+; SDAG-NEXT: v_smfmac_f32_32x32x64_bf8_bf8 v[0:15], v[24:27], v[16:23], v28 cbsz:1 abid:2
; SDAG-NEXT: v_mov_b32_e32 v16, 0
; SDAG-NEXT: s_nop 10
; SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[6:7] offset:32
@@ -3934,22 +3934,22 @@ define amdgpu_kernel void @test_smfmac_f32_32x32x64_bf8_bf8__vgpr(ptr addrspace(
; SDAG-VGPR-NEXT: global_load_dwordx4 v[8:11], v16, s[6:7] offset:32
; SDAG-VGPR-NEXT: global_load_dwordx4 v[4:7], v16, s[6:7] offset:16
; SDAG-VGPR-NEXT: global_load_dwordx4 v[0:3], v16, s[6:7]
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v26, s8
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v27, s9
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v28, s10
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v29, s11
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v18, s12
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v19, s13
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v20, s14
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v21, s15
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v22, s0
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v23, s1
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v24, s2
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v25, s3
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, s16
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v24, s8
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v25, s9
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v26, s10
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v27, s11
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, s12
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v17, s13
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v18, s14
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v19, s15
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v20, s0
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v21, s1
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v22, s2
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v23, s3
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v28, s16
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-VGPR-NEXT: s_nop 0
-; SDAG-VGPR-NEXT: v_smfmac_f32_32x32x64_bf8_bf8 v[0:15], v[26:29], v[18:25], v16 cbsz:1 abid:2
+; SDAG-VGPR-NEXT: v_smfmac_f32_32x32x64_bf8_bf8 v[0:15], v[24:27], v[16:23], v28 cbsz:1 abid:2
; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, 0
; SDAG-VGPR-NEXT: s_nop 10
; SDAG-VGPR-NEXT: global_store_dwordx4 v16, v[8:11], s[6:7] offset:32
@@ -4414,22 +4414,22 @@ define amdgpu_kernel void @test_smfmac_f32_32x32x64_bf8_fp8__vgpr(ptr addrspace(
; SDAG-NEXT: global_load_dwordx4 v[8:11], v16, s[6:7] offset:32
; SDAG-NEXT: global_load_dwordx4 v[4:7], v16, s[6:7] offset:16
; SDAG-NEXT: global_load_dwordx4 v[0:3], v16, s[6:7]
-; SDAG-NEXT: v_mov_b32_e32 v26, s8
-; SDAG-NEXT: v_mov_b32_e32 v27, s9
-; SDAG-NEXT: v_mov_b32_e32 v28, s10
-; SDAG-NEXT: v_mov_b32_e32 v29, s11
-; SDAG-NEXT: v_mov_b32_e32 v18, s12
-; SDAG-NEXT: v_mov_b32_e32 v19, s13
-; SDAG-NEXT: v_mov_b32_e32 v20, s14
-; SDAG-NEXT: v_mov_b32_e32 v21, s15
-; SDAG-NEXT: v_mov_b32_e32 v22, s0
-; SDAG-NEXT: v_mov_b32_e32 v23, s1
-; SDAG-NEXT: v_mov_b32_e32 v24, s2
-; SDAG-NEXT: v_mov_b32_e32 v25, s3
-; SDAG-NEXT: v_mov_b32_e32 v16, s16
+; SDAG-NEXT: v_mov_b32_e32 v24, s8
+; SDAG-NEXT: v_mov_b32_e32 v25, s9
+; SDAG-NEXT: v_mov_b32_e32 v26, s10
+; SDAG-NEXT: v_mov_b32_e32 v27, s11
+; SDAG-NEXT: v_mov_b32_e32 v16, s12
+; SDAG-NEXT: v_mov_b32_e32 v17, s13
+; SDAG-NEXT: v_mov_b32_e32 v18, s14
+; SDAG-NEXT: v_mov_b32_e32 v19, s15
+; SDAG-NEXT: v_mov_b32_e32 v20, s0
+; SDAG-NEXT: v_mov_b32_e32 v21, s1
+; SDAG-NEXT: v_mov_b32_e32 v22, s2
+; SDAG-NEXT: v_mov_b32_e32 v23, s3
+; SDAG-NEXT: v_mov_b32_e32 v28, s16
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
-; SDAG-NEXT: v_smfmac_f32_32x32x64_bf8_fp8 v[0:15], v[26:29], v[18:25], v16 cbsz:1 abid:2
+; SDAG-NEXT: v_smfmac_f32_32x32x64_bf8_fp8 v[0:15], v[24:27], v[16:23], v28 cbsz:1 abid:2
; SDAG-NEXT: v_mov_b32_e32 v16, 0
; SDAG-NEXT: s_nop 10
; SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[6:7] offset:32
@@ -4482,22 +4482,22 @@ define amdgpu_kernel void @test_smfmac_f32_32x32x64_bf8_fp8__vgpr(ptr addrspace(
; SDAG-VGPR-NEXT: global_load_dwordx4 v[8:11], v16, s[6:7] offset:32
; SDAG-VGPR-NEXT: global_load_dwordx4 v[4:7], v16, s[6:7] offset:16
; SDAG-VGPR-NEXT: global_load_dwordx4 v[0:3], v16, s[6:7]
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v26, s8
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v27, s9
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v28, s10
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v29, s11
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v18, s12
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v19, s13
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v20, s14
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v21, s15
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v22, s0
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v23, s1
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v24, s2
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v25, s3
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, s16
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v24, s8
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v25, s9
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v26, s10
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v27, s11
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, s12
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v17, s13
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v18, s14
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v19, s15
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v20, s0
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v21, s1
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v22, s2
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v23, s3
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v28, s16
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-VGPR-NEXT: s_nop 0
-; SDAG-VGPR-NEXT: v_smfmac_f32_32x32x64_bf8_fp8 v[0:15], v[26:29], v[18:25], v16 cbsz:1 abid:2
+; SDAG-VGPR-NEXT: v_smfmac_f32_32x32x64_bf8_fp8 v[0:15], v[24:27], v[16:23], v28 cbsz:1 abid:2
; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, 0
; SDAG-VGPR-NEXT: s_nop 10
; SDAG-VGPR-NEXT: global_store_dwordx4 v16, v[8:11], s[6:7] offset:32
@@ -4962,22 +4962,22 @@ define amdgpu_kernel void @test_smfmac_f32_32x32x64_fp8_bf8__vgpr(ptr addrspace(
; SDAG-NEXT: global_load_dwordx4 v[8:11], v16, s[6:7] offset:32
; SDAG-NEXT: global_load_dwordx4 v[4:7], v16, s[6:7] offset:16
; SDAG-NEXT: global_load_dwordx4 v[0:3], v16, s[6:7]
-; SDAG-NEXT: v_mov_b32_e32 v26, s8
-; SDAG-NEXT: v_mov_b32_e32 v27, s9
-; SDAG-NEXT: v_mov_b32_e32 v28, s10
-; SDAG-NEXT: v_mov_b32_e32 v29, s11
-; SDAG-NEXT: v_mov_b32_e32 v18, s12
-; SDAG-NEXT: v_mov_b32_e32 v19, s13
-; SDAG-NEXT: v_mov_b32_e32 v20, s14
-; SDAG-NEXT: v_mov_b32_e32 v21, s15
-; SDAG-NEXT: v_mov_b32_e32 v22, s0
-; SDAG-NEXT: v_mov_b32_e32 v23, s1
-; SDAG-NEXT: v_mov_b32_e32 v24, s2
-; SDAG-NEXT: v_mov_b32_e32 v25, s3
-; SDAG-NEXT: v_mov_b32_e32 v16, s16
+; SDAG-NEXT: v_mov_b32_e32 v24, s8
+; SDAG-NEXT: v_mov_b32_e32 v25, s9
+; SDAG-NEXT: v_mov_b32_e32 v26, s10
+; SDAG-NEXT: v_mov_b32_e32 v27, s11
+; SDAG-NEXT: v_mov_b32_e32 v16, s12
+; SDAG-NEXT: v_mov_b32_e32 v17, s13
+; SDAG-NEXT: v_mov_b32_e32 v18, s14
+; SDAG-NEXT: v_mov_b32_e32 v19, s15
+; SDAG-NEXT: v_mov_b32_e32 v20, s0
+; SDAG-NEXT: v_mov_b32_e32 v21, s1
+; SDAG-NEXT: v_mov_b32_e32 v22, s2
+; SDAG-NEXT: v_mov_b32_e32 v23, s3
+; SDAG-NEXT: v_mov_b32_e32 v28, s16
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
-; SDAG-NEXT: v_smfmac_f32_32x32x64_fp8_bf8 v[0:15], v[26:29], v[18:25], v16 cbsz:1 abid:2
+; SDAG-NEXT: v_smfmac_f32_32x32x64_fp8_bf8 v[0:15], v[24:27], v[16:23], v28 cbsz:1 abid:2
; SDAG-NEXT: v_mov_b32_e32 v16, 0
; SDAG-NEXT: s_nop 10
; SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[6:7] offset:32
@@ -5030,22 +5030,22 @@ define amdgpu_kernel void @test_smfmac_f32_32x32x64_fp8_bf8__vgpr(ptr addrspace(
; SDAG-VGPR-NEXT: global_load_dwordx4 v[8:11], v16, s[6:7] offset:32
; SDAG-VGPR-NEXT: global_load_dwordx4 v[4:7], v16, s[6:7] offset:16
; SDAG-VGPR-NEXT: global_load_dwordx4 v[0:3], v16, s[6:7]
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v26, s8
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v27, s9
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v28, s10
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v29, s11
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v18, s12
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v19, s13
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v20, s14
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v21, s15
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v22, s0
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v23, s1
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v24, s2
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v25, s3
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, s16
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v24, s8
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v25, s9
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v26, s10
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v27, s11
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, s12
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v17, s13
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v18, s14
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v19, s15
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v20, s0
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v21, s1
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v22, s2
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v23, s3
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v28, s16
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-VGPR-NEXT: s_nop 0
-; SDAG-VGPR-NEXT: v_smfmac_f32_32x32x64_fp8_bf8 v[0:15], v[26:29], v[18:25], v16 cbsz:1 abid:2
+; SDAG-VGPR-NEXT: v_smfmac_f32_32x32x64_fp8_bf8 v[0:15], v[24:27], v[16:23], v28 cbsz:1 abid:2
; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, 0
; SDAG-VGPR-NEXT: s_nop 10
; SDAG-VGPR-NEXT: global_store_dwordx4 v16, v[8:11], s[6:7] offset:32
@@ -5510,22 +5510,22 @@ define amdgpu_kernel void @test_smfmac_f32_32x32x64_fp8_fp8__vgpr(ptr addrspace(
; SDAG-NEXT: global_load_dwordx4 v[8:11], v16, s[6:7] offset:32
; SDAG-NEXT: global_load_dwordx4 v[4:7], v16, s[6:7] offset:16
; SDAG-NEXT: global_load_dwordx4 v[0:3], v16, s[6:7]
-; SDAG-NEXT: v_mov_b32_e32 v26, s8
-; SDAG-NEXT: v_mov_b32_e32 v27, s9
-; SDAG-NEXT: v_mov_b32_e32 v28, s10
-; SDAG-NEXT: v_mov_b32_e32 v29, s11
-; SDAG-NEXT: v_mov_b32_e32 v18, s12
-; SDAG-NEXT: v_mov_b32_e32 v19, s13
-; SDAG-NEXT: v_mov_b32_e32 v20, s14
-; SDAG-NEXT: v_mov_b32_e32 v21, s15
-; SDAG-NEXT: v_mov_b32_e32 v22, s0
-; SDAG-NEXT: v_mov_b32_e32 v23, s1
-; SDAG-NEXT: v_mov_b32_e32 v24, s2
-; SDAG-NEXT: v_mov_b32_e32 v25, s3
-; SDAG-NEXT: v_mov_b32_e32 v16, s16
+; SDAG-NEXT: v_mov_b32_e32 v24, s8
+; SDAG-NEXT: v_mov_b32_e32 v25, s9
+; SDAG-NEXT: v_mov_b32_e32 v26, s10
+; SDAG-NEXT: v_mov_b32_e32 v27, s11
+; SDAG-NEXT: v_mov_b32_e32 v16, s12
+; SDAG-NEXT: v_mov_b32_e32 v17, s13
+; SDAG-NEXT: v_mov_b32_e32 v18, s14
+; SDAG-NEXT: v_mov_b32_e32 v19, s15
+; SDAG-NEXT: v_mov_b32_e32 v20, s0
+; SDAG-NEXT: v_mov_b32_e32 v21, s1
+; SDAG-NEXT: v_mov_b32_e32 v22, s2
+; SDAG-NEXT: v_mov_b32_e32 v23, s3
+; SDAG-NEXT: v_mov_b32_e32 v28, s16
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
-; SDAG-NEXT: v_smfmac_f32_32x32x64_fp8_fp8 v[0:15], v[26:29], v[18:25], v16 cbsz:1 abid:2
+; SDAG-NEXT: v_smfmac_f32_32x32x64_fp8_fp8 v[0:15], v[24:27], v[16:23], v28 cbsz:1 abid:2
; SDAG-NEXT: v_mov_b32_e32 v16, 0
; SDAG-NEXT: s_nop 10
; SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[6:7] offset:32
@@ -5578,22 +5578,22 @@ define amdgpu_kernel void @test_smfmac_f32_32x32x64_fp8_fp8__vgpr(ptr addrspace(
; SDAG-VGPR-NEXT: global_load_dwordx4 v[8:11], v16, s[6:7] offset:32
; SDAG-VGPR-NEXT: global_load_dwordx4 v[4:7], v16, s[6:7] offset:16
; SDAG-VGPR-NEXT: global_load_dwordx4 v[0:3], v16, s[6:7]
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v26, s8
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v27, s9
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v28, s10
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v29, s11
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v18, s12
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v19, s13
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v20, s14
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v21, s15
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v22, s0
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v23, s1
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v24, s2
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v25, s3
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, s16
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v24, s8
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v25, s9
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v26, s10
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v27, s11
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, s12
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v17, s13
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v18, s14
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v19, s15
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v20, s0
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v21, s1
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v22, s2
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v23, s3
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v28, s16
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-VGPR-NEXT: s_nop 0
-; SDAG-VGPR-NEXT: v_smfmac_f32_32x32x64_fp8_fp8 v[0:15], v[26:29], v[18:25], v16 cbsz:1 abid:2
+; SDAG-VGPR-NEXT: v_smfmac_f32_32x32x64_fp8_fp8 v[0:15], v[24:27], v[16:23], v28 cbsz:1 abid:2
; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, 0
; SDAG-VGPR-NEXT: s_nop 10
; SDAG-VGPR-NEXT: global_store_dwordx4 v16, v[8:11], s[6:7] offset:32
diff --git a/llvm/test/CodeGen/AMDGPU/lo16-hi16-illegal-copy.mir b/llvm/test/CodeGen/AMDGPU/lo16-hi16-illegal-copy.mir
index b9a134df8238e..ba3df99adc909 100644
--- a/llvm/test/CodeGen/AMDGPU/lo16-hi16-illegal-copy.mir
+++ b/llvm/test/CodeGen/AMDGPU/lo16-hi16-illegal-copy.mir
@@ -1,7 +1,7 @@
-# RUN: not llc -mtriple=amdgpu8.02 -start-before postrapseudos -asm-verbose=0 -verify-machineinstrs %s -o - 2>&1 | FileCheck -check-prefixes=ERR,GFX8-ERR %s
-# RUN: not llc -mtriple=amdgpu8.02 -start-before postrapseudos -asm-verbose=0 -verify-machineinstrs %s -o - 2>&1 | FileCheck -check-prefix=GCN %s
-# RUN: not llc -mtriple=amdgpu9.00 -start-before postrapseudos -asm-verbose=0 -verify-machineinstrs %s -o - 2>&1 | FileCheck -check-prefix=ERR %s
-# RUN: not llc -mtriple=amdgpu9.00 -start-before postrapseudos -asm-verbose=0 -verify-machineinstrs %s -o - 2>&1 | FileCheck -check-prefixes=GCN,GFX9 %s
+# RUN: not llc -mtriple=amdgpu8.02 -start-before post-ra-pseudos -asm-verbose=0 -verify-machineinstrs %s -o - 2>&1 | FileCheck -check-prefixes=ERR,GFX8-ERR %s
+# RUN: not llc -mtriple=amdgpu8.02 -start-before post-ra-pseudos -asm-verbose=0 -verify-machineinstrs %s -o - 2>&1 | FileCheck -check-prefix=GCN %s
+# RUN: not llc -mtriple=amdgpu9.00 -start-before post-ra-pseudos -asm-verbose=0 -verify-machineinstrs %s -o - 2>&1 | FileCheck -check-prefix=ERR %s
+# RUN: not llc -mtriple=amdgpu9.00 -start-before post-ra-pseudos -asm-verbose=0 -verify-machineinstrs %s -o - 2>&1 | FileCheck -check-prefixes=GCN,GFX9 %s
# Note: GFX8 did not allow SDWA SGPR sources. Therefor no HI16 subregs can be used there.
diff --git a/llvm/test/CodeGen/AMDGPU/lo16-hi16-physreg-copy.mir b/llvm/test/CodeGen/AMDGPU/lo16-hi16-physreg-copy.mir
index 1601eaf8f0679..eda00f65c9ae7 100644
--- a/llvm/test/CodeGen/AMDGPU/lo16-hi16-physreg-copy.mir
+++ b/llvm/test/CodeGen/AMDGPU/lo16-hi16-physreg-copy.mir
@@ -1,6 +1,6 @@
-# RUN: llc -mtriple=amdgpu8.02 -start-before postrapseudos -asm-verbose=0 -verify-machineinstrs %s -o - | FileCheck -check-prefix=GCN %s
-# RUN: llc -mtriple=amdgpu9.00 -start-before postrapseudos -asm-verbose=0 -verify-machineinstrs %s -o - | FileCheck -check-prefix=GCN %s
-# RUN: llc -mtriple=amdgpu10.10 -start-before postrapseudos -asm-verbose=0 -verify-machineinstrs %s -o - | FileCheck -check-prefix=GCN %s
+# RUN: llc -mtriple=amdgpu8.02 -start-before post-ra-pseudos -asm-verbose=0 -verify-machineinstrs %s -o - | FileCheck -check-prefix=GCN %s
+# RUN: llc -mtriple=amdgpu9.00 -start-before post-ra-pseudos -asm-verbose=0 -verify-machineinstrs %s -o - | FileCheck -check-prefix=GCN %s
+# RUN: llc -mtriple=amdgpu10.10 -start-before post-ra-pseudos -asm-verbose=0 -verify-machineinstrs %s -o - | FileCheck -check-prefix=GCN %s
# GCN-LABEL: {{^}}lo_to_lo:
# GCN: v_mov_b32_sdwa v1, v0 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
diff --git a/llvm/test/CodeGen/AMDGPU/lo16-lo16-physreg-copy-agpr.mir b/llvm/test/CodeGen/AMDGPU/lo16-lo16-physreg-copy-agpr.mir
index 30b8b8f264bad..180b6c52309eb 100644
--- a/llvm/test/CodeGen/AMDGPU/lo16-lo16-physreg-copy-agpr.mir
+++ b/llvm/test/CodeGen/AMDGPU/lo16-lo16-physreg-copy-agpr.mir
@@ -1,4 +1,4 @@
-# RUN: llc -mtriple=amdgpu9.08 -start-before postrapseudos -asm-verbose=0 -verify-machineinstrs %s -o - | FileCheck -check-prefix=GCN %s
+# RUN: llc -mtriple=amdgpu9.08 -start-before post-ra-pseudos -asm-verbose=0 -verify-machineinstrs %s -o - | FileCheck -check-prefix=GCN %s
# GCN-LABEL: {{^}}lo_to_lo_agpr_to_agpr:
# GCN: v_accvgpr_read_b32 [[TMP:v[0-9]+]], a0
diff --git a/llvm/test/CodeGen/AMDGPU/lo16-lo16-physreg-copy-sgpr.mir b/llvm/test/CodeGen/AMDGPU/lo16-lo16-physreg-copy-sgpr.mir
index 87a29e75ed6b1..e19fefac74cbb 100644
--- a/llvm/test/CodeGen/AMDGPU/lo16-lo16-physreg-copy-sgpr.mir
+++ b/llvm/test/CodeGen/AMDGPU/lo16-lo16-physreg-copy-sgpr.mir
@@ -1,5 +1,5 @@
-# RUN: llc -mtriple=amdgpu9.00 -start-before postrapseudos -asm-verbose=0 -verify-machineinstrs %s -o - | FileCheck -check-prefix=GCN %s
-# RUN: llc -mtriple=amdgpu10.10 -start-before postrapseudos -asm-verbose=0 -verify-machineinstrs %s -o - | FileCheck -check-prefix=GCN %s
+# RUN: llc -mtriple=amdgpu9.00 -start-before post-ra-pseudos -asm-verbose=0 -verify-machineinstrs %s -o - | FileCheck -check-prefix=GCN %s
+# RUN: llc -mtriple=amdgpu10.10 -start-before post-ra-pseudos -asm-verbose=0 -verify-machineinstrs %s -o - | FileCheck -check-prefix=GCN %s
# Note: GFX8 did not allow SDWA SGPR sources. Therefor no HI16 subregs can be used there.
diff --git a/llvm/test/CodeGen/AMDGPU/lower-control-flow-live-variables-update.mir b/llvm/test/CodeGen/AMDGPU/lower-control-flow-live-variables-update.mir
index 67b971ed23f4f..e911d074fb52e 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-control-flow-live-variables-update.mir
+++ b/llvm/test/CodeGen/AMDGPU/lower-control-flow-live-variables-update.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu7.00-amd-amdhsa -start-before=livevars -stop-after=twoaddressinstruction -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu7.00-amd-amdhsa -start-before=unreachable-mbb-elimination -stop-after=two-address-instruction -verify-machineinstrs -o - %s | FileCheck %s
# FIXME: update_mir_test_checks tries to incorrectly re-use a variable
# name used for a copy, so some of the check variable names were
diff --git a/llvm/test/CodeGen/AMDGPU/lower-control-flow-live-variables-update.xfail.mir b/llvm/test/CodeGen/AMDGPU/lower-control-flow-live-variables-update.xfail.mir
index 7a6fb07b598bb..c675879cf622f 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-control-flow-live-variables-update.xfail.mir
+++ b/llvm/test/CodeGen/AMDGPU/lower-control-flow-live-variables-update.xfail.mir
@@ -1,4 +1,4 @@
-# RUN: not --crash llc -mtriple=amdgpu7.00-amd-amdhsa -start-before=livevars -stop-after=twoaddressinstruction -verify-machineinstrs -o - %s 2>&1 | FileCheck %s
+# RUN: not --crash llc -mtriple=amdgpu7.00-amd-amdhsa -start-before=unreachable-mbb-elimination -stop-after=two-address-instruction -verify-each -o - %s 2>&1 | FileCheck %s
# CHECK: *** Bad machine code: LiveVariables: Block missing from AliveBlocks ***
# CHECK-NEXT: function: live_variables_update_block_split_split_def_before_si_end_cf_live_out
diff --git a/llvm/test/CodeGen/AMDGPU/lto-lower-module-lds.ll b/llvm/test/CodeGen/AMDGPU/lto-lower-module-lds.ll
index b81cc4ef3c9c0..6c553955652d2 100644
--- a/llvm/test/CodeGen/AMDGPU/lto-lower-module-lds.ll
+++ b/llvm/test/CodeGen/AMDGPU/lto-lower-module-lds.ll
@@ -25,13 +25,13 @@
; Unified O3
; RUN: llvm-lto2 run -unified-lto=full -O3 -cg-opt-level 3 %t.unified.bc -o %t.s -r %t.unified.bc,test,px -debug-pass-manager -debug-pass=Structure 2>&1 | FileCheck %s
-; First print will be from the New PM during the full LTO pipeline.
-; Second print will be from the legacy PM during the CG pipeline.
-; These will be updated when NPM becomes default for AMDGPU CG pipeline.
+; NPM is default for both full LTO and CG pipelines.
+; CHECK-NOT: ModulePass Manager
; CHECK: Running pass: AMDGPULowerModuleLDSPass on [module]
-; CHECK: ModulePass Manager
-; CHECK: Lower uses of LDS variables from non-kernel functions
+; CHECK: Running pass: SelectionDAGISelPass on test
+; CHECK: Running pass: PrologEpilogInserterPass on test
+; CHECK: Running pass: AMDGPUAsmPrinterPass on test
; Test -force-new-pm-codegen=true.
diff --git a/llvm/test/CodeGen/AMDGPU/machine-sink-cycle.mir b/llvm/test/CodeGen/AMDGPU/machine-sink-cycle.mir
index f228e579763f1..ed517470da28a 100644
--- a/llvm/test/CodeGen/AMDGPU/machine-sink-cycle.mir
+++ b/llvm/test/CodeGen/AMDGPU/machine-sink-cycle.mir
@@ -1,6 +1,6 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 3
-# RUN: llc -mtriple=amdgpu9.42-amd-amdhsa --sink-insts-to-avoid-spills=1 --stop-after=machine-sink -o - %s | FileCheck -check-prefixes=GFX9-SUNK %s
-# RUN: llc -mtriple=amdgpu10.31-amd-amdhsa --sink-insts-to-avoid-spills=1 -mattr=+wavefrontsize64 --stop-after=machine-sink -o - %s | FileCheck -check-prefixes=GFX10-SUNK %s
+# RUN: llc -mtriple=amdgpu9.42-amd-amdhsa --sink-insts-to-avoid-spills=1 --start-before=amdgpu-isel --stop-after=machine-sink -o - %s | FileCheck -check-prefixes=GFX9-SUNK %s
+# RUN: llc -mtriple=amdgpu10.31-amd-amdhsa --sink-insts-to-avoid-spills=1 -mattr=+wavefrontsize64 --start-before=amdgpu-isel --stop-after=machine-sink -o - %s | FileCheck -check-prefixes=GFX10-SUNK %s
---
name: test_sink_copy
diff --git a/llvm/test/CodeGen/AMDGPU/memcpy-crash-issue63986.ll b/llvm/test/CodeGen/AMDGPU/memcpy-crash-issue63986.ll
index d90c9771c3e4a..862b55d93179a 100644
--- a/llvm/test/CodeGen/AMDGPU/memcpy-crash-issue63986.ll
+++ b/llvm/test/CodeGen/AMDGPU/memcpy-crash-issue63986.ll
@@ -190,6 +190,7 @@ define void @issue63986_reduced_expanded(i64 %idxprom) {
; CHECK-NEXT: v_mov_b32_e32 v0, s4
; CHECK-NEXT: v_mov_b32_e32 v1, s5
; CHECK-NEXT: .LBB1_6: ; %loop-memcpy-residual
+; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: s_add_i32 s6, s8, 1
; CHECK-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[0:1]
; CHECK-NEXT: s_mov_b64 s[8:9], 1
@@ -205,6 +206,7 @@ define void @issue63986_reduced_expanded(i64 %idxprom) {
; CHECK-NEXT: s_and_b64 vcc, exec, 0
; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[2:5]
; CHECK-NEXT: .LBB1_9: ; %loop-memcpy-expansion2
+; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: s_mov_b64 vcc, vcc
; CHECK-NEXT: s_cbranch_vccz .LBB1_9
; CHECK-NEXT: ; %bb.10: ; %DummyReturnBlock
diff --git a/llvm/test/CodeGen/AMDGPU/memcpy_const_compare.ll b/llvm/test/CodeGen/AMDGPU/memcpy_const_compare.ll
index bb9011a3c43d7..b2373fd33fd54 100644
--- a/llvm/test/CodeGen/AMDGPU/memcpy_const_compare.ll
+++ b/llvm/test/CodeGen/AMDGPU/memcpy_const_compare.ll
@@ -40,26 +40,23 @@ define amdgpu_kernel void @_start() {
; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[2:5]
; CHECK-NEXT: s_cbranch_vccnz .LBB0_1
; CHECK-NEXT: ; %bb.2: ; %dynamic-memcpy-expansion-residual-cond
-; FIXME: Compare should be evaluated at compile time
; CHECK-NEXT: s_cmp_eq_u64 13, 0
; CHECK-NEXT: s_cbranch_scc1 .LBB0_5
; CHECK-NEXT: ; %bb.3: ; %dynamic-memcpy-expansion-residual-body.preheader
-; CHECK-NEXT: s_sub_u32 s4, 29, 13
-; CHECK-NEXT: s_subb_u32 s5, 0, 0
; CHECK-NEXT: s_getpc_b64 s[0:1]
; CHECK-NEXT: s_add_u32 s0, s0, src_array at gotpcrel32@lo+4
; CHECK-NEXT: s_addc_u32 s1, s1, src_array at gotpcrel32@hi+12
; CHECK-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: s_add_u32 s2, s0, s4
-; CHECK-NEXT: s_addc_u32 s3, s1, s5
+; CHECK-NEXT: s_add_u32 s2, s0, 16
+; CHECK-NEXT: s_addc_u32 s3, s1, 0
; CHECK-NEXT: s_getpc_b64 s[0:1]
; CHECK-NEXT: s_add_u32 s0, s0, dst_array at gotpcrel32@lo+4
; CHECK-NEXT: s_addc_u32 s1, s1, dst_array at gotpcrel32@hi+12
; CHECK-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: s_add_u32 s4, s0, s4
-; CHECK-NEXT: s_addc_u32 s5, s1, s5
+; CHECK-NEXT: s_add_u32 s4, s0, 16
+; CHECK-NEXT: s_addc_u32 s5, s1, 0
; CHECK-NEXT: s_mov_b64 s[0:1], 0
; CHECK-NEXT: .LBB0_4: ; %dynamic-memcpy-expansion-residual-body
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -79,6 +76,7 @@ define amdgpu_kernel void @_start() {
; CHECK-NEXT: s_cbranch_vccnz .LBB0_4
; CHECK-NEXT: .LBB0_5: ; %dynamic-memcpy-post-expansion
; CHECK-NEXT: s_endpgm
+; FIXME: Compare should be evaluated at compile time
%src_ptr = getelementptr inbounds [128 x i8], ptr @src_array, i64 0, i64 0
%dst_ptr = getelementptr inbounds [128 x i8], ptr @dst_array, i64 0, i64 0
call void @llvm.memcpy.p0.p0.i64(ptr %dst_ptr, ptr %src_ptr, i64 add (i64 sub (i64 16, i64 ptrtoint (ptr addrspacecast (ptr addrspace(4) null to ptr) to i64)), i64 13), i1 false)
diff --git a/llvm/test/CodeGen/AMDGPU/memmove-var-size.ll b/llvm/test/CodeGen/AMDGPU/memmove-var-size.ll
index f3698433cb85e..9d1072b06838b 100644
--- a/llvm/test/CodeGen/AMDGPU/memmove-var-size.ll
+++ b/llvm/test/CodeGen/AMDGPU/memmove-var-size.ll
@@ -1044,7 +1044,7 @@ entry:
define void @memmove_p1_p3(ptr addrspace(1) align 1 %dst, ptr addrspace(3) align 1 readonly %src, i64 %sz) {
; CHECK-LABEL: memmove_p1_p3:
-; CHECK: ; %bb.0: ; %entry
+; CHECK: ; %bb.0:
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CHECK-NEXT: v_mov_b32_e32 v5, v3
; CHECK-NEXT: v_mov_b32_e32 v6, 0
@@ -1099,7 +1099,6 @@ define void @memmove_p1_p3(ptr addrspace(1) align 1 %dst, ptr addrspace(3) align
; CHECK-NEXT: .LBB7_6: ; %Flow7
; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s6
; CHECK-NEXT: s_setpc_b64 s[30:31]
-entry:
tail call void @llvm.memmove.p1.p3.i64(ptr addrspace(1) noundef nonnull align 1 %dst, ptr addrspace(3) noundef nonnull align 1 %src, i64 %sz, i1 false)
ret void
}
diff --git a/llvm/test/CodeGen/AMDGPU/memset-param-combinations.ll b/llvm/test/CodeGen/AMDGPU/memset-param-combinations.ll
index e68570f5630e7..fe8b7f1a21fc1 100644
--- a/llvm/test/CodeGen/AMDGPU/memset-param-combinations.ll
+++ b/llvm/test/CodeGen/AMDGPU/memset-param-combinations.ll
@@ -59,50 +59,49 @@ define void @memset_p0_varsize_align_4_varsetval(ptr addrspace(0) align 4 %dst,
; GFX942-GISEL-LABEL: memset_p0_varsize_align_4_varsetval:
; GFX942-GISEL: ; %bb.0: ; %entry
; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], 15
-; GFX942-GISEL-NEXT: v_and_b32_e32 v10, 15, v3
-; GFX942-GISEL-NEXT: v_sub_co_u32_e32 v12, vcc, v3, v10
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], 15
+; GFX942-GISEL-NEXT: v_and_b32_e32 v8, 15, v3
+; GFX942-GISEL-NEXT: v_sub_co_u32_e32 v10, vcc, v3, v8
; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], 0
; GFX942-GISEL-NEXT: s_nop 0
-; GFX942-GISEL-NEXT: v_subbrev_co_u32_e32 v13, vcc, 0, v4, vcc
-; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[12:13]
+; GFX942-GISEL-NEXT: v_subbrev_co_u32_e32 v11, vcc, 0, v4, vcc
+; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-GISEL-NEXT: s_cbranch_execz .LBB0_3
; GFX942-GISEL-NEXT: ; %bb.1: ; %dynamic-memset-expansion-main-body.preheader
-; GFX942-GISEL-NEXT: v_and_b32_e32 v5, 0xff, v2
-; GFX942-GISEL-NEXT: v_lshl_or_b32 v6, v5, 8, v5
-; GFX942-GISEL-NEXT: v_lshlrev_b32_e32 v7, 16, v5
-; GFX942-GISEL-NEXT: v_lshlrev_b32_e32 v5, 24, v5
-; GFX942-GISEL-NEXT: v_or3_b32 v6, v6, v7, v5
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v7, v6
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v8, v6
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v9, v6
+; GFX942-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v2
+; GFX942-GISEL-NEXT: v_lshl_or_b32 v4, v3, 8, v3
+; GFX942-GISEL-NEXT: v_lshlrev_b32_e32 v5, 16, v3
+; GFX942-GISEL-NEXT: v_lshlrev_b32_e32 v3, 24, v3
+; GFX942-GISEL-NEXT: v_or3_b32 v4, v4, v5, v3
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, v4
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v6, v4
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v7, v4
; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], 0
; GFX942-GISEL-NEXT: .LBB0_2: ; %dynamic-memset-expansion-main-body
; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[0:1]
; GFX942-GISEL-NEXT: s_add_u32 s0, s0, 16
-; GFX942-GISEL-NEXT: v_add_co_u32_e32 v14, vcc, v0, v14
+; GFX942-GISEL-NEXT: v_add_co_u32_e32 v12, vcc, v0, v12
; GFX942-GISEL-NEXT: s_addc_u32 s1, s1, 0
; GFX942-GISEL-NEXT: s_nop 0
-; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v15, vcc, v1, v15, vcc
-; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[12:13]
+; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v13, vcc, v1, v13, vcc
+; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[10:11]
; GFX942-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX942-GISEL-NEXT: flat_store_dwordx4 v[14:15], v[6:9]
+; GFX942-GISEL-NEXT: flat_store_dwordx4 v[12:13], v[4:7]
; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX942-GISEL-NEXT: s_cbranch_execnz .LBB0_2
; GFX942-GISEL-NEXT: .LBB0_3: ; %Flow4
; GFX942-GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
+; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[8:9]
; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-GISEL-NEXT: s_cbranch_execz .LBB0_6
; GFX942-GISEL-NEXT: ; %bb.4: ; %dynamic-memset-expansion-residual-body.preheader
-; GFX942-GISEL-NEXT: v_and_b32_e32 v3, -16, v3
-; GFX942-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v0, v3
+; GFX942-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v0, v10
; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], 0
; GFX942-GISEL-NEXT: s_nop 0
-; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v4, vcc
+; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v11, vcc
; GFX942-GISEL-NEXT: .LBB0_5: ; %dynamic-memset-expansion-residual-body
; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
@@ -111,7 +110,7 @@ define void @memset_p0_varsize_align_4_varsetval(ptr addrspace(0) align 4 %dst,
; GFX942-GISEL-NEXT: s_addc_u32 s1, s1, 0
; GFX942-GISEL-NEXT: s_nop 0
; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v5, vcc, v1, v5, vcc
-; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[10:11]
+; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[8:9]
; GFX942-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX942-GISEL-NEXT: flat_store_byte v[4:5], v2
; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -181,50 +180,49 @@ define void @memset_p1_varsize_align_4_varsetval(ptr addrspace(1) align 4 %dst,
; GFX942-GISEL-LABEL: memset_p1_varsize_align_4_varsetval:
; GFX942-GISEL: ; %bb.0: ; %entry
; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], 15
-; GFX942-GISEL-NEXT: v_and_b32_e32 v10, 15, v3
-; GFX942-GISEL-NEXT: v_sub_co_u32_e32 v12, vcc, v3, v10
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], 15
+; GFX942-GISEL-NEXT: v_and_b32_e32 v8, 15, v3
+; GFX942-GISEL-NEXT: v_sub_co_u32_e32 v10, vcc, v3, v8
; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], 0
; GFX942-GISEL-NEXT: s_nop 0
-; GFX942-GISEL-NEXT: v_subbrev_co_u32_e32 v13, vcc, 0, v4, vcc
-; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[12:13]
+; GFX942-GISEL-NEXT: v_subbrev_co_u32_e32 v11, vcc, 0, v4, vcc
+; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-GISEL-NEXT: s_cbranch_execz .LBB1_3
; GFX942-GISEL-NEXT: ; %bb.1: ; %dynamic-memset-expansion-main-body.preheader
-; GFX942-GISEL-NEXT: v_and_b32_e32 v5, 0xff, v2
-; GFX942-GISEL-NEXT: v_lshl_or_b32 v6, v5, 8, v5
-; GFX942-GISEL-NEXT: v_lshlrev_b32_e32 v7, 16, v5
-; GFX942-GISEL-NEXT: v_lshlrev_b32_e32 v5, 24, v5
-; GFX942-GISEL-NEXT: v_or3_b32 v6, v6, v7, v5
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v7, v6
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v8, v6
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v9, v6
+; GFX942-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v2
+; GFX942-GISEL-NEXT: v_lshl_or_b32 v4, v3, 8, v3
+; GFX942-GISEL-NEXT: v_lshlrev_b32_e32 v5, 16, v3
+; GFX942-GISEL-NEXT: v_lshlrev_b32_e32 v3, 24, v3
+; GFX942-GISEL-NEXT: v_or3_b32 v4, v4, v5, v3
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, v4
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v6, v4
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v7, v4
; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], 0
; GFX942-GISEL-NEXT: .LBB1_2: ; %dynamic-memset-expansion-main-body
; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[0:1]
; GFX942-GISEL-NEXT: s_add_u32 s0, s0, 16
-; GFX942-GISEL-NEXT: v_add_co_u32_e32 v14, vcc, v0, v14
+; GFX942-GISEL-NEXT: v_add_co_u32_e32 v12, vcc, v0, v12
; GFX942-GISEL-NEXT: s_addc_u32 s1, s1, 0
; GFX942-GISEL-NEXT: s_nop 0
-; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v15, vcc, v1, v15, vcc
-; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[12:13]
+; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v13, vcc, v1, v13, vcc
+; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[10:11]
; GFX942-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX942-GISEL-NEXT: global_store_dwordx4 v[14:15], v[6:9], off
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[12:13], v[4:7], off
; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX942-GISEL-NEXT: s_cbranch_execnz .LBB1_2
; GFX942-GISEL-NEXT: .LBB1_3: ; %Flow4
; GFX942-GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
+; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[8:9]
; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-GISEL-NEXT: s_cbranch_execz .LBB1_6
; GFX942-GISEL-NEXT: ; %bb.4: ; %dynamic-memset-expansion-residual-body.preheader
-; GFX942-GISEL-NEXT: v_and_b32_e32 v3, -16, v3
-; GFX942-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v0, v3
+; GFX942-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v0, v10
; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], 0
; GFX942-GISEL-NEXT: s_nop 0
-; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v4, vcc
+; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v11, vcc
; GFX942-GISEL-NEXT: .LBB1_5: ; %dynamic-memset-expansion-residual-body
; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
@@ -233,7 +231,7 @@ define void @memset_p1_varsize_align_4_varsetval(ptr addrspace(1) align 4 %dst,
; GFX942-GISEL-NEXT: s_addc_u32 s1, s1, 0
; GFX942-GISEL-NEXT: s_nop 0
; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v5, vcc, v1, v5, vcc
-; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[10:11]
+; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[8:9]
; GFX942-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX942-GISEL-NEXT: global_store_byte v[4:5], v2, off
; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -1406,13 +1404,13 @@ define void @memset_p1_varsz_align_4_set40(ptr addrspace(1) align 4 %dst, i64 %s
; GFX942-GISEL-LABEL: memset_p1_varsz_align_4_set40:
; GFX942-GISEL: ; %bb.0: ; %entry
; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], 15
-; GFX942-GISEL-NEXT: v_and_b32_e32 v8, 15, v2
-; GFX942-GISEL-NEXT: v_sub_co_u32_e32 v10, vcc, v2, v8
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], 15
+; GFX942-GISEL-NEXT: v_and_b32_e32 v6, 15, v2
+; GFX942-GISEL-NEXT: v_sub_co_u32_e32 v8, vcc, v2, v6
; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], 0
; GFX942-GISEL-NEXT: s_nop 0
-; GFX942-GISEL-NEXT: v_subbrev_co_u32_e32 v11, vcc, 0, v3, vcc
-; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
+; GFX942-GISEL-NEXT: v_subbrev_co_u32_e32 v9, vcc, 0, v3, vcc
+; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[8:9]
; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-GISEL-NEXT: s_cbranch_execz .LBB12_3
; GFX942-GISEL-NEXT: ; %bb.1: ; %dynamic-memset-expansion-main-body.preheader
@@ -1420,34 +1418,33 @@ define void @memset_p1_varsz_align_4_set40(ptr addrspace(1) align 4 %dst, i64 %s
; GFX942-GISEL-NEXT: s_mov_b32 s5, s4
; GFX942-GISEL-NEXT: s_mov_b32 s6, s4
; GFX942-GISEL-NEXT: s_mov_b32 s7, s4
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[4:5]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[6:7]
; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], 0
; GFX942-GISEL-NEXT: .LBB12_2: ; %dynamic-memset-expansion-main-body
; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[0:1]
; GFX942-GISEL-NEXT: s_add_u32 s0, s0, 16
-; GFX942-GISEL-NEXT: v_add_co_u32_e32 v12, vcc, v0, v12
+; GFX942-GISEL-NEXT: v_add_co_u32_e32 v10, vcc, v0, v10
; GFX942-GISEL-NEXT: s_addc_u32 s1, s1, 0
; GFX942-GISEL-NEXT: s_nop 0
-; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v13, vcc, v1, v13, vcc
-; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[10:11]
+; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v11, vcc, v1, v11, vcc
+; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[8:9]
; GFX942-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX942-GISEL-NEXT: global_store_dwordx4 v[12:13], v[4:7], off
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[10:11], v[2:5], off
; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX942-GISEL-NEXT: s_cbranch_execnz .LBB12_2
; GFX942-GISEL-NEXT: .LBB12_3: ; %Flow4
; GFX942-GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[8:9]
+; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-GISEL-NEXT: s_cbranch_execz .LBB12_6
; GFX942-GISEL-NEXT: ; %bb.4: ; %dynamic-memset-expansion-residual-body.preheader
-; GFX942-GISEL-NEXT: v_and_b32_e32 v2, -16, v2
-; GFX942-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v0, v2
+; GFX942-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v0, v8
; GFX942-GISEL-NEXT: v_mov_b32_e32 v2, 40
; GFX942-GISEL-NEXT: s_nop 0
-; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc
+; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v9, vcc
; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], 0
; GFX942-GISEL-NEXT: .LBB12_5: ; %dynamic-memset-expansion-residual-body
; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -1457,7 +1454,7 @@ define void @memset_p1_varsz_align_4_set40(ptr addrspace(1) align 4 %dst, i64 %s
; GFX942-GISEL-NEXT: s_addc_u32 s1, s1, 0
; GFX942-GISEL-NEXT: s_nop 0
; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v5, vcc, v1, v5, vcc
-; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[8:9]
+; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[6:7]
; GFX942-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX942-GISEL-NEXT: global_store_byte v[4:5], v2, off
; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -1526,13 +1523,13 @@ define void @memset_p1_varsz_align_4_set0(ptr addrspace(1) align 4 %dst, i64 %si
; GFX942-GISEL-LABEL: memset_p1_varsz_align_4_set0:
; GFX942-GISEL: ; %bb.0: ; %entry
; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], 15
-; GFX942-GISEL-NEXT: v_and_b32_e32 v8, 15, v2
-; GFX942-GISEL-NEXT: v_sub_co_u32_e32 v10, vcc, v2, v8
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], 15
+; GFX942-GISEL-NEXT: v_and_b32_e32 v6, 15, v2
+; GFX942-GISEL-NEXT: v_sub_co_u32_e32 v8, vcc, v2, v6
; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], 0
; GFX942-GISEL-NEXT: s_nop 0
-; GFX942-GISEL-NEXT: v_subbrev_co_u32_e32 v11, vcc, 0, v3, vcc
-; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
+; GFX942-GISEL-NEXT: v_subbrev_co_u32_e32 v9, vcc, 0, v3, vcc
+; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[8:9]
; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-GISEL-NEXT: s_cbranch_execz .LBB13_3
; GFX942-GISEL-NEXT: ; %bb.1: ; %dynamic-memset-expansion-main-body.preheader
@@ -1540,34 +1537,33 @@ define void @memset_p1_varsz_align_4_set0(ptr addrspace(1) align 4 %dst, i64 %si
; GFX942-GISEL-NEXT: s_mov_b32 s5, s4
; GFX942-GISEL-NEXT: s_mov_b32 s6, s4
; GFX942-GISEL-NEXT: s_mov_b32 s7, s4
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[4:5]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[6:7]
; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], 0
; GFX942-GISEL-NEXT: .LBB13_2: ; %dynamic-memset-expansion-main-body
; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[0:1]
; GFX942-GISEL-NEXT: s_add_u32 s0, s0, 16
-; GFX942-GISEL-NEXT: v_add_co_u32_e32 v12, vcc, v0, v12
+; GFX942-GISEL-NEXT: v_add_co_u32_e32 v10, vcc, v0, v10
; GFX942-GISEL-NEXT: s_addc_u32 s1, s1, 0
; GFX942-GISEL-NEXT: s_nop 0
-; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v13, vcc, v1, v13, vcc
-; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[10:11]
+; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v11, vcc, v1, v11, vcc
+; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[8:9]
; GFX942-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX942-GISEL-NEXT: global_store_dwordx4 v[12:13], v[4:7], off
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[10:11], v[2:5], off
; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX942-GISEL-NEXT: s_cbranch_execnz .LBB13_2
; GFX942-GISEL-NEXT: .LBB13_3: ; %Flow4
; GFX942-GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[8:9]
+; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-GISEL-NEXT: s_cbranch_execz .LBB13_6
; GFX942-GISEL-NEXT: ; %bb.4: ; %dynamic-memset-expansion-residual-body.preheader
-; GFX942-GISEL-NEXT: v_and_b32_e32 v2, -16, v2
-; GFX942-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v0, v2
+; GFX942-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v0, v8
; GFX942-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX942-GISEL-NEXT: s_nop 0
-; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc
+; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v9, vcc
; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], 0
; GFX942-GISEL-NEXT: .LBB13_5: ; %dynamic-memset-expansion-residual-body
; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -1577,7 +1573,7 @@ define void @memset_p1_varsz_align_4_set0(ptr addrspace(1) align 4 %dst, i64 %si
; GFX942-GISEL-NEXT: s_addc_u32 s1, s1, 0
; GFX942-GISEL-NEXT: s_nop 0
; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v5, vcc, v1, v5, vcc
-; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[8:9]
+; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[6:7]
; GFX942-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX942-GISEL-NEXT: global_store_byte v[4:5], v2, off
; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
diff --git a/llvm/test/CodeGen/AMDGPU/mfma-no-register-aliasing.ll b/llvm/test/CodeGen/AMDGPU/mfma-no-register-aliasing.ll
index 7f787568b5a0f..42a1e18fa1223 100644
--- a/llvm/test/CodeGen/AMDGPU/mfma-no-register-aliasing.ll
+++ b/llvm/test/CodeGen/AMDGPU/mfma-no-register-aliasing.ll
@@ -4,7 +4,7 @@
; RUN: llc -mtriple=amdgpu9.0a -early-live-intervals < %s | FileCheck -enable-var-scope --check-prefixes=GREEDY90A %s
; RUN: llc -mtriple=amdgpu9.42 < %s | FileCheck -enable-var-scope --check-prefixes=GREEDY942 %s
; RUN: llc -global-isel -mtriple=amdgpu9.0a < %s | FileCheck -enable-var-scope --check-prefixes=GREEDY90A-GISEL %s
-; RUN: llc -mtriple=amdgpu9.0a -sgpr-regalloc=fast -vgpr-regalloc=fast -wwm-regalloc=fast < %s | FileCheck -enable-var-scope --check-prefixes=FAST90A %s
+; RUN: llc -mtriple=amdgpu9.0a -sgpr-regalloc-npm=fast -vgpr-regalloc-npm=fast -wwm-regalloc-npm=fast < %s | FileCheck -enable-var-scope --check-prefixes=FAST90A %s
; This is better with 90a
diff --git a/llvm/test/CodeGen/AMDGPU/opt-exec-masking-pre-ra-update-liveness-wave32.mir b/llvm/test/CodeGen/AMDGPU/opt-exec-masking-pre-ra-update-liveness-wave32.mir
index 09d9e90e3d710..bbee2c4f107a1 100644
--- a/llvm/test/CodeGen/AMDGPU/opt-exec-masking-pre-ra-update-liveness-wave32.mir
+++ b/llvm/test/CodeGen/AMDGPU/opt-exec-masking-pre-ra-update-liveness-wave32.mir
@@ -1,6 +1,6 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
# REQUIRES: asserts
-# RUN: llc -mtriple=amdgpu10.10-amd-amdhsa -verify-regalloc -misched-only-block=999 -start-before=machine-scheduler -stop-after=greedy,0 -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu10.10-amd-amdhsa -verify-regalloc -misched-only-block=999 -start-before=machine-scheduler -stop-after=greedy,1 -o - %s | FileCheck %s
# This run line is a total hack to get the live intervals to make it
# to the verifier. This requires asserts to use
diff --git a/llvm/test/CodeGen/AMDGPU/opt-exec-masking-pre-ra-update-liveness.mir b/llvm/test/CodeGen/AMDGPU/opt-exec-masking-pre-ra-update-liveness.mir
index 4d32e22218cfc..3a416cd43fce6 100644
--- a/llvm/test/CodeGen/AMDGPU/opt-exec-masking-pre-ra-update-liveness.mir
+++ b/llvm/test/CodeGen/AMDGPU/opt-exec-masking-pre-ra-update-liveness.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -verify-regalloc -start-before=machine-scheduler -stop-after=greedy,0 -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -verify-regalloc -start-before=machine-scheduler -stop-after=greedy,1 -o - %s | FileCheck %s
# Make sure liveness is correctly updated when folding the cndmask and
# compare.
diff --git a/llvm/test/CodeGen/AMDGPU/optimize-exec-mask-pre-ra-alloc-failure.mir b/llvm/test/CodeGen/AMDGPU/optimize-exec-mask-pre-ra-alloc-failure.mir
index df83b74042ad2..91a9219548dc9 100644
--- a/llvm/test/CodeGen/AMDGPU/optimize-exec-mask-pre-ra-alloc-failure.mir
+++ b/llvm/test/CodeGen/AMDGPU/optimize-exec-mask-pre-ra-alloc-failure.mir
@@ -1,4 +1,4 @@
-# RUN: llc -mtriple=amdgpu10.30-amd-amdhsa -start-before=machine-scheduler -stop-after=virtregrewriter,0 -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu10.30-amd-amdhsa -start-before=machine-scheduler -stop-after=virt-reg-rewriter,1 -o - %s | FileCheck %s
---
# If optimize-exec-mask-pre-ra over approximates live intervals (not replicating splits)
diff --git a/llvm/test/CodeGen/AMDGPU/partial-regcopy-and-spill-missed-at-regalloc.ll b/llvm/test/CodeGen/AMDGPU/partial-regcopy-and-spill-missed-at-regalloc.ll
index 9431658b0e6b5..b9bd46bc0ae9f 100644
--- a/llvm/test/CodeGen/AMDGPU/partial-regcopy-and-spill-missed-at-regalloc.ll
+++ b/llvm/test/CodeGen/AMDGPU/partial-regcopy-and-spill-missed-at-regalloc.ll
@@ -1,7 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-;RUN: llc -mtriple=amdgpu9.08-amd-amdhsa --stop-after=greedy,2 < %s | FileCheck -check-prefix=REGALLOC-GFX908 %s
+;RUN: llc -mtriple=amdgpu9.08-amd-amdhsa --stop-after=greedy,3 < %s | FileCheck -check-prefix=REGALLOC-GFX908 %s
;RUN: llc -mtriple=amdgpu9.08-amd-amdhsa --stop-after=prolog-epilog < %s | FileCheck -check-prefix=PEI-GFX908 %s
-;RUN: llc -mtriple=amdgpu9.0a-amd-amdhsa -amdgpu-mfma-vgpr-form=0 --stop-after=greedy,2 < %s | FileCheck -check-prefix=REGALLOC-GFX90A %s
+;RUN: llc -mtriple=amdgpu9.0a-amd-amdhsa -amdgpu-mfma-vgpr-form=0 --stop-after=greedy,3 < %s | FileCheck -check-prefix=REGALLOC-GFX90A %s
;RUN: llc -mtriple=amdgpu9.0a-amd-amdhsa -amdgpu-mfma-vgpr-form=0 --stop-after=prolog-epilog < %s | FileCheck -check-prefix=PEI-GFX90A %s
; Partial reg copy and spill missed during regalloc handled later at frame lowering.
diff --git a/llvm/test/CodeGen/AMDGPU/pr51516.mir b/llvm/test/CodeGen/AMDGPU/pr51516.mir
index 033656354f2c1..404e4cdc13340 100644
--- a/llvm/test/CodeGen/AMDGPU/pr51516.mir
+++ b/llvm/test/CodeGen/AMDGPU/pr51516.mir
@@ -1,6 +1,6 @@
# NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-# RUN: llc -mtriple=amdgpu9.00--amdhsa -amdgpu-disable-unclustered-high-rp-reschedule -verify-misched -start-before=machine-scheduler -stop-after=virtregrewriter,2 -o - %s | FileCheck -check-prefix=GCN %s
-# RUN: llc -mtriple=amdgpu9.00--amdhsa -amdgpu-disable-unclustered-high-rp-reschedule -amdgpu-use-amdgpu-trackers=1 -verify-misched -start-before=machine-scheduler -stop-after=virtregrewriter,2 -o - %s | FileCheck -check-prefix=GCN-GCNTRACKER %s
+# RUN: llc -mtriple=amdgpu9.00--amdhsa -amdgpu-disable-unclustered-high-rp-reschedule -verify-misched -start-before=machine-scheduler -stop-after=virt-reg-rewriter,3 -o - %s | FileCheck -check-prefix=GCN %s
+# RUN: llc -mtriple=amdgpu9.00--amdhsa -amdgpu-disable-unclustered-high-rp-reschedule -amdgpu-use-amdgpu-trackers=1 -verify-misched -start-before=machine-scheduler -stop-after=virt-reg-rewriter,3 -o - %s | FileCheck -check-prefix=GCN-GCNTRACKER %s
# Check that %3 was not rematerialized before the last store since its operand %1
# is killed by that store.
diff --git a/llvm/test/CodeGen/AMDGPU/ran-out-of-registers-error-all-regs-reserved.ll b/llvm/test/CodeGen/AMDGPU/ran-out-of-registers-error-all-regs-reserved.ll
index 0825688cb03c0..c6624a1dbf93e 100644
--- a/llvm/test/CodeGen/AMDGPU/ran-out-of-registers-error-all-regs-reserved.ll
+++ b/llvm/test/CodeGen/AMDGPU/ran-out-of-registers-error-all-regs-reserved.ll
@@ -1,6 +1,6 @@
-; RUN: not llc -mtriple=amdgpu9.08-amd-amdhsa -vgpr-regalloc=greedy -filetype=null %s 2>&1 | FileCheck -implicit-check-not=error %s
-; RUN: not llc -mtriple=amdgpu9.08-amd-amdhsa -vgpr-regalloc=basic -filetype=null %s 2>&1 | FileCheck -implicit-check-not=error %s
-; RUN: not llc -mtriple=amdgpu9.08-amd-amdhsa -vgpr-regalloc=fast -filetype=null %s 2>&1 | FileCheck -implicit-check-not=error %s
+; RUN: not llc -mtriple=amdgpu9.08-amd-amdhsa -vgpr-regalloc-npm=greedy -filetype=null %s 2>&1 | FileCheck -implicit-check-not=error %s
+; RUN: not llc -enable-new-pm=0 -mtriple=amdgpu9.08-amd-amdhsa -vgpr-regalloc=basic -filetype=null %s 2>&1 | FileCheck -implicit-check-not=error %s
+; RUN: not llc -mtriple=amdgpu9.08-amd-amdhsa -vgpr-regalloc-npm=fast -filetype=null %s 2>&1 | FileCheck -implicit-check-not=error %s
declare <32 x i32> @llvm.amdgcn.mfma.i32.32x32x4i8(i32, i32, <32 x i32>, i32 immarg, i32 immarg, i32 immarg)
diff --git a/llvm/test/CodeGen/AMDGPU/ran-out-of-registers-errors.ll b/llvm/test/CodeGen/AMDGPU/ran-out-of-registers-errors.ll
index 11b5363aa7ca1..dc2f3fb6bd447 100644
--- a/llvm/test/CodeGen/AMDGPU/ran-out-of-registers-errors.ll
+++ b/llvm/test/CodeGen/AMDGPU/ran-out-of-registers-errors.ll
@@ -1,9 +1,9 @@
-; RUN: not llc -mtriple=amdgpu9.08-amd-amdhsa -stress-regalloc=1 -vgpr-regalloc=greedy -filetype=null %s 2>&1 | FileCheck -check-prefixes=CHECK,GREEDY -implicit-check-not=error %s
-; RUN: not llc -mtriple=amdgpu9.08-amd-amdhsa -stress-regalloc=1 -vgpr-regalloc=basic -filetype=null %s 2>&1 | FileCheck -implicit-check-not=error -check-prefixes=CHECK,BASIC %s
-; RUN: not llc -mtriple=amdgpu9.08-amd-amdhsa -stress-regalloc=1 -vgpr-regalloc=fast -filetype=null %s 2>&1 | FileCheck -implicit-check-not=error -check-prefixes=CHECK,FAST %s
+; RUN: not llc -mtriple=amdgpu9.08-amd-amdhsa -stress-regalloc=1 -vgpr-regalloc-npm=greedy -filetype=null %s 2>&1 | FileCheck -check-prefixes=CHECK,GREEDY -implicit-check-not=error %s
+; RUN: not llc -enable-new-pm=0 -mtriple=amdgpu9.08-amd-amdhsa -stress-regalloc=1 -vgpr-regalloc=basic -filetype=null %s 2>&1 | FileCheck -implicit-check-not=error -check-prefixes=CHECK,BASIC %s
+; RUN: not llc -mtriple=amdgpu9.08-amd-amdhsa -stress-regalloc=1 -vgpr-regalloc-npm=fast -filetype=null %s 2>&1 | FileCheck -implicit-check-not=error -check-prefixes=CHECK,FAST %s
; RUN: opt -passes=debugify -o %t.bc %s
-; RUN: not llc -mtriple=amdgpu9.08-amd-amdhsa -stress-regalloc=1 -vgpr-regalloc=greedy -filetype=null %t.bc 2>&1 | FileCheck -implicit-check-not=error -check-prefixes=DBGINFO-CHECK,DBGINFO-GREEDY %s
-; RUN: not llc -mtriple=amdgpu9.08-amd-amdhsa -stress-regalloc=1 -vgpr-regalloc=basic -filetype=null %t.bc 2>&1 | FileCheck -implicit-check-not=error -check-prefixes=DBGINFO-CHECK,DBGINFO-BASIC %s
+; RUN: not llc -mtriple=amdgpu9.08-amd-amdhsa -stress-regalloc=1 -vgpr-regalloc-npm=greedy -filetype=null %t.bc 2>&1 | FileCheck -implicit-check-not=error -check-prefixes=DBGINFO-CHECK,DBGINFO-GREEDY %s
+; RUN: not llc -enable-new-pm=0 -mtriple=amdgpu9.08-amd-amdhsa -stress-regalloc=1 -vgpr-regalloc=basic -filetype=null %t.bc 2>&1 | FileCheck -implicit-check-not=error -check-prefixes=DBGINFO-CHECK,DBGINFO-BASIC %s
; FIXME: Asserts when using -O2 + -vgpr-regalloc=fast
; RUN: not llc -mtriple=amdgpu9.08-amd-amdhsa -stress-regalloc=1 -O0 -filetype=null %t.bc 2>&1 | FileCheck -implicit-check-not=error -check-prefixes=DBGINFO-CHECK,DBGINFO-FAST %s
diff --git a/llvm/test/CodeGen/AMDGPU/ran-out-of-sgprs-allocation-failure.mir b/llvm/test/CodeGen/AMDGPU/ran-out-of-sgprs-allocation-failure.mir
index 794d96dfa2365..7784f2cd6dc1a 100644
--- a/llvm/test/CodeGen/AMDGPU/ran-out-of-sgprs-allocation-failure.mir
+++ b/llvm/test/CodeGen/AMDGPU/ran-out-of-sgprs-allocation-failure.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu9.0a-amd-amdhsa -verify-regalloc -start-before=greedy,0 -stop-after=virtregrewriter,0 -greedy-regclass-priority-trumps-globalness=1 -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.0a-amd-amdhsa -verify-regalloc -start-before=greedy,1 -stop-after=virt-reg-rewriter,1 -greedy-regclass-priority-trumps-globalness=1 -o - %s | FileCheck %s
# The allocation would previously fail due to poor ordering based on
# register class. The super wide tuples should be allocated first so
diff --git a/llvm/test/CodeGen/AMDGPU/recursive-resource-usage-mcexpr.ll b/llvm/test/CodeGen/AMDGPU/recursive-resource-usage-mcexpr.ll
index b26e2784fab17..329b3d4d2ceab 100644
--- a/llvm/test/CodeGen/AMDGPU/recursive-resource-usage-mcexpr.ll
+++ b/llvm/test/CodeGen/AMDGPU/recursive-resource-usage-mcexpr.ll
@@ -2,27 +2,19 @@
; Recursion: foo -> bar -> baz -> qux -> foo
-; CHECK-LABEL: {{^}}qux
-; CHECK: .set .Lqux.num_vgpr, max(71, .Lfoo.num_vgpr)
-; CHECK: .set .Lqux.num_agpr, max(0, .Lfoo.num_agpr)
-; CHECK: .set .Lqux.numbered_sgpr, max(46, .Lfoo.numbered_sgpr)
-; CHECK: .set .Lqux.private_seg_size, 16
-; CHECK: .set .Lqux.uses_vcc, or(1, .Lfoo.uses_vcc)
-; CHECK: .set .Lqux.uses_flat_scratch, or(0, .Lfoo.uses_flat_scratch)
-; CHECK: .set .Lqux.has_dyn_sized_stack, or(0, .Lfoo.has_dyn_sized_stack)
-; CHECK: .set .Lqux.has_recursion, or(1, .Lfoo.has_recursion)
-; CHECK: .set .Lqux.has_indirect_call, or(0, .Lfoo.has_indirect_call)
-
-; CHECK-LABEL: {{^}}baz
-; CHECK: .set .Lbaz.num_vgpr, max(61, .Lqux.num_vgpr)
-; CHECK: .set .Lbaz.num_agpr, max(0, .Lqux.num_agpr)
-; CHECK: .set .Lbaz.numbered_sgpr, max(51, .Lqux.numbered_sgpr)
-; CHECK: .set .Lbaz.private_seg_size, 16+max(.Lqux.private_seg_size)
-; CHECK: .set .Lbaz.uses_vcc, or(1, .Lqux.uses_vcc)
-; CHECK: .set .Lbaz.uses_flat_scratch, or(0, .Lqux.uses_flat_scratch)
-; CHECK: .set .Lbaz.has_dyn_sized_stack, or(0, .Lqux.has_dyn_sized_stack)
-; CHECK: .set .Lbaz.has_recursion, or(1, .Lqux.has_recursion)
-; CHECK: .set .Lbaz.has_indirect_call, or(0, .Lqux.has_indirect_call)
+; The new pass manager emits an SCC in reverse order compared to the legacy
+; pass manager: the entry (foo) is emitted first and refers to its successors
+; symbolically, while the last-emitted member (qux) is the fully-resolved base.
+; CHECK-LABEL: {{^}}foo
+; CHECK: .set .Lfoo.num_vgpr, max(46, .Lbar.num_vgpr)
+; CHECK: .set .Lfoo.num_agpr, max(0, .Lbar.num_agpr)
+; CHECK: .set .Lfoo.numbered_sgpr, max(71, .Lbar.numbered_sgpr)
+; CHECK: .set .Lfoo.private_seg_size, 16+max(.Lbar.private_seg_size)
+; CHECK: .set .Lfoo.uses_vcc, or(1, .Lbar.uses_vcc)
+; CHECK: .set .Lfoo.uses_flat_scratch, or(0, .Lbar.uses_flat_scratch)
+; CHECK: .set .Lfoo.has_dyn_sized_stack, or(0, .Lbar.has_dyn_sized_stack)
+; CHECK: .set .Lfoo.has_recursion, or(1, .Lbar.has_recursion)
+; CHECK: .set .Lfoo.has_indirect_call, or(0, .Lbar.has_indirect_call)
; CHECK-LABEL: {{^}}bar
; CHECK: .set .Lbar.num_vgpr, max(51, .Lbaz.num_vgpr)
@@ -35,16 +27,27 @@
; CHECK: .set .Lbar.has_recursion, or(1, .Lbaz.has_recursion)
; CHECK: .set .Lbar.has_indirect_call, or(0, .Lbaz.has_indirect_call)
-; CHECK-LABEL: {{^}}foo
-; CHECK: .set .Lfoo.num_vgpr, max(46, 71)
-; CHECK: .set .Lfoo.num_agpr, max(0, 0)
-; CHECK: .set .Lfoo.numbered_sgpr, max(71, 61)
-; CHECK: .set .Lfoo.private_seg_size, 16
-; CHECK: .set .Lfoo.uses_vcc, 1
-; CHECK: .set .Lfoo.uses_flat_scratch, 0
-; CHECK: .set .Lfoo.has_dyn_sized_stack, 0
-; CHECK: .set .Lfoo.has_recursion, 1
-; CHECK: .set .Lfoo.has_indirect_call, 0
+; CHECK-LABEL: {{^}}baz
+; CHECK: .set .Lbaz.num_vgpr, max(61, .Lqux.num_vgpr)
+; CHECK: .set .Lbaz.num_agpr, max(0, .Lqux.num_agpr)
+; CHECK: .set .Lbaz.numbered_sgpr, max(51, .Lqux.numbered_sgpr)
+; CHECK: .set .Lbaz.private_seg_size, 16+max(.Lqux.private_seg_size)
+; CHECK: .set .Lbaz.uses_vcc, or(1, .Lqux.uses_vcc)
+; CHECK: .set .Lbaz.uses_flat_scratch, or(0, .Lqux.uses_flat_scratch)
+; CHECK: .set .Lbaz.has_dyn_sized_stack, or(0, .Lqux.has_dyn_sized_stack)
+; CHECK: .set .Lbaz.has_recursion, or(1, .Lqux.has_recursion)
+; CHECK: .set .Lbaz.has_indirect_call, or(0, .Lqux.has_indirect_call)
+
+; CHECK-LABEL: {{^}}qux
+; CHECK: .set .Lqux.num_vgpr, max(71, 61)
+; CHECK: .set .Lqux.num_agpr, max(0, 0)
+; CHECK: .set .Lqux.numbered_sgpr, max(71, 71)
+; CHECK: .set .Lqux.private_seg_size, 16
+; CHECK: .set .Lqux.uses_vcc, 1
+; CHECK: .set .Lqux.uses_flat_scratch, 0
+; CHECK: .set .Lqux.has_dyn_sized_stack, 0
+; CHECK: .set .Lqux.has_recursion, 1
+; CHECK: .set .Lqux.has_indirect_call, 0
define void @foo() {
entry:
@@ -95,32 +98,21 @@ define amdgpu_kernel void @usefoo() {
; Recursion: A -> B -> C -> A && C -> D -> C
-; CHECK-LABEL: {{^}}D
-; CHECK: .set .LD.num_vgpr, max(71, .LC.num_vgpr)
-; CHECK: .set .LD.num_agpr, max(0, .LC.num_agpr)
-; CHECK: .set .LD.numbered_sgpr, max(71, .LC.numbered_sgpr)
-; CHECK: .set .LD.private_seg_size, 16+max(.LC.private_seg_size)
-; CHECK: .set .LD.uses_vcc, or(1, .LC.uses_vcc)
-; CHECK: .set .LD.uses_flat_scratch, or(0, .LC.uses_flat_scratch)
-; CHECK: .set .LD.has_dyn_sized_stack, or(0, .LC.has_dyn_sized_stack)
-; CHECK: .set .LD.has_recursion, or(1, .LC.has_recursion)
-; CHECK: .set .LD.has_indirect_call, or(0, .LC.has_indirect_call)
-
-; CHECK-LABEL: {{^}}C
-; CHECK: .set .LC.num_vgpr, max(42, .LA.num_vgpr, 71)
-; CHECK: .set .LC.num_agpr, max(0, .LA.num_agpr, 0)
-; CHECK: .set .LC.numbered_sgpr, max(71, .LA.numbered_sgpr, 71)
-; CHECK: .set .LC.private_seg_size, 16+max(.LA.private_seg_size)
-; CHECK: .set .LC.uses_vcc, or(1, .LA.uses_vcc)
-; CHECK: .set .LC.uses_flat_scratch, or(0, .LA.uses_flat_scratch)
-; CHECK: .set .LC.has_dyn_sized_stack, or(0, .LA.has_dyn_sized_stack)
-; CHECK: .set .LC.has_recursion, or(1, .LA.has_recursion)
-; CHECK: .set .LC.has_indirect_call, or(0, .LA.has_indirect_call)
+; CHECK-LABEL: {{^}}A
+; CHECK: .set .LA.num_vgpr, max(41, .LB.num_vgpr)
+; CHECK: .set .LA.num_agpr, max(0, .LB.num_agpr)
+; CHECK: .set .LA.numbered_sgpr, max(51, .LB.numbered_sgpr)
+; CHECK: .set .LA.private_seg_size, 16+max(.LB.private_seg_size)
+; CHECK: .set .LA.uses_vcc, or(1, .LB.uses_vcc)
+; CHECK: .set .LA.uses_flat_scratch, or(0, .LB.uses_flat_scratch)
+; CHECK: .set .LA.has_dyn_sized_stack, or(0, .LB.has_dyn_sized_stack)
+; CHECK: .set .LA.has_recursion, or(1, .LB.has_recursion)
+; CHECK: .set .LA.has_indirect_call, or(0, .LB.has_indirect_call)
; CHECK-LABEL: {{^}}B
-; CHECK: .set .LB.num_vgpr, max(42, .LC.num_vgpr)
+; CHECK: .set .LB.num_vgpr, max(41, .LC.num_vgpr)
; CHECK: .set .LB.num_agpr, max(0, .LC.num_agpr)
-; CHECK: .set .LB.numbered_sgpr, max(71, .LC.numbered_sgpr)
+; CHECK: .set .LB.numbered_sgpr, max(34, .LC.numbered_sgpr)
; CHECK: .set .LB.private_seg_size, 16+max(.LC.private_seg_size)
; CHECK: .set .LB.uses_vcc, or(1, .LC.uses_vcc)
; CHECK: .set .LB.uses_flat_scratch, or(0, .LC.uses_flat_scratch)
@@ -128,16 +120,27 @@ define amdgpu_kernel void @usefoo() {
; CHECK: .set .LB.has_recursion, or(1, .LC.has_recursion)
; CHECK: .set .LB.has_indirect_call, or(0, .LC.has_indirect_call)
-; CHECK-LABEL: {{^}}A
-; CHECK: .set .LA.num_vgpr, max(42, 71)
-; CHECK: .set .LA.num_agpr, max(0, 0)
-; CHECK: .set .LA.numbered_sgpr, max(71, 71)
-; CHECK: .set .LA.private_seg_size, 16
-; CHECK: .set .LA.uses_vcc, 1
-; CHECK: .set .LA.uses_flat_scratch, 0
-; CHECK: .set .LA.has_dyn_sized_stack, 0
-; CHECK: .set .LA.has_recursion, 1
-; CHECK: .set .LA.has_indirect_call, 0
+; CHECK-LABEL: {{^}}C
+; CHECK: .set .LC.num_vgpr, max(43, 41, .LD.num_vgpr)
+; CHECK: .set .LC.num_agpr, max(0, 0, .LD.num_agpr)
+; CHECK: .set .LC.numbered_sgpr, max(55, 51, .LD.numbered_sgpr)
+; CHECK: .set .LC.private_seg_size, 16+max(.LD.private_seg_size)
+; CHECK: .set .LC.uses_vcc, or(1, .LD.uses_vcc)
+; CHECK: .set .LC.uses_flat_scratch, or(0, .LD.uses_flat_scratch)
+; CHECK: .set .LC.has_dyn_sized_stack, or(0, .LD.has_dyn_sized_stack)
+; CHECK: .set .LC.has_recursion, or(1, .LD.has_recursion)
+; CHECK: .set .LC.has_indirect_call, or(0, .LD.has_indirect_call)
+
+; CHECK-LABEL: {{^}}D
+; CHECK: .set .LD.num_vgpr, max(71, 43)
+; CHECK: .set .LD.num_agpr, max(0, 0)
+; CHECK: .set .LD.numbered_sgpr, max(71, 55)
+; CHECK: .set .LD.private_seg_size, 16
+; CHECK: .set .LD.uses_vcc, 1
+; CHECK: .set .LD.uses_flat_scratch, 0
+; CHECK: .set .LD.has_dyn_sized_stack, 0
+; CHECK: .set .LD.has_recursion, 1
+; CHECK: .set .LD.has_indirect_call, 0
define void @A() {
call void @B()
diff --git a/llvm/test/CodeGen/AMDGPU/regalloc-failure-overlapping-insert-assert.mir b/llvm/test/CodeGen/AMDGPU/regalloc-failure-overlapping-insert-assert.mir
index 4f702f1356588..7dc4b5336b193 100644
--- a/llvm/test/CodeGen/AMDGPU/regalloc-failure-overlapping-insert-assert.mir
+++ b/llvm/test/CodeGen/AMDGPU/regalloc-failure-overlapping-insert-assert.mir
@@ -1,4 +1,4 @@
-# RUN: not llc -mtriple=amdgpu9.08 -verify-machineinstrs -start-before=greedy,1 -stop-after=virtregrewriter,2 %s -filetype=null 2>&1 | FileCheck -check-prefix=ERR %s
+# RUN: not llc -mtriple=amdgpu9.08 -verify-machineinstrs -start-before=greedy,2 -stop-after=virt-reg-rewriter,3 %s -filetype=null 2>&1 | FileCheck -check-prefix=ERR %s
# Make sure there's no machine verifier error after failure.
diff --git a/llvm/test/CodeGen/AMDGPU/regalloc-fast-dont-drop-subreg-index-issue61134.mir b/llvm/test/CodeGen/AMDGPU/regalloc-fast-dont-drop-subreg-index-issue61134.mir
index 27e0747fdd232..3f7b79275930a 100644
--- a/llvm/test/CodeGen/AMDGPU/regalloc-fast-dont-drop-subreg-index-issue61134.mir
+++ b/llvm/test/CodeGen/AMDGPU/regalloc-fast-dont-drop-subreg-index-issue61134.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -sgpr-regalloc=fast -start-before=regallocfast,0 -stop-before=greedy -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -sgpr-regalloc-npm=fast -start-before=regallocfast,1 -stop-before=greedy -o - %s | FileCheck %s
# RegAllocFast was incorrectly dropping subregister indexes on
# unassigned virtual registers. Make sure they are passed through
diff --git a/llvm/test/CodeGen/AMDGPU/regalloc-introduces-copy-sgpr-to-agpr.mir b/llvm/test/CodeGen/AMDGPU/regalloc-introduces-copy-sgpr-to-agpr.mir
index 578a1f8e261e6..f0349ddff548a 100644
--- a/llvm/test/CodeGen/AMDGPU/regalloc-introduces-copy-sgpr-to-agpr.mir
+++ b/llvm/test/CodeGen/AMDGPU/regalloc-introduces-copy-sgpr-to-agpr.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu9.08-amd-amdhsa -start-before=greedy,0 -stop-after=postrapseudos -o - %s | FileCheck -check-prefix=GFX908 %s
+# RUN: llc -mtriple=amdgpu9.08-amd-amdhsa -start-before=greedy,1 -stop-after=post-ra-pseudos -o - %s | FileCheck -check-prefix=GFX908 %s
# This testcase has a long sequence of sgpr to vgpr copies with a lot of vgpr
# pressure, encouraging the allocator to displace some into AGPRs. This
diff --git a/llvm/test/CodeGen/AMDGPU/regalloc-sgpr128-partial-def.mir b/llvm/test/CodeGen/AMDGPU/regalloc-sgpr128-partial-def.mir
index 16dd33326213b..45e20180058e2 100644
--- a/llvm/test/CodeGen/AMDGPU/regalloc-sgpr128-partial-def.mir
+++ b/llvm/test/CodeGen/AMDGPU/regalloc-sgpr128-partial-def.mir
@@ -1,4 +1,4 @@
-# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -verify-machineinstrs --stress-regalloc=2 -start-before=greedy -stop-after=virtregrewriter \
+# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -verify-machineinstrs --stress-regalloc=2 -start-before=greedy -stop-after=virt-reg-rewriter \
# RUN: %s -o - | FileCheck %s
# Test that S_MOV_B64 rematerialization is shrunk to S_MOV_B32 when only a
diff --git a/llvm/test/CodeGen/AMDGPU/regalloc-undef-copy-fold.mir b/llvm/test/CodeGen/AMDGPU/regalloc-undef-copy-fold.mir
index f9ad9ecf51a6f..4f7380565ab89 100644
--- a/llvm/test/CodeGen/AMDGPU/regalloc-undef-copy-fold.mir
+++ b/llvm/test/CodeGen/AMDGPU/regalloc-undef-copy-fold.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
-# RUN: llc -mtriple=amdgpu9.08 -verify-machineinstrs --start-before=greedy,2 --stop-after=greedy,2 %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.08 -verify-machineinstrs --start-before=greedy,3 --stop-after=greedy,3 %s -o - | FileCheck %s
# Make sure there's no machine verifier error
diff --git a/llvm/test/CodeGen/AMDGPU/register-killed-error-after-alloc-failure0.mir b/llvm/test/CodeGen/AMDGPU/register-killed-error-after-alloc-failure0.mir
index fb1866c379e5c..c92df2e1baed6 100644
--- a/llvm/test/CodeGen/AMDGPU/register-killed-error-after-alloc-failure0.mir
+++ b/llvm/test/CodeGen/AMDGPU/register-killed-error-after-alloc-failure0.mir
@@ -1,5 +1,5 @@
-# RUN: not llc -mtriple=amdgpu9.00-amd-amdhsa -vgpr-regalloc=basic -sgpr-regalloc=basic -start-before=regallocbasic,0 -stop-after=virtregrewriter,2 -verify-regalloc -o - %s 2> %t.basic.err | FileCheck -check-prefix=BASIC %s
-# RUN: not llc -mtriple=amdgpu9.00-amd-amdhsa -start-before=greedy,0 -stop-after=virtregrewriter,2 -verify-regalloc -o - %s 2> %t.greedy.err | FileCheck -check-prefix=GREEDY %s
+# RUN: not llc -enable-new-pm=0 -mtriple=amdgpu9.00-amd-amdhsa -vgpr-regalloc=basic -sgpr-regalloc=basic -start-before=regallocbasic,0 -stop-after=virtregrewriter,2 -verify-regalloc -o - %s 2> %t.basic.err | FileCheck -check-prefix=BASIC %s
+# RUN: not llc -mtriple=amdgpu9.00-amd-amdhsa -start-before=greedy,1 -stop-after=virt-reg-rewriter,3 -verify-regalloc -o - %s 2> %t.greedy.err | FileCheck -check-prefix=GREEDY %s
# RUN: FileCheck -check-prefix=ERR -implicit-check-not=error %s < %t.basic.err
# RUN: FileCheck -check-prefix=ERR -implicit-check-not=error %s < %t.greedy.err
diff --git a/llvm/test/CodeGen/AMDGPU/remaining-virtual-register-operands.mir b/llvm/test/CodeGen/AMDGPU/remaining-virtual-register-operands.mir
index 3476342177d5e..4bc178f3c5bdb 100644
--- a/llvm/test/CodeGen/AMDGPU/remaining-virtual-register-operands.mir
+++ b/llvm/test/CodeGen/AMDGPU/remaining-virtual-register-operands.mir
@@ -1,4 +1,4 @@
-# RUN: not llc -mtriple=amdgpu9.08-amd-amdhsa -verify-machineinstrs -start-before=greedy,2 -filetype=null %s 2>&1 | FileCheck %s
+# RUN: not llc -mtriple=amdgpu9.08-amd-amdhsa -verify-machineinstrs -start-before=greedy,3 -filetype=null %s 2>&1 | FileCheck %s
# This testcase fails register allocation at the same time it performs
# virtual register splitting (by introducing VGPR to AGPR copies). We
diff --git a/llvm/test/CodeGen/AMDGPU/remat-dead-subreg.mir b/llvm/test/CodeGen/AMDGPU/remat-dead-subreg.mir
index 29373f9df5d30..ba4c6e9ebb5c5 100644
--- a/llvm/test/CodeGen/AMDGPU/remat-dead-subreg.mir
+++ b/llvm/test/CodeGen/AMDGPU/remat-dead-subreg.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu6.00 -start-before=greedy -stop-after=virtregrewriter -stress-regalloc=3 -verify-machineinstrs -o - %s | FileCheck -check-prefix=GCN %s
+# RUN: llc -mtriple=amdgpu6.00 -start-before=greedy -stop-after=virt-reg-rewriter -stress-regalloc=3 -verify-machineinstrs -o - %s | FileCheck -check-prefix=GCN %s
# Check that subreg use is live at the point of materialization, not just the main range.
# Do not rematerialize if used subreg is dead at a new index.
diff --git a/llvm/test/CodeGen/AMDGPU/remat-smrd.mir b/llvm/test/CodeGen/AMDGPU/remat-smrd.mir
index efd9893c604fd..61d57e829d98b 100644
--- a/llvm/test/CodeGen/AMDGPU/remat-smrd.mir
+++ b/llvm/test/CodeGen/AMDGPU/remat-smrd.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -verify-machineinstrs --stress-regalloc=2 -start-before=greedy -stop-after=virtregrewriter -o - %s | FileCheck -check-prefix=GCN %s
+# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -verify-machineinstrs --stress-regalloc=2 -start-before=greedy -stop-after=virt-reg-rewriter -o - %s | FileCheck -check-prefix=GCN %s
# Case that should really rematerialize
---
diff --git a/llvm/test/CodeGen/AMDGPU/remat-sop.mir b/llvm/test/CodeGen/AMDGPU/remat-sop.mir
index 4cf0f7a2722bd..cedd70580d5ca 100644
--- a/llvm/test/CodeGen/AMDGPU/remat-sop.mir
+++ b/llvm/test/CodeGen/AMDGPU/remat-sop.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -verify-machineinstrs --stress-regalloc=2 -start-before=greedy -stop-after=virtregrewriter -o - %s | FileCheck -check-prefix=GCN %s
+# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -verify-machineinstrs --stress-regalloc=2 -start-before=greedy -stop-after=virt-reg-rewriter -o - %s | FileCheck -check-prefix=GCN %s
---
name: test_remat_s_mov_b32
diff --git a/llvm/test/CodeGen/AMDGPU/remat-vop.mir b/llvm/test/CodeGen/AMDGPU/remat-vop.mir
index 418b6e68a24e9..7720ee85ef6c9 100644
--- a/llvm/test/CodeGen/AMDGPU/remat-vop.mir
+++ b/llvm/test/CodeGen/AMDGPU/remat-vop.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu9.0a-amd-amdhsa -verify-regalloc --stress-regalloc=2 -start-before=greedy,0 -stop-after=virtregrewriter,1 -o - %s | FileCheck -check-prefix=GCN %s
+# RUN: llc -mtriple=amdgpu9.0a-amd-amdhsa -verify-regalloc --stress-regalloc=2 -start-before=greedy,1 -stop-after=virt-reg-rewriter,2 -o - %s | FileCheck -check-prefix=GCN %s
---
name: test_remat_v_mov_b32_e32
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-partial-reg-uses-dbg.mir b/llvm/test/CodeGen/AMDGPU/rewrite-partial-reg-uses-dbg.mir
index 8eb8050dbd9f2..8d54bb894eb97 100644
--- a/llvm/test/CodeGen/AMDGPU/rewrite-partial-reg-uses-dbg.mir
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-partial-reg-uses-dbg.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 2
-# RUN: llc -mtriple=amdgpu7.00-amd-amdhsa -amdgpu-enable-rewrite-partial-reg-uses=true -verify-machineinstrs -start-before=rename-independent-subregs -stop-after=rewrite-partial-reg-uses %s -o - | FileCheck -check-prefix=CHECK %s
+# RUN: llc -mtriple=amdgpu7.00-amd-amdhsa -amdgpu-enable-rewrite-partial-reg-uses=true -verify-machineinstrs -start-before=rename-independent-subregs -stop-after=amdgpu-rewrite-partial-reg-uses %s -o - | FileCheck -check-prefix=CHECK %s
# RUN: llc -mtriple=amdgpu7.00-amd-amdhsa -passes="rename-independent-subregs,amdgpu-rewrite-partial-reg-uses" %s -o - | FileCheck -check-prefix=CHECK %s
--- |
define void @test_vreg_96_w64() !dbg !5 {
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-partial-reg-uses-gen.mir b/llvm/test/CodeGen/AMDGPU/rewrite-partial-reg-uses-gen.mir
index 8c6e4c400b44a..78951bae47ee5 100644
--- a/llvm/test/CodeGen/AMDGPU/rewrite-partial-reg-uses-gen.mir
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-partial-reg-uses-gen.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu7.00-amd-amdhsa -amdgpu-enable-rewrite-partial-reg-uses=true -verify-machineinstrs -start-before=rename-independent-subregs -stop-after=rewrite-partial-reg-uses %s -o - | FileCheck -check-prefix=CHECK %s
+# RUN: llc -mtriple=amdgpu7.00-amd-amdhsa -amdgpu-enable-rewrite-partial-reg-uses=true -verify-machineinstrs -start-before=rename-independent-subregs -stop-after=amdgpu-rewrite-partial-reg-uses %s -o - | FileCheck -check-prefix=CHECK %s
# RUN: llc -mtriple=amdgpu7.00-amd-amdhsa -passes="rename-independent-subregs,amdgpu-rewrite-partial-reg-uses" %s -o - | FileCheck -check-prefix=CHECK %s
---
name: test_subregs_composition_vreg_1024
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-partial-reg-uses.mir b/llvm/test/CodeGen/AMDGPU/rewrite-partial-reg-uses.mir
index 67c5797c872fa..f8df0253fe5ab 100644
--- a/llvm/test/CodeGen/AMDGPU/rewrite-partial-reg-uses.mir
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-partial-reg-uses.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu7.00-amd-amdhsa -amdgpu-enable-rewrite-partial-reg-uses=true -verify-machineinstrs -start-before=rename-independent-subregs -stop-after=rewrite-partial-reg-uses %s -o - | FileCheck -check-prefix=CHECK %s
+# RUN: llc -mtriple=amdgpu7.00-amd-amdhsa -amdgpu-enable-rewrite-partial-reg-uses=true -verify-machineinstrs -start-before=rename-independent-subregs -stop-after=amdgpu-rewrite-partial-reg-uses %s -o - | FileCheck -check-prefix=CHECK %s
# RUN: llc -mtriple=amdgpu7.00-amd-amdhsa -passes="rename-independent-subregs,amdgpu-rewrite-partial-reg-uses" %s -o - | FileCheck -check-prefix=CHECK %s
---
name: test_subregs_composition_vreg_1024
diff --git a/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-tracker-physreg-crash.ll b/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-tracker-physreg-crash.ll
index c1ca7609df9f1..067d9c290cd84 100644
--- a/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-tracker-physreg-crash.ll
+++ b/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-tracker-physreg-crash.ll
@@ -1,4 +1,5 @@
-; RUN: not --crash llc -mtriple=amdgpu9.00-amd-amdhsa --amdgpu-xnack=true -amdgpu-use-amdgpu-trackers=1 -verify-machineinstrs 2>&1 < %s | FileCheck -check-prefixes=ERR-GCNTRACKERS %s
+; FIXME: NPM's "verify-each" errors out earlier.
+; RUN: not --crash llc -enable-new-pm=0 -mtriple=amdgpu9.00-amd-amdhsa --amdgpu-xnack=true -amdgpu-use-amdgpu-trackers=1 -verify-machineinstrs 2>&1 < %s | FileCheck -check-prefixes=ERR-GCNTRACKERS %s
; RUN: llc -mtriple=amdgpu9.00-amd-amdhsa --amdgpu-xnack=true 2>&1 < %s | FileCheck -check-prefixes=GCN %s
%asm.output = type { <16 x i32>, <16 x i32>, <16 x i32>, <8 x i32>, <2 x i32>, i32, ; sgprs
diff --git a/llvm/test/CodeGen/AMDGPU/schedule-regpressure-ilp-metric-spills.mir b/llvm/test/CodeGen/AMDGPU/schedule-regpressure-ilp-metric-spills.mir
index 6d9f8b6b3c2c5..37112359130bb 100644
--- a/llvm/test/CodeGen/AMDGPU/schedule-regpressure-ilp-metric-spills.mir
+++ b/llvm/test/CodeGen/AMDGPU/schedule-regpressure-ilp-metric-spills.mir
@@ -1,5 +1,5 @@
-# RUN: llc -mtriple=amdgpu11.00 -start-before=machine-scheduler -stop-after=greedy,2 -verify-machineinstrs -o - %s | FileCheck -check-prefix=GCN %s
-# RUN: llc -mtriple=amdgpu11.00 -start-before=machine-scheduler -stop-after=greedy,2 -amdgpu-use-amdgpu-trackers=1 -verify-machineinstrs -o - %s | FileCheck -check-prefix=GCN-GCNTRACKER %s
+# RUN: llc -mtriple=amdgpu11.00 -start-before=machine-scheduler -stop-after=greedy,3 -verify-machineinstrs -o - %s | FileCheck -check-prefix=GCN %s
+# RUN: llc -mtriple=amdgpu11.00 -start-before=machine-scheduler -stop-after=greedy,3 -amdgpu-use-amdgpu-trackers=1 -verify-machineinstrs -o - %s | FileCheck -check-prefix=GCN-GCNTRACKER %s
--- |
define amdgpu_kernel void @no_sched_metric_due_to_spills() #0 {
diff --git a/llvm/test/CodeGen/AMDGPU/schedule-regpressure-no-unclustered-regions.mir b/llvm/test/CodeGen/AMDGPU/schedule-regpressure-no-unclustered-regions.mir
index 8b70b5c320eb2..7e7615f052cc1 100644
--- a/llvm/test/CodeGen/AMDGPU/schedule-regpressure-no-unclustered-regions.mir
+++ b/llvm/test/CodeGen/AMDGPU/schedule-regpressure-no-unclustered-regions.mir
@@ -1,5 +1,5 @@
# REQUIRES: asserts
-# RUN: llc -mtriple=amdgpu11.00 -start-before=machine-scheduler -stop-after=greedy,2 -stress-regalloc=4 -debug-only=machine-scheduler %s -o - 2>&1 | FileCheck %s
+# RUN: llc -mtriple=amdgpu11.00 -start-before=machine-scheduler -stop-after=greedy,3 -stress-regalloc=4 -debug-only=machine-scheduler %s -o - 2>&1 | FileCheck %s
--- |
define amdgpu_kernel void @no_sched_metric_due_to_spills() #0 {
diff --git a/llvm/test/CodeGen/AMDGPU/sgpr-regalloc-flags.ll b/llvm/test/CodeGen/AMDGPU/sgpr-regalloc-flags.ll
index 1cc640a947a4f..b9a9116bc6a9f 100644
--- a/llvm/test/CodeGen/AMDGPU/sgpr-regalloc-flags.ll
+++ b/llvm/test/CodeGen/AMDGPU/sgpr-regalloc-flags.ll
@@ -1,16 +1,16 @@
; REQUIRES: asserts
-; RUN: llc -verify-machineinstrs=0 -mtriple=amdgpu7.00-amd-amdhsa -debug-pass=Structure -filetype=null %s 2>&1 | FileCheck -check-prefix=DEFAULT %s
-; RUN: llc -verify-machineinstrs=0 -sgpr-regalloc=greedy -wwm-regalloc=greedy -vgpr-regalloc=greedy -mtriple=amdgpu7.00-amd-amdhsa -debug-pass=Structure -filetype=null %s 2>&1 | FileCheck -check-prefix=DEFAULT %s
+; RUN: llc -enable-new-pm=0 -verify-machineinstrs=0 -mtriple=amdgpu7.00-amd-amdhsa -debug-pass=Structure -filetype=null %s 2>&1 | FileCheck -check-prefix=DEFAULT %s
+; RUN: llc -enable-new-pm=0 -verify-machineinstrs=0 -sgpr-regalloc=greedy -wwm-regalloc=greedy -vgpr-regalloc=greedy -mtriple=amdgpu7.00-amd-amdhsa -debug-pass=Structure -filetype=null %s 2>&1 | FileCheck -check-prefix=DEFAULT %s
-; RUN: llc -verify-machineinstrs=0 -O0 -mtriple=amdgpu7.00-amd-amdhsa -debug-pass=Structure -filetype=null %s 2>&1 | FileCheck -check-prefix=O0 %s
+; RUN: llc -enable-new-pm=0 -verify-machineinstrs=0 -O0 -mtriple=amdgpu7.00-amd-amdhsa -debug-pass=Structure -filetype=null %s 2>&1 | FileCheck -check-prefix=O0 %s
-; RUN: llc -verify-machineinstrs=0 -wwm-regalloc=basic -vgpr-regalloc=basic -mtriple=amdgpu7.00-amd-amdhsa -debug-pass=Structure -filetype=null %s 2>&1 | FileCheck -check-prefix=DEFAULT-BASIC %s
-; RUN: llc -verify-machineinstrs=0 -sgpr-regalloc=basic -mtriple=amdgpu7.00-amd-amdhsa -debug-pass=Structure -filetype=null %s 2>&1 | FileCheck -check-prefix=BASIC-DEFAULT %s
-; RUN: llc -verify-machineinstrs=0 -sgpr-regalloc=basic -wwm-regalloc=basic -vgpr-regalloc=basic -mtriple=amdgpu7.00-amd-amdhsa -debug-pass=Structure -filetype=null %s 2>&1 | FileCheck -check-prefix=BASIC-BASIC %s
+; RUN: llc -enable-new-pm=0 -verify-machineinstrs=0 -wwm-regalloc=basic -vgpr-regalloc=basic -mtriple=amdgpu7.00-amd-amdhsa -debug-pass=Structure -filetype=null %s 2>&1 | FileCheck -check-prefix=DEFAULT-BASIC %s
+; RUN: llc -enable-new-pm=0 -verify-machineinstrs=0 -sgpr-regalloc=basic -mtriple=amdgpu7.00-amd-amdhsa -debug-pass=Structure -filetype=null %s 2>&1 | FileCheck -check-prefix=BASIC-DEFAULT %s
+; RUN: llc -enable-new-pm=0 -verify-machineinstrs=0 -sgpr-regalloc=basic -wwm-regalloc=basic -vgpr-regalloc=basic -mtriple=amdgpu7.00-amd-amdhsa -debug-pass=Structure -filetype=null %s 2>&1 | FileCheck -check-prefix=BASIC-BASIC %s
-; RUN: not llc -verify-machineinstrs=0 -regalloc=basic -mtriple=amdgpu7.00-amd-amdhsa -debug-pass=Structure -filetype=null %s 2>&1 | FileCheck -check-prefix=REGALLOC %s
-; RUN: not llc -verify-machineinstrs=0 -regalloc=fast -O0 -mtriple=amdgpu7.00-amd-amdhsa -debug-pass=Structure -filetype=null %s 2>&1 | FileCheck -check-prefix=REGALLOC %s
+; RUN: not llc -enable-new-pm=0 -verify-machineinstrs=0 -regalloc=basic -mtriple=amdgpu7.00-amd-amdhsa -debug-pass=Structure -filetype=null %s 2>&1 | FileCheck -check-prefix=REGALLOC %s
+; RUN: not llc -enable-new-pm=0 -verify-machineinstrs=0 -regalloc=fast -O0 -mtriple=amdgpu7.00-amd-amdhsa -debug-pass=Structure -filetype=null %s 2>&1 | FileCheck -check-prefix=REGALLOC %s
; REGALLOC: -regalloc not supported with amdgcn. Use -sgpr-regalloc, -wwm-regalloc, and -vgpr-regalloc
diff --git a/llvm/test/CodeGen/AMDGPU/sgpr-spill-update-only-slot-indexes.ll b/llvm/test/CodeGen/AMDGPU/sgpr-spill-update-only-slot-indexes.ll
index 34260a0e7ea02..fe43eeeef86ee 100644
--- a/llvm/test/CodeGen/AMDGPU/sgpr-spill-update-only-slot-indexes.ll
+++ b/llvm/test/CodeGen/AMDGPU/sgpr-spill-update-only-slot-indexes.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=amdgpu9.00 -verify-machineinstrs -sgpr-regalloc=fast -vgpr-regalloc=fast -wwm-regalloc=fast < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -mtriple=amdgpu9.00 -verify-each -sgpr-regalloc-npm=fast -vgpr-regalloc-npm=fast -wwm-regalloc-npm=fast < %s | FileCheck -check-prefix=GCN %s
; Make sure there's no verifier error from improperly updated
; SlotIndexes if regalloc fast is manually used.
diff --git a/llvm/test/CodeGen/AMDGPU/shift-select.ll b/llvm/test/CodeGen/AMDGPU/shift-select.ll
index 69786d0a16786..dbc262a37f8e8 100644
--- a/llvm/test/CodeGen/AMDGPU/shift-select.ll
+++ b/llvm/test/CodeGen/AMDGPU/shift-select.ll
@@ -1,7 +1,8 @@
-; RUN: llc -mtriple=amdgpu6.00 -stop-after=instruction-select < %s | FileCheck -check-prefixes=GCN,GFX6 %s
-; RUN: llc -mtriple=amdgpu8.03 -stop-after=instruction-select < %s | FileCheck -check-prefixes=GCN,GFX8PLUS %s
-; RUN: llc -mtriple=amdgpu10.10 -stop-after=instruction-select < %s | FileCheck -check-prefixes=GCN,GFX8PLUS %s
-; RUN: llc -mtriple=amdgpu11.00 -amdgpu-enable-vopd=0 -stop-after=instruction-select < %s | FileCheck -check-prefixes=GCN,GFX8PLUS %s
+; FIXME: use NPM when GIsel passes have been ported.
+; RUN: llc -enable-new-pm=0 -mtriple=amdgpu6.00 -stop-after=instruction-select < %s | FileCheck -check-prefixes=GCN,GFX6 %s
+; RUN: llc -enable-new-pm=0 -mtriple=amdgpu8.03 -stop-after=instruction-select < %s | FileCheck -check-prefixes=GCN,GFX8PLUS %s
+; RUN: llc -enable-new-pm=0 -mtriple=amdgpu10.10 -stop-after=instruction-select < %s | FileCheck -check-prefixes=GCN,GFX8PLUS %s
+; RUN: llc -enable-new-pm=0 -mtriple=amdgpu11.00 -amdgpu-enable-vopd=0 -stop-after=instruction-select < %s | FileCheck -check-prefixes=GCN,GFX8PLUS %s
; GCN-LABEL: name: s_shl_i32
; GCN: S_LSHL_B32
diff --git a/llvm/test/CodeGen/AMDGPU/si-lower-sgpr-spills-cycle-header.mir b/llvm/test/CodeGen/AMDGPU/si-lower-sgpr-spills-cycle-header.mir
index 469a10d47fc1e..20dfa7537f4df 100644
--- a/llvm/test/CodeGen/AMDGPU/si-lower-sgpr-spills-cycle-header.mir
+++ b/llvm/test/CodeGen/AMDGPU/si-lower-sgpr-spills-cycle-header.mir
@@ -1,7 +1,7 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
# RUN: llc -mtriple=amdgpu7.00-amd-amdhsa -verify-machineinstrs -run-pass=si-lower-sgpr-spills -o - %s | FileCheck -check-prefix=SGPR-SPILL %s
-# RUN: llc -mtriple=amdgpu7.00-amd-amdhsa -O0 -start-before=si-lower-sgpr-spills -stop-after=regallocfast,1 -verify-machineinstrs %s -o - | FileCheck -check-prefix=WWM-REGALLOC %s
+# RUN: llc -mtriple=amdgpu7.00-amd-amdhsa -O0 -start-before=si-lower-sgpr-spills -stop-after=regallocfast,2 -verify-machineinstrs %s -o - | FileCheck -check-prefix=WWM-REGALLOC %s
# When SGPR spills to a virtual VGPR lane occur in both a loop header and the latch,
diff --git a/llvm/test/CodeGen/AMDGPU/si-lower-sgpr-spills-multi-entry-cycle.mir b/llvm/test/CodeGen/AMDGPU/si-lower-sgpr-spills-multi-entry-cycle.mir
index 63939ff5d69ae..e544210ed5815 100644
--- a/llvm/test/CodeGen/AMDGPU/si-lower-sgpr-spills-multi-entry-cycle.mir
+++ b/llvm/test/CodeGen/AMDGPU/si-lower-sgpr-spills-multi-entry-cycle.mir
@@ -1,6 +1,6 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
# RUN: llc -mtriple=amdgpu7.00-amd-amdhsa -verify-machineinstrs -run-pass=si-lower-sgpr-spills -o - %s | FileCheck -check-prefix=SGPR-SPILL %s
-# RUN: llc -mtriple=amdgpu7.00-amd-amdhsa -O0 -start-before=si-lower-sgpr-spills -stop-after=regallocfast,1 -verify-machineinstrs %s -o - | FileCheck -check-prefix=WWM-REGALLOC %s
+# RUN: llc -mtriple=amdgpu7.00-amd-amdhsa -O0 -start-before=si-lower-sgpr-spills -stop-after=regallocfast,2 -verify-machineinstrs %s -o - | FileCheck -check-prefix=WWM-REGALLOC %s
# Ensure that for a multi-entry cycle si-lower-sgpr-spills inserts
# IMPLICIT_DEF into the NCD of the cycle's entries.
diff --git a/llvm/test/CodeGen/AMDGPU/si-lower-sgpr-spills-vgpr-lanes-usage.mir b/llvm/test/CodeGen/AMDGPU/si-lower-sgpr-spills-vgpr-lanes-usage.mir
index 570469d0955ce..d156bbc8a2919 100644
--- a/llvm/test/CodeGen/AMDGPU/si-lower-sgpr-spills-vgpr-lanes-usage.mir
+++ b/llvm/test/CodeGen/AMDGPU/si-lower-sgpr-spills-vgpr-lanes-usage.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
-# RUN: llc -mtriple=amdgpu9.08-amd-amdhsa -verify-machineinstrs -stress-regalloc=3 -start-before=greedy,0 -stop-after=si-lower-sgpr-spills -o - %s | FileCheck -check-prefix=SGPR_SPILLED %s
+# RUN: llc -mtriple=amdgpu9.08-amd-amdhsa -verify-machineinstrs -stress-regalloc=3 -start-before=greedy,1 -stop-after=si-lower-sgpr-spills -o - %s | FileCheck -check-prefix=SGPR_SPILLED %s
# INFO : The test starts from the sgpr-regalloc pipeline.
# INFO : Now, StackSlotColoring pass comes just after sgpr-regalloc pipeline.
diff --git a/llvm/test/CodeGen/AMDGPU/snippet-copy-bundle-regression.mir b/llvm/test/CodeGen/AMDGPU/snippet-copy-bundle-regression.mir
index 3c99fab308ad0..50789cffb4cfa 100644
--- a/llvm/test/CodeGen/AMDGPU/snippet-copy-bundle-regression.mir
+++ b/llvm/test/CodeGen/AMDGPU/snippet-copy-bundle-regression.mir
@@ -1,6 +1,6 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 2
# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa \
-# RUN: -start-before=greedy,0 -stop-after=virtregrewriter,1 -verify-regalloc -o - %s | FileCheck %s
+# RUN: -start-before=greedy,1 -stop-after=virt-reg-rewriter,2 -verify-regalloc -o - %s | FileCheck %s
# This hit an assert when identifying snippet copy bundles from
# running off the end of the block due to using
diff --git a/llvm/test/CodeGen/AMDGPU/spill-empty-live-interval.mir b/llvm/test/CodeGen/AMDGPU/spill-empty-live-interval.mir
index 75b68900a10a7..2b7f1af29e41e 100644
--- a/llvm/test/CodeGen/AMDGPU/spill-empty-live-interval.mir
+++ b/llvm/test/CodeGen/AMDGPU/spill-empty-live-interval.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu7.00-amd-amdhsa -verify-machineinstrs -amdgpu-enable-rewrite-partial-reg-uses=false -amdgpu-dce-in-ra=0 -stress-regalloc=1 -start-before=register-coalescer -stop-after=greedy,1 -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu7.00-amd-amdhsa -verify-machineinstrs -amdgpu-enable-rewrite-partial-reg-uses=false -amdgpu-dce-in-ra=0 -stress-regalloc=1 -start-before=register-coalescer -stop-after=greedy,2 -o - %s | FileCheck %s
# https://bugs.llvm.org/show_bug.cgi?id=33620
---
diff --git a/llvm/test/CodeGen/AMDGPU/spill-scavenge-offset.ll b/llvm/test/CodeGen/AMDGPU/spill-scavenge-offset.ll
index d45833b1fc262..4a9b556935841 100644
--- a/llvm/test/CodeGen/AMDGPU/spill-scavenge-offset.ll
+++ b/llvm/test/CodeGen/AMDGPU/spill-scavenge-offset.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=amdgpu6.01 -enable-misched=0 -post-RA-scheduler=0 -amdgpu-spill-sgpr-to-vgpr=0 < %s | FileCheck -check-prefixes=CHECK,GFX6 %s
-; RUN: llc -sgpr-regalloc=basic -vgpr-regalloc=basic -mtriple=amdgpu8.02 -enable-misched=0 -post-RA-scheduler=0 -amdgpu-spill-sgpr-to-vgpr=0 < %s | FileCheck --check-prefix=CHECK %s
+; RUN: llc -enable-new-pm=0 -sgpr-regalloc=basic -vgpr-regalloc=basic -mtriple=amdgpu8.02 -enable-misched=0 -post-RA-scheduler=0 -amdgpu-spill-sgpr-to-vgpr=0 < %s | FileCheck --check-prefix=CHECK %s
; RUN: llc -mtriple=amdgpu9.00 --amdgpu-xnack=false -mattr=+enable-flat-scratch -enable-misched=0 -post-RA-scheduler=0 -amdgpu-spill-sgpr-to-vgpr=0 < %s | FileCheck -check-prefixes=CHECK,GFX9-FLATSCR,FLATSCR %s
; RUN: llc -mtriple=amdgpu10.30 -enable-misched=0 -post-RA-scheduler=0 -amdgpu-spill-sgpr-to-vgpr=0 -mattr=+enable-flat-scratch < %s | FileCheck -check-prefixes=CHECK,GFX10-FLATSCR,FLATSCR %s
;
diff --git a/llvm/test/CodeGen/AMDGPU/spill-vector-superclass.ll b/llvm/test/CodeGen/AMDGPU/spill-vector-superclass.ll
index 29c5092609a82..ae3bbd6e3eded 100644
--- a/llvm/test/CodeGen/AMDGPU/spill-vector-superclass.ll
+++ b/llvm/test/CodeGen/AMDGPU/spill-vector-superclass.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -mtriple=amdgpu9.08-amd-amdhsa -stop-after=greedy,1 -o - %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -mtriple=amdgpu9.08-amd-amdhsa -stop-after=greedy,2 -o - %s | FileCheck -check-prefix=GCN %s
; Convert AV spills into VGPR spills by introducing appropriate copies in between.
define amdgpu_kernel void @test_spill_av_class(<4 x i32> %arg) #0 {
diff --git a/llvm/test/CodeGen/AMDGPU/splitkit-do-not-undo-subclass-split-with-remat.mir b/llvm/test/CodeGen/AMDGPU/splitkit-do-not-undo-subclass-split-with-remat.mir
index 70d6975dad8f3..c01688ae02c52 100644
--- a/llvm/test/CodeGen/AMDGPU/splitkit-do-not-undo-subclass-split-with-remat.mir
+++ b/llvm/test/CodeGen/AMDGPU/splitkit-do-not-undo-subclass-split-with-remat.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
-# RUN: llc -mtriple=amdgpu9.0a-amd-amdhsa -verify-regalloc -start-before=greedy,2 -stop-before=virtregrewriter,2 -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.0a-amd-amdhsa -verify-regalloc -start-before=greedy,3 -stop-before=virt-reg-rewriter,3 -o - %s | FileCheck %s
# FIXME: Assert if run to end
# The V_MFMA_F32_32X32X1F32_vgprcd_e64 as written requires 66 VGPRs
diff --git a/llvm/test/CodeGen/AMDGPU/stress-regalloc-limits-regalloc.mir b/llvm/test/CodeGen/AMDGPU/stress-regalloc-limits-regalloc.mir
index 2595d1e80b39d..d7f5ce90c1b3d 100644
--- a/llvm/test/CodeGen/AMDGPU/stress-regalloc-limits-regalloc.mir
+++ b/llvm/test/CodeGen/AMDGPU/stress-regalloc-limits-regalloc.mir
@@ -1,6 +1,6 @@
-# RUN: llc -mtriple=amdgpu9.50-amd-amdhsa %s -o - | FileCheck -check-prefix=DEFAULT %s
-# RUN: llc -mtriple=amdgpu9.50-amd-amdhsa -amdgpu-stress-vgpr=2 %s -o - | FileCheck -check-prefix=VGPR2 %s
-# RUN: llc -mtriple=amdgpu9.50-amd-amdhsa -amdgpu-stress-vgpr=4 -amdgpu-stress-agpr=0 %s -o - | FileCheck -check-prefix=VGPR4-AGPR0 %s
+# RUN: llc -mtriple=amdgpu9.50-amd-amdhsa -start-before=amdgpu-isel %s -o - | FileCheck -check-prefix=DEFAULT %s
+# RUN: llc -mtriple=amdgpu9.50-amd-amdhsa -start-before=amdgpu-isel -amdgpu-stress-vgpr=2 %s -o - | FileCheck -check-prefix=VGPR2 %s
+# RUN: llc -mtriple=amdgpu9.50-amd-amdhsa -start-before=amdgpu-isel -amdgpu-stress-vgpr=4 -amdgpu-stress-agpr=0 %s -o - | FileCheck -check-prefix=VGPR4-AGPR0 %s
# Test that -amdgpu-stress-vgpr and -amdgpu-stress-agpr limit register
# availability during register allocation.
diff --git a/llvm/test/CodeGen/AMDGPU/swdev502267-use-after-free-last-chance-recoloring-alloc-succeeds.mir b/llvm/test/CodeGen/AMDGPU/swdev502267-use-after-free-last-chance-recoloring-alloc-succeeds.mir
index 2ef6908d3701d..57b65681557fa 100644
--- a/llvm/test/CodeGen/AMDGPU/swdev502267-use-after-free-last-chance-recoloring-alloc-succeeds.mir
+++ b/llvm/test/CodeGen/AMDGPU/swdev502267-use-after-free-last-chance-recoloring-alloc-succeeds.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
-# RUN: llc -mtriple=amdgpu9.42-amd-amdhsa -stress-regalloc=4 -verify-regalloc -start-before=greedy,2 -stop-after=virtregrewriter,2 -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.42-amd-amdhsa -stress-regalloc=4 -verify-regalloc -start-before=greedy,3 -stop-after=virt-reg-rewriter,3 -o - %s | FileCheck %s
# This testcase hit a situation where greedy would hit a use after
# free during last chance recoloring. This case successfully allocates
diff --git a/llvm/test/CodeGen/AMDGPU/threeaddr-wmma.mir b/llvm/test/CodeGen/AMDGPU/threeaddr-wmma.mir
index 695619300e298..af244ac580aa4 100644
--- a/llvm/test/CodeGen/AMDGPU/threeaddr-wmma.mir
+++ b/llvm/test/CodeGen/AMDGPU/threeaddr-wmma.mir
@@ -1,4 +1,4 @@
-# RUN: llc -mtriple=amdgpu11.00 %s -start-after postrapseudos -verify-machineinstrs -o - | FileCheck -check-prefix=GCN %s
+# RUN: llc -mtriple=amdgpu11.00 %s -start-after post-ra-pseudos -verify-machineinstrs -o - | FileCheck -check-prefix=GCN %s
# GCN-LABEL: test_V_WMMA_F32_16X16X16_F16_threeaddr_w32:
# GCN: v_wmma_f32_16x16x16_f16 v[34:41], v[0:7], v[8:15], v[16:23]
diff --git a/llvm/test/CodeGen/AMDGPU/true16-ra-pre-gfx11-regression-test.mir b/llvm/test/CodeGen/AMDGPU/true16-ra-pre-gfx11-regression-test.mir
index 90cbbaef3ab2e..d97415df573e3 100644
--- a/llvm/test/CodeGen/AMDGPU/true16-ra-pre-gfx11-regression-test.mir
+++ b/llvm/test/CodeGen/AMDGPU/true16-ra-pre-gfx11-regression-test.mir
@@ -1,4 +1,4 @@
-# RUN: llc -mtriple=amdgpu10.10 -start-before=greedy,0 -stop-after=virtregrewriter,2 -verify-machineinstrs -o - %s | FileCheck --check-prefixes=GCN %s
+# RUN: llc -mtriple=amdgpu10.10 -start-before=greedy,1 -stop-after=virt-reg-rewriter,3 -verify-machineinstrs -o - %s | FileCheck --check-prefixes=GCN %s
--- |
define amdgpu_ps void @e32() #0 {
diff --git a/llvm/test/CodeGen/AMDGPU/twoaddr-constrain.ll b/llvm/test/CodeGen/AMDGPU/twoaddr-constrain.ll
index fa12292d047d4..d15015d6f16cc 100644
--- a/llvm/test/CodeGen/AMDGPU/twoaddr-constrain.ll
+++ b/llvm/test/CodeGen/AMDGPU/twoaddr-constrain.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -global-isel -mtriple=amdgpu9.00 -stop-after twoaddressinstruction < %s | FileCheck %s
+; RUN: llc -global-isel -mtriple=amdgpu9.00 -stop-after two-address-instruction < %s | FileCheck %s
; Check that %16 gets constrained to register class sgpr_96_with_sub0_sub1.
define amdgpu_ps <3 x i32> @s_load_constant_v3i32_align4(ptr addrspace(4) inreg %ptr) {
diff --git a/llvm/test/CodeGen/AMDGPU/vgpr-agpr-limit-gfx90a.ll b/llvm/test/CodeGen/AMDGPU/vgpr-agpr-limit-gfx90a.ll
index 12ffc853188e7..6138ba6df8771 100644
--- a/llvm/test/CodeGen/AMDGPU/vgpr-agpr-limit-gfx90a.ll
+++ b/llvm/test/CodeGen/AMDGPU/vgpr-agpr-limit-gfx90a.ll
@@ -1,6 +1,6 @@
; -enable-misched=false makes the register usage more predictable
; -regalloc=fast just makes the test run faster
-; RUN: llc -mtriple=amdgpu9.0a -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc=fast -vgpr-regalloc=fast -wwm-regalloc=fast < %s | FileCheck %s --check-prefixes=GCN,GFX90A
+; RUN: llc -mtriple=amdgpu9.0a -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc-npm=fast -vgpr-regalloc-npm=fast -wwm-regalloc-npm=fast < %s | FileCheck %s --check-prefixes=GCN,GFX90A
define internal void @use256vgprs() {
%v0 = call i32 asm sideeffect "; def $0", "=v"()
diff --git a/llvm/test/CodeGen/AMDGPU/vgpr-limit-gfx1250.ll b/llvm/test/CodeGen/AMDGPU/vgpr-limit-gfx1250.ll
index 8c690f6579d89..416e89554ee89 100644
--- a/llvm/test/CodeGen/AMDGPU/vgpr-limit-gfx1250.ll
+++ b/llvm/test/CodeGen/AMDGPU/vgpr-limit-gfx1250.ll
@@ -1,6 +1,6 @@
; -enable-misched=false makes the register usage more predictable
; -regalloc=fast just makes the test run faster
-; RUN: llc -mtriple=amdgpu12.50 -O0 -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc=fast -vgpr-regalloc=fast -wwm-regalloc=fast < %s | FileCheck %s --check-prefixes=GCN,GFX1250
+; RUN: llc -mtriple=amdgpu12.50 -O0 -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc-npm=fast -vgpr-regalloc-npm=fast -wwm-regalloc-npm=fast < %s | FileCheck %s --check-prefixes=GCN,GFX1250
define internal void @use256vgprs_asm() {
%v0 = call i32 asm sideeffect "; def $0", "=v"()
diff --git a/llvm/test/CodeGen/AMDGPU/virtregrewrite-undef-identity-copy.mir b/llvm/test/CodeGen/AMDGPU/virtregrewrite-undef-identity-copy.mir
index 6d57b9dae6e32..ace8bd0e2a554 100644
--- a/llvm/test/CodeGen/AMDGPU/virtregrewrite-undef-identity-copy.mir
+++ b/llvm/test/CodeGen/AMDGPU/virtregrewrite-undef-identity-copy.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu7.00-amd-amdhsa -start-before=greedy,0 -stop-after=virtregrewriter,1 -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu7.00-amd-amdhsa -start-before=greedy,1 -stop-after=virt-reg-rewriter,2 -verify-machineinstrs -o - %s | FileCheck %s
# The undef copy of %4 is allocated to $vgpr3, and the identity copy
# was deleted, and $vgpr3 was considered undef. The code to replace
diff --git a/llvm/test/CodeGen/AMDGPU/vopc-remat.mir b/llvm/test/CodeGen/AMDGPU/vopc-remat.mir
index d15ed36b1b1fe..4e6a57ad8056a 100644
--- a/llvm/test/CodeGen/AMDGPU/vopc-remat.mir
+++ b/llvm/test/CodeGen/AMDGPU/vopc-remat.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu9.42-amd-amdhsa --stress-regalloc=2 -start-before=greedy -stop-after=virtregrewriter -o - %s | FileCheck -check-prefix=GCN %s
+# RUN: llc -mtriple=amdgpu9.42-amd-amdhsa --stress-regalloc=2 -start-before=greedy -stop-after=virt-reg-rewriter -o - %s | FileCheck -check-prefix=GCN %s
# Test that V_CMP_*_e64 instructions are rematerialized instead of spilled.
diff --git a/llvm/test/CodeGen/AMDGPU/wave32.ll b/llvm/test/CodeGen/AMDGPU/wave32.ll
index 2e3ff14b299e3..347c4f0df64dc 100644
--- a/llvm/test/CodeGen/AMDGPU/wave32.ll
+++ b/llvm/test/CodeGen/AMDGPU/wave32.ll
@@ -2196,11 +2196,12 @@ define amdgpu_kernel void @test_branch_true() #2 {
; GFX1032-NEXT: s_cbranch_execnz .LBB41_2
; GFX1032-NEXT: ; %bb.1: ; %for.body.lr.ph
; GFX1032-NEXT: s_branch .LBB41_3
-; GFX1032-NEXT: .LBB41_2: ; %Flow
+; GFX1032-NEXT: .LBB41_2: ; %Flow1
; GFX1032-NEXT: s_branch .LBB41_5
; GFX1032-NEXT: .LBB41_3: ; %for.body
-; GFX1032-NEXT: s_mov_b32 vcc_lo, 0
-; GFX1032-NEXT: ; %bb.4: ; %for.end.loopexit
+; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_cbranch_scc1 .LBB41_3
+; GFX1032-NEXT: ; %bb.4: ; %Flow
; GFX1032-NEXT: s_branch .LBB41_2
; GFX1032-NEXT: .LBB41_5: ; %for.end
; GFX1032-NEXT: s_endpgm
@@ -2211,11 +2212,12 @@ define amdgpu_kernel void @test_branch_true() #2 {
; GFX1064-NEXT: s_cbranch_execnz .LBB41_2
; GFX1064-NEXT: ; %bb.1: ; %for.body.lr.ph
; GFX1064-NEXT: s_branch .LBB41_3
-; GFX1064-NEXT: .LBB41_2: ; %Flow
+; GFX1064-NEXT: .LBB41_2: ; %Flow1
; GFX1064-NEXT: s_branch .LBB41_5
; GFX1064-NEXT: .LBB41_3: ; %for.body
-; GFX1064-NEXT: s_mov_b64 vcc, 0
-; GFX1064-NEXT: ; %bb.4: ; %for.end.loopexit
+; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_cbranch_scc1 .LBB41_3
+; GFX1064-NEXT: ; %bb.4: ; %Flow
; GFX1064-NEXT: s_branch .LBB41_2
; GFX1064-NEXT: .LBB41_5: ; %for.end
; GFX1064-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/wwm-regalloc-partial-pool.mir b/llvm/test/CodeGen/AMDGPU/wwm-regalloc-partial-pool.mir
index 6c79c594771b9..a092907c0e41c 100644
--- a/llvm/test/CodeGen/AMDGPU/wwm-regalloc-partial-pool.mir
+++ b/llvm/test/CodeGen/AMDGPU/wwm-regalloc-partial-pool.mir
@@ -1,5 +1,5 @@
-# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -O0 -amdgpu-stress-vgpr=1 -amdgpu-num-vgprs-for-wwm-alloc=2 -start-before=si-lower-sgpr-spills -stop-after=regallocfast,1 -verify-machineinstrs -o - %s | FileCheck %s
-# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -O0 -amdgpu-num-vgprs-for-wwm-alloc=0 -start-before=si-lower-sgpr-spills -stop-after=regallocfast,1 -verify-machineinstrs -o - %s | FileCheck --check-prefix=ZERO %s
+# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -O0 -amdgpu-stress-vgpr=1 -amdgpu-num-vgprs-for-wwm-alloc=2 -start-before=si-lower-sgpr-spills -stop-after=regallocfast,2 -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -O0 -amdgpu-num-vgprs-for-wwm-alloc=0 -start-before=si-lower-sgpr-spills -stop-after=regallocfast,2 -verify-machineinstrs -o - %s | FileCheck --check-prefix=ZERO %s
# CHECK-LABEL: name: partial_pool
# CHECK-COUNT-2: flags: [ WWM_REG ]
diff --git a/llvm/test/CodeGen/AMDGPU/wwm-spill-superclass-pseudo.mir b/llvm/test/CodeGen/AMDGPU/wwm-spill-superclass-pseudo.mir
index ce8aebe7458ed..3c445919d7a26 100644
--- a/llvm/test/CodeGen/AMDGPU/wwm-spill-superclass-pseudo.mir
+++ b/llvm/test/CodeGen/AMDGPU/wwm-spill-superclass-pseudo.mir
@@ -1,5 +1,5 @@
-# RUN: llc -mtriple=amdgpu9.08-amd-amdhsa -start-before=si-lower-sgpr-spills -stop-after=greedy,1 -verify-machineinstrs --stress-regalloc=2 -o - %s | FileCheck -check-prefix GCN-REGALLOC %s
-# RUN: llc -mtriple=amdgpu9.08-amd-amdhsa -start-before=si-lower-sgpr-spills -stop-after=virtregrewriter,1 -verify-machineinstrs --stress-regalloc=2 -o - %s | FileCheck -check-prefix GCN-REWRITER %s
+# RUN: llc -mtriple=amdgpu9.08-amd-amdhsa -start-before=si-lower-sgpr-spills -stop-after=greedy,2 -verify-machineinstrs --stress-regalloc=2 -o - %s | FileCheck -check-prefix GCN-REGALLOC %s
+# RUN: llc -mtriple=amdgpu9.08-amd-amdhsa -start-before=si-lower-sgpr-spills -stop-after=virt-reg-rewriter,2 -verify-machineinstrs --stress-regalloc=2 -o - %s | FileCheck -check-prefix GCN-REWRITER %s
name: test_wwm_reg_superclass_spill
tracksRegLiveness: true
diff --git a/llvm/test/CodeGen/Generic/GlobalISel/irtranslator-byte-type.ll b/llvm/test/CodeGen/Generic/GlobalISel/irtranslator-byte-type.ll
index 9d91ff55d6916..f4750c0da645e 100644
--- a/llvm/test/CodeGen/Generic/GlobalISel/irtranslator-byte-type.ll
+++ b/llvm/test/CodeGen/Generic/GlobalISel/irtranslator-byte-type.ll
@@ -4,7 +4,7 @@
; REQUIRES: x86-registered-target
; RUN: llc -mtriple=aarch64-linux-gnu -O0 -stop-after=irtranslator -global-isel %s -o - 2>&1 | FileCheck %s --check-prefix=AARCH64
-; RUN: llc -mtriple=amdgpu6.00--mesa-mesa3d -O0 -stop-after=irtranslator -global-isel %s -o - 2>&1 | FileCheck %s --check-prefix=AMDGPU
+; RUN: llc -mtriple=amdgpu6.00--mesa-mesa3d -O0 -stop-after=ir-translator -global-isel %s -o - 2>&1 | FileCheck %s --check-prefix=AMDGPU
; RUN: llc -mtriple=x86_64-linux-gnu -O0 -stop-after=irtranslator -global-isel %s -o - 2>&1 | FileCheck %s --check-prefix=X86
; IRTranslator handling of LLVM IR's byte type (bN), constants and loads /
diff --git a/llvm/test/DebugInfo/AMDGPU/debug-loc-copy.ll b/llvm/test/DebugInfo/AMDGPU/debug-loc-copy.ll
index f81a57f12173b..d35b727edb3ef 100644
--- a/llvm/test/DebugInfo/AMDGPU/debug-loc-copy.ll
+++ b/llvm/test/DebugInfo/AMDGPU/debug-loc-copy.ll
@@ -24,7 +24,6 @@ define void @_Z12lane_pc_testj() #0 !dbg !9 {
; GCN-NEXT: .cfi_undefined 39
; GCN-NEXT: .cfi_undefined 40
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: ; %bb.1: ; %lab
; GCN-NEXT: .Ltmp0:
; GCN-NEXT: .loc 0 12 1 prologue_end ; t.cpp:12:1
; GCN-NEXT: s_mov_b64 s[4:5], 0
@@ -34,13 +33,14 @@ define void @_Z12lane_pc_testj() #0 !dbg !9 {
; GCN-NEXT: s_cmp_lg_u32 s8, s6
; GCN-NEXT: s_cselect_b32 s5, s7, s5
; GCN-NEXT: s_cselect_b32 s4, s8, s4
-; GCN-NEXT: v_mov_b32_e32 v2, 0
+; GCN-NEXT: ; %bb.1: ; %lab
; GCN-NEXT: .loc 0 13 1 ; t.cpp:13:1
+; GCN-NEXT: v_mov_b32_e32 v2, 0
; GCN-NEXT: v_mov_b32_e32 v0, s4
; GCN-NEXT: v_mov_b32_e32 v1, s5
; GCN-NEXT: flat_store_dword v[0:1], v2
-; GCN-NEXT: v_mov_b32_e32 v2, 1
; GCN-NEXT: .loc 0 14 1 ; t.cpp:14:1
+; GCN-NEXT: v_mov_b32_e32 v2, 1
; GCN-NEXT: v_mov_b32_e32 v0, s4
; GCN-NEXT: v_mov_b32_e32 v1, s5
; GCN-NEXT: flat_store_dword v[0:1], v2
More information about the llvm-branch-commits
mailing list