[llvm-branch-commits] [llvm] [LLC][AMDGPU] start using new pass manager for backend codegen (PR #210251)
Vikram Hegde via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Mon Jul 27 02:11:22 PDT 2026
https://github.com/vikramRH updated https://github.com/llvm/llvm-project/pull/210251
>From a2d12d55a4de6be51b68c63167c8c34d65a5e9df Mon Sep 17 00:00:00 2001
From: vikhegde <vikram.hegde at amd.com>
Date: Thu, 16 Jul 2026 17:15:53 +0530
Subject: [PATCH] [LLC][AMDGPU] start using new pass manager for backend
codegen
---
...ee-registers-assertion-after-ra-failure.ll | 2 +-
.../AMDGPU/alloc-aligned-tuples-gfx1250.mir | 2 +-
.../AMDGPU/alloc-aligned-tuples-gfx908.mir | 2 +-
.../AMDGPU/alloc-aligned-tuples-gfx90a.mir | 2 +-
...-amdgpu-flat-work-group-size-vgpr-limit.ll | 20 +-
.../AMDGPU/av_spill_cross_bb_usage.mir | 2 +-
.../bad-agpr-vgpr-regalloc-priority.mir | 2 +-
.../AMDGPU/bb-prolog-spill-during-regalloc.ll | 2 +-
.../block-should-not-be-in-alive-blocks.mir | 2 +-
.../AMDGPU/call-alias-register-usage-agpr.ll | 36 +-
.../AMDGPU/call-alias-register-usage0.ll | 18 +-
.../AMDGPU/call-alias-register-usage1.ll | 22 +-
.../AMDGPU/call-alias-register-usage2.ll | 22 +-
.../AMDGPU/call-alias-register-usage3.ll | 22 +-
.../CodeGen/AMDGPU/call-constexpr-gisel.ll | 243 +++++++
.../CodeGen/AMDGPU/call-constexpr-sdag.ll | 238 +++++++
llvm/test/CodeGen/AMDGPU/call-constexpr.ll | 388 -----------
.../CodeGen/AMDGPU/convergence-laneops.ll | 4 +-
llvm/test/CodeGen/AMDGPU/dead_bundle.mir | 2 +-
.../dummy-regalloc-priority-advisor.mir | 4 +-
...gpr-vgpr-count-propagation-direct-chain.ll | 42 +-
.../dvgpr-vgpr-count-propagation-indirect.ll | 26 +-
.../AMDGPU/dvgpr-vgpr-count-propagation.ll | 50 +-
llvm/test/CodeGen/AMDGPU/emit-high-vgprs.mir | 2 +-
.../CodeGen/AMDGPU/expand-si-indirect.mir | 2 +-
.../AMDGPU/extend-wwm-virt-reg-liveness.mir | 2 +-
.../CodeGen/AMDGPU/fastregalloc-sgpr-only.mir | 2 +-
llvm/test/CodeGen/AMDGPU/fmamk_fmaak-t16.mir | 2 +-
.../CodeGen/AMDGPU/fold-reload-into-exec.mir | 2 +-
.../CodeGen/AMDGPU/fold-reload-into-m0.mir | 2 +-
.../CodeGen/AMDGPU/fold-restore-undef-use.mir | 2 +-
.../CodeGen/AMDGPU/function-resource-usage.ll | 92 +--
.../greedy-alloc-fail-sgpr1024-spill.mir | 2 +-
.../greedy-instruction-split-subrange.mir | 2 +-
.../greedy-remark-crash-unassigned-reg.mir | 2 +-
.../AMDGPU/illegal-eviction-assert.mir | 2 +-
.../AMDGPU/indirect-call-known-callees.ll | 73 +--
.../AMDGPU/indirect-reg-read-imm-idx.mir | 2 +-
.../CodeGen/AMDGPU/inflate-av-remat-imm.mir | 2 +-
...class-vgpr-mfma-to-agpr-negative-tests.mir | 2 +-
...class-vgpr-mfma-to-av-with-load-source.mir | 2 +-
.../inflate-reg-class-vgpr-mfma-to-av.mir | 2 +-
...-reg-class-snippet-copy-use-after-free.mir | 2 +-
...nfloop-subrange-spill-inspect-subrange.mir | 2 +-
.../CodeGen/AMDGPU/infloop-subrange-spill.mir | 2 +-
llvm/test/CodeGen/AMDGPU/issue48473.mir | 2 +-
...sue98474-assigned-physreg-interference.mir | 2 +-
...egrewriter-live-out-undef-subregisters.mir | 2 +-
.../AMDGPU/large-avgpr-assign-last.mir | 2 +-
llvm/test/CodeGen/AMDGPU/llc-pipeline.ll | 10 +-
.../llvm.amdgcn.ds.gws.barrier-bundle.ll | 4 +-
...llvm.amdgcn.ds.gws.barrier-fastregalloc.ll | 2 +-
.../AMDGPU/llvm.amdgcn.mfma.gfx950.bf16.ll | 144 ++--
.../CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll | 612 ++++++++---------
....amdgcn.mfma.scale.f32.16x16x128.f8f6f4.ll | 460 ++++++-------
...m.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll | 456 ++++++-------
.../AMDGPU/llvm.amdgcn.smfmac.gfx950.ll | 620 +++++++++---------
.../CodeGen/AMDGPU/lo16-hi16-illegal-copy.mir | 8 +-
.../CodeGen/AMDGPU/lo16-hi16-physreg-copy.mir | 6 +-
.../AMDGPU/lo16-lo16-physreg-copy-agpr.mir | 2 +-
.../AMDGPU/lo16-lo16-physreg-copy-sgpr.mir | 4 +-
...wer-control-flow-live-variables-update.mir | 2 +-
...ntrol-flow-live-variables-update.xfail.mir | 2 +-
.../CodeGen/AMDGPU/machine-sink-cycle.mir | 4 +-
.../CodeGen/AMDGPU/memcpy-crash-issue63986.ll | 2 +
.../CodeGen/AMDGPU/memcpy_const_compare.ll | 12 +-
llvm/test/CodeGen/AMDGPU/memmove-var-size.ll | 230 +++----
.../AMDGPU/memset-param-combinations.ll | 58 +-
.../AMDGPU/mfma-no-register-aliasing.ll | 2 +-
...-masking-pre-ra-update-liveness-wave32.mir | 2 +-
...pt-exec-masking-pre-ra-update-liveness.mir | 2 +-
...ptimize-exec-mask-pre-ra-alloc-failure.mir | 2 +-
...al-regcopy-and-spill-missed-at-regalloc.ll | 4 +-
llvm/test/CodeGen/AMDGPU/pr51516.mir | 4 +-
...ut-of-registers-error-all-regs-reserved.ll | 6 +-
.../AMDGPU/ran-out-of-registers-errors.ll | 10 +-
.../ran-out-of-sgprs-allocation-failure.mir | 2 +-
.../AMDGPU/recursive-resource-usage-mcexpr.ll | 131 ++--
...lloc-failure-overlapping-insert-assert.mir | 2 +-
...fast-dont-drop-subreg-index-issue61134.mir | 2 +-
.../regalloc-introduces-copy-sgpr-to-agpr.mir | 2 +-
.../AMDGPU/regalloc-sgpr128-partial-def.mir | 2 +-
.../AMDGPU/regalloc-undef-copy-fold.mir | 2 +-
...ster-killed-error-after-alloc-failure0.mir | 4 +-
.../remaining-virtual-register-operands.mir | 2 +-
.../test/CodeGen/AMDGPU/remat-dead-subreg.mir | 2 +-
llvm/test/CodeGen/AMDGPU/remat-smrd.mir | 2 +-
llvm/test/CodeGen/AMDGPU/remat-sop.mir | 2 +-
llvm/test/CodeGen/AMDGPU/remat-vop.mir | 2 +-
.../AMDGPU/rewrite-partial-reg-uses-dbg.mir | 2 +-
.../AMDGPU/rewrite-partial-reg-uses-gen.mir | 2 +-
.../AMDGPU/rewrite-partial-reg-uses.mir | 2 +-
...schedule-regpressure-ilp-metric-spills.mir | 4 +-
...ule-regpressure-no-unclustered-regions.mir | 2 +-
.../CodeGen/AMDGPU/sgpr-regalloc-flags.ll | 16 +-
.../sgpr-spill-update-only-slot-indexes.ll | 2 +-
llvm/test/CodeGen/AMDGPU/shift-select.ll | 9 +-
.../si-lower-sgpr-spills-cycle-header.mir | 2 +-
...si-lower-sgpr-spills-multi-entry-cycle.mir | 2 +-
.../si-lower-sgpr-spills-vgpr-lanes-usage.mir | 2 +-
.../AMDGPU/snippet-copy-bundle-regression.mir | 2 +-
.../AMDGPU/spill-empty-live-interval.mir | 2 +-
.../CodeGen/AMDGPU/spill-scavenge-offset.ll | 2 +-
.../CodeGen/AMDGPU/spill-vector-superclass.ll | 2 +-
...-do-not-undo-subclass-split-with-remat.mir | 2 +-
...-last-chance-recoloring-alloc-succeeds.mir | 2 +-
llvm/test/CodeGen/AMDGPU/threeaddr-wmma.mir | 2 +-
.../true16-ra-pre-gfx11-regression-test.mir | 2 +-
.../CodeGen/AMDGPU/vgpr-agpr-limit-gfx90a.ll | 2 +-
.../test/CodeGen/AMDGPU/vgpr-limit-gfx1250.ll | 2 +-
.../virtregrewrite-undef-identity-copy.mir | 2 +-
llvm/test/CodeGen/AMDGPU/vopc-remat.mir | 2 +-
llvm/test/CodeGen/AMDGPU/wave32.ll | 14 +-
.../AMDGPU/wwm-spill-superclass-pseudo.mir | 4 +-
llvm/test/DebugInfo/AMDGPU/debug-loc-copy.ll | 6 +-
llvm/tools/llc/NewPMDriver.cpp | 11 +-
llvm/tools/llc/NewPMDriver.h | 14 +-
llvm/tools/llc/llc.cpp | 27 +-
118 files changed, 2224 insertions(+), 2114 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/call-constexpr-gisel.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/call-constexpr-sdag.ll
delete mode 100644 llvm/test/CodeGen/AMDGPU/call-constexpr.ll
diff --git a/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers-assertion-after-ra-failure.ll b/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers-assertion-after-ra-failure.ll
index d35f04c64ac89..f3b0a315d6166 100644
--- a/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers-assertion-after-ra-failure.ll
+++ b/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers-assertion-after-ra-failure.ll
@@ -1,5 +1,5 @@
; RUN: not llc -mtriple=amdgpu9.0a-amd-amdhsa -filetype=null %s 2>&1 | FileCheck -check-prefix=ERR %s
-; RUN: not llc -mtriple=amdgpu9.0a-amd-amdhsa -vgpr-regalloc=basic -filetype=null %s 2>&1 | FileCheck -check-prefix=ERR %s
+; RUN: not llc -enable-new-pm=0 -mtriple=amdgpu9.0a-amd-amdhsa -vgpr-regalloc=basic -filetype=null %s 2>&1 | FileCheck -check-prefix=ERR %s
; ERR: error: <unknown>:0:0: no registers from class available to allocate in function 'no_free_vgprs_at_agpr_to_agpr_copy'
diff --git a/llvm/test/CodeGen/AMDGPU/alloc-aligned-tuples-gfx1250.mir b/llvm/test/CodeGen/AMDGPU/alloc-aligned-tuples-gfx1250.mir
index 9b4e364063f36..3eb221620587f 100644
--- a/llvm/test/CodeGen/AMDGPU/alloc-aligned-tuples-gfx1250.mir
+++ b/llvm/test/CodeGen/AMDGPU/alloc-aligned-tuples-gfx1250.mir
@@ -1,4 +1,4 @@
-# RUN: llc -mtriple=amdgpu12.50 -start-before=greedy,0 -stop-after=virtregrewriter,2 -verify-machineinstrs -o - %s | FileCheck --check-prefixes=GCN,GFX1250 %s
+# RUN: llc -mtriple=amdgpu12.50 -start-before=greedy,1 -stop-after=virt-reg-rewriter,3 -verify-machineinstrs -o - %s | FileCheck --check-prefixes=GCN,GFX1250 %s
# Using the unaligned vector tuples are OK as long as they aren't used
# in a real instruction.
diff --git a/llvm/test/CodeGen/AMDGPU/alloc-aligned-tuples-gfx908.mir b/llvm/test/CodeGen/AMDGPU/alloc-aligned-tuples-gfx908.mir
index cb042fa6bc261..76c807aaeb0da 100644
--- a/llvm/test/CodeGen/AMDGPU/alloc-aligned-tuples-gfx908.mir
+++ b/llvm/test/CodeGen/AMDGPU/alloc-aligned-tuples-gfx908.mir
@@ -1,4 +1,4 @@
-# RUN: llc -mtriple=amdgpu9.08 -start-before=greedy,0 -stop-after=virtregrewriter,2 -verify-machineinstrs -o - %s | FileCheck --check-prefixes=GCN,GFX908 %s
+# RUN: llc -mtriple=amdgpu9.08 -start-before=greedy,1 -stop-after=virt-reg-rewriter,3 -verify-machineinstrs -o - %s | FileCheck --check-prefixes=GCN,GFX908 %s
---
# GCN-LABEL: name: alloc_vgpr_64
diff --git a/llvm/test/CodeGen/AMDGPU/alloc-aligned-tuples-gfx90a.mir b/llvm/test/CodeGen/AMDGPU/alloc-aligned-tuples-gfx90a.mir
index c6593c0dbeb45..6bebdb59a1548 100644
--- a/llvm/test/CodeGen/AMDGPU/alloc-aligned-tuples-gfx90a.mir
+++ b/llvm/test/CodeGen/AMDGPU/alloc-aligned-tuples-gfx90a.mir
@@ -1,4 +1,4 @@
-# RUN: llc -mtriple=amdgpu9.0a -start-before=greedy,0 -stop-after=virtregrewriter,2 -verify-machineinstrs -o - %s | FileCheck --check-prefixes=GCN,GFX90A %s
+# RUN: llc -mtriple=amdgpu9.0a -start-before=greedy,1 -stop-after=virt-reg-rewriter,3 -verify-machineinstrs -o - %s | FileCheck --check-prefixes=GCN,GFX90A %s
# RUN: llc -enable-new-pm -mtriple=amdgpu9.0a -passes='greedy<sgpr>,virt-reg-rewriter<no-clear-vregs>,stack-slot-coloring,si-lower-sgpr-spills,si-pre-allocate-wwm-regs,greedy<wwm>,si-lower-wwm-copies,virt-reg-rewriter<no-clear-vregs>,amdgpu-reserve-wwm-regs,greedy<vgpr>,amdgpu-nsa-reassign,virt-reg-rewriter' -o - %s | FileCheck --check-prefixes=GCN,GFX90A %s
# Using the unaligned vector tuples are OK as long as they aren't used
diff --git a/llvm/test/CodeGen/AMDGPU/attr-amdgpu-flat-work-group-size-vgpr-limit.ll b/llvm/test/CodeGen/AMDGPU/attr-amdgpu-flat-work-group-size-vgpr-limit.ll
index 412933f78660b..332f237d8d43e 100644
--- a/llvm/test/CodeGen/AMDGPU/attr-amdgpu-flat-work-group-size-vgpr-limit.ll
+++ b/llvm/test/CodeGen/AMDGPU/attr-amdgpu-flat-work-group-size-vgpr-limit.ll
@@ -1,15 +1,15 @@
; -enable-misched=false makes the register usage more predictable
; -regalloc=fast just makes the test run faster
-; RUN: llc -mtriple=amdgpu9.00 -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc=fast -vgpr-regalloc=fast -wwm-regalloc=fast < %s | FileCheck %s --check-prefixes=GCN,GFX9
-; RUN: llc -mtriple=amdgpu9.0a -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc=fast -vgpr-regalloc=fast -wwm-regalloc=fast < %s | FileCheck %s --check-prefixes=GCN,GFX90A
-; RUN: llc -mtriple=amdgpu10.10 -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc=fast -vgpr-regalloc=fast -wwm-regalloc=fast < %s | FileCheck %s --check-prefixes=GCN,GFX10WGP-WAVE32
-; RUN: llc -mtriple=amdgpu10.10 -mattr=+wavefrontsize64 -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc=fast -vgpr-regalloc=fast -wwm-regalloc=fast < %s | FileCheck %s --check-prefixes=GCN,GFX10WGP-WAVE64
-; RUN: llc -mtriple=amdgpu10.10 -mattr=+cumode -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc=fast -vgpr-regalloc=fast -wwm-regalloc=fast < %s | FileCheck %s --check-prefixes=GCN,GFX10CU-WAVE32
-; RUN: llc -mtriple=amdgpu10.10 -mattr=+cumode,+wavefrontsize64 -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc=fast -vgpr-regalloc=fast -wwm-regalloc=fast < %s | FileCheck %s --check-prefixes=GCN,GFX10CU-WAVE64
-; RUN: llc -mtriple=amdgpu11.00 -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc=fast -vgpr-regalloc=fast -wwm-regalloc=fast < %s | FileCheck %s --check-prefixes=GCN,GFX11WGP-WAVE32
-; RUN: llc -mtriple=amdgpu11.00 -mattr=+wavefrontsize64 -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc=fast -vgpr-regalloc=fast -wwm-regalloc=fast < %s | FileCheck %s --check-prefixes=GCN,GFX11WGP-WAVE64
-; RUN: llc -mtriple=amdgpu11.00 -mattr=+cumode -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc=fast -vgpr-regalloc=fast -wwm-regalloc=fast < %s | FileCheck %s --check-prefixes=GCN,GFX11CU-WAVE32
-; RUN: llc -mtriple=amdgpu11.00 -mattr=+cumode,+wavefrontsize64 -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc=fast -vgpr-regalloc=fast -wwm-regalloc=fast < %s | FileCheck %s --check-prefixes=GCN,GFX11CU-WAVE64
+; RUN: llc -mtriple=amdgpu9.00 -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc-npm=fast -vgpr-regalloc-npm=fast -wwm-regalloc-npm=fast < %s | FileCheck %s --check-prefixes=GCN,GFX9
+; RUN: llc -mtriple=amdgpu9.0a -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc-npm=fast -vgpr-regalloc-npm=fast -wwm-regalloc-npm=fast < %s | FileCheck %s --check-prefixes=GCN,GFX90A
+; RUN: llc -mtriple=amdgpu10.10 -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc-npm=fast -vgpr-regalloc-npm=fast -wwm-regalloc-npm=fast < %s | FileCheck %s --check-prefixes=GCN,GFX10WGP-WAVE32
+; RUN: llc -mtriple=amdgpu10.10 -mattr=+wavefrontsize64 -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc-npm=fast -vgpr-regalloc-npm=fast -wwm-regalloc-npm=fast < %s | FileCheck %s --check-prefixes=GCN,GFX10WGP-WAVE64
+; RUN: llc -mtriple=amdgpu10.10 -mattr=+cumode -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc-npm=fast -vgpr-regalloc-npm=fast -wwm-regalloc-npm=fast < %s | FileCheck %s --check-prefixes=GCN,GFX10CU-WAVE32
+; RUN: llc -mtriple=amdgpu10.10 -mattr=+cumode,+wavefrontsize64 -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc-npm=fast -vgpr-regalloc-npm=fast -wwm-regalloc-npm=fast < %s | FileCheck %s --check-prefixes=GCN,GFX10CU-WAVE64
+; RUN: llc -mtriple=amdgpu11.00 -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc-npm=fast -vgpr-regalloc-npm=fast -wwm-regalloc-npm=fast < %s | FileCheck %s --check-prefixes=GCN,GFX11WGP-WAVE32
+; RUN: llc -mtriple=amdgpu11.00 -mattr=+wavefrontsize64 -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc-npm=fast -vgpr-regalloc-npm=fast -wwm-regalloc-npm=fast < %s | FileCheck %s --check-prefixes=GCN,GFX11WGP-WAVE64
+; RUN: llc -mtriple=amdgpu11.00 -mattr=+cumode -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc-npm=fast -vgpr-regalloc-npm=fast -wwm-regalloc-npm=fast < %s | FileCheck %s --check-prefixes=GCN,GFX11CU-WAVE32
+; RUN: llc -mtriple=amdgpu11.00 -mattr=+cumode,+wavefrontsize64 -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc-npm=fast -vgpr-regalloc-npm=fast -wwm-regalloc-npm=fast < %s | FileCheck %s --check-prefixes=GCN,GFX11CU-WAVE64
define internal void @use256vgprs() {
%v0 = call i32 asm sideeffect "; def $0", "=v"()
diff --git a/llvm/test/CodeGen/AMDGPU/av_spill_cross_bb_usage.mir b/llvm/test/CodeGen/AMDGPU/av_spill_cross_bb_usage.mir
index 11b009307ce1e..6eb7181d99c53 100644
--- a/llvm/test/CodeGen/AMDGPU/av_spill_cross_bb_usage.mir
+++ b/llvm/test/CodeGen/AMDGPU/av_spill_cross_bb_usage.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu9.08-amd-amdhsa -start-before=greedy,1 -stop-after=prolog-epilog -verify-machineinstrs -verify-regalloc -o - %s | FileCheck --check-prefixes=GCN %s
+# RUN: llc -mtriple=amdgpu9.08-amd-amdhsa -start-before=greedy,2 -stop-after=prolog-epilog -verify-machineinstrs -verify-regalloc -o - %s | FileCheck --check-prefixes=GCN %s
# The VGPR pair spilled and restored around the callsite is used in the next basic block.
#
diff --git a/llvm/test/CodeGen/AMDGPU/bad-agpr-vgpr-regalloc-priority.mir b/llvm/test/CodeGen/AMDGPU/bad-agpr-vgpr-regalloc-priority.mir
index a2763a1a1ccfd..7d0ced376f1cb 100644
--- a/llvm/test/CodeGen/AMDGPU/bad-agpr-vgpr-regalloc-priority.mir
+++ b/llvm/test/CodeGen/AMDGPU/bad-agpr-vgpr-regalloc-priority.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
-# RUN: llc -mtriple=amdgpu9.08-amd-amdhsa -start-before=greedy,2 -stop-after=virtregrewriter,2 -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.08-amd-amdhsa -start-before=greedy,3 -stop-after=virt-reg-rewriter,3 -o - %s | FileCheck %s
---
name: bad_ra
diff --git a/llvm/test/CodeGen/AMDGPU/bb-prolog-spill-during-regalloc.ll b/llvm/test/CodeGen/AMDGPU/bb-prolog-spill-during-regalloc.ll
index 7694e30a612af..71134ed6b67fb 100644
--- a/llvm/test/CodeGen/AMDGPU/bb-prolog-spill-during-regalloc.ll
+++ b/llvm/test/CodeGen/AMDGPU/bb-prolog-spill-during-regalloc.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 3
-; RUN: llc -mtriple=amdgpu9.06-amd-amdhsa -O0 --stop-after=regallocfast,2 -o - %s | FileCheck -check-prefix=REGALLOC %s
+; RUN: llc -mtriple=amdgpu9.06-amd-amdhsa -O0 --stop-after=regallocfast,3 -o - %s | FileCheck -check-prefix=REGALLOC %s
; Test to check if the bb prolog spills are inserted correctly during regalloc.
define i32 @prolog_spill(i32 %arg0, i32 %arg1, i32 %arg2) {
diff --git a/llvm/test/CodeGen/AMDGPU/block-should-not-be-in-alive-blocks.mir b/llvm/test/CodeGen/AMDGPU/block-should-not-be-in-alive-blocks.mir
index 7ecb1cd7a5343..e5e436c8fc97a 100644
--- a/llvm/test/CodeGen/AMDGPU/block-should-not-be-in-alive-blocks.mir
+++ b/llvm/test/CodeGen/AMDGPU/block-should-not-be-in-alive-blocks.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu10.31-amd-amdhsa -verify-machineinstrs -start-after=unreachable-mbb-elimination -stop-after=phi-node-elimination -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu10.31-amd-amdhsa -verify-machineinstrs -start-after=unreachable-mbb-elimination -stop-after=si-lower-control-flow -o - %s | FileCheck %s
# FIXME: Should be able to just use run-pass, but need to keep
# LiveVariables live after for the verifier. Also -start-before
diff --git a/llvm/test/CodeGen/AMDGPU/call-alias-register-usage-agpr.ll b/llvm/test/CodeGen/AMDGPU/call-alias-register-usage-agpr.ll
index 4ee6585c0a84a..b39b6cc4712de 100644
--- a/llvm/test/CodeGen/AMDGPU/call-alias-register-usage-agpr.ll
+++ b/llvm/test/CodeGen/AMDGPU/call-alias-register-usage-agpr.ll
@@ -1,16 +1,25 @@
-; RUN: llc -O0 -mtriple=amdgpu9.08-amd-amdhsa < %s | FileCheck -check-prefix=ALL %s
+; RUN: llc -O0 -mtriple=amdgpu9.08-amd-amdhsa < %s | FileCheck -check-prefix=ALL,GFX908 %s
; RUN: llc -O0 -mtriple=amdgpu9.0a-amd-amdhsa < %s | FileCheck -check-prefixes=ALL,GFX90A %s
-; CallGraphAnalysis, which CodeGenSCC order depends on, does not look
-; through aliases. If GlobalOpt is never run, we do not see direct
-; calls,
-
@alias = hidden alias void (), ptr @aliasee_default
-; ALL-LABEL: {{^}}kernel:
-; ALL: .amdhsa_next_free_vgpr max(totalnumvgprs(.Lkernel.num_agpr, .Lkernel.num_vgpr), 1, 0)
-; ALL-NEXT: .amdhsa_next_free_sgpr max(.Lkernel.numbered_sgpr+extrasgprs(.Lkernel.uses_vcc, .Lkernel.uses_flat_scratch, 1), 1, 0)-extrasgprs(.Lkernel.uses_vcc, .Lkernel.uses_flat_scratch, 1)
-; GFX90A-NEXT: .amdhsa_accum_offset (((((alignto(max(1, .Lkernel.num_vgpr), 4)/4)-1)&~65536)&63)+1)*4
+define internal void @aliasee_default() #1 {
+bb:
+ call void asm sideeffect "; clobber a26 ", "~{a26}"()
+ ret void
+}
+; ALL: .set .Laliasee_default.num_vgpr, 0
+; ALL-NEXT: .set .Laliasee_default.num_agpr, 27
+; ALL-NEXT: .set .Laliasee_default.numbered_sgpr, 32
+
+; GFX908-LABEL: {{^}}kernel:
+; GFX908: .amdhsa_next_free_vgpr 41
+; GFX908-NEXT: .amdhsa_next_free_sgpr 33
+
+; GFX90A-LABEL: {{^}}kernel:
+; GFX90A: .amdhsa_next_free_vgpr 71
+; GFX90A-NEXT: .amdhsa_next_free_sgpr 33
+; GFX90A-NEXT: .amdhsa_accum_offset 44
; ALL: .set .Lkernel.num_vgpr, max(41, .Laliasee_default.num_vgpr)
; ALL-NEXT: .set .Lkernel.num_agpr, max(0, .Laliasee_default.num_agpr)
@@ -21,15 +30,6 @@ bb:
ret void
}
-define internal void @aliasee_default() #1 {
-bb:
- call void asm sideeffect "; clobber a26 ", "~{a26}"()
- ret void
-}
-; ALL: .set .Laliasee_default.num_vgpr, 0
-; ALL-NEXT: .set .Laliasee_default.num_agpr, 27
-; ALL-NEXT: .set .Laliasee_default.numbered_sgpr, 32
-
attributes #0 = { noinline norecurse nounwind optnone }
attributes #1 = { noinline norecurse nounwind readnone willreturn }
attributes #2 = { nounwind readnone willreturn }
diff --git a/llvm/test/CodeGen/AMDGPU/call-alias-register-usage0.ll b/llvm/test/CodeGen/AMDGPU/call-alias-register-usage0.ll
index 8e74dcdd9490f..296507e2c2bbf 100644
--- a/llvm/test/CodeGen/AMDGPU/call-alias-register-usage0.ll
+++ b/llvm/test/CodeGen/AMDGPU/call-alias-register-usage0.ll
@@ -6,6 +6,15 @@
@alias0 = hidden alias void (), ptr @aliasee_default_vgpr64_sgpr102
+; CHECK: .set .Laliasee_default_vgpr64_sgpr102.num_vgpr, 53
+; CHECK-NEXT: .set .Laliasee_default_vgpr64_sgpr102.num_agpr, 0
+; CHECK-NEXT: .set .Laliasee_default_vgpr64_sgpr102.numbered_sgpr, 32
+define internal void @aliasee_default_vgpr64_sgpr102() #1 {
+bb:
+ call void asm sideeffect "; clobber v52 ", "~{v52}"()
+ ret void
+}
+
; CHECK-LABEL: {{^}}kernel0:
; CHECK: .set .Lkernel0.num_vgpr, max(41, .Laliasee_default_vgpr64_sgpr102.num_vgpr)
; CHECK-NEXT: .set .Lkernel0.num_agpr, max(0, .Laliasee_default_vgpr64_sgpr102.num_agpr)
@@ -16,15 +25,6 @@ bb:
ret void
}
-; CHECK: .set .Laliasee_default_vgpr64_sgpr102.num_vgpr, 53
-; CHECK-NEXT: .set .Laliasee_default_vgpr64_sgpr102.num_agpr, 0
-; CHECK-NEXT: .set .Laliasee_default_vgpr64_sgpr102.numbered_sgpr, 32
-define internal void @aliasee_default_vgpr64_sgpr102() #1 {
-bb:
- call void asm sideeffect "; clobber v52 ", "~{v52}"()
- ret void
-}
-
attributes #0 = { noinline norecurse nounwind optnone }
attributes #1 = { noinline norecurse nounwind readnone willreturn }
attributes #2 = { nounwind readnone willreturn }
diff --git a/llvm/test/CodeGen/AMDGPU/call-alias-register-usage1.ll b/llvm/test/CodeGen/AMDGPU/call-alias-register-usage1.ll
index 05412b343ecb7..81ee5b96a297f 100644
--- a/llvm/test/CodeGen/AMDGPU/call-alias-register-usage1.ll
+++ b/llvm/test/CodeGen/AMDGPU/call-alias-register-usage1.ll
@@ -8,9 +8,18 @@
; The parent kernel has a higher VGPR usage than the possible callees.
+; CHECK: .set .Laliasee_vgpr32_sgpr76.num_vgpr, 27
+; CHECK-NEXT: .set .Laliasee_vgpr32_sgpr76.num_agpr, 0
+; CHECK-NEXT: .set .Laliasee_vgpr32_sgpr76.numbered_sgpr, 32
+define internal void @aliasee_vgpr32_sgpr76() #1 {
+bb:
+ call void asm sideeffect "; clobber v26 ", "~{v26}"()
+ ret void
+}
+
; CHECK-LABEL: {{^}}kernel1:
-; CHECK: .amdhsa_next_free_vgpr max(totalnumvgprs(.Lkernel1.num_agpr, .Lkernel1.num_vgpr), 1, 0)
-; CHECK-NEXT: .amdhsa_next_free_sgpr max(.Lkernel1.numbered_sgpr+extrasgprs(.Lkernel1.uses_vcc, .Lkernel1.uses_flat_scratch, 1), 1, 0)-extrasgprs(.Lkernel1.uses_vcc, .Lkernel1.uses_flat_scratch, 1)
+; CHECK: .amdhsa_next_free_vgpr 42
+; CHECK-NEXT: .amdhsa_next_free_sgpr 33
; CHECK: .set .Lkernel1.num_vgpr, max(42, .Laliasee_vgpr32_sgpr76.num_vgpr)
; CHECK-NEXT: .set .Lkernel1.num_agpr, max(0, .Laliasee_vgpr32_sgpr76.num_agpr)
@@ -22,15 +31,6 @@ bb:
ret void
}
-; CHECK: .set .Laliasee_vgpr32_sgpr76.num_vgpr, 27
-; CHECK-NEXT: .set .Laliasee_vgpr32_sgpr76.num_agpr, 0
-; CHECK-NEXT: .set .Laliasee_vgpr32_sgpr76.numbered_sgpr, 32
-define internal void @aliasee_vgpr32_sgpr76() #1 {
-bb:
- call void asm sideeffect "; clobber v26 ", "~{v26}"()
- ret void
-}
-
attributes #0 = { noinline norecurse nounwind optnone }
attributes #1 = { noinline norecurse nounwind readnone willreturn "amdgpu-waves-per-eu"="8,10" }
attributes #2 = { nounwind readnone willreturn }
diff --git a/llvm/test/CodeGen/AMDGPU/call-alias-register-usage2.ll b/llvm/test/CodeGen/AMDGPU/call-alias-register-usage2.ll
index 5735ef21b084d..afcac9cb4327d 100644
--- a/llvm/test/CodeGen/AMDGPU/call-alias-register-usage2.ll
+++ b/llvm/test/CodeGen/AMDGPU/call-alias-register-usage2.ll
@@ -6,9 +6,18 @@
@alias2 = hidden alias void (), ptr @aliasee_vgpr64_sgpr102
+; CHECK: .set .Laliasee_vgpr64_sgpr102.num_vgpr, 53
+; CHECK-NEXT: .set .Laliasee_vgpr64_sgpr102.num_agpr, 0
+; CHECK-NEXT: .set .Laliasee_vgpr64_sgpr102.numbered_sgpr, 32
+define internal void @aliasee_vgpr64_sgpr102() #1 {
+bb:
+ call void asm sideeffect "; clobber v52 ", "~{v52}"()
+ ret void
+}
+
; CHECK-LABEL: {{^}}kernel2:
-; CHECK: .amdhsa_next_free_vgpr max(totalnumvgprs(.Lkernel2.num_agpr, .Lkernel2.num_vgpr), 1, 0)
-; CHECK-NEXT: .amdhsa_next_free_sgpr max(.Lkernel2.numbered_sgpr+extrasgprs(.Lkernel2.uses_vcc, .Lkernel2.uses_flat_scratch, 1), 1, 0)-extrasgprs(.Lkernel2.uses_vcc, .Lkernel2.uses_flat_scratch, 1)
+; CHECK: .amdhsa_next_free_vgpr 53
+; CHECK-NEXT: .amdhsa_next_free_sgpr 33
; CHECK: .set .Lkernel2.num_vgpr, max(41, .Laliasee_vgpr64_sgpr102.num_vgpr)
; CHECK-NEXT: .set .Lkernel2.num_agpr, max(0, .Laliasee_vgpr64_sgpr102.num_agpr)
@@ -19,15 +28,6 @@ bb:
ret void
}
-; CHECK: .set .Laliasee_vgpr64_sgpr102.num_vgpr, 53
-; CHECK-NEXT: .set .Laliasee_vgpr64_sgpr102.num_agpr, 0
-; CHECK-NEXT: .set .Laliasee_vgpr64_sgpr102.numbered_sgpr, 32
-define internal void @aliasee_vgpr64_sgpr102() #1 {
-bb:
- call void asm sideeffect "; clobber v52 ", "~{v52}"()
- ret void
-}
-
attributes #0 = { noinline norecurse nounwind optnone }
attributes #1 = { noinline norecurse nounwind readnone willreturn "amdgpu-waves-per-eu"="4,10" }
attributes #2 = { nounwind readnone willreturn }
diff --git a/llvm/test/CodeGen/AMDGPU/call-alias-register-usage3.ll b/llvm/test/CodeGen/AMDGPU/call-alias-register-usage3.ll
index 7e6be8593e3c5..4c094d75ecd70 100644
--- a/llvm/test/CodeGen/AMDGPU/call-alias-register-usage3.ll
+++ b/llvm/test/CodeGen/AMDGPU/call-alias-register-usage3.ll
@@ -6,9 +6,18 @@
@alias3 = hidden alias void (), ptr @aliasee_vgpr256_sgpr102
+; CHECK: .set .Laliasee_vgpr256_sgpr102.num_vgpr, 253
+; CHECK-NEXT: .set .Laliasee_vgpr256_sgpr102.num_agpr, 0
+; CHECK-NEXT: .set .Laliasee_vgpr256_sgpr102.numbered_sgpr, 33
+define internal void @aliasee_vgpr256_sgpr102() #1 {
+bb:
+ call void asm sideeffect "; clobber v252 ", "~{v252}"()
+ ret void
+}
+
; CHECK-LABEL: {{^}}kernel3:
-; CHECK: .amdhsa_next_free_vgpr max(totalnumvgprs(.Lkernel3.num_agpr, .Lkernel3.num_vgpr), 1, 0)
-; CHECK-NEXT: .amdhsa_next_free_sgpr max(.Lkernel3.numbered_sgpr+extrasgprs(.Lkernel3.uses_vcc, .Lkernel3.uses_flat_scratch, 1), 1, 0)-extrasgprs(.Lkernel3.uses_vcc, .Lkernel3.uses_flat_scratch, 1)
+; CHECK: .amdhsa_next_free_vgpr 253
+; CHECK-NEXT: .amdhsa_next_free_sgpr 33
; CHECK: .set .Lkernel3.num_vgpr, max(41, .Laliasee_vgpr256_sgpr102.num_vgpr)
; CHECK-NEXT: .set .Lkernel3.num_agpr, max(0, .Laliasee_vgpr256_sgpr102.num_agpr)
@@ -19,15 +28,6 @@ bb:
ret void
}
-; CHECK: .set .Laliasee_vgpr256_sgpr102.num_vgpr, 253
-; CHECK-NEXT: .set .Laliasee_vgpr256_sgpr102.num_agpr, 0
-; CHECK-NEXT: .set .Laliasee_vgpr256_sgpr102.numbered_sgpr, 33
-define internal void @aliasee_vgpr256_sgpr102() #1 {
-bb:
- call void asm sideeffect "; clobber v252 ", "~{v252}"()
- ret void
-}
-
attributes #0 = { noinline norecurse nounwind optnone }
attributes #1 = { noinline norecurse nounwind readnone willreturn "amdgpu-flat-work-group-size"="1,256" "amdgpu-waves-per-eu"="1,1" }
attributes #2 = { nounwind readnone willreturn }
diff --git a/llvm/test/CodeGen/AMDGPU/call-constexpr-gisel.ll b/llvm/test/CodeGen/AMDGPU/call-constexpr-gisel.ll
new file mode 100644
index 0000000000000..0b99b44f61483
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/call-constexpr-gisel.ll
@@ -0,0 +1,243 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
+; RUN: llc -global-isel=1 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 < %s | FileCheck -check-prefixes=GCN %s
+
+; TODO: merge this test with call-constexpr-sdag.ll when GIsel is ported to NPM.
+
+define amdgpu_kernel void @test_bitcast_return_type_noinline() #0 {
+; GCN-LABEL: test_bitcast_return_type_noinline:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_add_i32 s12, s12, s17
+; GCN-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-NEXT: s_add_u32 s0, s0, s17
+; GCN-NEXT: v_lshlrev_b32_e32 v1, 10, v1
+; GCN-NEXT: s_addc_u32 s1, s1, 0
+; GCN-NEXT: v_or_b32_e32 v0, v0, v1
+; GCN-NEXT: v_lshlrev_b32_e32 v1, 20, v2
+; GCN-NEXT: s_mov_b32 flat_scratch_lo, s13
+; GCN-NEXT: s_mov_b32 s13, s15
+; GCN-NEXT: s_mov_b32 s12, s14
+; GCN-NEXT: v_or_b32_e32 v31, v0, v1
+; GCN-NEXT: s_getpc_b64 s[18:19]
+; GCN-NEXT: s_add_u32 s18, s18, ret_i32_noinline at rel32@lo+4
+; GCN-NEXT: s_addc_u32 s19, s19, ret_i32_noinline at rel32@hi+12
+; GCN-NEXT: s_mov_b32 s14, s16
+; GCN-NEXT: s_mov_b32 s32, 0
+; GCN-NEXT: s_swappc_b64 s[30:31], s[18:19]
+; GCN-NEXT: v_add_f32_e32 v0, 1.0, v0
+; GCN-NEXT: flat_store_dword v[0:1], v0
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: s_endpgm
+ %val = call float @ret_i32_noinline()
+ %op = fadd float %val, 1.0
+ store volatile float %op, ptr addrspace(1) poison
+ ret void
+}
+
+define amdgpu_kernel void @test_bitcast_return_type_alwaysinline() #0 {
+; GCN-LABEL: test_bitcast_return_type_alwaysinline:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_add_i32 s12, s12, s17
+; GCN-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-NEXT: s_add_u32 s0, s0, s17
+; GCN-NEXT: v_lshlrev_b32_e32 v1, 10, v1
+; GCN-NEXT: s_addc_u32 s1, s1, 0
+; GCN-NEXT: v_or_b32_e32 v0, v0, v1
+; GCN-NEXT: v_lshlrev_b32_e32 v1, 20, v2
+; GCN-NEXT: s_mov_b32 flat_scratch_lo, s13
+; GCN-NEXT: s_mov_b32 s13, s15
+; GCN-NEXT: s_mov_b32 s12, s14
+; GCN-NEXT: v_or_b32_e32 v31, v0, v1
+; GCN-NEXT: s_getpc_b64 s[18:19]
+; GCN-NEXT: s_add_u32 s18, s18, ret_i32_alwaysinline at rel32@lo+4
+; GCN-NEXT: s_addc_u32 s19, s19, ret_i32_alwaysinline at rel32@hi+12
+; GCN-NEXT: s_mov_b32 s14, s16
+; GCN-NEXT: s_mov_b32 s32, 0
+; GCN-NEXT: s_swappc_b64 s[30:31], s[18:19]
+; GCN-NEXT: v_add_f32_e32 v0, 1.0, v0
+; GCN-NEXT: flat_store_dword v[0:1], v0
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: s_endpgm
+ %val = call float @ret_i32_alwaysinline()
+ %op = fadd float %val, 1.0
+ store volatile float %op, ptr addrspace(1) poison
+ ret void
+}
+
+define amdgpu_kernel void @test_bitcast_argument_type() #0 {
+; GCN-LABEL: test_bitcast_argument_type:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_add_i32 s12, s12, s17
+; GCN-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-NEXT: s_add_u32 s0, s0, s17
+; GCN-NEXT: v_lshlrev_b32_e32 v1, 10, v1
+; GCN-NEXT: s_addc_u32 s1, s1, 0
+; GCN-NEXT: v_or_b32_e32 v0, v0, v1
+; GCN-NEXT: v_lshlrev_b32_e32 v1, 20, v2
+; GCN-NEXT: s_mov_b32 flat_scratch_lo, s13
+; GCN-NEXT: s_mov_b32 s13, s15
+; GCN-NEXT: s_mov_b32 s12, s14
+; GCN-NEXT: v_or_b32_e32 v31, v0, v1
+; GCN-NEXT: s_getpc_b64 s[18:19]
+; GCN-NEXT: s_add_u32 s18, s18, ident_i32 at rel32@lo+4
+; GCN-NEXT: s_addc_u32 s19, s19, ident_i32 at rel32@hi+12
+; GCN-NEXT: v_mov_b32_e32 v0, 2.0
+; GCN-NEXT: s_mov_b32 s14, s16
+; GCN-NEXT: s_mov_b32 s32, 0
+; GCN-NEXT: s_swappc_b64 s[30:31], s[18:19]
+; GCN-NEXT: v_add_i32_e32 v0, vcc, 1, v0
+; GCN-NEXT: flat_store_dword v[0:1], v0
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: s_endpgm
+ %val = call i32 @ident_i32(float 2.0)
+ %op = add i32 %val, 1
+ store volatile i32 %op, ptr addrspace(1) poison
+ ret void
+}
+
+define amdgpu_kernel void @test_bitcast_argument_and_return_types() #0 {
+; GCN-LABEL: test_bitcast_argument_and_return_types:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_add_i32 s12, s12, s17
+; GCN-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-NEXT: s_add_u32 s0, s0, s17
+; GCN-NEXT: v_lshlrev_b32_e32 v1, 10, v1
+; GCN-NEXT: s_addc_u32 s1, s1, 0
+; GCN-NEXT: v_or_b32_e32 v0, v0, v1
+; GCN-NEXT: v_lshlrev_b32_e32 v1, 20, v2
+; GCN-NEXT: s_mov_b32 flat_scratch_lo, s13
+; GCN-NEXT: s_mov_b32 s13, s15
+; GCN-NEXT: s_mov_b32 s12, s14
+; GCN-NEXT: v_or_b32_e32 v31, v0, v1
+; GCN-NEXT: s_getpc_b64 s[18:19]
+; GCN-NEXT: s_add_u32 s18, s18, ident_i32 at rel32@lo+4
+; GCN-NEXT: s_addc_u32 s19, s19, ident_i32 at rel32@hi+12
+; GCN-NEXT: v_mov_b32_e32 v0, 2.0
+; GCN-NEXT: s_mov_b32 s14, s16
+; GCN-NEXT: s_mov_b32 s32, 0
+; GCN-NEXT: s_swappc_b64 s[30:31], s[18:19]
+; GCN-NEXT: v_add_f32_e32 v0, 1.0, v0
+; GCN-NEXT: flat_store_dword v[0:1], v0
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: s_endpgm
+ %val = call float @ident_i32(float 2.0)
+ %op = fadd float %val, 1.0
+ store volatile float %op, ptr addrspace(1) poison
+ ret void
+}
+
+define hidden i32 @use_workitem_id_x(i32 %arg0) #3 {
+; GCN-LABEL: use_workitem_id_x:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_and_b32_e32 v1, 0x3ff, v31
+; GCN-NEXT: v_add_i32_e32 v0, vcc, v1, v0
+; GCN-NEXT: s_setpc_b64 s[30:31]
+ %id = call i32 @llvm.amdgcn.workitem.id.x()
+ %op = add i32 %id, %arg0
+ ret i32 %op
+}
+
+define amdgpu_kernel void @test_bitcast_use_workitem_id_x() #3 {
+; GCN-LABEL: test_bitcast_use_workitem_id_x:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_add_i32 s12, s12, s17
+; GCN-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-NEXT: s_add_u32 s0, s0, s17
+; GCN-NEXT: s_addc_u32 s1, s1, 0
+; GCN-NEXT: s_mov_b32 flat_scratch_lo, s13
+; GCN-NEXT: v_mov_b32_e32 v31, v0
+; GCN-NEXT: s_mov_b32 s13, s15
+; GCN-NEXT: s_mov_b32 s12, s14
+; GCN-NEXT: s_getpc_b64 s[18:19]
+; GCN-NEXT: s_add_u32 s18, s18, use_workitem_id_x at rel32@lo+4
+; GCN-NEXT: s_addc_u32 s19, s19, use_workitem_id_x at rel32@hi+12
+; GCN-NEXT: v_mov_b32_e32 v0, 9
+; GCN-NEXT: s_mov_b32 s14, s16
+; GCN-NEXT: s_mov_b32 s32, 0
+; GCN-NEXT: s_swappc_b64 s[30:31], s[18:19]
+; GCN-NEXT: v_add_f32_e32 v0, 1.0, v0
+; GCN-NEXT: flat_store_dword v[0:1], v0
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: s_endpgm
+ %val = call float @use_workitem_id_x(i32 9)
+ %op = fadd float %val, 1.0
+ store volatile float %op, ptr addrspace(1) poison
+ ret void
+}
+
+ at _ZTIi = external global ptr
+declare i32 @__gxx_personality_v0(...)
+define amdgpu_kernel void @test_invoke() #0 personality ptr @__gxx_personality_v0 {
+; GCN-LABEL: test_invoke:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_add_i32 s12, s12, s17
+; GCN-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-NEXT: s_add_u32 s0, s0, s17
+; GCN-NEXT: v_lshlrev_b32_e32 v1, 10, v1
+; GCN-NEXT: s_addc_u32 s1, s1, 0
+; GCN-NEXT: v_or_b32_e32 v0, v0, v1
+; GCN-NEXT: v_lshlrev_b32_e32 v1, 20, v2
+; GCN-NEXT: s_mov_b32 flat_scratch_lo, s13
+; GCN-NEXT: s_mov_b32 s13, s15
+; GCN-NEXT: s_mov_b32 s12, s14
+; GCN-NEXT: v_or_b32_e32 v31, v0, v1
+; GCN-NEXT: s_getpc_b64 s[18:19]
+; GCN-NEXT: s_add_u32 s18, s18, ident_i32 at rel32@lo+4
+; GCN-NEXT: s_addc_u32 s19, s19, ident_i32 at rel32@hi+12
+; GCN-NEXT: v_mov_b32_e32 v0, 2.0
+; GCN-NEXT: s_mov_b32 s14, s16
+; GCN-NEXT: s_mov_b32 s32, 0
+; GCN-NEXT: ; implicit-def: $sgpr15
+; GCN-NEXT: s_swappc_b64 s[30:31], s[18:19]
+; GCN-NEXT: v_add_f32_e32 v0, 1.0, v0
+; GCN-NEXT: flat_store_dword v[0:1], v0
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: s_endpgm
+ %val = invoke float @ident_i32(float 2.0)
+ to label %continue unwind label %broken
+
+broken:
+ landingpad { ptr, i32 } catch ptr @_ZTIi
+ ret void
+
+continue:
+ %op = fadd float %val, 1.0
+ store volatile float %op, ptr addrspace(1) poison
+ ret void
+}
+
+; Callees appears last in source file to test that we still lower their
+; arguments before we lower any calls to them.
+
+define hidden i32 @ret_i32_noinline() #0 {
+; GCN-LABEL: ret_i32_noinline:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mov_b32_e32 v0, 4
+; GCN-NEXT: s_setpc_b64 s[30:31]
+ ret i32 4
+}
+
+define hidden i32 @ret_i32_alwaysinline() #1 {
+; GCN-LABEL: ret_i32_alwaysinline:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mov_b32_e32 v0, 4
+; GCN-NEXT: s_setpc_b64 s[30:31]
+ ret i32 4
+}
+
+define hidden i32 @ident_i32(i32 %i) #0 {
+; GCN-LABEL: ident_i32:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: s_setpc_b64 s[30:31]
+ ret i32 %i
+}
+
+declare i32 @llvm.amdgcn.workitem.id.x() #2
+
+attributes #0 = { nounwind noinline }
+attributes #1 = { alwaysinline nounwind }
+attributes #2 = { nounwind readnone speculatable }
+attributes #3 = { nounwind noinline "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" }
diff --git a/llvm/test/CodeGen/AMDGPU/call-constexpr-sdag.ll b/llvm/test/CodeGen/AMDGPU/call-constexpr-sdag.ll
new file mode 100644
index 0000000000000..2660e6c85cedf
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/call-constexpr-sdag.ll
@@ -0,0 +1,238 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
+; RUN: llc -global-isel=0 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 < %s | FileCheck -check-prefixes=GCN %s
+
+define hidden i32 @ret_i32_noinline() #0 {
+; GCN-LABEL: ret_i32_noinline:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mov_b32_e32 v0, 4
+; GCN-NEXT: s_setpc_b64 s[30:31]
+ ret i32 4
+}
+
+define hidden i32 @ret_i32_alwaysinline() #1 {
+; GCN-LABEL: ret_i32_alwaysinline:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mov_b32_e32 v0, 4
+; GCN-NEXT: s_setpc_b64 s[30:31]
+ ret i32 4
+}
+
+define hidden i32 @ident_i32(i32 %i) #0 {
+; GCN-LABEL: ident_i32:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: s_setpc_b64 s[30:31]
+ ret i32 %i
+}
+
+define amdgpu_kernel void @test_bitcast_return_type_noinline() #0 {
+; GCN-LABEL: test_bitcast_return_type_noinline:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_add_i32 s12, s12, s17
+; GCN-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-NEXT: s_add_u32 s0, s0, s17
+; GCN-NEXT: v_lshlrev_b32_e32 v1, 10, v1
+; GCN-NEXT: s_addc_u32 s1, s1, 0
+; GCN-NEXT: v_lshlrev_b32_e32 v2, 20, v2
+; GCN-NEXT: v_or_b32_e32 v0, v0, v1
+; GCN-NEXT: s_mov_b32 flat_scratch_lo, s13
+; GCN-NEXT: s_mov_b32 s13, s15
+; GCN-NEXT: s_mov_b32 s12, s14
+; GCN-NEXT: s_getpc_b64 s[18:19]
+; GCN-NEXT: s_add_u32 s18, s18, ret_i32_noinline at rel32@lo+4
+; GCN-NEXT: s_addc_u32 s19, s19, ret_i32_noinline at rel32@hi+12
+; GCN-NEXT: v_or_b32_e32 v31, v0, v2
+; GCN-NEXT: s_mov_b32 s14, s16
+; GCN-NEXT: s_mov_b32 s32, 0
+; GCN-NEXT: s_swappc_b64 s[30:31], s[18:19]
+; GCN-NEXT: v_add_f32_e32 v0, 1.0, v0
+; GCN-NEXT: flat_store_dword v[0:1], v0
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: s_endpgm
+ %val = call float @ret_i32_noinline()
+ %op = fadd float %val, 1.0
+ store volatile float %op, ptr addrspace(1) poison
+ ret void
+}
+
+define amdgpu_kernel void @test_bitcast_return_type_alwaysinline() #0 {
+; GCN-LABEL: test_bitcast_return_type_alwaysinline:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_add_i32 s12, s12, s17
+; GCN-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-NEXT: s_add_u32 s0, s0, s17
+; GCN-NEXT: v_lshlrev_b32_e32 v1, 10, v1
+; GCN-NEXT: s_addc_u32 s1, s1, 0
+; GCN-NEXT: v_lshlrev_b32_e32 v2, 20, v2
+; GCN-NEXT: v_or_b32_e32 v0, v0, v1
+; GCN-NEXT: s_mov_b32 flat_scratch_lo, s13
+; GCN-NEXT: s_mov_b32 s13, s15
+; GCN-NEXT: s_mov_b32 s12, s14
+; GCN-NEXT: s_getpc_b64 s[18:19]
+; GCN-NEXT: s_add_u32 s18, s18, ret_i32_alwaysinline at rel32@lo+4
+; GCN-NEXT: s_addc_u32 s19, s19, ret_i32_alwaysinline at rel32@hi+12
+; GCN-NEXT: v_or_b32_e32 v31, v0, v2
+; GCN-NEXT: s_mov_b32 s14, s16
+; GCN-NEXT: s_mov_b32 s32, 0
+; GCN-NEXT: s_swappc_b64 s[30:31], s[18:19]
+; GCN-NEXT: v_add_f32_e32 v0, 1.0, v0
+; GCN-NEXT: flat_store_dword v[0:1], v0
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: s_endpgm
+ %val = call float @ret_i32_alwaysinline()
+ %op = fadd float %val, 1.0
+ store volatile float %op, ptr addrspace(1) poison
+ ret void
+}
+
+define amdgpu_kernel void @test_bitcast_argument_type() #0 {
+; GCN-LABEL: test_bitcast_argument_type:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_add_i32 s12, s12, s17
+; GCN-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-NEXT: s_add_u32 s0, s0, s17
+; GCN-NEXT: v_lshlrev_b32_e32 v1, 10, v1
+; GCN-NEXT: s_addc_u32 s1, s1, 0
+; GCN-NEXT: v_lshlrev_b32_e32 v2, 20, v2
+; GCN-NEXT: v_or_b32_e32 v0, v0, v1
+; GCN-NEXT: s_mov_b32 flat_scratch_lo, s13
+; GCN-NEXT: s_mov_b32 s13, s15
+; GCN-NEXT: s_mov_b32 s12, s14
+; GCN-NEXT: s_getpc_b64 s[18:19]
+; GCN-NEXT: s_add_u32 s18, s18, ident_i32 at rel32@lo+4
+; GCN-NEXT: s_addc_u32 s19, s19, ident_i32 at rel32@hi+12
+; GCN-NEXT: v_or_b32_e32 v31, v0, v2
+; GCN-NEXT: s_mov_b32 s14, s16
+; GCN-NEXT: v_mov_b32_e32 v0, 2.0
+; GCN-NEXT: s_mov_b32 s32, 0
+; GCN-NEXT: s_swappc_b64 s[30:31], s[18:19]
+; GCN-NEXT: v_add_i32_e32 v0, vcc, 1, v0
+; GCN-NEXT: flat_store_dword v[0:1], v0
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: s_endpgm
+ %val = call i32 @ident_i32(float 2.0)
+ %op = add i32 %val, 1
+ store volatile i32 %op, ptr addrspace(1) poison
+ ret void
+}
+
+define amdgpu_kernel void @test_bitcast_argument_and_return_types() #0 {
+; GCN-LABEL: test_bitcast_argument_and_return_types:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_add_i32 s12, s12, s17
+; GCN-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-NEXT: s_add_u32 s0, s0, s17
+; GCN-NEXT: v_lshlrev_b32_e32 v1, 10, v1
+; GCN-NEXT: s_addc_u32 s1, s1, 0
+; GCN-NEXT: v_lshlrev_b32_e32 v2, 20, v2
+; GCN-NEXT: v_or_b32_e32 v0, v0, v1
+; GCN-NEXT: s_mov_b32 flat_scratch_lo, s13
+; GCN-NEXT: s_mov_b32 s13, s15
+; GCN-NEXT: s_mov_b32 s12, s14
+; GCN-NEXT: s_getpc_b64 s[18:19]
+; GCN-NEXT: s_add_u32 s18, s18, ident_i32 at rel32@lo+4
+; GCN-NEXT: s_addc_u32 s19, s19, ident_i32 at rel32@hi+12
+; GCN-NEXT: v_or_b32_e32 v31, v0, v2
+; GCN-NEXT: s_mov_b32 s14, s16
+; GCN-NEXT: v_mov_b32_e32 v0, 2.0
+; GCN-NEXT: s_mov_b32 s32, 0
+; GCN-NEXT: s_swappc_b64 s[30:31], s[18:19]
+; GCN-NEXT: v_add_f32_e32 v0, 1.0, v0
+; GCN-NEXT: flat_store_dword v[0:1], v0
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: s_endpgm
+ %val = call float @ident_i32(float 2.0)
+ %op = fadd float %val, 1.0
+ store volatile float %op, ptr addrspace(1) poison
+ ret void
+}
+
+define hidden i32 @use_workitem_id_x(i32 %arg0) #3 {
+; GCN-LABEL: use_workitem_id_x:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_and_b32_e32 v1, 0x3ff, v31
+; GCN-NEXT: v_add_i32_e32 v0, vcc, v1, v0
+; GCN-NEXT: s_setpc_b64 s[30:31]
+ %id = call i32 @llvm.amdgcn.workitem.id.x()
+ %op = add i32 %id, %arg0
+ ret i32 %op
+}
+
+define amdgpu_kernel void @test_bitcast_use_workitem_id_x() #3 {
+; GCN-LABEL: test_bitcast_use_workitem_id_x:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_add_i32 s12, s12, s17
+; GCN-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-NEXT: s_add_u32 s0, s0, s17
+; GCN-NEXT: s_addc_u32 s1, s1, 0
+; GCN-NEXT: s_mov_b32 flat_scratch_lo, s13
+; GCN-NEXT: s_mov_b32 s13, s15
+; GCN-NEXT: s_mov_b32 s12, s14
+; GCN-NEXT: s_getpc_b64 s[18:19]
+; GCN-NEXT: s_add_u32 s18, s18, use_workitem_id_x at rel32@lo+4
+; GCN-NEXT: s_addc_u32 s19, s19, use_workitem_id_x at rel32@hi+12
+; GCN-NEXT: s_mov_b32 s14, s16
+; GCN-NEXT: v_mov_b32_e32 v31, v0
+; GCN-NEXT: v_mov_b32_e32 v0, 9
+; GCN-NEXT: s_mov_b32 s32, 0
+; GCN-NEXT: s_swappc_b64 s[30:31], s[18:19]
+; GCN-NEXT: v_add_f32_e32 v0, 1.0, v0
+; GCN-NEXT: flat_store_dword v[0:1], v0
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: s_endpgm
+ %val = call float @use_workitem_id_x(i32 9)
+ %op = fadd float %val, 1.0
+ store volatile float %op, ptr addrspace(1) poison
+ ret void
+}
+
+ at _ZTIi = external global ptr
+declare i32 @__gxx_personality_v0(...)
+define amdgpu_kernel void @test_invoke() #0 personality ptr @__gxx_personality_v0 {
+; GCN-LABEL: test_invoke:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_add_i32 s12, s12, s17
+; GCN-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-NEXT: s_add_u32 s0, s0, s17
+; GCN-NEXT: v_lshlrev_b32_e32 v1, 10, v1
+; GCN-NEXT: s_addc_u32 s1, s1, 0
+; GCN-NEXT: v_lshlrev_b32_e32 v2, 20, v2
+; GCN-NEXT: v_or_b32_e32 v0, v0, v1
+; GCN-NEXT: s_mov_b32 flat_scratch_lo, s13
+; GCN-NEXT: s_mov_b32 s13, s15
+; GCN-NEXT: s_mov_b32 s12, s14
+; GCN-NEXT: s_getpc_b64 s[18:19]
+; GCN-NEXT: s_add_u32 s18, s18, ident_i32 at rel32@lo+4
+; GCN-NEXT: s_addc_u32 s19, s19, ident_i32 at rel32@hi+12
+; GCN-NEXT: v_or_b32_e32 v31, v0, v2
+; GCN-NEXT: s_mov_b32 s14, s16
+; GCN-NEXT: v_mov_b32_e32 v0, 2.0
+; GCN-NEXT: s_mov_b32 s32, 0
+; GCN-NEXT: ; implicit-def: $sgpr15
+; GCN-NEXT: s_swappc_b64 s[30:31], s[18:19]
+; GCN-NEXT: v_add_f32_e32 v0, 1.0, v0
+; GCN-NEXT: flat_store_dword v[0:1], v0
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: s_endpgm
+ %val = invoke float @ident_i32(float 2.0)
+ to label %continue unwind label %broken
+
+broken:
+ landingpad { ptr, i32 } catch ptr @_ZTIi
+ ret void
+
+continue:
+ %op = fadd float %val, 1.0
+ store volatile float %op, ptr addrspace(1) poison
+ ret void
+}
+
+declare i32 @llvm.amdgcn.workitem.id.x() #2
+
+attributes #0 = { nounwind noinline }
+attributes #1 = { alwaysinline nounwind }
+attributes #2 = { nounwind readnone speculatable }
+attributes #3 = { nounwind noinline "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" }
diff --git a/llvm/test/CodeGen/AMDGPU/call-constexpr.ll b/llvm/test/CodeGen/AMDGPU/call-constexpr.ll
deleted file mode 100644
index 53867b9031f6b..0000000000000
--- a/llvm/test/CodeGen/AMDGPU/call-constexpr.ll
+++ /dev/null
@@ -1,388 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
-; RUN: llc -global-isel=0 -mtriple=amdgpu7.00-amd-amdhsa < %s | FileCheck -check-prefixes=GCN,SDAG %s
-; RUN: llc -global-isel=1 -mtriple=amdgpu7.00-amd-amdhsa < %s | FileCheck -check-prefixes=GCN,GISEL %s
-
-define amdgpu_kernel void @test_bitcast_return_type_noinline() #0 {
-; SDAG-LABEL: test_bitcast_return_type_noinline:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: s_add_i32 s12, s12, s17
-; SDAG-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; SDAG-NEXT: s_add_u32 s0, s0, s17
-; SDAG-NEXT: v_lshlrev_b32_e32 v1, 10, v1
-; SDAG-NEXT: s_addc_u32 s1, s1, 0
-; SDAG-NEXT: v_lshlrev_b32_e32 v2, 20, v2
-; SDAG-NEXT: v_or_b32_e32 v0, v0, v1
-; SDAG-NEXT: s_mov_b32 flat_scratch_lo, s13
-; SDAG-NEXT: s_mov_b32 s13, s15
-; SDAG-NEXT: s_mov_b32 s12, s14
-; SDAG-NEXT: s_getpc_b64 s[18:19]
-; SDAG-NEXT: s_add_u32 s18, s18, ret_i32_noinline at rel32@lo+4
-; SDAG-NEXT: s_addc_u32 s19, s19, ret_i32_noinline at rel32@hi+12
-; SDAG-NEXT: v_or_b32_e32 v31, v0, v2
-; SDAG-NEXT: s_mov_b32 s14, s16
-; SDAG-NEXT: s_mov_b32 s32, 0
-; SDAG-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; SDAG-NEXT: v_add_f32_e32 v0, 1.0, v0
-; SDAG-NEXT: flat_store_dword v[0:1], v0
-; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: test_bitcast_return_type_noinline:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: s_add_i32 s12, s12, s17
-; GISEL-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; GISEL-NEXT: s_add_u32 s0, s0, s17
-; GISEL-NEXT: v_lshlrev_b32_e32 v1, 10, v1
-; GISEL-NEXT: s_addc_u32 s1, s1, 0
-; GISEL-NEXT: v_or_b32_e32 v0, v0, v1
-; GISEL-NEXT: v_lshlrev_b32_e32 v1, 20, v2
-; GISEL-NEXT: s_mov_b32 flat_scratch_lo, s13
-; GISEL-NEXT: s_mov_b32 s13, s15
-; GISEL-NEXT: s_mov_b32 s12, s14
-; GISEL-NEXT: v_or_b32_e32 v31, v0, v1
-; GISEL-NEXT: s_getpc_b64 s[18:19]
-; GISEL-NEXT: s_add_u32 s18, s18, ret_i32_noinline at rel32@lo+4
-; GISEL-NEXT: s_addc_u32 s19, s19, ret_i32_noinline at rel32@hi+12
-; GISEL-NEXT: s_mov_b32 s14, s16
-; GISEL-NEXT: s_mov_b32 s32, 0
-; GISEL-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GISEL-NEXT: v_add_f32_e32 v0, 1.0, v0
-; GISEL-NEXT: flat_store_dword v[0:1], v0
-; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: s_endpgm
- %val = call float @ret_i32_noinline()
- %op = fadd float %val, 1.0
- store volatile float %op, ptr addrspace(1) poison
- ret void
-}
-
-define amdgpu_kernel void @test_bitcast_return_type_alwaysinline() #0 {
-; SDAG-LABEL: test_bitcast_return_type_alwaysinline:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: s_add_i32 s12, s12, s17
-; SDAG-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; SDAG-NEXT: s_add_u32 s0, s0, s17
-; SDAG-NEXT: v_lshlrev_b32_e32 v1, 10, v1
-; SDAG-NEXT: s_addc_u32 s1, s1, 0
-; SDAG-NEXT: v_lshlrev_b32_e32 v2, 20, v2
-; SDAG-NEXT: v_or_b32_e32 v0, v0, v1
-; SDAG-NEXT: s_mov_b32 flat_scratch_lo, s13
-; SDAG-NEXT: s_mov_b32 s13, s15
-; SDAG-NEXT: s_mov_b32 s12, s14
-; SDAG-NEXT: s_getpc_b64 s[18:19]
-; SDAG-NEXT: s_add_u32 s18, s18, ret_i32_alwaysinline at rel32@lo+4
-; SDAG-NEXT: s_addc_u32 s19, s19, ret_i32_alwaysinline at rel32@hi+12
-; SDAG-NEXT: v_or_b32_e32 v31, v0, v2
-; SDAG-NEXT: s_mov_b32 s14, s16
-; SDAG-NEXT: s_mov_b32 s32, 0
-; SDAG-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; SDAG-NEXT: v_add_f32_e32 v0, 1.0, v0
-; SDAG-NEXT: flat_store_dword v[0:1], v0
-; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: test_bitcast_return_type_alwaysinline:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: s_add_i32 s12, s12, s17
-; GISEL-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; GISEL-NEXT: s_add_u32 s0, s0, s17
-; GISEL-NEXT: v_lshlrev_b32_e32 v1, 10, v1
-; GISEL-NEXT: s_addc_u32 s1, s1, 0
-; GISEL-NEXT: v_or_b32_e32 v0, v0, v1
-; GISEL-NEXT: v_lshlrev_b32_e32 v1, 20, v2
-; GISEL-NEXT: s_mov_b32 flat_scratch_lo, s13
-; GISEL-NEXT: s_mov_b32 s13, s15
-; GISEL-NEXT: s_mov_b32 s12, s14
-; GISEL-NEXT: v_or_b32_e32 v31, v0, v1
-; GISEL-NEXT: s_getpc_b64 s[18:19]
-; GISEL-NEXT: s_add_u32 s18, s18, ret_i32_alwaysinline at rel32@lo+4
-; GISEL-NEXT: s_addc_u32 s19, s19, ret_i32_alwaysinline at rel32@hi+12
-; GISEL-NEXT: s_mov_b32 s14, s16
-; GISEL-NEXT: s_mov_b32 s32, 0
-; GISEL-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GISEL-NEXT: v_add_f32_e32 v0, 1.0, v0
-; GISEL-NEXT: flat_store_dword v[0:1], v0
-; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: s_endpgm
- %val = call float @ret_i32_alwaysinline()
- %op = fadd float %val, 1.0
- store volatile float %op, ptr addrspace(1) poison
- ret void
-}
-
-define amdgpu_kernel void @test_bitcast_argument_type() #0 {
-; SDAG-LABEL: test_bitcast_argument_type:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: s_add_i32 s12, s12, s17
-; SDAG-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; SDAG-NEXT: s_add_u32 s0, s0, s17
-; SDAG-NEXT: v_lshlrev_b32_e32 v1, 10, v1
-; SDAG-NEXT: s_addc_u32 s1, s1, 0
-; SDAG-NEXT: v_lshlrev_b32_e32 v2, 20, v2
-; SDAG-NEXT: v_or_b32_e32 v0, v0, v1
-; SDAG-NEXT: s_mov_b32 flat_scratch_lo, s13
-; SDAG-NEXT: s_mov_b32 s13, s15
-; SDAG-NEXT: s_mov_b32 s12, s14
-; SDAG-NEXT: s_getpc_b64 s[18:19]
-; SDAG-NEXT: s_add_u32 s18, s18, ident_i32 at rel32@lo+4
-; SDAG-NEXT: s_addc_u32 s19, s19, ident_i32 at rel32@hi+12
-; SDAG-NEXT: v_or_b32_e32 v31, v0, v2
-; SDAG-NEXT: s_mov_b32 s14, s16
-; SDAG-NEXT: v_mov_b32_e32 v0, 2.0
-; SDAG-NEXT: s_mov_b32 s32, 0
-; SDAG-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; SDAG-NEXT: v_add_i32_e32 v0, vcc, 1, v0
-; SDAG-NEXT: flat_store_dword v[0:1], v0
-; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: test_bitcast_argument_type:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: s_add_i32 s12, s12, s17
-; GISEL-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; GISEL-NEXT: s_add_u32 s0, s0, s17
-; GISEL-NEXT: v_lshlrev_b32_e32 v1, 10, v1
-; GISEL-NEXT: s_addc_u32 s1, s1, 0
-; GISEL-NEXT: v_or_b32_e32 v0, v0, v1
-; GISEL-NEXT: v_lshlrev_b32_e32 v1, 20, v2
-; GISEL-NEXT: s_mov_b32 flat_scratch_lo, s13
-; GISEL-NEXT: s_mov_b32 s13, s15
-; GISEL-NEXT: s_mov_b32 s12, s14
-; GISEL-NEXT: v_or_b32_e32 v31, v0, v1
-; GISEL-NEXT: s_getpc_b64 s[18:19]
-; GISEL-NEXT: s_add_u32 s18, s18, ident_i32 at rel32@lo+4
-; GISEL-NEXT: s_addc_u32 s19, s19, ident_i32 at rel32@hi+12
-; GISEL-NEXT: v_mov_b32_e32 v0, 2.0
-; GISEL-NEXT: s_mov_b32 s14, s16
-; GISEL-NEXT: s_mov_b32 s32, 0
-; GISEL-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, 1, v0
-; GISEL-NEXT: flat_store_dword v[0:1], v0
-; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: s_endpgm
- %val = call i32 @ident_i32(float 2.0)
- %op = add i32 %val, 1
- store volatile i32 %op, ptr addrspace(1) poison
- ret void
-}
-
-define amdgpu_kernel void @test_bitcast_argument_and_return_types() #0 {
-; SDAG-LABEL: test_bitcast_argument_and_return_types:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: s_add_i32 s12, s12, s17
-; SDAG-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; SDAG-NEXT: s_add_u32 s0, s0, s17
-; SDAG-NEXT: v_lshlrev_b32_e32 v1, 10, v1
-; SDAG-NEXT: s_addc_u32 s1, s1, 0
-; SDAG-NEXT: v_lshlrev_b32_e32 v2, 20, v2
-; SDAG-NEXT: v_or_b32_e32 v0, v0, v1
-; SDAG-NEXT: s_mov_b32 flat_scratch_lo, s13
-; SDAG-NEXT: s_mov_b32 s13, s15
-; SDAG-NEXT: s_mov_b32 s12, s14
-; SDAG-NEXT: s_getpc_b64 s[18:19]
-; SDAG-NEXT: s_add_u32 s18, s18, ident_i32 at rel32@lo+4
-; SDAG-NEXT: s_addc_u32 s19, s19, ident_i32 at rel32@hi+12
-; SDAG-NEXT: v_or_b32_e32 v31, v0, v2
-; SDAG-NEXT: s_mov_b32 s14, s16
-; SDAG-NEXT: v_mov_b32_e32 v0, 2.0
-; SDAG-NEXT: s_mov_b32 s32, 0
-; SDAG-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; SDAG-NEXT: v_add_f32_e32 v0, 1.0, v0
-; SDAG-NEXT: flat_store_dword v[0:1], v0
-; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: test_bitcast_argument_and_return_types:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: s_add_i32 s12, s12, s17
-; GISEL-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; GISEL-NEXT: s_add_u32 s0, s0, s17
-; GISEL-NEXT: v_lshlrev_b32_e32 v1, 10, v1
-; GISEL-NEXT: s_addc_u32 s1, s1, 0
-; GISEL-NEXT: v_or_b32_e32 v0, v0, v1
-; GISEL-NEXT: v_lshlrev_b32_e32 v1, 20, v2
-; GISEL-NEXT: s_mov_b32 flat_scratch_lo, s13
-; GISEL-NEXT: s_mov_b32 s13, s15
-; GISEL-NEXT: s_mov_b32 s12, s14
-; GISEL-NEXT: v_or_b32_e32 v31, v0, v1
-; GISEL-NEXT: s_getpc_b64 s[18:19]
-; GISEL-NEXT: s_add_u32 s18, s18, ident_i32 at rel32@lo+4
-; GISEL-NEXT: s_addc_u32 s19, s19, ident_i32 at rel32@hi+12
-; GISEL-NEXT: v_mov_b32_e32 v0, 2.0
-; GISEL-NEXT: s_mov_b32 s14, s16
-; GISEL-NEXT: s_mov_b32 s32, 0
-; GISEL-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GISEL-NEXT: v_add_f32_e32 v0, 1.0, v0
-; GISEL-NEXT: flat_store_dword v[0:1], v0
-; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: s_endpgm
- %val = call float @ident_i32(float 2.0)
- %op = fadd float %val, 1.0
- store volatile float %op, ptr addrspace(1) poison
- ret void
-}
-
-define hidden i32 @use_workitem_id_x(i32 %arg0) #3 {
-; GCN-LABEL: use_workitem_id_x:
-; GCN: ; %bb.0:
-; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v1, 0x3ff, v31
-; GCN-NEXT: v_add_i32_e32 v0, vcc, v1, v0
-; GCN-NEXT: s_setpc_b64 s[30:31]
- %id = call i32 @llvm.amdgcn.workitem.id.x()
- %op = add i32 %id, %arg0
- ret i32 %op
-}
-
-define amdgpu_kernel void @test_bitcast_use_workitem_id_x() #3 {
-; SDAG-LABEL: test_bitcast_use_workitem_id_x:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: s_add_i32 s12, s12, s17
-; SDAG-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; SDAG-NEXT: s_add_u32 s0, s0, s17
-; SDAG-NEXT: s_addc_u32 s1, s1, 0
-; SDAG-NEXT: s_mov_b32 flat_scratch_lo, s13
-; SDAG-NEXT: s_mov_b32 s13, s15
-; SDAG-NEXT: s_mov_b32 s12, s14
-; SDAG-NEXT: s_getpc_b64 s[18:19]
-; SDAG-NEXT: s_add_u32 s18, s18, use_workitem_id_x at rel32@lo+4
-; SDAG-NEXT: s_addc_u32 s19, s19, use_workitem_id_x at rel32@hi+12
-; SDAG-NEXT: s_mov_b32 s14, s16
-; SDAG-NEXT: v_mov_b32_e32 v31, v0
-; SDAG-NEXT: v_mov_b32_e32 v0, 9
-; SDAG-NEXT: s_mov_b32 s32, 0
-; SDAG-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; SDAG-NEXT: v_add_f32_e32 v0, 1.0, v0
-; SDAG-NEXT: flat_store_dword v[0:1], v0
-; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: test_bitcast_use_workitem_id_x:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: s_add_i32 s12, s12, s17
-; GISEL-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; GISEL-NEXT: s_add_u32 s0, s0, s17
-; GISEL-NEXT: s_addc_u32 s1, s1, 0
-; GISEL-NEXT: s_mov_b32 flat_scratch_lo, s13
-; GISEL-NEXT: v_mov_b32_e32 v31, v0
-; GISEL-NEXT: s_mov_b32 s13, s15
-; GISEL-NEXT: s_mov_b32 s12, s14
-; GISEL-NEXT: s_getpc_b64 s[18:19]
-; GISEL-NEXT: s_add_u32 s18, s18, use_workitem_id_x at rel32@lo+4
-; GISEL-NEXT: s_addc_u32 s19, s19, use_workitem_id_x at rel32@hi+12
-; GISEL-NEXT: v_mov_b32_e32 v0, 9
-; GISEL-NEXT: s_mov_b32 s14, s16
-; GISEL-NEXT: s_mov_b32 s32, 0
-; GISEL-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GISEL-NEXT: v_add_f32_e32 v0, 1.0, v0
-; GISEL-NEXT: flat_store_dword v[0:1], v0
-; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: s_endpgm
- %val = call float @use_workitem_id_x(i32 9)
- %op = fadd float %val, 1.0
- store volatile float %op, ptr addrspace(1) poison
- ret void
-}
-
- at _ZTIi = external global ptr
-declare i32 @__gxx_personality_v0(...)
-define amdgpu_kernel void @test_invoke() #0 personality ptr @__gxx_personality_v0 {
-; SDAG-LABEL: test_invoke:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: s_add_i32 s12, s12, s17
-; SDAG-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; SDAG-NEXT: s_add_u32 s0, s0, s17
-; SDAG-NEXT: v_lshlrev_b32_e32 v1, 10, v1
-; SDAG-NEXT: s_addc_u32 s1, s1, 0
-; SDAG-NEXT: v_lshlrev_b32_e32 v2, 20, v2
-; SDAG-NEXT: v_or_b32_e32 v0, v0, v1
-; SDAG-NEXT: s_mov_b32 flat_scratch_lo, s13
-; SDAG-NEXT: s_mov_b32 s13, s15
-; SDAG-NEXT: s_mov_b32 s12, s14
-; SDAG-NEXT: s_getpc_b64 s[18:19]
-; SDAG-NEXT: s_add_u32 s18, s18, ident_i32 at rel32@lo+4
-; SDAG-NEXT: s_addc_u32 s19, s19, ident_i32 at rel32@hi+12
-; SDAG-NEXT: v_or_b32_e32 v31, v0, v2
-; SDAG-NEXT: s_mov_b32 s14, s16
-; SDAG-NEXT: v_mov_b32_e32 v0, 2.0
-; SDAG-NEXT: s_mov_b32 s32, 0
-; SDAG-NEXT: ; implicit-def: $sgpr15
-; SDAG-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; SDAG-NEXT: v_add_f32_e32 v0, 1.0, v0
-; SDAG-NEXT: flat_store_dword v[0:1], v0
-; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: test_invoke:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: s_add_i32 s12, s12, s17
-; GISEL-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; GISEL-NEXT: s_add_u32 s0, s0, s17
-; GISEL-NEXT: v_lshlrev_b32_e32 v1, 10, v1
-; GISEL-NEXT: s_addc_u32 s1, s1, 0
-; GISEL-NEXT: v_or_b32_e32 v0, v0, v1
-; GISEL-NEXT: v_lshlrev_b32_e32 v1, 20, v2
-; GISEL-NEXT: s_mov_b32 flat_scratch_lo, s13
-; GISEL-NEXT: s_mov_b32 s13, s15
-; GISEL-NEXT: s_mov_b32 s12, s14
-; GISEL-NEXT: v_or_b32_e32 v31, v0, v1
-; GISEL-NEXT: s_getpc_b64 s[18:19]
-; GISEL-NEXT: s_add_u32 s18, s18, ident_i32 at rel32@lo+4
-; GISEL-NEXT: s_addc_u32 s19, s19, ident_i32 at rel32@hi+12
-; GISEL-NEXT: v_mov_b32_e32 v0, 2.0
-; GISEL-NEXT: s_mov_b32 s14, s16
-; GISEL-NEXT: s_mov_b32 s32, 0
-; GISEL-NEXT: ; implicit-def: $sgpr15
-; GISEL-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GISEL-NEXT: v_add_f32_e32 v0, 1.0, v0
-; GISEL-NEXT: flat_store_dword v[0:1], v0
-; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: s_endpgm
- %val = invoke float @ident_i32(float 2.0)
- to label %continue unwind label %broken
-
-broken:
- landingpad { ptr, i32 } catch ptr @_ZTIi
- ret void
-
-continue:
- %op = fadd float %val, 1.0
- store volatile float %op, ptr addrspace(1) poison
- ret void
-}
-
-; Callees appears last in source file to test that we still lower their
-; arguments before we lower any calls to them.
-
-define hidden i32 @ret_i32_noinline() #0 {
-; GCN-LABEL: ret_i32_noinline:
-; GCN: ; %bb.0:
-; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v0, 4
-; GCN-NEXT: s_setpc_b64 s[30:31]
- ret i32 4
-}
-
-define hidden i32 @ret_i32_alwaysinline() #1 {
-; GCN-LABEL: ret_i32_alwaysinline:
-; GCN: ; %bb.0:
-; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v0, 4
-; GCN-NEXT: s_setpc_b64 s[30:31]
- ret i32 4
-}
-
-define hidden i32 @ident_i32(i32 %i) #0 {
-; GCN-LABEL: ident_i32:
-; GCN: ; %bb.0:
-; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: s_setpc_b64 s[30:31]
- ret i32 %i
-}
-
-declare i32 @llvm.amdgcn.workitem.id.x() #2
-
-attributes #0 = { nounwind noinline }
-attributes #1 = { alwaysinline nounwind }
-attributes #2 = { nounwind readnone speculatable }
-attributes #3 = { nounwind noinline "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" }
diff --git a/llvm/test/CodeGen/AMDGPU/convergence-laneops.ll b/llvm/test/CodeGen/AMDGPU/convergence-laneops.ll
index 64a6edcc72c93..3353f200629dc 100644
--- a/llvm/test/CodeGen/AMDGPU/convergence-laneops.ll
+++ b/llvm/test/CodeGen/AMDGPU/convergence-laneops.ll
@@ -1,5 +1,5 @@
-; RUN: llc -stop-after=amdgpu-isel -mtriple=amdgpu11.00-- -verify-machineinstrs -o - %s | FileCheck --check-prefixes=CHECK,ISEL %s
-; RUN: not --crash llc -mtriple=amdgpu11.00--amdhsa -verify-machineinstrs < %s 2>&1 | FileCheck --check-prefix=CHECK-ERROR %s
+; RUN: llc -stop-after=amdgpu-isel -mtriple=amdgpu11.00-- -verify-each -o - %s | FileCheck --check-prefixes=CHECK,ISEL %s
+; RUN: not --crash llc -mtriple=amdgpu11.00--amdhsa -verify-each < %s 2>&1 | FileCheck --check-prefix=CHECK-ERROR %s
; FIXME: Merge these tests with existing lane op tests (llvm.amdgcn.readlane.ll, llvm.amdgcn.writelane.ll ...) once the crash is fixed.
diff --git a/llvm/test/CodeGen/AMDGPU/dead_bundle.mir b/llvm/test/CodeGen/AMDGPU/dead_bundle.mir
index 77d4a437d167d..b761deb2f4cf7 100644
--- a/llvm/test/CodeGen/AMDGPU/dead_bundle.mir
+++ b/llvm/test/CodeGen/AMDGPU/dead_bundle.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu11.00--amdpal -verify-machineinstrs=1 -start-before=greedy,0 -stop-after=virtregrewriter,0 -stress-regalloc=5 %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgpu11.00--amdpal -verify-machineinstrs=1 -start-before=greedy,1 -stop-after=virt-reg-rewriter,1 -stress-regalloc=5 %s -o - | FileCheck %s
# This test checks that dead bundles are handled correctly.
---
diff --git a/llvm/test/CodeGen/AMDGPU/dummy-regalloc-priority-advisor.mir b/llvm/test/CodeGen/AMDGPU/dummy-regalloc-priority-advisor.mir
index 3f034d709b102..f32ebfaeb4a78 100644
--- a/llvm/test/CodeGen/AMDGPU/dummy-regalloc-priority-advisor.mir
+++ b/llvm/test/CodeGen/AMDGPU/dummy-regalloc-priority-advisor.mir
@@ -1,6 +1,6 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
-# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -start-before=greedy,2 -stress-regalloc=4 -stop-after=virtregrewriter,2 -regalloc-enable-priority-advisor=default -o - %s | FileCheck -check-prefixes=CHECK,DEFAULT %s
-# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -start-before=greedy,2 -stress-regalloc=4 -stop-after=virtregrewriter,2 -regalloc-enable-priority-advisor=dummy -o - %s | FileCheck -check-prefixes=CHECK,DUMMY %s
+# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -start-before=greedy,3 -stress-regalloc=4 -stop-after=virt-reg-rewriter,3 -regalloc-enable-priority-advisor=default -o - %s | FileCheck -check-prefixes=CHECK,DEFAULT %s
+# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -start-before=greedy,3 -stress-regalloc=4 -stop-after=virt-reg-rewriter,3 -regalloc-enable-priority-advisor=dummy -o - %s | FileCheck -check-prefixes=CHECK,DUMMY %s
# Check that the regalloc-enable-priority-advisor=dummy option works
# and the result is different from the default. Ordinarily %1 would be
diff --git a/llvm/test/CodeGen/AMDGPU/dvgpr-vgpr-count-propagation-direct-chain.ll b/llvm/test/CodeGen/AMDGPU/dvgpr-vgpr-count-propagation-direct-chain.ll
index 88683131078cf..08e2cb9332ede 100644
--- a/llvm/test/CodeGen/AMDGPU/dvgpr-vgpr-count-propagation-direct-chain.ll
+++ b/llvm/test/CodeGen/AMDGPU/dvgpr-vgpr-count-propagation-direct-chain.ll
@@ -10,27 +10,27 @@
; func.2 and func.4 have DVGPRs disabled
-; DVGPR: .set .Lgfx_func_a.num_vgpr, 40
-; DVGPR: .set .Lamdgpu_cs_main.num_vgpr, max(42, .Lgfx_func_a.num_vgpr)
-; DVGPR: .set .Lfunc.0.num_vgpr, max(11, .Lgfx_func_a.num_vgpr)
-; DVGPR: .set .Lfunc.1.num_vgpr, 82
-; DVGPR: .set .Lfunc.2.num_vgpr, max(11, amdgpu.max_num_vgpr)
-; DVGPR: .set .Lfunc.3.num_vgpr, 11
-; DVGPR: .set .Lfunc.4.num_vgpr, max(82, amdgpu.max_num_vgpr)
-; DVGPR: .set .Lretry_vgpr_alloc.num_vgpr, 11
-; DVGPR: .set .Lfirst_retry_wrapper.num_vgpr, 11
-; DVGPR: .set amdgpu.max_num_vgpr, 82
-
-; NODVGPR: .set .Lgfx_func_a.num_vgpr, 40
-; NODVGPR: .set .Lamdgpu_cs_main.num_vgpr, max(42, amdgpu.max_num_vgpr)
-; NODVGPR: .set .Lfunc.0.num_vgpr, max(11, amdgpu.max_num_vgpr)
-; NODVGPR: .set .Lfunc.1.num_vgpr, max(82, amdgpu.max_num_vgpr)
-; NODVGPR: .set .Lfunc.2.num_vgpr, max(11, amdgpu.max_num_vgpr)
-; NODVGPR: .set .Lfunc.3.num_vgpr, max(11, amdgpu.max_num_vgpr)
-; NODVGPR: .set .Lfunc.4.num_vgpr, max(82, amdgpu.max_num_vgpr)
-; NODVGPR: .set .Lretry_vgpr_alloc.num_vgpr, max(11, amdgpu.max_num_vgpr)
-; NODVGPR: .set .Lfirst_retry_wrapper.num_vgpr, max(11, amdgpu.max_num_vgpr)
-; NODVGPR: .set amdgpu.max_num_vgpr, 82
+; DVGPR-DAG: .set .Lgfx_func_a.num_vgpr, 40
+; DVGPR-DAG: .set .Lamdgpu_cs_main.num_vgpr, max(42, .Lgfx_func_a.num_vgpr)
+; DVGPR-DAG: .set .Lfunc.0.num_vgpr, max(11, .Lgfx_func_a.num_vgpr)
+; DVGPR-DAG: .set .Lfunc.1.num_vgpr, 82
+; DVGPR-DAG: .set .Lfunc.2.num_vgpr, max(11, amdgpu.max_num_vgpr)
+; DVGPR-DAG: .set .Lfunc.3.num_vgpr, 11
+; DVGPR-DAG: .set .Lfunc.4.num_vgpr, max(82, amdgpu.max_num_vgpr)
+; DVGPR-DAG: .set .Lretry_vgpr_alloc.num_vgpr, 11
+; DVGPR-DAG: .set .Lfirst_retry_wrapper.num_vgpr, 11
+; DVGPR-DAG: .set amdgpu.max_num_vgpr, 82
+
+; NODVGPR-DAG: .set .Lgfx_func_a.num_vgpr, 40
+; NODVGPR-DAG: .set .Lamdgpu_cs_main.num_vgpr, max(42, amdgpu.max_num_vgpr)
+; NODVGPR-DAG: .set .Lfunc.0.num_vgpr, max(11, amdgpu.max_num_vgpr)
+; NODVGPR-DAG: .set .Lfunc.1.num_vgpr, max(82, amdgpu.max_num_vgpr)
+; NODVGPR-DAG: .set .Lfunc.2.num_vgpr, max(11, amdgpu.max_num_vgpr)
+; NODVGPR-DAG: .set .Lfunc.3.num_vgpr, max(11, amdgpu.max_num_vgpr)
+; NODVGPR-DAG: .set .Lfunc.4.num_vgpr, max(82, amdgpu.max_num_vgpr)
+; NODVGPR-DAG: .set .Lretry_vgpr_alloc.num_vgpr, max(11, amdgpu.max_num_vgpr)
+; NODVGPR-DAG: .set .Lfirst_retry_wrapper.num_vgpr, max(11, amdgpu.max_num_vgpr)
+; NODVGPR-DAG: .set amdgpu.max_num_vgpr, 82
; DVGPR: - .hardware_stages:
; DVGPR: .vgpr_count: 0x2a
diff --git a/llvm/test/CodeGen/AMDGPU/dvgpr-vgpr-count-propagation-indirect.ll b/llvm/test/CodeGen/AMDGPU/dvgpr-vgpr-count-propagation-indirect.ll
index 21028afc34003..b9f27d01205eb 100644
--- a/llvm/test/CodeGen/AMDGPU/dvgpr-vgpr-count-propagation-indirect.ll
+++ b/llvm/test/CodeGen/AMDGPU/dvgpr-vgpr-count-propagation-indirect.ll
@@ -6,19 +6,19 @@
; their `num_vgpr` count set to the max of their local count and the module-wide
; max VGPR count of all non-chain functions.
-; DVGPR: .set .Lgfx_func_a.num_vgpr, 40
-; DVGPR: .set .Lgfx_func_b.num_vgpr, 80
-; DVGPR: .set .Lfunc_with_indirect_call.num_vgpr, max(11, amdgpu.max_num_vgpr)
-; DVGPR: .set .Lfunc_direct_only.num_vgpr, max(11, .Lgfx_func_a.num_vgpr)
-; DVGPR: .set .Lfunc_chain_only.num_vgpr, 11
-; DVGPR: .set amdgpu.max_num_vgpr, 80
-
-; NODVGPR: .set .Lgfx_func_a.num_vgpr, 40
-; NODVGPR: .set .Lgfx_func_b.num_vgpr, 80
-; NODVGPR: .set .Lfunc_with_indirect_call.num_vgpr, max(11, amdgpu.max_num_vgpr)
-; NODVGPR: .set .Lfunc_direct_only.num_vgpr, max(11, amdgpu.max_num_vgpr)
-; NODVGPR: .set .Lfunc_chain_only.num_vgpr, max(11, amdgpu.max_num_vgpr)
-; NODVGPR: .set amdgpu.max_num_vgpr, 80
+; DVGPR-DAG: .set .Lgfx_func_a.num_vgpr, 40
+; DVGPR-DAG: .set .Lgfx_func_b.num_vgpr, 80
+; DVGPR-DAG: .set .Lfunc_with_indirect_call.num_vgpr, max(11, amdgpu.max_num_vgpr)
+; DVGPR-DAG: .set .Lfunc_direct_only.num_vgpr, max(11, .Lgfx_func_a.num_vgpr)
+; DVGPR-DAG: .set .Lfunc_chain_only.num_vgpr, 11
+; DVGPR-DAG: .set amdgpu.max_num_vgpr, 80
+
+; NODVGPR-DAG: .set .Lgfx_func_a.num_vgpr, 40
+; NODVGPR-DAG: .set .Lgfx_func_b.num_vgpr, 80
+; NODVGPR-DAG: .set .Lfunc_with_indirect_call.num_vgpr, max(11, amdgpu.max_num_vgpr)
+; NODVGPR-DAG: .set .Lfunc_direct_only.num_vgpr, max(11, amdgpu.max_num_vgpr)
+; NODVGPR-DAG: .set .Lfunc_chain_only.num_vgpr, max(11, amdgpu.max_num_vgpr)
+; NODVGPR-DAG: .set amdgpu.max_num_vgpr, 80
define amdgpu_gfx void @gfx_func_a() #0 {
call void asm sideeffect "", "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7},~{v8},~{v9},~{v10},~{v11},~{v12},~{v13},~{v14},~{v15},~{v16},~{v17},~{v18},~{v19},~{v20},~{v21},~{v22},~{v23},~{v24},~{v25},~{v26},~{v27},~{v28},~{v29},~{v30},~{v31},~{v32},~{v33},~{v34},~{v35},~{v36},~{v37},~{v38},~{v39}"()
diff --git a/llvm/test/CodeGen/AMDGPU/dvgpr-vgpr-count-propagation.ll b/llvm/test/CodeGen/AMDGPU/dvgpr-vgpr-count-propagation.ll
index 22ea185019cf9..43f107611f4b8 100644
--- a/llvm/test/CodeGen/AMDGPU/dvgpr-vgpr-count-propagation.ll
+++ b/llvm/test/CodeGen/AMDGPU/dvgpr-vgpr-count-propagation.ll
@@ -11,31 +11,31 @@
; non-chain functions. With DVGPRs disabled, it represents module-wide max VGPR
; count.
-; DVGPR: .set .Lgfx_func_a.num_vgpr, 40
-; DVGPR: .set .Lgfx_func_b2.num_vgpr, 80
-; DVGPR: .set .Lgfx_func_b.num_vgpr, max(61, .Lgfx_func_b2.num_vgpr)
-; DVGPR: .set .Lamdgpu_cs_main.num_vgpr, max(42, .Lgfx_func_a.num_vgpr)
-; DVGPR: .set .Lfunc.0.num_vgpr, 13
-; DVGPR: .set .Lfunc.1.num_vgpr, max(14, .Lgfx_func_a.num_vgpr, .Lgfx_func_b.num_vgpr)
-; DVGPR: .set .Lfunc.2.num_vgpr, max(16, .Lgfx_func_a.num_vgpr)
-; DVGPR: .set .Lfunc.3.num_vgpr, max(15, .Lgfx_func_b.num_vgpr)
-; DVGPR: .set .Lfunc.4.num_vgpr, max(100, .Lgfx_func_b.num_vgpr)
-; DVGPR: .set .Lretry_vgpr_alloc.num_vgpr, 11
-; DVGPR: .set .Lfirst_retry_wrapper.num_vgpr, 11
-; DVGPR: .set amdgpu.max_num_vgpr, 80
-
-; NODVGPR: .set .Lgfx_func_a.num_vgpr, 40
-; NODVGPR: .set .Lgfx_func_b2.num_vgpr, 80
-; NODVGPR: .set .Lgfx_func_b.num_vgpr, max(61, .Lgfx_func_b2.num_vgpr)
-; NODVGPR: .set .Lamdgpu_cs_main.num_vgpr, max(42, amdgpu.max_num_vgpr)
-; NODVGPR: .set .Lfunc.0.num_vgpr, max(13, amdgpu.max_num_vgpr)
-; NODVGPR: .set .Lfunc.1.num_vgpr, max(14, amdgpu.max_num_vgpr)
-; NODVGPR: .set .Lfunc.2.num_vgpr, max(16, amdgpu.max_num_vgpr)
-; NODVGPR: .set .Lfunc.3.num_vgpr, max(15, amdgpu.max_num_vgpr)
-; NODVGPR: .set .Lfunc.4.num_vgpr, max(100, amdgpu.max_num_vgpr)
-; NODVGPR: .set .Lretry_vgpr_alloc.num_vgpr, max(11, amdgpu.max_num_vgpr)
-; NODVGPR: .set .Lfirst_retry_wrapper.num_vgpr, max(11, amdgpu.max_num_vgpr)
-; NODVGPR: .set amdgpu.max_num_vgpr, 100
+; DVGPR-DAG: .set .Lgfx_func_a.num_vgpr, 40
+; DVGPR-DAG: .set .Lgfx_func_b2.num_vgpr, 80
+; DVGPR-DAG: .set .Lgfx_func_b.num_vgpr, max(61, .Lgfx_func_b2.num_vgpr)
+; DVGPR-DAG: .set .Lamdgpu_cs_main.num_vgpr, max(42, .Lgfx_func_a.num_vgpr)
+; DVGPR-DAG: .set .Lfunc.0.num_vgpr, 13
+; DVGPR-DAG: .set .Lfunc.1.num_vgpr, max(14, .Lgfx_func_a.num_vgpr, .Lgfx_func_b.num_vgpr)
+; DVGPR-DAG: .set .Lfunc.2.num_vgpr, max(16, .Lgfx_func_a.num_vgpr)
+; DVGPR-DAG: .set .Lfunc.3.num_vgpr, max(15, .Lgfx_func_b.num_vgpr)
+; DVGPR-DAG: .set .Lfunc.4.num_vgpr, max(100, .Lgfx_func_b.num_vgpr)
+; DVGPR-DAG: .set .Lretry_vgpr_alloc.num_vgpr, 11
+; DVGPR-DAG: .set .Lfirst_retry_wrapper.num_vgpr, 11
+; DVGPR-DAG: .set amdgpu.max_num_vgpr, 80
+
+; NODVGPR-DAG: .set .Lgfx_func_a.num_vgpr, 40
+; NODVGPR-DAG: .set .Lgfx_func_b2.num_vgpr, 80
+; NODVGPR-DAG: .set .Lgfx_func_b.num_vgpr, max(61, .Lgfx_func_b2.num_vgpr)
+; NODVGPR-DAG: .set .Lamdgpu_cs_main.num_vgpr, max(42, amdgpu.max_num_vgpr)
+; NODVGPR-DAG: .set .Lfunc.0.num_vgpr, max(13, amdgpu.max_num_vgpr)
+; NODVGPR-DAG: .set .Lfunc.1.num_vgpr, max(14, amdgpu.max_num_vgpr)
+; NODVGPR-DAG: .set .Lfunc.2.num_vgpr, max(16, amdgpu.max_num_vgpr)
+; NODVGPR-DAG: .set .Lfunc.3.num_vgpr, max(15, amdgpu.max_num_vgpr)
+; NODVGPR-DAG: .set .Lfunc.4.num_vgpr, max(100, amdgpu.max_num_vgpr)
+; NODVGPR-DAG: .set .Lretry_vgpr_alloc.num_vgpr, max(11, amdgpu.max_num_vgpr)
+; NODVGPR-DAG: .set .Lfirst_retry_wrapper.num_vgpr, max(11, amdgpu.max_num_vgpr)
+; NODVGPR-DAG: .set amdgpu.max_num_vgpr, 100
; DVGPR: - .hardware_stages:
; DVGPR: .vgpr_count: 0x2a
diff --git a/llvm/test/CodeGen/AMDGPU/emit-high-vgprs.mir b/llvm/test/CodeGen/AMDGPU/emit-high-vgprs.mir
index df569571c6045..2fbd5c2a7b644 100644
--- a/llvm/test/CodeGen/AMDGPU/emit-high-vgprs.mir
+++ b/llvm/test/CodeGen/AMDGPU/emit-high-vgprs.mir
@@ -1,4 +1,4 @@
-# RUN: llc -mtriple=amdgpu12.50 -show-mc-encoding -start-before=amdgpu-resource-usage -o - %s | FileCheck -check-prefix=GCN %s
+# RUN: llc -mtriple=amdgpu12.50 -show-mc-encoding -start-before=amdgpu-asm-printer -o - %s | FileCheck -check-prefix=GCN %s
# GCN-LABEL: {{^}}high_vgprs:
---
diff --git a/llvm/test/CodeGen/AMDGPU/expand-si-indirect.mir b/llvm/test/CodeGen/AMDGPU/expand-si-indirect.mir
index 8b4bf8cf8492b..247a71ee0e905 100644
--- a/llvm/test/CodeGen/AMDGPU/expand-si-indirect.mir
+++ b/llvm/test/CodeGen/AMDGPU/expand-si-indirect.mir
@@ -1,4 +1,4 @@
-# RUN: llc -mtriple=amdgpu9.00 -start-before=twoaddressinstruction %s -o - | FileCheck %s -check-prefix=GCN
+# RUN: llc -mtriple=amdgpu9.00 -start-before=two-address-instruction %s -o - | FileCheck %s -check-prefix=GCN
# Wait to sxpand SI_INDIRECT sequences that use VGPR indexing until after
# register allocation. We don’t want to reschedule the mode switching or to
diff --git a/llvm/test/CodeGen/AMDGPU/extend-wwm-virt-reg-liveness.mir b/llvm/test/CodeGen/AMDGPU/extend-wwm-virt-reg-liveness.mir
index 3433534698dd4..0b7ad9256c71d 100644
--- a/llvm/test/CodeGen/AMDGPU/extend-wwm-virt-reg-liveness.mir
+++ b/llvm/test/CodeGen/AMDGPU/extend-wwm-virt-reg-liveness.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu9.06-amd-amdhsa -start-before=si-lower-sgpr-spills -stop-after=virtregrewriter,1 -verify-machineinstrs %s -o - | FileCheck -check-prefix=GCN %s
+# RUN: llc -mtriple=amdgpu9.06-amd-amdhsa -start-before=si-lower-sgpr-spills -stop-after=virt-reg-rewriter,2 -verify-machineinstrs %s -o - | FileCheck -check-prefix=GCN %s
# Tests to check the conservative lieness extension for the wwm registers during SGPR spill lowering.
diff --git a/llvm/test/CodeGen/AMDGPU/fastregalloc-sgpr-only.mir b/llvm/test/CodeGen/AMDGPU/fastregalloc-sgpr-only.mir
index 22908c4834b63..305ec1ff8708c 100644
--- a/llvm/test/CodeGen/AMDGPU/fastregalloc-sgpr-only.mir
+++ b/llvm/test/CodeGen/AMDGPU/fastregalloc-sgpr-only.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -O0 -mtriple=amdgpu9.00-amd-amdhsa -start-before=regallocfast,0 --stop-after=regallocfast,0 -o - %s | FileCheck -check-prefix=GCN %s
+# RUN: llc -O0 -mtriple=amdgpu9.00-amd-amdhsa -start-before=regallocfast,1 --stop-after=regallocfast,1 -o - %s | FileCheck -check-prefix=GCN %s
---
name: copy_vgpr_allocation
diff --git a/llvm/test/CodeGen/AMDGPU/fmamk_fmaak-t16.mir b/llvm/test/CodeGen/AMDGPU/fmamk_fmaak-t16.mir
index 24bfc46380e6a..d6f0e129782f6 100644
--- a/llvm/test/CodeGen/AMDGPU/fmamk_fmaak-t16.mir
+++ b/llvm/test/CodeGen/AMDGPU/fmamk_fmaak-t16.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
-# RUN: llc -mtriple=amdgpu11.00 -mattr=+real-true16 -start-before=greedy,0 -stop-after=virtregrewriter,2 -o - %s | FileCheck -check-prefix=GFX11 %s
+# RUN: llc -mtriple=amdgpu11.00 -mattr=+real-true16 -start-before=greedy,1 -stop-after=virt-reg-rewriter,3 -o - %s | FileCheck -check-prefix=GFX11 %s
---
name: v_fmamk_f16
diff --git a/llvm/test/CodeGen/AMDGPU/fold-reload-into-exec.mir b/llvm/test/CodeGen/AMDGPU/fold-reload-into-exec.mir
index aa8b2e5b3f604..8d07d9ff5d90b 100644
--- a/llvm/test/CodeGen/AMDGPU/fold-reload-into-exec.mir
+++ b/llvm/test/CodeGen/AMDGPU/fold-reload-into-exec.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -verify-machineinstrs -stress-regalloc=2 -start-before=greedy -stop-after=virtregmap -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -verify-machineinstrs -stress-regalloc=2 -start-before=greedy -stop-after=amdgpu-preload-kern-arg-prolog -o - %s | FileCheck %s
# Test that a spill of a copy of exec is not folded to be a spill of exec directly.
diff --git a/llvm/test/CodeGen/AMDGPU/fold-reload-into-m0.mir b/llvm/test/CodeGen/AMDGPU/fold-reload-into-m0.mir
index ab1bc8641842f..604e7e4cc1e97 100644
--- a/llvm/test/CodeGen/AMDGPU/fold-reload-into-m0.mir
+++ b/llvm/test/CodeGen/AMDGPU/fold-reload-into-m0.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -verify-machineinstrs -stress-regalloc=2 -start-before=greedy -stop-after=virtregmap -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -verify-machineinstrs -stress-regalloc=2 -start-before=greedy -stop-after=amdgpu-preload-kern-arg-prolog -o - %s | FileCheck %s
# Test that a spill of a copy of m0 is not folded to be a spill of m0 directly.
diff --git a/llvm/test/CodeGen/AMDGPU/fold-restore-undef-use.mir b/llvm/test/CodeGen/AMDGPU/fold-restore-undef-use.mir
index 9297b44695c68..06725af984963 100644
--- a/llvm/test/CodeGen/AMDGPU/fold-restore-undef-use.mir
+++ b/llvm/test/CodeGen/AMDGPU/fold-restore-undef-use.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu9.0a-amd-amdhsa -stress-regalloc=4 -verify-regalloc -start-before=greedy,0 -stop-after=virtregrewriter,0 %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.0a-amd-amdhsa -stress-regalloc=4 -verify-regalloc -start-before=greedy,1 -stop-after=virt-reg-rewriter,1 %s -o - | FileCheck %s
# RUN: llc -enable-new-pm -mtriple=amdgpu9.0a-amd-amdhsa -stress-regalloc=4 -verify-regalloc -passes="greedy<sgpr>,virt-reg-rewriter<no-clear-vregs>" %s -o - | FileCheck %s
diff --git a/llvm/test/CodeGen/AMDGPU/function-resource-usage.ll b/llvm/test/CodeGen/AMDGPU/function-resource-usage.ll
index 6db3a35aaae72..f721cca014554 100644
--- a/llvm/test/CodeGen/AMDGPU/function-resource-usage.ll
+++ b/llvm/test/CodeGen/AMDGPU/function-resource-usage.ll
@@ -482,29 +482,29 @@ define amdgpu_kernel void @usage_direct_recursion(i32 %n) #0 {
ret void
}
-; GCN-LABEL: {{^}}multi_stage_recurse2:
-; GCN: .set .Lmulti_stage_recurse2.num_vgpr, max(43, .Lmulti_stage_recurse1.num_vgpr)
-; GCN: .set .Lmulti_stage_recurse2.num_agpr, max(0, .Lmulti_stage_recurse1.num_agpr)
-; GCN: .set .Lmulti_stage_recurse2.numbered_sgpr, max(34, .Lmulti_stage_recurse1.numbered_sgpr)
-; GCN: .set .Lmulti_stage_recurse2.private_seg_size, 16+max(.Lmulti_stage_recurse1.private_seg_size)
-; GCN: .set .Lmulti_stage_recurse2.uses_vcc, or(1, .Lmulti_stage_recurse1.uses_vcc)
-; GCN: .set .Lmulti_stage_recurse2.uses_flat_scratch, or(0, .Lmulti_stage_recurse1.uses_flat_scratch)
-; GCN: .set .Lmulti_stage_recurse2.has_dyn_sized_stack, or(0, .Lmulti_stage_recurse1.has_dyn_sized_stack)
-; GCN: .set .Lmulti_stage_recurse2.has_recursion, or(1, .Lmulti_stage_recurse1.has_recursion)
-; GCN: .set .Lmulti_stage_recurse2.has_indirect_call, or(0, .Lmulti_stage_recurse1.has_indirect_call)
-; GCN: TotalNumSgprs: .Lmulti_stage_recurse2.numbered_sgpr+extrasgprs(.Lmulti_stage_recurse2.uses_vcc, .Lmulti_stage_recurse2.uses_flat_scratch, 1)
-; GCN: NumVgprs: max(43, .Lmulti_stage_recurse1.num_vgpr)
-; GCN: ScratchSize: 16+max(.Lmulti_stage_recurse1.private_seg_size)
; GCN-LABEL: {{^}}multi_stage_recurse1:
-; GCN: .set .Lmulti_stage_recurse1.num_vgpr, max(48, 43)
-; GCN: .set .Lmulti_stage_recurse1.num_agpr, max(0, 0)
-; GCN: .set .Lmulti_stage_recurse1.numbered_sgpr, max(34, 34)
-; GCN: .set .Lmulti_stage_recurse1.private_seg_size, 16
-; GCN: .set .Lmulti_stage_recurse1.uses_vcc, 1
-; GCN: .set .Lmulti_stage_recurse1.uses_flat_scratch, 0
-; GCN: .set .Lmulti_stage_recurse1.has_dyn_sized_stack, 0
-; GCN: .set .Lmulti_stage_recurse1.has_recursion, 1
-; GCN: .set .Lmulti_stage_recurse1.has_indirect_call, 0
+; GCN: .set .Lmulti_stage_recurse1.num_vgpr, max(48, .Lmulti_stage_recurse2.num_vgpr)
+; GCN: .set .Lmulti_stage_recurse1.num_agpr, max(0, .Lmulti_stage_recurse2.num_agpr)
+; GCN: .set .Lmulti_stage_recurse1.numbered_sgpr, max(34, .Lmulti_stage_recurse2.numbered_sgpr)
+; GCN: .set .Lmulti_stage_recurse1.private_seg_size, 16+max(.Lmulti_stage_recurse2.private_seg_size)
+; GCN: .set .Lmulti_stage_recurse1.uses_vcc, or(1, .Lmulti_stage_recurse2.uses_vcc)
+; GCN: .set .Lmulti_stage_recurse1.uses_flat_scratch, or(0, .Lmulti_stage_recurse2.uses_flat_scratch)
+; GCN: .set .Lmulti_stage_recurse1.has_dyn_sized_stack, or(0, .Lmulti_stage_recurse2.has_dyn_sized_stack)
+; GCN: .set .Lmulti_stage_recurse1.has_recursion, or(1, .Lmulti_stage_recurse2.has_recursion)
+; GCN: .set .Lmulti_stage_recurse1.has_indirect_call, or(0, .Lmulti_stage_recurse2.has_indirect_call)
+; GCN: TotalNumSgprs: .Lmulti_stage_recurse1.numbered_sgpr+extrasgprs(.Lmulti_stage_recurse1.uses_vcc, .Lmulti_stage_recurse1.uses_flat_scratch, 1)
+; GCN: NumVgprs: max(48, .Lmulti_stage_recurse2.num_vgpr)
+; GCN: ScratchSize: 16+max(.Lmulti_stage_recurse2.private_seg_size)
+; GCN-LABEL: {{^}}multi_stage_recurse2:
+; GCN: .set .Lmulti_stage_recurse2.num_vgpr, max(43, 48)
+; GCN: .set .Lmulti_stage_recurse2.num_agpr, max(0, 0)
+; GCN: .set .Lmulti_stage_recurse2.numbered_sgpr, max(34, 34)
+; GCN: .set .Lmulti_stage_recurse2.private_seg_size, 16
+; GCN: .set .Lmulti_stage_recurse2.uses_vcc, 1
+; GCN: .set .Lmulti_stage_recurse2.uses_flat_scratch, 0
+; GCN: .set .Lmulti_stage_recurse2.has_dyn_sized_stack, 0
+; GCN: .set .Lmulti_stage_recurse2.has_recursion, 1
+; GCN: .set .Lmulti_stage_recurse2.has_indirect_call, 0
; GCN: TotalNumSgprs: 38
; GCN: NumVgprs: 48
; GCN: ScratchSize: 16
@@ -531,35 +531,35 @@ define void @multi_stage_recurse2(i32 %val) #2 {
; GCN: .set .Lusage_multi_stage_recurse.has_indirect_call, or(0, .Lmulti_stage_recurse1.has_indirect_call)
; GCN: TotalNumSgprs: 40
; GCN: NumVgprs: 48
-; GCN: ScratchSize: 16
+; GCN: ScratchSize: 32
define amdgpu_kernel void @usage_multi_stage_recurse(i32 %n) #0 {
call void @multi_stage_recurse1(i32 %n)
ret void
}
-; GCN-LABEL: {{^}}multi_stage_recurse_noattr2:
-; GCN: .set .Lmulti_stage_recurse_noattr2.num_vgpr, max(41, .Lmulti_stage_recurse_noattr1.num_vgpr)
-; GCN: .set .Lmulti_stage_recurse_noattr2.num_agpr, max(0, .Lmulti_stage_recurse_noattr1.num_agpr)
-; GCN: .set .Lmulti_stage_recurse_noattr2.numbered_sgpr, max(54, .Lmulti_stage_recurse_noattr1.numbered_sgpr)
-; GCN: .set .Lmulti_stage_recurse_noattr2.private_seg_size, 16+max(.Lmulti_stage_recurse_noattr1.private_seg_size)
-; GCN: .set .Lmulti_stage_recurse_noattr2.uses_vcc, or(1, .Lmulti_stage_recurse_noattr1.uses_vcc)
-; GCN: .set .Lmulti_stage_recurse_noattr2.uses_flat_scratch, or(0, .Lmulti_stage_recurse_noattr1.uses_flat_scratch)
-; GCN: .set .Lmulti_stage_recurse_noattr2.has_dyn_sized_stack, or(0, .Lmulti_stage_recurse_noattr1.has_dyn_sized_stack)
-; GCN: .set .Lmulti_stage_recurse_noattr2.has_recursion, or(0, .Lmulti_stage_recurse_noattr1.has_recursion)
-; GCN: .set .Lmulti_stage_recurse_noattr2.has_indirect_call, or(0, .Lmulti_stage_recurse_noattr1.has_indirect_call)
-; GCN: TotalNumSgprs: .Lmulti_stage_recurse_noattr2.numbered_sgpr+extrasgprs(.Lmulti_stage_recurse_noattr2.uses_vcc, .Lmulti_stage_recurse_noattr2.uses_flat_scratch, 1)
-; GCN: NumVgprs: max(41, .Lmulti_stage_recurse_noattr1.num_vgpr)
-; GCN: ScratchSize: 16+max(.Lmulti_stage_recurse_noattr1.private_seg_size)
; GCN-LABEL: {{^}}multi_stage_recurse_noattr1:
-; GCN: .set .Lmulti_stage_recurse_noattr1.num_vgpr, max(41, 41)
-; GCN: .set .Lmulti_stage_recurse_noattr1.num_agpr, max(0, 0)
-; GCN: .set .Lmulti_stage_recurse_noattr1.numbered_sgpr, max(57, 54)
-; GCN: .set .Lmulti_stage_recurse_noattr1.private_seg_size, 16
-; GCN: .set .Lmulti_stage_recurse_noattr1.uses_vcc, 1
-; GCN: .set .Lmulti_stage_recurse_noattr1.uses_flat_scratch, 0
-; GCN: .set .Lmulti_stage_recurse_noattr1.has_dyn_sized_stack, 0
-; GCN: .set .Lmulti_stage_recurse_noattr1.has_recursion, 0
-; GCN: .set .Lmulti_stage_recurse_noattr1.has_indirect_call, 0
+; GCN: .set .Lmulti_stage_recurse_noattr1.num_vgpr, max(41, .Lmulti_stage_recurse_noattr2.num_vgpr)
+; GCN: .set .Lmulti_stage_recurse_noattr1.num_agpr, max(0, .Lmulti_stage_recurse_noattr2.num_agpr)
+; GCN: .set .Lmulti_stage_recurse_noattr1.numbered_sgpr, max(57, .Lmulti_stage_recurse_noattr2.numbered_sgpr)
+; GCN: .set .Lmulti_stage_recurse_noattr1.private_seg_size, 16+max(.Lmulti_stage_recurse_noattr2.private_seg_size)
+; GCN: .set .Lmulti_stage_recurse_noattr1.uses_vcc, or(1, .Lmulti_stage_recurse_noattr2.uses_vcc)
+; GCN: .set .Lmulti_stage_recurse_noattr1.uses_flat_scratch, or(0, .Lmulti_stage_recurse_noattr2.uses_flat_scratch)
+; GCN: .set .Lmulti_stage_recurse_noattr1.has_dyn_sized_stack, or(0, .Lmulti_stage_recurse_noattr2.has_dyn_sized_stack)
+; GCN: .set .Lmulti_stage_recurse_noattr1.has_recursion, or(0, .Lmulti_stage_recurse_noattr2.has_recursion)
+; GCN: .set .Lmulti_stage_recurse_noattr1.has_indirect_call, or(0, .Lmulti_stage_recurse_noattr2.has_indirect_call)
+; GCN: TotalNumSgprs: .Lmulti_stage_recurse_noattr1.numbered_sgpr+extrasgprs(.Lmulti_stage_recurse_noattr1.uses_vcc, .Lmulti_stage_recurse_noattr1.uses_flat_scratch, 1)
+; GCN: NumVgprs: max(41, .Lmulti_stage_recurse_noattr2.num_vgpr)
+; GCN: ScratchSize: 16+max(.Lmulti_stage_recurse_noattr2.private_seg_size)
+; GCN-LABEL: {{^}}multi_stage_recurse_noattr2:
+; GCN: .set .Lmulti_stage_recurse_noattr2.num_vgpr, max(41, 41)
+; GCN: .set .Lmulti_stage_recurse_noattr2.num_agpr, max(0, 0)
+; GCN: .set .Lmulti_stage_recurse_noattr2.numbered_sgpr, max(54, 57)
+; GCN: .set .Lmulti_stage_recurse_noattr2.private_seg_size, 16
+; GCN: .set .Lmulti_stage_recurse_noattr2.uses_vcc, 1
+; GCN: .set .Lmulti_stage_recurse_noattr2.uses_flat_scratch, 0
+; GCN: .set .Lmulti_stage_recurse_noattr2.has_dyn_sized_stack, 0
+; GCN: .set .Lmulti_stage_recurse_noattr2.has_recursion, 0
+; GCN: .set .Lmulti_stage_recurse_noattr2.has_indirect_call, 0
; GCN: TotalNumSgprs: 61
; GCN: NumVgprs: 41
; GCN: ScratchSize: 16
@@ -586,7 +586,7 @@ define void @multi_stage_recurse_noattr2(i32 %val) #0 {
; GCN: .set .Lusage_multi_stage_recurse_noattrs.has_indirect_call, or(0, .Lmulti_stage_recurse_noattr1.has_indirect_call)
; GCN: TotalNumSgprs: 63
; GCN: NumVgprs: 41
-; GCN: ScratchSize: 16
+; GCN: ScratchSize: 32
define amdgpu_kernel void @usage_multi_stage_recurse_noattrs(i32 %n) #0 {
call void @multi_stage_recurse_noattr1(i32 %n)
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/greedy-alloc-fail-sgpr1024-spill.mir b/llvm/test/CodeGen/AMDGPU/greedy-alloc-fail-sgpr1024-spill.mir
index 94c22b1aa8664..20ec57429619b 100644
--- a/llvm/test/CodeGen/AMDGPU/greedy-alloc-fail-sgpr1024-spill.mir
+++ b/llvm/test/CodeGen/AMDGPU/greedy-alloc-fail-sgpr1024-spill.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu9.08-amd-amdhsa -start-before=greedy,0 -stop-after=virtregrewriter,0 -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.08-amd-amdhsa -start-before=greedy,1 -stop-after=virt-reg-rewriter,1 -o - %s | FileCheck %s
# This testcase used to fail due to introducing a spill of an SGPR
# 1024 for every subregister use inside the loop. With overlapping
diff --git a/llvm/test/CodeGen/AMDGPU/greedy-instruction-split-subrange.mir b/llvm/test/CodeGen/AMDGPU/greedy-instruction-split-subrange.mir
index a3523221c7637..dceb62e0907c9 100644
--- a/llvm/test/CodeGen/AMDGPU/greedy-instruction-split-subrange.mir
+++ b/llvm/test/CodeGen/AMDGPU/greedy-instruction-split-subrange.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -verify-regalloc -stress-regalloc=3 -start-before=greedy,1 -stop-before=virtregrewriter,1 -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -verify-regalloc -stress-regalloc=3 -start-before=greedy,2 -stop-before=virt-reg-rewriter,2 -o - %s | FileCheck %s
---
name: split_instruction_subranges
alignment: 1
diff --git a/llvm/test/CodeGen/AMDGPU/greedy-remark-crash-unassigned-reg.mir b/llvm/test/CodeGen/AMDGPU/greedy-remark-crash-unassigned-reg.mir
index 71e99a6589882..1f3aa4030d623 100644
--- a/llvm/test/CodeGen/AMDGPU/greedy-remark-crash-unassigned-reg.mir
+++ b/llvm/test/CodeGen/AMDGPU/greedy-remark-crash-unassigned-reg.mir
@@ -1,5 +1,5 @@
# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa \
-# RUN: -start-before=greedy,0 -stop-after=virtregrewriter,0 -pass-remarks='.*' -pass-remarks-output=%t.yaml -filetype=null %s
+# RUN: -start-before=greedy,1 -stop-after=virt-reg-rewriter,1 -pass-remarks='.*' -pass-remarks-output=%t.yaml -filetype=null %s
# RUN: FileCheck %s < %t.yaml
# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa \
diff --git a/llvm/test/CodeGen/AMDGPU/illegal-eviction-assert.mir b/llvm/test/CodeGen/AMDGPU/illegal-eviction-assert.mir
index 7666aa35a4214..cdad480b3aaaa 100644
--- a/llvm/test/CodeGen/AMDGPU/illegal-eviction-assert.mir
+++ b/llvm/test/CodeGen/AMDGPU/illegal-eviction-assert.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: not llc -mtriple=amdgpu9.00 -start-before=greedy,0 -stop-after=virtregrewriter,2 -o - 2>%t.err %s | FileCheck %s
+# RUN: not llc -mtriple=amdgpu9.00 -start-before=greedy,1 -stop-after=virt-reg-rewriter,3 -o - 2>%t.err %s | FileCheck %s
# RUN: FileCheck -check-prefix=ERR %s < %t.err
# This testcase cannot be compiled. An attempted eviction legality
diff --git a/llvm/test/CodeGen/AMDGPU/indirect-call-known-callees.ll b/llvm/test/CodeGen/AMDGPU/indirect-call-known-callees.ll
index 4166f5a2ef3b2..2e54c5a68453b 100644
--- a/llvm/test/CodeGen/AMDGPU/indirect-call-known-callees.ll
+++ b/llvm/test/CodeGen/AMDGPU/indirect-call-known-callees.ll
@@ -8,6 +8,42 @@
; FIXME: Passing real values for workitem ID, and 0s that can be undef
+define void @snork() {
+; GFX9-LABEL: snork:
+; GFX9: ; %bb.0: ; %bb
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: snork:
+; GFX12: ; %bb.0: ; %bb
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+bb:
+ ret void
+}
+
+define void @wobble() {
+; GFX9-LABEL: wobble:
+; GFX9: ; %bb.0: ; %bb
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: wobble:
+; GFX12: ; %bb.0: ; %bb
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+bb:
+ ret void
+}
+
define amdgpu_kernel void @indirect_call_known_no_special_inputs() {
; GFX9-LABEL: indirect_call_known_no_special_inputs:
; GFX9: ; %bb.0: ; %bb
@@ -71,46 +107,9 @@ define amdgpu_kernel void @indirect_call_known_no_special_inputs() {
; GFX12-NEXT: s_mov_b64 s[6:7], s[2:3]
; GFX12-NEXT: s_swappc_b64 s[30:31], s[12:13]
; GFX12-NEXT: s_endpgm
-
bb:
%cond = load i1, ptr addrspace(4) null
%tmp = select i1 %cond, ptr @wobble, ptr @snork
call void %tmp(ptr poison, i32 poison, ptr poison)
ret void
}
-
-define void @wobble() {
-; GFX9-LABEL: wobble:
-; GFX9: ; %bb.0: ; %bb
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-LABEL: wobble:
-; GFX12: ; %bb.0: ; %bb
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_setpc_b64 s[30:31]
-bb:
- ret void
-}
-
-define void @snork() {
-; GFX9-LABEL: snork:
-; GFX9: ; %bb.0: ; %bb
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-LABEL: snork:
-; GFX12: ; %bb.0: ; %bb
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_setpc_b64 s[30:31]
-bb:
- ret void
-}
diff --git a/llvm/test/CodeGen/AMDGPU/indirect-reg-read-imm-idx.mir b/llvm/test/CodeGen/AMDGPU/indirect-reg-read-imm-idx.mir
index 5a7572fa4a1a6..f693f286375d5 100644
--- a/llvm/test/CodeGen/AMDGPU/indirect-reg-read-imm-idx.mir
+++ b/llvm/test/CodeGen/AMDGPU/indirect-reg-read-imm-idx.mir
@@ -1,4 +1,4 @@
-# RUN: llc -mtriple=amdgpu9.0a -start-before=twoaddressinstruction %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.0a -start-before=two-address-instruction %s -o - | FileCheck %s
# Test that V_INDIRECT_REG_READ_GPR_IDX expansion handles immediate index operands.
diff --git a/llvm/test/CodeGen/AMDGPU/inflate-av-remat-imm.mir b/llvm/test/CodeGen/AMDGPU/inflate-av-remat-imm.mir
index 498a5cea9c3f2..6bca79c818280 100644
--- a/llvm/test/CodeGen/AMDGPU/inflate-av-remat-imm.mir
+++ b/llvm/test/CodeGen/AMDGPU/inflate-av-remat-imm.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
-# RUN: llc -mtriple=amdgpu9.0a-amd-amdhsa -simplify-mir -start-before=greedy,2 -stress-regalloc=3 -stop-after=postrapseudos -o - -verify-regalloc %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.0a-amd-amdhsa -simplify-mir -start-before=greedy,3 -stress-regalloc=3 -stop-after=post-ra-pseudos -o - -verify-regalloc %s | FileCheck %s
# Compare results of using V_MOV_B32 vs. AV_MOV_B32_IMM_PSEUDO during
# allocation.
diff --git a/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-agpr-negative-tests.mir b/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-agpr-negative-tests.mir
index 462714a57e30e..5e0e62b367261 100644
--- a/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-agpr-negative-tests.mir
+++ b/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-agpr-negative-tests.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
-# RUN: llc -mtriple=amdgpu9.42-amd-amdhsa -start-before=greedy,2 -stop-after=virtregrewriter,2 -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.42-amd-amdhsa -start-before=greedy,3 -stop-after=virt-reg-rewriter,3 -o - %s | FileCheck %s
--- |
diff --git a/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-av-with-load-source.mir b/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-av-with-load-source.mir
index 5bc25d5994ff6..c35ccca162e2b 100644
--- a/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-av-with-load-source.mir
+++ b/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-av-with-load-source.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
-# RUN: llc -mtriple=amdgpu9.42-amd-amdhsa -start-before=greedy,2 -stop-after=virtregrewriter,2 -verify-regalloc -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.42-amd-amdhsa -start-before=greedy,3 -stop-after=virt-reg-rewriter,3 -verify-regalloc -verify-machineinstrs -o - %s | FileCheck %s
# There aren't enough VGPRs for %0 to stay in a VGPR. %0 should be
# split and inflated to AV_512. The VGPR version of the instruction
diff --git a/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-av.mir b/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-av.mir
index 3cb539d3a423a..7cf0ee7fcb0ed 100644
--- a/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-av.mir
+++ b/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-av.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
-# RUN: llc -mtriple=amdgpu9.42-amd-amdhsa -start-before=greedy,2 -stop-after=virtregrewriter,2 -verify-regalloc -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.42-amd-amdhsa -start-before=greedy,3 -stop-after=virt-reg-rewriter,3 -verify-regalloc -verify-machineinstrs -o - %s | FileCheck %s
# There aren't enough VGPRs for %0 to stay in a VGPR. %0 should be
# split and inflated to AV_512. The VGPR version of the instruction
diff --git a/llvm/test/CodeGen/AMDGPU/inflated-reg-class-snippet-copy-use-after-free.mir b/llvm/test/CodeGen/AMDGPU/inflated-reg-class-snippet-copy-use-after-free.mir
index 3ed98899d6019..97cef18bd6254 100644
--- a/llvm/test/CodeGen/AMDGPU/inflated-reg-class-snippet-copy-use-after-free.mir
+++ b/llvm/test/CodeGen/AMDGPU/inflated-reg-class-snippet-copy-use-after-free.mir
@@ -1,4 +1,4 @@
-# RUN: not llc -mtriple=amdgpu9.0a-amd-amdhsa -simplify-mir -start-before=greedy,2 -stress-regalloc=4 -stop-before=virtregrewriter,2 -o - -verify-regalloc %s 2> %t.err | FileCheck %s
+# RUN: not llc -mtriple=amdgpu9.0a-amd-amdhsa -simplify-mir -start-before=greedy,3 -stress-regalloc=4 -stop-before=virt-reg-rewriter,3 -o - -verify-regalloc %s 2> %t.err | FileCheck %s
# RUN: FileCheck -check-prefix=ERR %s < %t.err
# To allocate the vreg_512_align2, the allocation will attempt to
diff --git a/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill-inspect-subrange.mir b/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill-inspect-subrange.mir
index 63ee4473a56e5..befc33e18aa94 100644
--- a/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill-inspect-subrange.mir
+++ b/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill-inspect-subrange.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 4
-# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -verify-regalloc -start-before=greedy,0 -stop-after=virtregrewriter,0 -simplify-mir -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -verify-regalloc -start-before=greedy,1 -stop-after=virt-reg-rewriter,1 -simplify-mir -o - %s | FileCheck %s
--- |
diff --git a/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill.mir b/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill.mir
index a8419a4bfc749..e6f29a8c11afc 100644
--- a/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill.mir
+++ b/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 3
-# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -start-before=greedy,0 -stop-after=virtregrewriter,0 -simplify-mir -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -start-before=greedy,1 -stop-after=virt-reg-rewriter,1 -simplify-mir -o - %s | FileCheck %s
--- |
define void @main() #0 {
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/issue48473.mir b/llvm/test/CodeGen/AMDGPU/issue48473.mir
index 7a8fde7bef6b5..097a11ee88493 100644
--- a/llvm/test/CodeGen/AMDGPU/issue48473.mir
+++ b/llvm/test/CodeGen/AMDGPU/issue48473.mir
@@ -1,4 +1,4 @@
-# RUN: not llc -mtriple=amdgpu9.08-amd-amdhsa -start-before=greedy,0 -stop-after=virtregrewriter,1 -verify-machineinstrs -o - 2> %t.err %s | FileCheck %s
+# RUN: not llc -mtriple=amdgpu9.08-amd-amdhsa -start-before=greedy,1 -stop-after=virt-reg-rewriter,2 -verify-machineinstrs -o - 2> %t.err %s | FileCheck %s
# RUN: FileCheck -check-prefix=ERR %s < %t.err
# ERR: error: register allocation failed: maximum depth for recoloring reached. Use -fexhaustive-register-search to skip cutoffs
diff --git a/llvm/test/CodeGen/AMDGPU/issue98474-assigned-physreg-interference.mir b/llvm/test/CodeGen/AMDGPU/issue98474-assigned-physreg-interference.mir
index f8d7635d8a6c1..a078979e8b33b 100644
--- a/llvm/test/CodeGen/AMDGPU/issue98474-assigned-physreg-interference.mir
+++ b/llvm/test/CodeGen/AMDGPU/issue98474-assigned-physreg-interference.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
-# RUN: llc -mtriple=amdgpu9.00-amd-amdpal -start-before=greedy,2 -stop-after=tailduplication -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.00-amd-amdpal -start-before=greedy,3 -stop-after=tailduplication -verify-machineinstrs -o - %s | FileCheck %s
---
name: undef_subreg_def_live_out_tailduplicate_vreg96_undef_sub1_sub2_assigned_physreg_interference
diff --git a/llvm/test/CodeGen/AMDGPU/issue98474-virtregrewriter-live-out-undef-subregisters.mir b/llvm/test/CodeGen/AMDGPU/issue98474-virtregrewriter-live-out-undef-subregisters.mir
index 886c8e6494be9..5e5aa5782ef82 100644
--- a/llvm/test/CodeGen/AMDGPU/issue98474-virtregrewriter-live-out-undef-subregisters.mir
+++ b/llvm/test/CodeGen/AMDGPU/issue98474-virtregrewriter-live-out-undef-subregisters.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
-# RUN: llc -mtriple=amdgpu9.00-amd-amdpal -start-before=greedy,2 -stop-after=tailduplication -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.00-amd-amdpal -start-before=greedy,3 -stop-after=tailduplication -verify-machineinstrs -o - %s | FileCheck %s
# The partial def of %0 introduces a live out undef def of %0.sub1
# into bb.3. We need to maintain this liveness with an explicit def of
diff --git a/llvm/test/CodeGen/AMDGPU/large-avgpr-assign-last.mir b/llvm/test/CodeGen/AMDGPU/large-avgpr-assign-last.mir
index fcb8461963dbb..854677b970e61 100644
--- a/llvm/test/CodeGen/AMDGPU/large-avgpr-assign-last.mir
+++ b/llvm/test/CodeGen/AMDGPU/large-avgpr-assign-last.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
-# RUN: llc -mtriple=amdgpu9.50-amd-amdhsa -verify-regalloc -greedy-regclass-priority-trumps-globalness=1 -start-after=machine-scheduler -stop-after=virtregrewriter,2 -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.50-amd-amdhsa -verify-regalloc -greedy-regclass-priority-trumps-globalness=1 -start-after=machine-scheduler -stop-after=virt-reg-rewriter,3 -o - %s | FileCheck %s
--- |
define void @temp_vgpr_to_agpr_should_not_undo_split_with_remat() #0 {
diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
index 4d68b98d6d590..12147e70d6883 100644
--- a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
+++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
@@ -1,14 +1,14 @@
; UNSUPPORTED: expensive_checks
-; RUN: llc -O0 -mtriple=amdgpu7.00--amdhsa -disable-verify -debug-pass=Structure < %s 2>&1 \
+; RUN: llc -O0 -enable-new-pm=0 -mtriple=amdgpu7.00--amdhsa -disable-verify -debug-pass=Structure < %s 2>&1 \
; RUN: | FileCheck -match-full-lines -strict-whitespace -check-prefix=GCN-O0 %s
-; RUN: llc -O1 -mtriple=amdgpu7.00--amdhsa -disable-verify -debug-pass=Structure < %s 2>&1 \
+; RUN: llc -O1 -enable-new-pm=0 -mtriple=amdgpu7.00--amdhsa -disable-verify -debug-pass=Structure < %s 2>&1 \
; RUN: | FileCheck -match-full-lines -strict-whitespace -check-prefix=GCN-O1 %s
-; RUN: llc -O1 -mtriple=amdgpu7.00--amdhsa -disable-verify -amdgpu-scalar-ir-passes -amdgpu-sdwa-peephole \
+; RUN: llc -O1 -enable-new-pm=0 -mtriple=amdgpu7.00--amdhsa -disable-verify -amdgpu-scalar-ir-passes -amdgpu-sdwa-peephole \
; RUN: -amdgpu-load-store-vectorizer -amdgpu-enable-pre-ra-optimizations -amdgpu-loop-prefetch -debug-pass=Structure < %s 2>&1 \
; RUN: | FileCheck -match-full-lines -strict-whitespace -check-prefix=GCN-O1-OPTS %s
-; RUN: llc -O2 -mtriple=amdgpu7.00--amdhsa -disable-verify -debug-pass=Structure < %s 2>&1 \
+; RUN: llc -O2 -enable-new-pm=0 -mtriple=amdgpu7.00--amdhsa -disable-verify -debug-pass=Structure < %s 2>&1 \
; RUN: | FileCheck -match-full-lines -strict-whitespace -check-prefix=GCN-O2 %s
-; RUN: llc -O3 -mtriple=amdgpu7.00--amdhsa -disable-verify -debug-pass=Structure < %s 2>&1 \
+; RUN: llc -O3 -enable-new-pm=0 -mtriple=amdgpu7.00--amdhsa -disable-verify -debug-pass=Structure < %s 2>&1 \
; RUN: | FileCheck -match-full-lines -strict-whitespace -check-prefix=GCN-O3 %s
; REQUIRES: asserts
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.barrier-bundle.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.barrier-bundle.ll
index d5fca840bd4e6..53ffe5d63fbdc 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.barrier-bundle.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.barrier-bundle.ll
@@ -1,7 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -global-isel=0 -mtriple=amdgpu6.00-mesa-mesa3d -stop-after=postrapseudos < %s | FileCheck -check-prefix=GFX6-SDAG %s
+; RUN: llc -global-isel=0 -mtriple=amdgpu6.00-mesa-mesa3d -stop-after=post-ra-pseudos < %s | FileCheck -check-prefix=GFX6-SDAG %s
; RUN: llc -global-isel=1 -mtriple=amdgpu6.00-mesa-mesa3d -stop-after=postrapseudos < %s | FileCheck -check-prefix=GFX6-GISEL %s
-; RUN: llc -global-isel=0 -mtriple=amdgpu9.00-mesa-mesa3d -stop-after=postrapseudos < %s | FileCheck -check-prefix=GFX9-SDAG %s
+; RUN: llc -global-isel=0 -mtriple=amdgpu9.00-mesa-mesa3d -stop-after=post-ra-pseudos < %s | FileCheck -check-prefix=GFX9-SDAG %s
; RUN: llc -global-isel=1 -mtriple=amdgpu9.00-mesa-mesa3d -stop-after=postrapseudos < %s | FileCheck -check-prefix=GFX9-GISEL %s
; Make sure the op is emitted bundled with a waitcnt with and without the retry loop, and the bundle is not removed by ExpandPostRAPseudos.
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.barrier-fastregalloc.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.barrier-fastregalloc.ll
index 62aceaf290e94..bf56e181cda51 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.barrier-fastregalloc.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.barrier-fastregalloc.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -mtriple=amdgpu9.00-mesa-mesa3d -O0 -stop-after=postrapseudos -o - < %s | FileCheck -enable-var-scope -check-prefix=MIR %s
+; RUN: llc -mtriple=amdgpu9.00-mesa-mesa3d -O0 -stop-after=post-ra-pseudos -o - < %s | FileCheck -enable-var-scope -check-prefix=MIR %s
define amdgpu_kernel void @gws_barrier_offset0(i32 %val) #0 {
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.bf16.ll
index c28536adace87..50a70f86398ef 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.bf16.ll
@@ -682,12 +682,12 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16__vgprcd(<8 x bfloat> %arg
; AGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; AGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; AGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
-; AGPR-NEXT: v_mov_b32_e32 v32, 0
+; AGPR-NEXT: v_mov_b32_e32 v36, 0
; AGPR-NEXT: s_waitcnt lgkmcnt(0)
-; AGPR-NEXT: v_mov_b64_e32 v[36:37], s[26:27]
-; AGPR-NEXT: v_mov_b64_e32 v[34:35], s[24:25]
-; AGPR-NEXT: v_mov_b64_e32 v[40:41], s[30:31]
-; AGPR-NEXT: v_mov_b64_e32 v[38:39], s[28:29]
+; AGPR-NEXT: v_mov_b64_e32 v[40:41], s[26:27]
+; AGPR-NEXT: v_mov_b64_e32 v[38:39], s[24:25]
+; AGPR-NEXT: v_mov_b64_e32 v[44:45], s[30:31]
+; AGPR-NEXT: v_mov_b64_e32 v[42:43], s[28:29]
; AGPR-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; AGPR-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; AGPR-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
@@ -696,41 +696,41 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16__vgprcd(<8 x bfloat> %arg
; AGPR-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
; AGPR-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; AGPR-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
-; AGPR-NEXT: v_mov_b32_e32 v42, s20
-; AGPR-NEXT: v_mov_b32_e32 v43, s21
-; AGPR-NEXT: v_mfma_f32_32x32x16_bf16 v[0:15], v[34:37], v[38:41], v[16:31]
-; AGPR-NEXT: v_mov_b32_e32 v44, s22
-; AGPR-NEXT: v_mov_b32_e32 v45, s23
-; AGPR-NEXT: global_store_dwordx4 v32, v[42:45], s[0:1] offset:48 sc0 sc1
+; AGPR-NEXT: v_mov_b32_e32 v32, s20
+; AGPR-NEXT: v_mov_b32_e32 v33, s21
+; AGPR-NEXT: v_mfma_f32_32x32x16_bf16 v[0:15], v[38:41], v[42:45], v[16:31]
+; AGPR-NEXT: v_mov_b32_e32 v34, s22
+; AGPR-NEXT: v_mov_b32_e32 v35, s23
+; AGPR-NEXT: global_store_dwordx4 v36, v[32:35], s[0:1] offset:48 sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: s_nop 2
; AGPR-NEXT: v_mov_b32_e32 v16, s16
; AGPR-NEXT: v_mov_b32_e32 v17, s17
; AGPR-NEXT: v_mov_b32_e32 v18, s18
; AGPR-NEXT: v_mov_b32_e32 v19, s19
-; AGPR-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1
+; AGPR-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] offset:32 sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: s_nop 0
; AGPR-NEXT: v_mov_b32_e32 v16, s12
; AGPR-NEXT: v_mov_b32_e32 v17, s13
; AGPR-NEXT: v_mov_b32_e32 v18, s14
; AGPR-NEXT: v_mov_b32_e32 v19, s15
-; AGPR-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1
+; AGPR-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] offset:16 sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: s_nop 0
; AGPR-NEXT: v_mov_b32_e32 v16, s8
; AGPR-NEXT: v_mov_b32_e32 v17, s9
; AGPR-NEXT: v_mov_b32_e32 v18, s10
; AGPR-NEXT: v_mov_b32_e32 v19, s11
-; AGPR-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1
+; AGPR-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
-; AGPR-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32 sc0 sc1
+; AGPR-NEXT: global_store_dwordx4 v36, v[8:11], s[0:1] offset:32 sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
-; AGPR-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48 sc0 sc1
+; AGPR-NEXT: global_store_dwordx4 v36, v[12:15], s[0:1] offset:48 sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
-; AGPR-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] sc0 sc1
+; AGPR-NEXT: global_store_dwordx4 v36, v[0:3], s[0:1] sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
-; AGPR-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16 sc0 sc1
+; AGPR-NEXT: global_store_dwordx4 v36, v[4:7], s[0:1] offset:16 sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: s_endpgm
;
@@ -739,12 +739,12 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16__vgprcd(<8 x bfloat> %arg
; VGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; VGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
-; VGPR-NEXT: v_mov_b32_e32 v32, 0
+; VGPR-NEXT: v_mov_b32_e32 v36, 0
; VGPR-NEXT: s_waitcnt lgkmcnt(0)
-; VGPR-NEXT: v_mov_b64_e32 v[36:37], s[26:27]
-; VGPR-NEXT: v_mov_b64_e32 v[34:35], s[24:25]
-; VGPR-NEXT: v_mov_b64_e32 v[40:41], s[30:31]
-; VGPR-NEXT: v_mov_b64_e32 v[38:39], s[28:29]
+; VGPR-NEXT: v_mov_b64_e32 v[40:41], s[26:27]
+; VGPR-NEXT: v_mov_b64_e32 v[38:39], s[24:25]
+; VGPR-NEXT: v_mov_b64_e32 v[44:45], s[30:31]
+; VGPR-NEXT: v_mov_b64_e32 v[42:43], s[28:29]
; VGPR-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; VGPR-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; VGPR-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
@@ -753,41 +753,41 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16__vgprcd(<8 x bfloat> %arg
; VGPR-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
; VGPR-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; VGPR-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
-; VGPR-NEXT: v_mov_b32_e32 v42, s20
-; VGPR-NEXT: v_mov_b32_e32 v43, s21
-; VGPR-NEXT: v_mfma_f32_32x32x16_bf16 v[0:15], v[34:37], v[38:41], v[16:31]
-; VGPR-NEXT: v_mov_b32_e32 v44, s22
-; VGPR-NEXT: v_mov_b32_e32 v45, s23
-; VGPR-NEXT: global_store_dwordx4 v32, v[42:45], s[0:1] offset:48 sc0 sc1
+; VGPR-NEXT: v_mov_b32_e32 v32, s20
+; VGPR-NEXT: v_mov_b32_e32 v33, s21
+; VGPR-NEXT: v_mfma_f32_32x32x16_bf16 v[0:15], v[38:41], v[42:45], v[16:31]
+; VGPR-NEXT: v_mov_b32_e32 v34, s22
+; VGPR-NEXT: v_mov_b32_e32 v35, s23
+; VGPR-NEXT: global_store_dwordx4 v36, v[32:35], s[0:1] offset:48 sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: s_nop 2
; VGPR-NEXT: v_mov_b32_e32 v16, s16
; VGPR-NEXT: v_mov_b32_e32 v17, s17
; VGPR-NEXT: v_mov_b32_e32 v18, s18
; VGPR-NEXT: v_mov_b32_e32 v19, s19
-; VGPR-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1
+; VGPR-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] offset:32 sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: s_nop 0
; VGPR-NEXT: v_mov_b32_e32 v16, s12
; VGPR-NEXT: v_mov_b32_e32 v17, s13
; VGPR-NEXT: v_mov_b32_e32 v18, s14
; VGPR-NEXT: v_mov_b32_e32 v19, s15
-; VGPR-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1
+; VGPR-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] offset:16 sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: s_nop 0
; VGPR-NEXT: v_mov_b32_e32 v16, s8
; VGPR-NEXT: v_mov_b32_e32 v17, s9
; VGPR-NEXT: v_mov_b32_e32 v18, s10
; VGPR-NEXT: v_mov_b32_e32 v19, s11
-; VGPR-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1
+; VGPR-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
-; VGPR-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32 sc0 sc1
+; VGPR-NEXT: global_store_dwordx4 v36, v[8:11], s[0:1] offset:32 sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
-; VGPR-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48 sc0 sc1
+; VGPR-NEXT: global_store_dwordx4 v36, v[12:15], s[0:1] offset:48 sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
-; VGPR-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] sc0 sc1
+; VGPR-NEXT: global_store_dwordx4 v36, v[0:3], s[0:1] sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
-; VGPR-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16 sc0 sc1
+; VGPR-NEXT: global_store_dwordx4 v36, v[4:7], s[0:1] offset:16 sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: s_endpgm
%result = call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> %arg0, <8 x bfloat> %arg1, <16 x float> %arg2, i32 0, i32 0, i32 0)
@@ -859,12 +859,12 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16__vgprcd__flags(<8 x bfloa
; AGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; AGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; AGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
-; AGPR-NEXT: v_mov_b32_e32 v32, 0
+; AGPR-NEXT: v_mov_b32_e32 v36, 0
; AGPR-NEXT: s_waitcnt lgkmcnt(0)
-; AGPR-NEXT: v_mov_b64_e32 v[36:37], s[26:27]
-; AGPR-NEXT: v_mov_b64_e32 v[34:35], s[24:25]
-; AGPR-NEXT: v_mov_b64_e32 v[40:41], s[30:31]
-; AGPR-NEXT: v_mov_b64_e32 v[38:39], s[28:29]
+; AGPR-NEXT: v_mov_b64_e32 v[40:41], s[26:27]
+; AGPR-NEXT: v_mov_b64_e32 v[38:39], s[24:25]
+; AGPR-NEXT: v_mov_b64_e32 v[44:45], s[30:31]
+; AGPR-NEXT: v_mov_b64_e32 v[42:43], s[28:29]
; AGPR-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; AGPR-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; AGPR-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
@@ -873,41 +873,41 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16__vgprcd__flags(<8 x bfloa
; AGPR-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
; AGPR-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; AGPR-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
-; AGPR-NEXT: v_mov_b32_e32 v42, s20
-; AGPR-NEXT: v_mov_b32_e32 v43, s21
-; AGPR-NEXT: v_mfma_f32_32x32x16_bf16 v[0:15], v[34:37], v[38:41], v[16:31] cbsz:1 abid:2 blgp:3
-; AGPR-NEXT: v_mov_b32_e32 v44, s22
-; AGPR-NEXT: v_mov_b32_e32 v45, s23
-; AGPR-NEXT: global_store_dwordx4 v32, v[42:45], s[0:1] offset:48 sc0 sc1
+; AGPR-NEXT: v_mov_b32_e32 v32, s20
+; AGPR-NEXT: v_mov_b32_e32 v33, s21
+; AGPR-NEXT: v_mfma_f32_32x32x16_bf16 v[0:15], v[38:41], v[42:45], v[16:31] cbsz:1 abid:2 blgp:3
+; AGPR-NEXT: v_mov_b32_e32 v34, s22
+; AGPR-NEXT: v_mov_b32_e32 v35, s23
+; AGPR-NEXT: global_store_dwordx4 v36, v[32:35], s[0:1] offset:48 sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: s_nop 2
; AGPR-NEXT: v_mov_b32_e32 v16, s16
; AGPR-NEXT: v_mov_b32_e32 v17, s17
; AGPR-NEXT: v_mov_b32_e32 v18, s18
; AGPR-NEXT: v_mov_b32_e32 v19, s19
-; AGPR-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1
+; AGPR-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] offset:32 sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: s_nop 0
; AGPR-NEXT: v_mov_b32_e32 v16, s12
; AGPR-NEXT: v_mov_b32_e32 v17, s13
; AGPR-NEXT: v_mov_b32_e32 v18, s14
; AGPR-NEXT: v_mov_b32_e32 v19, s15
-; AGPR-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1
+; AGPR-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] offset:16 sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: s_nop 0
; AGPR-NEXT: v_mov_b32_e32 v16, s8
; AGPR-NEXT: v_mov_b32_e32 v17, s9
; AGPR-NEXT: v_mov_b32_e32 v18, s10
; AGPR-NEXT: v_mov_b32_e32 v19, s11
-; AGPR-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1
+; AGPR-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
-; AGPR-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32 sc0 sc1
+; AGPR-NEXT: global_store_dwordx4 v36, v[8:11], s[0:1] offset:32 sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
-; AGPR-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48 sc0 sc1
+; AGPR-NEXT: global_store_dwordx4 v36, v[12:15], s[0:1] offset:48 sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
-; AGPR-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] sc0 sc1
+; AGPR-NEXT: global_store_dwordx4 v36, v[0:3], s[0:1] sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
-; AGPR-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16 sc0 sc1
+; AGPR-NEXT: global_store_dwordx4 v36, v[4:7], s[0:1] offset:16 sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: s_endpgm
;
@@ -916,12 +916,12 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16__vgprcd__flags(<8 x bfloa
; VGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; VGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
-; VGPR-NEXT: v_mov_b32_e32 v32, 0
+; VGPR-NEXT: v_mov_b32_e32 v36, 0
; VGPR-NEXT: s_waitcnt lgkmcnt(0)
-; VGPR-NEXT: v_mov_b64_e32 v[36:37], s[26:27]
-; VGPR-NEXT: v_mov_b64_e32 v[34:35], s[24:25]
-; VGPR-NEXT: v_mov_b64_e32 v[40:41], s[30:31]
-; VGPR-NEXT: v_mov_b64_e32 v[38:39], s[28:29]
+; VGPR-NEXT: v_mov_b64_e32 v[40:41], s[26:27]
+; VGPR-NEXT: v_mov_b64_e32 v[38:39], s[24:25]
+; VGPR-NEXT: v_mov_b64_e32 v[44:45], s[30:31]
+; VGPR-NEXT: v_mov_b64_e32 v[42:43], s[28:29]
; VGPR-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; VGPR-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; VGPR-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
@@ -930,41 +930,41 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16__vgprcd__flags(<8 x bfloa
; VGPR-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
; VGPR-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; VGPR-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
-; VGPR-NEXT: v_mov_b32_e32 v42, s20
-; VGPR-NEXT: v_mov_b32_e32 v43, s21
-; VGPR-NEXT: v_mfma_f32_32x32x16_bf16 v[0:15], v[34:37], v[38:41], v[16:31] cbsz:1 abid:2 blgp:3
-; VGPR-NEXT: v_mov_b32_e32 v44, s22
-; VGPR-NEXT: v_mov_b32_e32 v45, s23
-; VGPR-NEXT: global_store_dwordx4 v32, v[42:45], s[0:1] offset:48 sc0 sc1
+; VGPR-NEXT: v_mov_b32_e32 v32, s20
+; VGPR-NEXT: v_mov_b32_e32 v33, s21
+; VGPR-NEXT: v_mfma_f32_32x32x16_bf16 v[0:15], v[38:41], v[42:45], v[16:31] cbsz:1 abid:2 blgp:3
+; VGPR-NEXT: v_mov_b32_e32 v34, s22
+; VGPR-NEXT: v_mov_b32_e32 v35, s23
+; VGPR-NEXT: global_store_dwordx4 v36, v[32:35], s[0:1] offset:48 sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: s_nop 2
; VGPR-NEXT: v_mov_b32_e32 v16, s16
; VGPR-NEXT: v_mov_b32_e32 v17, s17
; VGPR-NEXT: v_mov_b32_e32 v18, s18
; VGPR-NEXT: v_mov_b32_e32 v19, s19
-; VGPR-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1
+; VGPR-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] offset:32 sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: s_nop 0
; VGPR-NEXT: v_mov_b32_e32 v16, s12
; VGPR-NEXT: v_mov_b32_e32 v17, s13
; VGPR-NEXT: v_mov_b32_e32 v18, s14
; VGPR-NEXT: v_mov_b32_e32 v19, s15
-; VGPR-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1
+; VGPR-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] offset:16 sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: s_nop 0
; VGPR-NEXT: v_mov_b32_e32 v16, s8
; VGPR-NEXT: v_mov_b32_e32 v17, s9
; VGPR-NEXT: v_mov_b32_e32 v18, s10
; VGPR-NEXT: v_mov_b32_e32 v19, s11
-; VGPR-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1
+; VGPR-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
-; VGPR-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32 sc0 sc1
+; VGPR-NEXT: global_store_dwordx4 v36, v[8:11], s[0:1] offset:32 sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
-; VGPR-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48 sc0 sc1
+; VGPR-NEXT: global_store_dwordx4 v36, v[12:15], s[0:1] offset:48 sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
-; VGPR-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] sc0 sc1
+; VGPR-NEXT: global_store_dwordx4 v36, v[0:3], s[0:1] sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
-; VGPR-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16 sc0 sc1
+; VGPR-NEXT: global_store_dwordx4 v36, v[4:7], s[0:1] offset:16 sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: s_endpgm
%result = call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> %arg0, <8 x bfloat> %arg1, <16 x float> %arg2, i32 1, i32 2, i32 3)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll
index cee17039bdf59..36b1c9f1ffbfc 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll
@@ -1401,12 +1401,12 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd(<8 x half> %arg0,
; SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
-; SDAG-NEXT: v_mov_b32_e32 v32, 0
+; SDAG-NEXT: v_mov_b32_e32 v36, 0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b64_e32 v[36:37], s[26:27]
-; SDAG-NEXT: v_mov_b64_e32 v[34:35], s[24:25]
-; SDAG-NEXT: v_mov_b64_e32 v[40:41], s[30:31]
-; SDAG-NEXT: v_mov_b64_e32 v[38:39], s[28:29]
+; SDAG-NEXT: v_mov_b64_e32 v[40:41], s[26:27]
+; SDAG-NEXT: v_mov_b64_e32 v[38:39], s[24:25]
+; SDAG-NEXT: v_mov_b64_e32 v[44:45], s[30:31]
+; SDAG-NEXT: v_mov_b64_e32 v[42:43], s[28:29]
; SDAG-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; SDAG-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; SDAG-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
@@ -1415,41 +1415,41 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd(<8 x half> %arg0,
; SDAG-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
-; SDAG-NEXT: v_mov_b32_e32 v42, s20
-; SDAG-NEXT: v_mov_b32_e32 v43, s21
-; SDAG-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[34:37], v[38:41], v[16:31]
-; SDAG-NEXT: v_mov_b32_e32 v44, s22
-; SDAG-NEXT: v_mov_b32_e32 v45, s23
-; SDAG-NEXT: global_store_dwordx4 v32, v[42:45], s[0:1] offset:48 sc0 sc1
+; SDAG-NEXT: v_mov_b32_e32 v32, s20
+; SDAG-NEXT: v_mov_b32_e32 v33, s21
+; SDAG-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[38:41], v[42:45], v[16:31]
+; SDAG-NEXT: v_mov_b32_e32 v34, s22
+; SDAG-NEXT: v_mov_b32_e32 v35, s23
+; SDAG-NEXT: global_store_dwordx4 v36, v[32:35], s[0:1] offset:48 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 2
; SDAG-NEXT: v_mov_b32_e32 v16, s16
; SDAG-NEXT: v_mov_b32_e32 v17, s17
; SDAG-NEXT: v_mov_b32_e32 v18, s18
; SDAG-NEXT: v_mov_b32_e32 v19, s19
-; SDAG-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] offset:32 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_mov_b32_e32 v16, s12
; SDAG-NEXT: v_mov_b32_e32 v17, s13
; SDAG-NEXT: v_mov_b32_e32 v18, s14
; SDAG-NEXT: v_mov_b32_e32 v19, s15
-; SDAG-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] offset:16 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_mov_b32_e32 v16, s8
; SDAG-NEXT: v_mov_b32_e32 v17, s9
; SDAG-NEXT: v_mov_b32_e32 v18, s10
; SDAG-NEXT: v_mov_b32_e32 v19, s11
-; SDAG-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32 sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v36, v[8:11], s[0:1] offset:32 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48 sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v36, v[12:15], s[0:1] offset:48 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v36, v[0:3], s[0:1] sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16 sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v36, v[4:7], s[0:1] offset:16 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_endpgm
;
@@ -1504,12 +1504,12 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd(<8 x half> %arg0,
; HEURRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; HEURRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
-; HEURRC-NEXT: v_mov_b32_e32 v32, 0
+; HEURRC-NEXT: v_mov_b32_e32 v36, 0
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
-; HEURRC-NEXT: v_mov_b64_e32 v[36:37], s[26:27]
-; HEURRC-NEXT: v_mov_b64_e32 v[34:35], s[24:25]
-; HEURRC-NEXT: v_mov_b64_e32 v[40:41], s[30:31]
-; HEURRC-NEXT: v_mov_b64_e32 v[38:39], s[28:29]
+; HEURRC-NEXT: v_mov_b64_e32 v[40:41], s[26:27]
+; HEURRC-NEXT: v_mov_b64_e32 v[38:39], s[24:25]
+; HEURRC-NEXT: v_mov_b64_e32 v[44:45], s[30:31]
+; HEURRC-NEXT: v_mov_b64_e32 v[42:43], s[28:29]
; HEURRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; HEURRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; HEURRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
@@ -1518,41 +1518,41 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd(<8 x half> %arg0,
; HEURRC-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
; HEURRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; HEURRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
-; HEURRC-NEXT: v_mov_b32_e32 v42, s20
-; HEURRC-NEXT: v_mov_b32_e32 v43, s21
-; HEURRC-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[34:37], v[38:41], v[16:31]
-; HEURRC-NEXT: v_mov_b32_e32 v44, s22
-; HEURRC-NEXT: v_mov_b32_e32 v45, s23
-; HEURRC-NEXT: global_store_dwordx4 v32, v[42:45], s[0:1] offset:48 sc0 sc1
+; HEURRC-NEXT: v_mov_b32_e32 v32, s20
+; HEURRC-NEXT: v_mov_b32_e32 v33, s21
+; HEURRC-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[38:41], v[42:45], v[16:31]
+; HEURRC-NEXT: v_mov_b32_e32 v34, s22
+; HEURRC-NEXT: v_mov_b32_e32 v35, s23
+; HEURRC-NEXT: global_store_dwordx4 v36, v[32:35], s[0:1] offset:48 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_nop 2
; HEURRC-NEXT: v_mov_b32_e32 v16, s16
; HEURRC-NEXT: v_mov_b32_e32 v17, s17
; HEURRC-NEXT: v_mov_b32_e32 v18, s18
; HEURRC-NEXT: v_mov_b32_e32 v19, s19
-; HEURRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] offset:32 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_nop 0
; HEURRC-NEXT: v_mov_b32_e32 v16, s12
; HEURRC-NEXT: v_mov_b32_e32 v17, s13
; HEURRC-NEXT: v_mov_b32_e32 v18, s14
; HEURRC-NEXT: v_mov_b32_e32 v19, s15
-; HEURRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] offset:16 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_nop 0
; HEURRC-NEXT: v_mov_b32_e32 v16, s8
; HEURRC-NEXT: v_mov_b32_e32 v17, s9
; HEURRC-NEXT: v_mov_b32_e32 v18, s10
; HEURRC-NEXT: v_mov_b32_e32 v19, s11
-; HEURRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32 sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v36, v[8:11], s[0:1] offset:32 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48 sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v36, v[12:15], s[0:1] offset:48 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v36, v[0:3], s[0:1] sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16 sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v36, v[4:7], s[0:1] offset:16 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_endpgm
;
@@ -1561,12 +1561,12 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd(<8 x half> %arg0,
; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPRRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
-; VGPRRC-NEXT: v_mov_b32_e32 v32, 0
+; VGPRRC-NEXT: v_mov_b32_e32 v36, 0
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
-; VGPRRC-NEXT: v_mov_b64_e32 v[36:37], s[26:27]
-; VGPRRC-NEXT: v_mov_b64_e32 v[34:35], s[24:25]
-; VGPRRC-NEXT: v_mov_b64_e32 v[40:41], s[30:31]
-; VGPRRC-NEXT: v_mov_b64_e32 v[38:39], s[28:29]
+; VGPRRC-NEXT: v_mov_b64_e32 v[40:41], s[26:27]
+; VGPRRC-NEXT: v_mov_b64_e32 v[38:39], s[24:25]
+; VGPRRC-NEXT: v_mov_b64_e32 v[44:45], s[30:31]
+; VGPRRC-NEXT: v_mov_b64_e32 v[42:43], s[28:29]
; VGPRRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; VGPRRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; VGPRRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
@@ -1575,41 +1575,41 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd(<8 x half> %arg0,
; VGPRRC-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
-; VGPRRC-NEXT: v_mov_b32_e32 v42, s20
-; VGPRRC-NEXT: v_mov_b32_e32 v43, s21
-; VGPRRC-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[34:37], v[38:41], v[16:31]
-; VGPRRC-NEXT: v_mov_b32_e32 v44, s22
-; VGPRRC-NEXT: v_mov_b32_e32 v45, s23
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[42:45], s[0:1] offset:48 sc0 sc1
+; VGPRRC-NEXT: v_mov_b32_e32 v32, s20
+; VGPRRC-NEXT: v_mov_b32_e32 v33, s21
+; VGPRRC-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[38:41], v[42:45], v[16:31]
+; VGPRRC-NEXT: v_mov_b32_e32 v34, s22
+; VGPRRC-NEXT: v_mov_b32_e32 v35, s23
+; VGPRRC-NEXT: global_store_dwordx4 v36, v[32:35], s[0:1] offset:48 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 2
; VGPRRC-NEXT: v_mov_b32_e32 v16, s16
; VGPRRC-NEXT: v_mov_b32_e32 v17, s17
; VGPRRC-NEXT: v_mov_b32_e32 v18, s18
; VGPRRC-NEXT: v_mov_b32_e32 v19, s19
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] offset:32 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b32_e32 v16, s12
; VGPRRC-NEXT: v_mov_b32_e32 v17, s13
; VGPRRC-NEXT: v_mov_b32_e32 v18, s14
; VGPRRC-NEXT: v_mov_b32_e32 v19, s15
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] offset:16 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b32_e32 v16, s8
; VGPRRC-NEXT: v_mov_b32_e32 v17, s9
; VGPRRC-NEXT: v_mov_b32_e32 v18, s10
; VGPRRC-NEXT: v_mov_b32_e32 v19, s11
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32 sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v36, v[8:11], s[0:1] offset:32 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48 sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v36, v[12:15], s[0:1] offset:48 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v36, v[0:3], s[0:1] sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16 sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v36, v[4:7], s[0:1] offset:16 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_endpgm
; AGPR-LABEL: test_mfma_f32_32x32x16_f16__vgprcd:
@@ -1743,12 +1743,12 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd__flags(<8 x half>
; SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
-; SDAG-NEXT: v_mov_b32_e32 v32, 0
+; SDAG-NEXT: v_mov_b32_e32 v36, 0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b64_e32 v[36:37], s[26:27]
-; SDAG-NEXT: v_mov_b64_e32 v[34:35], s[24:25]
-; SDAG-NEXT: v_mov_b64_e32 v[40:41], s[30:31]
-; SDAG-NEXT: v_mov_b64_e32 v[38:39], s[28:29]
+; SDAG-NEXT: v_mov_b64_e32 v[40:41], s[26:27]
+; SDAG-NEXT: v_mov_b64_e32 v[38:39], s[24:25]
+; SDAG-NEXT: v_mov_b64_e32 v[44:45], s[30:31]
+; SDAG-NEXT: v_mov_b64_e32 v[42:43], s[28:29]
; SDAG-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; SDAG-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; SDAG-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
@@ -1757,41 +1757,41 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd__flags(<8 x half>
; SDAG-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
-; SDAG-NEXT: v_mov_b32_e32 v42, s20
-; SDAG-NEXT: v_mov_b32_e32 v43, s21
-; SDAG-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[34:37], v[38:41], v[16:31] cbsz:1 abid:2 blgp:3
-; SDAG-NEXT: v_mov_b32_e32 v44, s22
-; SDAG-NEXT: v_mov_b32_e32 v45, s23
-; SDAG-NEXT: global_store_dwordx4 v32, v[42:45], s[0:1] offset:48 sc0 sc1
+; SDAG-NEXT: v_mov_b32_e32 v32, s20
+; SDAG-NEXT: v_mov_b32_e32 v33, s21
+; SDAG-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[38:41], v[42:45], v[16:31] cbsz:1 abid:2 blgp:3
+; SDAG-NEXT: v_mov_b32_e32 v34, s22
+; SDAG-NEXT: v_mov_b32_e32 v35, s23
+; SDAG-NEXT: global_store_dwordx4 v36, v[32:35], s[0:1] offset:48 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 2
; SDAG-NEXT: v_mov_b32_e32 v16, s16
; SDAG-NEXT: v_mov_b32_e32 v17, s17
; SDAG-NEXT: v_mov_b32_e32 v18, s18
; SDAG-NEXT: v_mov_b32_e32 v19, s19
-; SDAG-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] offset:32 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_mov_b32_e32 v16, s12
; SDAG-NEXT: v_mov_b32_e32 v17, s13
; SDAG-NEXT: v_mov_b32_e32 v18, s14
; SDAG-NEXT: v_mov_b32_e32 v19, s15
-; SDAG-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] offset:16 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_mov_b32_e32 v16, s8
; SDAG-NEXT: v_mov_b32_e32 v17, s9
; SDAG-NEXT: v_mov_b32_e32 v18, s10
; SDAG-NEXT: v_mov_b32_e32 v19, s11
-; SDAG-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32 sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v36, v[8:11], s[0:1] offset:32 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48 sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v36, v[12:15], s[0:1] offset:48 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v36, v[0:3], s[0:1] sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16 sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v36, v[4:7], s[0:1] offset:16 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_endpgm
;
@@ -1846,12 +1846,12 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd__flags(<8 x half>
; HEURRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; HEURRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
-; HEURRC-NEXT: v_mov_b32_e32 v32, 0
+; HEURRC-NEXT: v_mov_b32_e32 v36, 0
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
-; HEURRC-NEXT: v_mov_b64_e32 v[36:37], s[26:27]
-; HEURRC-NEXT: v_mov_b64_e32 v[34:35], s[24:25]
-; HEURRC-NEXT: v_mov_b64_e32 v[40:41], s[30:31]
-; HEURRC-NEXT: v_mov_b64_e32 v[38:39], s[28:29]
+; HEURRC-NEXT: v_mov_b64_e32 v[40:41], s[26:27]
+; HEURRC-NEXT: v_mov_b64_e32 v[38:39], s[24:25]
+; HEURRC-NEXT: v_mov_b64_e32 v[44:45], s[30:31]
+; HEURRC-NEXT: v_mov_b64_e32 v[42:43], s[28:29]
; HEURRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; HEURRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; HEURRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
@@ -1860,41 +1860,41 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd__flags(<8 x half>
; HEURRC-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
; HEURRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; HEURRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
-; HEURRC-NEXT: v_mov_b32_e32 v42, s20
-; HEURRC-NEXT: v_mov_b32_e32 v43, s21
-; HEURRC-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[34:37], v[38:41], v[16:31] cbsz:1 abid:2 blgp:3
-; HEURRC-NEXT: v_mov_b32_e32 v44, s22
-; HEURRC-NEXT: v_mov_b32_e32 v45, s23
-; HEURRC-NEXT: global_store_dwordx4 v32, v[42:45], s[0:1] offset:48 sc0 sc1
+; HEURRC-NEXT: v_mov_b32_e32 v32, s20
+; HEURRC-NEXT: v_mov_b32_e32 v33, s21
+; HEURRC-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[38:41], v[42:45], v[16:31] cbsz:1 abid:2 blgp:3
+; HEURRC-NEXT: v_mov_b32_e32 v34, s22
+; HEURRC-NEXT: v_mov_b32_e32 v35, s23
+; HEURRC-NEXT: global_store_dwordx4 v36, v[32:35], s[0:1] offset:48 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_nop 2
; HEURRC-NEXT: v_mov_b32_e32 v16, s16
; HEURRC-NEXT: v_mov_b32_e32 v17, s17
; HEURRC-NEXT: v_mov_b32_e32 v18, s18
; HEURRC-NEXT: v_mov_b32_e32 v19, s19
-; HEURRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] offset:32 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_nop 0
; HEURRC-NEXT: v_mov_b32_e32 v16, s12
; HEURRC-NEXT: v_mov_b32_e32 v17, s13
; HEURRC-NEXT: v_mov_b32_e32 v18, s14
; HEURRC-NEXT: v_mov_b32_e32 v19, s15
-; HEURRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] offset:16 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_nop 0
; HEURRC-NEXT: v_mov_b32_e32 v16, s8
; HEURRC-NEXT: v_mov_b32_e32 v17, s9
; HEURRC-NEXT: v_mov_b32_e32 v18, s10
; HEURRC-NEXT: v_mov_b32_e32 v19, s11
-; HEURRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32 sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v36, v[8:11], s[0:1] offset:32 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48 sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v36, v[12:15], s[0:1] offset:48 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v36, v[0:3], s[0:1] sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16 sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v36, v[4:7], s[0:1] offset:16 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_endpgm
;
@@ -1903,12 +1903,12 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd__flags(<8 x half>
; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPRRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
-; VGPRRC-NEXT: v_mov_b32_e32 v32, 0
+; VGPRRC-NEXT: v_mov_b32_e32 v36, 0
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
-; VGPRRC-NEXT: v_mov_b64_e32 v[36:37], s[26:27]
-; VGPRRC-NEXT: v_mov_b64_e32 v[34:35], s[24:25]
-; VGPRRC-NEXT: v_mov_b64_e32 v[40:41], s[30:31]
-; VGPRRC-NEXT: v_mov_b64_e32 v[38:39], s[28:29]
+; VGPRRC-NEXT: v_mov_b64_e32 v[40:41], s[26:27]
+; VGPRRC-NEXT: v_mov_b64_e32 v[38:39], s[24:25]
+; VGPRRC-NEXT: v_mov_b64_e32 v[44:45], s[30:31]
+; VGPRRC-NEXT: v_mov_b64_e32 v[42:43], s[28:29]
; VGPRRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; VGPRRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; VGPRRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
@@ -1917,41 +1917,41 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd__flags(<8 x half>
; VGPRRC-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
-; VGPRRC-NEXT: v_mov_b32_e32 v42, s20
-; VGPRRC-NEXT: v_mov_b32_e32 v43, s21
-; VGPRRC-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[34:37], v[38:41], v[16:31] cbsz:1 abid:2 blgp:3
-; VGPRRC-NEXT: v_mov_b32_e32 v44, s22
-; VGPRRC-NEXT: v_mov_b32_e32 v45, s23
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[42:45], s[0:1] offset:48 sc0 sc1
+; VGPRRC-NEXT: v_mov_b32_e32 v32, s20
+; VGPRRC-NEXT: v_mov_b32_e32 v33, s21
+; VGPRRC-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[38:41], v[42:45], v[16:31] cbsz:1 abid:2 blgp:3
+; VGPRRC-NEXT: v_mov_b32_e32 v34, s22
+; VGPRRC-NEXT: v_mov_b32_e32 v35, s23
+; VGPRRC-NEXT: global_store_dwordx4 v36, v[32:35], s[0:1] offset:48 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 2
; VGPRRC-NEXT: v_mov_b32_e32 v16, s16
; VGPRRC-NEXT: v_mov_b32_e32 v17, s17
; VGPRRC-NEXT: v_mov_b32_e32 v18, s18
; VGPRRC-NEXT: v_mov_b32_e32 v19, s19
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] offset:32 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b32_e32 v16, s12
; VGPRRC-NEXT: v_mov_b32_e32 v17, s13
; VGPRRC-NEXT: v_mov_b32_e32 v18, s14
; VGPRRC-NEXT: v_mov_b32_e32 v19, s15
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] offset:16 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b32_e32 v16, s8
; VGPRRC-NEXT: v_mov_b32_e32 v17, s9
; VGPRRC-NEXT: v_mov_b32_e32 v18, s10
; VGPRRC-NEXT: v_mov_b32_e32 v19, s11
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v36, v[16:19], s[0:1] sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32 sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v36, v[8:11], s[0:1] offset:32 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48 sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v36, v[12:15], s[0:1] offset:48 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v36, v[0:3], s[0:1] sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16 sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v36, v[4:7], s[0:1] offset:16 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_endpgm
; AGPR-LABEL: test_mfma_f32_32x32x16_f16__vgprcd__flags:
@@ -2557,24 +2557,24 @@ define amdgpu_kernel void @test_mfma_i32_16x16x64_i8_no_agpr__vgprcd(ptr addrspa
; SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
; SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
-; SDAG-NEXT: v_mov_b32_e32 v4, 0
+; SDAG-NEXT: v_mov_b32_e32 v12, 0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v6, s8
-; SDAG-NEXT: v_mov_b32_e32 v7, s9
-; SDAG-NEXT: v_mov_b32_e32 v8, s10
-; SDAG-NEXT: v_mov_b32_e32 v9, s11
-; SDAG-NEXT: v_mov_b32_e32 v10, s12
-; SDAG-NEXT: v_mov_b32_e32 v11, s13
-; SDAG-NEXT: v_mov_b32_e32 v12, s14
-; SDAG-NEXT: v_mov_b32_e32 v13, s15
-; SDAG-NEXT: v_mov_b32_e32 v0, s0
-; SDAG-NEXT: v_mov_b32_e32 v1, s1
-; SDAG-NEXT: v_mov_b32_e32 v2, s2
-; SDAG-NEXT: v_mov_b32_e32 v3, s3
+; SDAG-NEXT: v_mov_b32_e32 v0, s8
+; SDAG-NEXT: v_mov_b32_e32 v1, s9
+; SDAG-NEXT: v_mov_b32_e32 v2, s10
+; SDAG-NEXT: v_mov_b32_e32 v3, s11
+; SDAG-NEXT: v_mov_b32_e32 v4, s12
+; SDAG-NEXT: v_mov_b32_e32 v5, s13
+; SDAG-NEXT: v_mov_b32_e32 v6, s14
+; SDAG-NEXT: v_mov_b32_e32 v7, s15
+; SDAG-NEXT: v_mov_b32_e32 v8, s0
+; SDAG-NEXT: v_mov_b32_e32 v9, s1
+; SDAG-NEXT: v_mov_b32_e32 v10, s2
+; SDAG-NEXT: v_mov_b32_e32 v11, s3
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[6:9], v[10:13], v[0:3]
+; SDAG-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[0:3], v[4:7], v[8:11]
; SDAG-NEXT: s_nop 7
-; SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; SDAG-NEXT: global_store_dwordx4 v12, v[0:3], s[6:7]
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: test_mfma_i32_16x16x64_i8_no_agpr__vgprcd:
@@ -2601,24 +2601,24 @@ define amdgpu_kernel void @test_mfma_i32_16x16x64_i8_no_agpr__vgprcd(ptr addrspa
; HEURRC-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
; HEURRC-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; HEURRC-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
-; HEURRC-NEXT: v_mov_b32_e32 v4, 0
+; HEURRC-NEXT: v_mov_b32_e32 v12, 0
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
-; HEURRC-NEXT: v_mov_b32_e32 v6, s8
-; HEURRC-NEXT: v_mov_b32_e32 v7, s9
-; HEURRC-NEXT: v_mov_b32_e32 v8, s10
-; HEURRC-NEXT: v_mov_b32_e32 v9, s11
-; HEURRC-NEXT: v_mov_b32_e32 v10, s12
-; HEURRC-NEXT: v_mov_b32_e32 v11, s13
-; HEURRC-NEXT: v_mov_b32_e32 v12, s14
-; HEURRC-NEXT: v_mov_b32_e32 v13, s15
-; HEURRC-NEXT: v_mov_b32_e32 v0, s0
-; HEURRC-NEXT: v_mov_b32_e32 v1, s1
-; HEURRC-NEXT: v_mov_b32_e32 v2, s2
-; HEURRC-NEXT: v_mov_b32_e32 v3, s3
+; HEURRC-NEXT: v_mov_b32_e32 v0, s8
+; HEURRC-NEXT: v_mov_b32_e32 v1, s9
+; HEURRC-NEXT: v_mov_b32_e32 v2, s10
+; HEURRC-NEXT: v_mov_b32_e32 v3, s11
+; HEURRC-NEXT: v_mov_b32_e32 v4, s12
+; HEURRC-NEXT: v_mov_b32_e32 v5, s13
+; HEURRC-NEXT: v_mov_b32_e32 v6, s14
+; HEURRC-NEXT: v_mov_b32_e32 v7, s15
+; HEURRC-NEXT: v_mov_b32_e32 v8, s0
+; HEURRC-NEXT: v_mov_b32_e32 v9, s1
+; HEURRC-NEXT: v_mov_b32_e32 v10, s2
+; HEURRC-NEXT: v_mov_b32_e32 v11, s3
; HEURRC-NEXT: s_nop 1
-; HEURRC-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[6:9], v[10:13], v[0:3]
+; HEURRC-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[0:3], v[4:7], v[8:11]
; HEURRC-NEXT: s_nop 7
-; HEURRC-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; HEURRC-NEXT: global_store_dwordx4 v12, v[0:3], s[6:7]
; HEURRC-NEXT: s_endpgm
;
; VGPRRC-LABEL: test_mfma_i32_16x16x64_i8_no_agpr__vgprcd:
@@ -2626,24 +2626,24 @@ define amdgpu_kernel void @test_mfma_i32_16x16x64_i8_no_agpr__vgprcd(ptr addrspa
; VGPRRC-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
; VGPRRC-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; VGPRRC-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
-; VGPRRC-NEXT: v_mov_b32_e32 v4, 0
+; VGPRRC-NEXT: v_mov_b32_e32 v12, 0
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
-; VGPRRC-NEXT: v_mov_b32_e32 v6, s8
-; VGPRRC-NEXT: v_mov_b32_e32 v7, s9
-; VGPRRC-NEXT: v_mov_b32_e32 v8, s10
-; VGPRRC-NEXT: v_mov_b32_e32 v9, s11
-; VGPRRC-NEXT: v_mov_b32_e32 v10, s12
-; VGPRRC-NEXT: v_mov_b32_e32 v11, s13
-; VGPRRC-NEXT: v_mov_b32_e32 v12, s14
-; VGPRRC-NEXT: v_mov_b32_e32 v13, s15
-; VGPRRC-NEXT: v_mov_b32_e32 v0, s0
-; VGPRRC-NEXT: v_mov_b32_e32 v1, s1
-; VGPRRC-NEXT: v_mov_b32_e32 v2, s2
-; VGPRRC-NEXT: v_mov_b32_e32 v3, s3
+; VGPRRC-NEXT: v_mov_b32_e32 v0, s8
+; VGPRRC-NEXT: v_mov_b32_e32 v1, s9
+; VGPRRC-NEXT: v_mov_b32_e32 v2, s10
+; VGPRRC-NEXT: v_mov_b32_e32 v3, s11
+; VGPRRC-NEXT: v_mov_b32_e32 v4, s12
+; VGPRRC-NEXT: v_mov_b32_e32 v5, s13
+; VGPRRC-NEXT: v_mov_b32_e32 v6, s14
+; VGPRRC-NEXT: v_mov_b32_e32 v7, s15
+; VGPRRC-NEXT: v_mov_b32_e32 v8, s0
+; VGPRRC-NEXT: v_mov_b32_e32 v9, s1
+; VGPRRC-NEXT: v_mov_b32_e32 v10, s2
+; VGPRRC-NEXT: v_mov_b32_e32 v11, s3
; VGPRRC-NEXT: s_nop 1
-; VGPRRC-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[6:9], v[10:13], v[0:3]
+; VGPRRC-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[0:3], v[4:7], v[8:11]
; VGPRRC-NEXT: s_nop 7
-; VGPRRC-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; VGPRRC-NEXT: global_store_dwordx4 v12, v[0:3], s[6:7]
; VGPRRC-NEXT: s_endpgm
; AGPR-LABEL: test_mfma_i32_16x16x64_i8_no_agpr__vgprcd:
; AGPR: ; %bb.0:
@@ -2704,24 +2704,24 @@ define amdgpu_kernel void @test_mfma_i32_16x16x64_i8_no_agpr__vgprcd__flags(ptr
; SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
; SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
-; SDAG-NEXT: v_mov_b32_e32 v4, 0
+; SDAG-NEXT: v_mov_b32_e32 v12, 0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v6, s8
-; SDAG-NEXT: v_mov_b32_e32 v7, s9
-; SDAG-NEXT: v_mov_b32_e32 v8, s10
-; SDAG-NEXT: v_mov_b32_e32 v9, s11
-; SDAG-NEXT: v_mov_b32_e32 v10, s12
-; SDAG-NEXT: v_mov_b32_e32 v11, s13
-; SDAG-NEXT: v_mov_b32_e32 v12, s14
-; SDAG-NEXT: v_mov_b32_e32 v13, s15
-; SDAG-NEXT: v_mov_b32_e32 v0, s0
-; SDAG-NEXT: v_mov_b32_e32 v1, s1
-; SDAG-NEXT: v_mov_b32_e32 v2, s2
-; SDAG-NEXT: v_mov_b32_e32 v3, s3
+; SDAG-NEXT: v_mov_b32_e32 v0, s8
+; SDAG-NEXT: v_mov_b32_e32 v1, s9
+; SDAG-NEXT: v_mov_b32_e32 v2, s10
+; SDAG-NEXT: v_mov_b32_e32 v3, s11
+; SDAG-NEXT: v_mov_b32_e32 v4, s12
+; SDAG-NEXT: v_mov_b32_e32 v5, s13
+; SDAG-NEXT: v_mov_b32_e32 v6, s14
+; SDAG-NEXT: v_mov_b32_e32 v7, s15
+; SDAG-NEXT: v_mov_b32_e32 v8, s0
+; SDAG-NEXT: v_mov_b32_e32 v9, s1
+; SDAG-NEXT: v_mov_b32_e32 v10, s2
+; SDAG-NEXT: v_mov_b32_e32 v11, s3
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[6:9], v[10:13], v[0:3] cbsz:3 abid:2 blgp:1
+; SDAG-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[0:3], v[4:7], v[8:11] cbsz:3 abid:2 blgp:1
; SDAG-NEXT: s_nop 7
-; SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; SDAG-NEXT: global_store_dwordx4 v12, v[0:3], s[6:7]
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: test_mfma_i32_16x16x64_i8_no_agpr__vgprcd__flags:
@@ -2748,24 +2748,24 @@ define amdgpu_kernel void @test_mfma_i32_16x16x64_i8_no_agpr__vgprcd__flags(ptr
; HEURRC-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
; HEURRC-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; HEURRC-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
-; HEURRC-NEXT: v_mov_b32_e32 v4, 0
+; HEURRC-NEXT: v_mov_b32_e32 v12, 0
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
-; HEURRC-NEXT: v_mov_b32_e32 v6, s8
-; HEURRC-NEXT: v_mov_b32_e32 v7, s9
-; HEURRC-NEXT: v_mov_b32_e32 v8, s10
-; HEURRC-NEXT: v_mov_b32_e32 v9, s11
-; HEURRC-NEXT: v_mov_b32_e32 v10, s12
-; HEURRC-NEXT: v_mov_b32_e32 v11, s13
-; HEURRC-NEXT: v_mov_b32_e32 v12, s14
-; HEURRC-NEXT: v_mov_b32_e32 v13, s15
-; HEURRC-NEXT: v_mov_b32_e32 v0, s0
-; HEURRC-NEXT: v_mov_b32_e32 v1, s1
-; HEURRC-NEXT: v_mov_b32_e32 v2, s2
-; HEURRC-NEXT: v_mov_b32_e32 v3, s3
+; HEURRC-NEXT: v_mov_b32_e32 v0, s8
+; HEURRC-NEXT: v_mov_b32_e32 v1, s9
+; HEURRC-NEXT: v_mov_b32_e32 v2, s10
+; HEURRC-NEXT: v_mov_b32_e32 v3, s11
+; HEURRC-NEXT: v_mov_b32_e32 v4, s12
+; HEURRC-NEXT: v_mov_b32_e32 v5, s13
+; HEURRC-NEXT: v_mov_b32_e32 v6, s14
+; HEURRC-NEXT: v_mov_b32_e32 v7, s15
+; HEURRC-NEXT: v_mov_b32_e32 v8, s0
+; HEURRC-NEXT: v_mov_b32_e32 v9, s1
+; HEURRC-NEXT: v_mov_b32_e32 v10, s2
+; HEURRC-NEXT: v_mov_b32_e32 v11, s3
; HEURRC-NEXT: s_nop 1
-; HEURRC-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[6:9], v[10:13], v[0:3] cbsz:3 abid:2 blgp:1
+; HEURRC-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[0:3], v[4:7], v[8:11] cbsz:3 abid:2 blgp:1
; HEURRC-NEXT: s_nop 7
-; HEURRC-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; HEURRC-NEXT: global_store_dwordx4 v12, v[0:3], s[6:7]
; HEURRC-NEXT: s_endpgm
;
; VGPRRC-LABEL: test_mfma_i32_16x16x64_i8_no_agpr__vgprcd__flags:
@@ -2773,24 +2773,24 @@ define amdgpu_kernel void @test_mfma_i32_16x16x64_i8_no_agpr__vgprcd__flags(ptr
; VGPRRC-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
; VGPRRC-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; VGPRRC-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
-; VGPRRC-NEXT: v_mov_b32_e32 v4, 0
+; VGPRRC-NEXT: v_mov_b32_e32 v12, 0
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
-; VGPRRC-NEXT: v_mov_b32_e32 v6, s8
-; VGPRRC-NEXT: v_mov_b32_e32 v7, s9
-; VGPRRC-NEXT: v_mov_b32_e32 v8, s10
-; VGPRRC-NEXT: v_mov_b32_e32 v9, s11
-; VGPRRC-NEXT: v_mov_b32_e32 v10, s12
-; VGPRRC-NEXT: v_mov_b32_e32 v11, s13
-; VGPRRC-NEXT: v_mov_b32_e32 v12, s14
-; VGPRRC-NEXT: v_mov_b32_e32 v13, s15
-; VGPRRC-NEXT: v_mov_b32_e32 v0, s0
-; VGPRRC-NEXT: v_mov_b32_e32 v1, s1
-; VGPRRC-NEXT: v_mov_b32_e32 v2, s2
-; VGPRRC-NEXT: v_mov_b32_e32 v3, s3
+; VGPRRC-NEXT: v_mov_b32_e32 v0, s8
+; VGPRRC-NEXT: v_mov_b32_e32 v1, s9
+; VGPRRC-NEXT: v_mov_b32_e32 v2, s10
+; VGPRRC-NEXT: v_mov_b32_e32 v3, s11
+; VGPRRC-NEXT: v_mov_b32_e32 v4, s12
+; VGPRRC-NEXT: v_mov_b32_e32 v5, s13
+; VGPRRC-NEXT: v_mov_b32_e32 v6, s14
+; VGPRRC-NEXT: v_mov_b32_e32 v7, s15
+; VGPRRC-NEXT: v_mov_b32_e32 v8, s0
+; VGPRRC-NEXT: v_mov_b32_e32 v9, s1
+; VGPRRC-NEXT: v_mov_b32_e32 v10, s2
+; VGPRRC-NEXT: v_mov_b32_e32 v11, s3
; VGPRRC-NEXT: s_nop 1
-; VGPRRC-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[6:9], v[10:13], v[0:3] cbsz:3 abid:2 blgp:1
+; VGPRRC-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[0:3], v[4:7], v[8:11] cbsz:3 abid:2 blgp:1
; VGPRRC-NEXT: s_nop 7
-; VGPRRC-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; VGPRRC-NEXT: global_store_dwordx4 v12, v[0:3], s[6:7]
; VGPRRC-NEXT: s_endpgm
; AGPR-LABEL: test_mfma_i32_16x16x64_i8_no_agpr__vgprcd__flags:
; AGPR: ; %bb.0:
@@ -3952,17 +3952,17 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd(<4 x i32> %arg0, <4
; SDAG: ; %bb.0:
; SDAG-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
-; SDAG-NEXT: v_mov_b32_e32 v32, 0
+; SDAG-NEXT: v_mov_b32_e32 v40, 0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v34, s20
-; SDAG-NEXT: v_mov_b32_e32 v35, s21
-; SDAG-NEXT: v_mov_b32_e32 v36, s22
-; SDAG-NEXT: v_mov_b32_e32 v37, s23
+; SDAG-NEXT: v_mov_b32_e32 v32, s20
+; SDAG-NEXT: v_mov_b32_e32 v33, s21
+; SDAG-NEXT: v_mov_b32_e32 v34, s22
+; SDAG-NEXT: v_mov_b32_e32 v35, s23
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
-; SDAG-NEXT: v_mov_b32_e32 v38, s24
-; SDAG-NEXT: v_mov_b32_e32 v39, s25
-; SDAG-NEXT: v_mov_b32_e32 v40, s26
-; SDAG-NEXT: v_mov_b32_e32 v41, s27
+; SDAG-NEXT: v_mov_b32_e32 v36, s24
+; SDAG-NEXT: v_mov_b32_e32 v37, s25
+; SDAG-NEXT: v_mov_b32_e32 v38, s26
+; SDAG-NEXT: v_mov_b32_e32 v39, s27
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; SDAG-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
@@ -3973,42 +3973,42 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd(<4 x i32> %arg0, <4
; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[34:37], v[38:41], v[16:31]
+; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31]
; SDAG-NEXT: s_nop 6
; SDAG-NEXT: v_mov_b32_e32 v16, s20
; SDAG-NEXT: v_mov_b32_e32 v17, s21
; SDAG-NEXT: v_mov_b32_e32 v18, s22
; SDAG-NEXT: v_mov_b32_e32 v19, s23
-; SDAG-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:48 sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:48 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_mov_b32_e32 v16, s16
; SDAG-NEXT: v_mov_b32_e32 v17, s17
; SDAG-NEXT: v_mov_b32_e32 v18, s18
; SDAG-NEXT: v_mov_b32_e32 v19, s19
-; SDAG-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:32 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_mov_b32_e32 v16, s12
; SDAG-NEXT: v_mov_b32_e32 v17, s13
; SDAG-NEXT: v_mov_b32_e32 v18, s14
; SDAG-NEXT: v_mov_b32_e32 v19, s15
-; SDAG-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:16 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_mov_b32_e32 v16, s8
; SDAG-NEXT: v_mov_b32_e32 v17, s9
; SDAG-NEXT: v_mov_b32_e32 v18, s10
; SDAG-NEXT: v_mov_b32_e32 v19, s11
-; SDAG-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32 sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v40, v[8:11], s[0:1] offset:32 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48 sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v40, v[12:15], s[0:1] offset:48 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v40, v[0:3], s[0:1] sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16 sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v40, v[4:7], s[0:1] offset:16 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_endpgm
;
@@ -4062,17 +4062,17 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd(<4 x i32> %arg0, <4
; HEURRC: ; %bb.0:
; HEURRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
; HEURRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
-; HEURRC-NEXT: v_mov_b32_e32 v32, 0
+; HEURRC-NEXT: v_mov_b32_e32 v40, 0
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
-; HEURRC-NEXT: v_mov_b32_e32 v34, s20
-; HEURRC-NEXT: v_mov_b32_e32 v35, s21
-; HEURRC-NEXT: v_mov_b32_e32 v36, s22
-; HEURRC-NEXT: v_mov_b32_e32 v37, s23
+; HEURRC-NEXT: v_mov_b32_e32 v32, s20
+; HEURRC-NEXT: v_mov_b32_e32 v33, s21
+; HEURRC-NEXT: v_mov_b32_e32 v34, s22
+; HEURRC-NEXT: v_mov_b32_e32 v35, s23
; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
-; HEURRC-NEXT: v_mov_b32_e32 v38, s24
-; HEURRC-NEXT: v_mov_b32_e32 v39, s25
-; HEURRC-NEXT: v_mov_b32_e32 v40, s26
-; HEURRC-NEXT: v_mov_b32_e32 v41, s27
+; HEURRC-NEXT: v_mov_b32_e32 v36, s24
+; HEURRC-NEXT: v_mov_b32_e32 v37, s25
+; HEURRC-NEXT: v_mov_b32_e32 v38, s26
+; HEURRC-NEXT: v_mov_b32_e32 v39, s27
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
; HEURRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; HEURRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
@@ -4083,42 +4083,42 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd(<4 x i32> %arg0, <4
; HEURRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; HEURRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; HEURRC-NEXT: s_nop 1
-; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[34:37], v[38:41], v[16:31]
+; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31]
; HEURRC-NEXT: s_nop 6
; HEURRC-NEXT: v_mov_b32_e32 v16, s20
; HEURRC-NEXT: v_mov_b32_e32 v17, s21
; HEURRC-NEXT: v_mov_b32_e32 v18, s22
; HEURRC-NEXT: v_mov_b32_e32 v19, s23
-; HEURRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:48 sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:48 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_nop 0
; HEURRC-NEXT: v_mov_b32_e32 v16, s16
; HEURRC-NEXT: v_mov_b32_e32 v17, s17
; HEURRC-NEXT: v_mov_b32_e32 v18, s18
; HEURRC-NEXT: v_mov_b32_e32 v19, s19
-; HEURRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:32 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_nop 0
; HEURRC-NEXT: v_mov_b32_e32 v16, s12
; HEURRC-NEXT: v_mov_b32_e32 v17, s13
; HEURRC-NEXT: v_mov_b32_e32 v18, s14
; HEURRC-NEXT: v_mov_b32_e32 v19, s15
-; HEURRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:16 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_nop 0
; HEURRC-NEXT: v_mov_b32_e32 v16, s8
; HEURRC-NEXT: v_mov_b32_e32 v17, s9
; HEURRC-NEXT: v_mov_b32_e32 v18, s10
; HEURRC-NEXT: v_mov_b32_e32 v19, s11
-; HEURRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32 sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v40, v[8:11], s[0:1] offset:32 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48 sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v40, v[12:15], s[0:1] offset:48 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v40, v[0:3], s[0:1] sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16 sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v40, v[4:7], s[0:1] offset:16 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_endpgm
;
@@ -4126,17 +4126,17 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd(<4 x i32> %arg0, <4
; VGPRRC: ; %bb.0:
; VGPRRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
; VGPRRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
-; VGPRRC-NEXT: v_mov_b32_e32 v32, 0
+; VGPRRC-NEXT: v_mov_b32_e32 v40, 0
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
-; VGPRRC-NEXT: v_mov_b32_e32 v34, s20
-; VGPRRC-NEXT: v_mov_b32_e32 v35, s21
-; VGPRRC-NEXT: v_mov_b32_e32 v36, s22
-; VGPRRC-NEXT: v_mov_b32_e32 v37, s23
+; VGPRRC-NEXT: v_mov_b32_e32 v32, s20
+; VGPRRC-NEXT: v_mov_b32_e32 v33, s21
+; VGPRRC-NEXT: v_mov_b32_e32 v34, s22
+; VGPRRC-NEXT: v_mov_b32_e32 v35, s23
; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
-; VGPRRC-NEXT: v_mov_b32_e32 v38, s24
-; VGPRRC-NEXT: v_mov_b32_e32 v39, s25
-; VGPRRC-NEXT: v_mov_b32_e32 v40, s26
-; VGPRRC-NEXT: v_mov_b32_e32 v41, s27
+; VGPRRC-NEXT: v_mov_b32_e32 v36, s24
+; VGPRRC-NEXT: v_mov_b32_e32 v37, s25
+; VGPRRC-NEXT: v_mov_b32_e32 v38, s26
+; VGPRRC-NEXT: v_mov_b32_e32 v39, s27
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
; VGPRRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; VGPRRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
@@ -4147,42 +4147,42 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd(<4 x i32> %arg0, <4
; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; VGPRRC-NEXT: s_nop 1
-; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[34:37], v[38:41], v[16:31]
+; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31]
; VGPRRC-NEXT: s_nop 6
; VGPRRC-NEXT: v_mov_b32_e32 v16, s20
; VGPRRC-NEXT: v_mov_b32_e32 v17, s21
; VGPRRC-NEXT: v_mov_b32_e32 v18, s22
; VGPRRC-NEXT: v_mov_b32_e32 v19, s23
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:48 sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:48 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b32_e32 v16, s16
; VGPRRC-NEXT: v_mov_b32_e32 v17, s17
; VGPRRC-NEXT: v_mov_b32_e32 v18, s18
; VGPRRC-NEXT: v_mov_b32_e32 v19, s19
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:32 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b32_e32 v16, s12
; VGPRRC-NEXT: v_mov_b32_e32 v17, s13
; VGPRRC-NEXT: v_mov_b32_e32 v18, s14
; VGPRRC-NEXT: v_mov_b32_e32 v19, s15
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:16 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b32_e32 v16, s8
; VGPRRC-NEXT: v_mov_b32_e32 v17, s9
; VGPRRC-NEXT: v_mov_b32_e32 v18, s10
; VGPRRC-NEXT: v_mov_b32_e32 v19, s11
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32 sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v40, v[8:11], s[0:1] offset:32 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48 sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v40, v[12:15], s[0:1] offset:48 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v40, v[0:3], s[0:1] sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16 sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v40, v[4:7], s[0:1] offset:16 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_endpgm
; AGPR-LABEL: test_mfma_i32_32x32x32_i8__vgprcd:
@@ -4329,17 +4329,17 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd__flags(<4 x i32> %a
; SDAG: ; %bb.0:
; SDAG-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
-; SDAG-NEXT: v_mov_b32_e32 v32, 0
+; SDAG-NEXT: v_mov_b32_e32 v40, 0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v34, s20
-; SDAG-NEXT: v_mov_b32_e32 v35, s21
-; SDAG-NEXT: v_mov_b32_e32 v36, s22
-; SDAG-NEXT: v_mov_b32_e32 v37, s23
+; SDAG-NEXT: v_mov_b32_e32 v32, s20
+; SDAG-NEXT: v_mov_b32_e32 v33, s21
+; SDAG-NEXT: v_mov_b32_e32 v34, s22
+; SDAG-NEXT: v_mov_b32_e32 v35, s23
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
-; SDAG-NEXT: v_mov_b32_e32 v38, s24
-; SDAG-NEXT: v_mov_b32_e32 v39, s25
-; SDAG-NEXT: v_mov_b32_e32 v40, s26
-; SDAG-NEXT: v_mov_b32_e32 v41, s27
+; SDAG-NEXT: v_mov_b32_e32 v36, s24
+; SDAG-NEXT: v_mov_b32_e32 v37, s25
+; SDAG-NEXT: v_mov_b32_e32 v38, s26
+; SDAG-NEXT: v_mov_b32_e32 v39, s27
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; SDAG-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
@@ -4350,42 +4350,42 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd__flags(<4 x i32> %a
; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[34:37], v[38:41], v[16:31] cbsz:1 abid:2 blgp:3
+; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31] cbsz:1 abid:2 blgp:3
; SDAG-NEXT: s_nop 6
; SDAG-NEXT: v_mov_b32_e32 v16, s20
; SDAG-NEXT: v_mov_b32_e32 v17, s21
; SDAG-NEXT: v_mov_b32_e32 v18, s22
; SDAG-NEXT: v_mov_b32_e32 v19, s23
-; SDAG-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:48 sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:48 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_mov_b32_e32 v16, s16
; SDAG-NEXT: v_mov_b32_e32 v17, s17
; SDAG-NEXT: v_mov_b32_e32 v18, s18
; SDAG-NEXT: v_mov_b32_e32 v19, s19
-; SDAG-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:32 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_mov_b32_e32 v16, s12
; SDAG-NEXT: v_mov_b32_e32 v17, s13
; SDAG-NEXT: v_mov_b32_e32 v18, s14
; SDAG-NEXT: v_mov_b32_e32 v19, s15
-; SDAG-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:16 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_mov_b32_e32 v16, s8
; SDAG-NEXT: v_mov_b32_e32 v17, s9
; SDAG-NEXT: v_mov_b32_e32 v18, s10
; SDAG-NEXT: v_mov_b32_e32 v19, s11
-; SDAG-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32 sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v40, v[8:11], s[0:1] offset:32 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48 sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v40, v[12:15], s[0:1] offset:48 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v40, v[0:3], s[0:1] sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16 sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v40, v[4:7], s[0:1] offset:16 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_endpgm
;
@@ -4439,17 +4439,17 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd__flags(<4 x i32> %a
; HEURRC: ; %bb.0:
; HEURRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
; HEURRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
-; HEURRC-NEXT: v_mov_b32_e32 v32, 0
+; HEURRC-NEXT: v_mov_b32_e32 v40, 0
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
-; HEURRC-NEXT: v_mov_b32_e32 v34, s20
-; HEURRC-NEXT: v_mov_b32_e32 v35, s21
-; HEURRC-NEXT: v_mov_b32_e32 v36, s22
-; HEURRC-NEXT: v_mov_b32_e32 v37, s23
+; HEURRC-NEXT: v_mov_b32_e32 v32, s20
+; HEURRC-NEXT: v_mov_b32_e32 v33, s21
+; HEURRC-NEXT: v_mov_b32_e32 v34, s22
+; HEURRC-NEXT: v_mov_b32_e32 v35, s23
; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
-; HEURRC-NEXT: v_mov_b32_e32 v38, s24
-; HEURRC-NEXT: v_mov_b32_e32 v39, s25
-; HEURRC-NEXT: v_mov_b32_e32 v40, s26
-; HEURRC-NEXT: v_mov_b32_e32 v41, s27
+; HEURRC-NEXT: v_mov_b32_e32 v36, s24
+; HEURRC-NEXT: v_mov_b32_e32 v37, s25
+; HEURRC-NEXT: v_mov_b32_e32 v38, s26
+; HEURRC-NEXT: v_mov_b32_e32 v39, s27
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
; HEURRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; HEURRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
@@ -4460,42 +4460,42 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd__flags(<4 x i32> %a
; HEURRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; HEURRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; HEURRC-NEXT: s_nop 1
-; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[34:37], v[38:41], v[16:31] cbsz:1 abid:2 blgp:3
+; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31] cbsz:1 abid:2 blgp:3
; HEURRC-NEXT: s_nop 6
; HEURRC-NEXT: v_mov_b32_e32 v16, s20
; HEURRC-NEXT: v_mov_b32_e32 v17, s21
; HEURRC-NEXT: v_mov_b32_e32 v18, s22
; HEURRC-NEXT: v_mov_b32_e32 v19, s23
-; HEURRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:48 sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:48 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_nop 0
; HEURRC-NEXT: v_mov_b32_e32 v16, s16
; HEURRC-NEXT: v_mov_b32_e32 v17, s17
; HEURRC-NEXT: v_mov_b32_e32 v18, s18
; HEURRC-NEXT: v_mov_b32_e32 v19, s19
-; HEURRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:32 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_nop 0
; HEURRC-NEXT: v_mov_b32_e32 v16, s12
; HEURRC-NEXT: v_mov_b32_e32 v17, s13
; HEURRC-NEXT: v_mov_b32_e32 v18, s14
; HEURRC-NEXT: v_mov_b32_e32 v19, s15
-; HEURRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:16 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_nop 0
; HEURRC-NEXT: v_mov_b32_e32 v16, s8
; HEURRC-NEXT: v_mov_b32_e32 v17, s9
; HEURRC-NEXT: v_mov_b32_e32 v18, s10
; HEURRC-NEXT: v_mov_b32_e32 v19, s11
-; HEURRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32 sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v40, v[8:11], s[0:1] offset:32 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48 sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v40, v[12:15], s[0:1] offset:48 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v40, v[0:3], s[0:1] sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16 sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v40, v[4:7], s[0:1] offset:16 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_endpgm
;
@@ -4503,17 +4503,17 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd__flags(<4 x i32> %a
; VGPRRC: ; %bb.0:
; VGPRRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
; VGPRRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
-; VGPRRC-NEXT: v_mov_b32_e32 v32, 0
+; VGPRRC-NEXT: v_mov_b32_e32 v40, 0
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
-; VGPRRC-NEXT: v_mov_b32_e32 v34, s20
-; VGPRRC-NEXT: v_mov_b32_e32 v35, s21
-; VGPRRC-NEXT: v_mov_b32_e32 v36, s22
-; VGPRRC-NEXT: v_mov_b32_e32 v37, s23
+; VGPRRC-NEXT: v_mov_b32_e32 v32, s20
+; VGPRRC-NEXT: v_mov_b32_e32 v33, s21
+; VGPRRC-NEXT: v_mov_b32_e32 v34, s22
+; VGPRRC-NEXT: v_mov_b32_e32 v35, s23
; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
-; VGPRRC-NEXT: v_mov_b32_e32 v38, s24
-; VGPRRC-NEXT: v_mov_b32_e32 v39, s25
-; VGPRRC-NEXT: v_mov_b32_e32 v40, s26
-; VGPRRC-NEXT: v_mov_b32_e32 v41, s27
+; VGPRRC-NEXT: v_mov_b32_e32 v36, s24
+; VGPRRC-NEXT: v_mov_b32_e32 v37, s25
+; VGPRRC-NEXT: v_mov_b32_e32 v38, s26
+; VGPRRC-NEXT: v_mov_b32_e32 v39, s27
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
; VGPRRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; VGPRRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
@@ -4524,42 +4524,42 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd__flags(<4 x i32> %a
; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; VGPRRC-NEXT: s_nop 1
-; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[34:37], v[38:41], v[16:31] cbsz:1 abid:2 blgp:3
+; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31] cbsz:1 abid:2 blgp:3
; VGPRRC-NEXT: s_nop 6
; VGPRRC-NEXT: v_mov_b32_e32 v16, s20
; VGPRRC-NEXT: v_mov_b32_e32 v17, s21
; VGPRRC-NEXT: v_mov_b32_e32 v18, s22
; VGPRRC-NEXT: v_mov_b32_e32 v19, s23
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:48 sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:48 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b32_e32 v16, s16
; VGPRRC-NEXT: v_mov_b32_e32 v17, s17
; VGPRRC-NEXT: v_mov_b32_e32 v18, s18
; VGPRRC-NEXT: v_mov_b32_e32 v19, s19
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:32 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b32_e32 v16, s12
; VGPRRC-NEXT: v_mov_b32_e32 v17, s13
; VGPRRC-NEXT: v_mov_b32_e32 v18, s14
; VGPRRC-NEXT: v_mov_b32_e32 v19, s15
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:16 sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:16 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b32_e32 v16, s8
; VGPRRC-NEXT: v_mov_b32_e32 v17, s9
; VGPRRC-NEXT: v_mov_b32_e32 v18, s10
; VGPRRC-NEXT: v_mov_b32_e32 v19, s11
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32 sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v40, v[8:11], s[0:1] offset:32 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48 sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v40, v[12:15], s[0:1] offset:48 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v40, v[0:3], s[0:1] sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16 sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v40, v[4:7], s[0:1] offset:16 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_endpgm
; AGPR-LABEL: test_mfma_i32_32x32x32_i8__vgprcd__flags:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.ll
index aeeb5958e9ca3..0977caa3d677a 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.ll
@@ -2513,36 +2513,36 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd(<8 x i32
; SDAG-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd:
; SDAG: ; %bb.0:
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x0
-; SDAG-NEXT: v_mov_b32_e32 v4, 0
-; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v8, s8
-; SDAG-NEXT: v_mov_b32_e32 v9, s9
-; SDAG-NEXT: v_mov_b32_e32 v10, s10
-; SDAG-NEXT: v_mov_b32_e32 v11, s11
-; SDAG-NEXT: v_mov_b32_e32 v12, s12
-; SDAG-NEXT: v_mov_b32_e32 v13, s13
-; SDAG-NEXT: v_mov_b32_e32 v14, s14
-; SDAG-NEXT: v_mov_b32_e32 v15, s15
-; SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x40
-; SDAG-NEXT: v_mov_b32_e32 v16, s16
-; SDAG-NEXT: v_mov_b32_e32 v17, s17
-; SDAG-NEXT: v_mov_b32_e32 v18, s18
-; SDAG-NEXT: v_mov_b32_e32 v19, s19
-; SDAG-NEXT: v_mov_b32_e32 v20, s20
-; SDAG-NEXT: v_mov_b32_e32 v21, s21
-; SDAG-NEXT: v_mov_b32_e32 v22, s22
-; SDAG-NEXT: v_mov_b32_e32 v23, s23
+; SDAG-NEXT: v_mov_b32_e32 v20, 0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: v_mov_b32_e32 v0, s8
; SDAG-NEXT: v_mov_b32_e32 v1, s9
; SDAG-NEXT: v_mov_b32_e32 v2, s10
; SDAG-NEXT: v_mov_b32_e32 v3, s11
-; SDAG-NEXT: v_mov_b32_e32 v5, s12
-; SDAG-NEXT: v_mov_b32_e32 v6, s13
+; SDAG-NEXT: v_mov_b32_e32 v4, s12
+; SDAG-NEXT: v_mov_b32_e32 v5, s13
+; SDAG-NEXT: v_mov_b32_e32 v6, s14
+; SDAG-NEXT: v_mov_b32_e32 v7, s15
+; SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x40
+; SDAG-NEXT: v_mov_b32_e32 v8, s16
+; SDAG-NEXT: v_mov_b32_e32 v9, s17
+; SDAG-NEXT: v_mov_b32_e32 v10, s18
+; SDAG-NEXT: v_mov_b32_e32 v11, s19
+; SDAG-NEXT: v_mov_b32_e32 v12, s20
+; SDAG-NEXT: v_mov_b32_e32 v13, s21
+; SDAG-NEXT: v_mov_b32_e32 v14, s22
+; SDAG-NEXT: v_mov_b32_e32 v15, s23
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: v_mov_b32_e32 v16, s8
+; SDAG-NEXT: v_mov_b32_e32 v17, s9
+; SDAG-NEXT: v_mov_b32_e32 v18, s10
+; SDAG-NEXT: v_mov_b32_e32 v19, s11
+; SDAG-NEXT: v_mov_b32_e32 v21, s12
+; SDAG-NEXT: v_mov_b32_e32 v22, s13
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[8:15], v[16:23], v[0:3], v5, v6 op_sel:[1,1,0] op_sel_hi:[1,0,0] blgp:2
+; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v21, v22 op_sel:[1,1,0] op_sel_hi:[1,0,0] blgp:2
; SDAG-NEXT: s_nop 11
-; SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[14:15]
+; SDAG-NEXT: global_store_dwordx4 v20, v[0:3], s[14:15]
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd:
@@ -2572,36 +2572,36 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd(<8 x i32
; AGPR-SDAG-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd:
; AGPR-SDAG: ; %bb.0:
; AGPR-SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x0
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v4, 0
-; AGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v8, s8
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v9, s9
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v10, s10
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v11, s11
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v12, s12
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v13, s13
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v14, s14
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v15, s15
-; AGPR-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x40
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s16
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s17
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s18
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s19
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s20
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s21
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, s22
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v23, s23
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, 0
; AGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; AGPR-SDAG-NEXT: v_mov_b32_e32 v0, s8
; AGPR-SDAG-NEXT: v_mov_b32_e32 v1, s9
; AGPR-SDAG-NEXT: v_mov_b32_e32 v2, s10
; AGPR-SDAG-NEXT: v_mov_b32_e32 v3, s11
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v5, s12
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v6, s13
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v4, s12
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v5, s13
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v6, s14
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v7, s15
+; AGPR-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x40
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v8, s16
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v9, s17
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v10, s18
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v11, s19
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v12, s20
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v13, s21
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v14, s22
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v15, s23
+; AGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s8
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s9
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s10
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s11
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s12
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, s13
; AGPR-SDAG-NEXT: s_nop 1
-; AGPR-SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[8:15], v[16:23], v[0:3], v5, v6 op_sel:[1,1,0] op_sel_hi:[1,0,0] blgp:2
+; AGPR-SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v21, v22 op_sel:[1,1,0] op_sel_hi:[1,0,0] blgp:2
; AGPR-SDAG-NEXT: s_nop 11
-; AGPR-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[14:15]
+; AGPR-SDAG-NEXT: global_store_dwordx4 v20, v[0:3], s[14:15]
; AGPR-SDAG-NEXT: s_endpgm
;
; AGPR-GISEL-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd:
@@ -2638,32 +2638,32 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x0
; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x40
; SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x50
-; SDAG-NEXT: v_mov_b32_e32 v5, -2
-; SDAG-NEXT: v_mov_b32_e32 v6, 0x41
-; SDAG-NEXT: v_mov_b32_e32 v4, 0
+; SDAG-NEXT: v_mov_b32_e32 v21, -2
+; SDAG-NEXT: v_mov_b32_e32 v22, 0x41
+; SDAG-NEXT: v_mov_b32_e32 v20, 0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v8, s8
-; SDAG-NEXT: v_mov_b32_e32 v9, s9
-; SDAG-NEXT: v_mov_b32_e32 v10, s10
-; SDAG-NEXT: v_mov_b32_e32 v11, s11
-; SDAG-NEXT: v_mov_b32_e32 v12, s12
-; SDAG-NEXT: v_mov_b32_e32 v13, s13
-; SDAG-NEXT: v_mov_b32_e32 v14, s14
-; SDAG-NEXT: v_mov_b32_e32 v15, s15
-; SDAG-NEXT: v_mov_b32_e32 v16, s16
-; SDAG-NEXT: v_mov_b32_e32 v17, s17
-; SDAG-NEXT: v_mov_b32_e32 v18, s18
-; SDAG-NEXT: v_mov_b32_e32 v19, s19
-; SDAG-NEXT: v_mov_b32_e32 v20, s20
-; SDAG-NEXT: v_mov_b32_e32 v21, s21
-; SDAG-NEXT: v_mov_b32_e32 v22, s22
-; SDAG-NEXT: v_mov_b32_e32 v23, s23
-; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; SDAG-NEXT: v_mov_b32_e32 v0, s8
+; SDAG-NEXT: v_mov_b32_e32 v1, s9
+; SDAG-NEXT: v_mov_b32_e32 v2, s10
+; SDAG-NEXT: v_mov_b32_e32 v3, s11
+; SDAG-NEXT: v_mov_b32_e32 v4, s12
+; SDAG-NEXT: v_mov_b32_e32 v5, s13
+; SDAG-NEXT: v_mov_b32_e32 v6, s14
+; SDAG-NEXT: v_mov_b32_e32 v7, s15
+; SDAG-NEXT: v_mov_b32_e32 v8, s16
+; SDAG-NEXT: v_mov_b32_e32 v9, s17
+; SDAG-NEXT: v_mov_b32_e32 v10, s18
+; SDAG-NEXT: v_mov_b32_e32 v11, s19
+; SDAG-NEXT: v_mov_b32_e32 v12, s20
+; SDAG-NEXT: v_mov_b32_e32 v13, s21
+; SDAG-NEXT: v_mov_b32_e32 v14, s22
+; SDAG-NEXT: v_mov_b32_e32 v15, s23
+; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[2:3]
+; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[0:1]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[8:15], v[16:23], v[0:3], v6, v5 op_sel:[1,1,0] op_sel_hi:[1,0,0]
+; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v22, v21 op_sel:[1,1,0] op_sel_hi:[1,0,0]
; SDAG-NEXT: s_nop 11
-; SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; SDAG-NEXT: global_store_dwordx4 v20, v[0:3], s[6:7]
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA_kimm__scaleB__inlineimm:
@@ -2696,32 +2696,32 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; AGPR-SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x0
; AGPR-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x40
; AGPR-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x50
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v5, -2
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v6, 0x41
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v4, 0
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, -2
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, 0x41
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, 0
; AGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v8, s8
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v9, s9
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v10, s10
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v11, s11
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v12, s12
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v13, s13
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v14, s14
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v15, s15
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s16
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s17
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s18
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s19
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s20
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s21
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, s22
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v23, s23
-; AGPR-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; AGPR-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v0, s8
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v1, s9
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v2, s10
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v3, s11
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v4, s12
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v5, s13
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v6, s14
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v7, s15
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v8, s16
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v9, s17
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v10, s18
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v11, s19
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v12, s20
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v13, s21
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v14, s22
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v15, s23
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[18:19], s[2:3]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[16:17], s[0:1]
; AGPR-SDAG-NEXT: s_nop 1
-; AGPR-SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[8:15], v[16:23], v[0:3], v6, v5 op_sel:[1,1,0] op_sel_hi:[1,0,0]
+; AGPR-SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v22, v21 op_sel:[1,1,0] op_sel_hi:[1,0,0]
; AGPR-SDAG-NEXT: s_nop 11
-; AGPR-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; AGPR-SDAG-NEXT: global_store_dwordx4 v20, v[0:3], s[6:7]
; AGPR-SDAG-NEXT: s_endpgm
;
; AGPR-GISEL-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA_kimm__scaleB__inlineimm:
@@ -2759,32 +2759,32 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x0
; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x40
; SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x50
-; SDAG-NEXT: v_mov_b32_e32 v5, 1.0
-; SDAG-NEXT: v_mov_b32_e32 v6, 0x41
-; SDAG-NEXT: v_mov_b32_e32 v4, 0
+; SDAG-NEXT: v_mov_b32_e32 v21, 1.0
+; SDAG-NEXT: v_mov_b32_e32 v22, 0x41
+; SDAG-NEXT: v_mov_b32_e32 v20, 0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v8, s8
-; SDAG-NEXT: v_mov_b32_e32 v9, s9
-; SDAG-NEXT: v_mov_b32_e32 v10, s10
-; SDAG-NEXT: v_mov_b32_e32 v11, s11
-; SDAG-NEXT: v_mov_b32_e32 v12, s12
-; SDAG-NEXT: v_mov_b32_e32 v13, s13
-; SDAG-NEXT: v_mov_b32_e32 v14, s14
-; SDAG-NEXT: v_mov_b32_e32 v15, s15
-; SDAG-NEXT: v_mov_b32_e32 v16, s16
-; SDAG-NEXT: v_mov_b32_e32 v17, s17
-; SDAG-NEXT: v_mov_b32_e32 v18, s18
-; SDAG-NEXT: v_mov_b32_e32 v19, s19
-; SDAG-NEXT: v_mov_b32_e32 v20, s20
-; SDAG-NEXT: v_mov_b32_e32 v21, s21
-; SDAG-NEXT: v_mov_b32_e32 v22, s22
-; SDAG-NEXT: v_mov_b32_e32 v23, s23
-; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; SDAG-NEXT: v_mov_b32_e32 v0, s8
+; SDAG-NEXT: v_mov_b32_e32 v1, s9
+; SDAG-NEXT: v_mov_b32_e32 v2, s10
+; SDAG-NEXT: v_mov_b32_e32 v3, s11
+; SDAG-NEXT: v_mov_b32_e32 v4, s12
+; SDAG-NEXT: v_mov_b32_e32 v5, s13
+; SDAG-NEXT: v_mov_b32_e32 v6, s14
+; SDAG-NEXT: v_mov_b32_e32 v7, s15
+; SDAG-NEXT: v_mov_b32_e32 v8, s16
+; SDAG-NEXT: v_mov_b32_e32 v9, s17
+; SDAG-NEXT: v_mov_b32_e32 v10, s18
+; SDAG-NEXT: v_mov_b32_e32 v11, s19
+; SDAG-NEXT: v_mov_b32_e32 v12, s20
+; SDAG-NEXT: v_mov_b32_e32 v13, s21
+; SDAG-NEXT: v_mov_b32_e32 v14, s22
+; SDAG-NEXT: v_mov_b32_e32 v15, s23
+; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[2:3]
+; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[0:1]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[8:15], v[16:23], v[0:3], v6, v5 op_sel:[1,1,0] op_sel_hi:[1,0,0]
+; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v22, v21 op_sel:[1,1,0] op_sel_hi:[1,0,0]
; SDAG-NEXT: s_nop 11
-; SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; SDAG-NEXT: global_store_dwordx4 v20, v[0:3], s[6:7]
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA_kimm__scaleB__FP_literal:
@@ -2817,32 +2817,32 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; AGPR-SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x0
; AGPR-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x40
; AGPR-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x50
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v5, 1.0
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v6, 0x41
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v4, 0
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, 1.0
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, 0x41
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, 0
; AGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v8, s8
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v9, s9
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v10, s10
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v11, s11
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v12, s12
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v13, s13
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v14, s14
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v15, s15
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s16
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s17
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s18
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s19
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s20
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s21
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, s22
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v23, s23
-; AGPR-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; AGPR-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v0, s8
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v1, s9
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v2, s10
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v3, s11
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v4, s12
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v5, s13
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v6, s14
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v7, s15
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v8, s16
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v9, s17
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v10, s18
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v11, s19
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v12, s20
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v13, s21
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v14, s22
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v15, s23
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[18:19], s[2:3]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[16:17], s[0:1]
; AGPR-SDAG-NEXT: s_nop 1
-; AGPR-SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[8:15], v[16:23], v[0:3], v6, v5 op_sel:[1,1,0] op_sel_hi:[1,0,0]
+; AGPR-SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v22, v21 op_sel:[1,1,0] op_sel_hi:[1,0,0]
; AGPR-SDAG-NEXT: s_nop 11
-; AGPR-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; AGPR-SDAG-NEXT: global_store_dwordx4 v20, v[0:3], s[6:7]
; AGPR-SDAG-NEXT: s_endpgm
;
; AGPR-GISEL-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA_kimm__scaleB__FP_literal:
@@ -2880,32 +2880,32 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x0
; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x40
; SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x50
-; SDAG-NEXT: v_mov_b32_e32 v5, -2
-; SDAG-NEXT: v_mov_b32_e32 v6, 1.0
-; SDAG-NEXT: v_mov_b32_e32 v4, 0
+; SDAG-NEXT: v_mov_b32_e32 v21, -2
+; SDAG-NEXT: v_mov_b32_e32 v22, 1.0
+; SDAG-NEXT: v_mov_b32_e32 v20, 0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v8, s8
-; SDAG-NEXT: v_mov_b32_e32 v9, s9
-; SDAG-NEXT: v_mov_b32_e32 v10, s10
-; SDAG-NEXT: v_mov_b32_e32 v11, s11
-; SDAG-NEXT: v_mov_b32_e32 v12, s12
-; SDAG-NEXT: v_mov_b32_e32 v13, s13
-; SDAG-NEXT: v_mov_b32_e32 v14, s14
-; SDAG-NEXT: v_mov_b32_e32 v15, s15
-; SDAG-NEXT: v_mov_b32_e32 v16, s16
-; SDAG-NEXT: v_mov_b32_e32 v17, s17
-; SDAG-NEXT: v_mov_b32_e32 v18, s18
-; SDAG-NEXT: v_mov_b32_e32 v19, s19
-; SDAG-NEXT: v_mov_b32_e32 v20, s20
-; SDAG-NEXT: v_mov_b32_e32 v21, s21
-; SDAG-NEXT: v_mov_b32_e32 v22, s22
-; SDAG-NEXT: v_mov_b32_e32 v23, s23
-; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; SDAG-NEXT: v_mov_b32_e32 v0, s8
+; SDAG-NEXT: v_mov_b32_e32 v1, s9
+; SDAG-NEXT: v_mov_b32_e32 v2, s10
+; SDAG-NEXT: v_mov_b32_e32 v3, s11
+; SDAG-NEXT: v_mov_b32_e32 v4, s12
+; SDAG-NEXT: v_mov_b32_e32 v5, s13
+; SDAG-NEXT: v_mov_b32_e32 v6, s14
+; SDAG-NEXT: v_mov_b32_e32 v7, s15
+; SDAG-NEXT: v_mov_b32_e32 v8, s16
+; SDAG-NEXT: v_mov_b32_e32 v9, s17
+; SDAG-NEXT: v_mov_b32_e32 v10, s18
+; SDAG-NEXT: v_mov_b32_e32 v11, s19
+; SDAG-NEXT: v_mov_b32_e32 v12, s20
+; SDAG-NEXT: v_mov_b32_e32 v13, s21
+; SDAG-NEXT: v_mov_b32_e32 v14, s22
+; SDAG-NEXT: v_mov_b32_e32 v15, s23
+; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[2:3]
+; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[0:1]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[8:15], v[16:23], v[0:3], v6, v5 op_sel:[1,1,0] op_sel_hi:[1,0,0]
+; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v22, v21 op_sel:[1,1,0] op_sel_hi:[1,0,0]
; SDAG-NEXT: s_nop 11
-; SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; SDAG-NEXT: global_store_dwordx4 v20, v[0:3], s[6:7]
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA_FP_literal__scaleB__inline_imm:
@@ -2938,32 +2938,32 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; AGPR-SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x0
; AGPR-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x40
; AGPR-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x50
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v5, -2
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v6, 1.0
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v4, 0
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, -2
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, 1.0
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, 0
; AGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v8, s8
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v9, s9
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v10, s10
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v11, s11
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v12, s12
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v13, s13
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v14, s14
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v15, s15
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s16
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s17
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s18
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s19
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s20
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s21
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, s22
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v23, s23
-; AGPR-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; AGPR-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v0, s8
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v1, s9
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v2, s10
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v3, s11
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v4, s12
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v5, s13
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v6, s14
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v7, s15
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v8, s16
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v9, s17
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v10, s18
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v11, s19
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v12, s20
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v13, s21
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v14, s22
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v15, s23
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[18:19], s[2:3]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[16:17], s[0:1]
; AGPR-SDAG-NEXT: s_nop 1
-; AGPR-SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[8:15], v[16:23], v[0:3], v6, v5 op_sel:[1,1,0] op_sel_hi:[1,0,0]
+; AGPR-SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v22, v21 op_sel:[1,1,0] op_sel_hi:[1,0,0]
; AGPR-SDAG-NEXT: s_nop 11
-; AGPR-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; AGPR-SDAG-NEXT: global_store_dwordx4 v20, v[0:3], s[6:7]
; AGPR-SDAG-NEXT: s_endpgm
;
; AGPR-GISEL-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA_FP_literal__scaleB__inline_imm:
@@ -3001,32 +3001,32 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x0
; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x40
; SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x50
-; SDAG-NEXT: v_mov_b32_e32 v5, 0.15915494
-; SDAG-NEXT: v_mov_b32_e32 v6, 1.0
-; SDAG-NEXT: v_mov_b32_e32 v4, 0
+; SDAG-NEXT: v_mov_b32_e32 v21, 0.15915494
+; SDAG-NEXT: v_mov_b32_e32 v22, 1.0
+; SDAG-NEXT: v_mov_b32_e32 v20, 0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v8, s8
-; SDAG-NEXT: v_mov_b32_e32 v9, s9
-; SDAG-NEXT: v_mov_b32_e32 v10, s10
-; SDAG-NEXT: v_mov_b32_e32 v11, s11
-; SDAG-NEXT: v_mov_b32_e32 v12, s12
-; SDAG-NEXT: v_mov_b32_e32 v13, s13
-; SDAG-NEXT: v_mov_b32_e32 v14, s14
-; SDAG-NEXT: v_mov_b32_e32 v15, s15
-; SDAG-NEXT: v_mov_b32_e32 v16, s16
-; SDAG-NEXT: v_mov_b32_e32 v17, s17
-; SDAG-NEXT: v_mov_b32_e32 v18, s18
-; SDAG-NEXT: v_mov_b32_e32 v19, s19
-; SDAG-NEXT: v_mov_b32_e32 v20, s20
-; SDAG-NEXT: v_mov_b32_e32 v21, s21
-; SDAG-NEXT: v_mov_b32_e32 v22, s22
-; SDAG-NEXT: v_mov_b32_e32 v23, s23
-; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; SDAG-NEXT: v_mov_b32_e32 v0, s8
+; SDAG-NEXT: v_mov_b32_e32 v1, s9
+; SDAG-NEXT: v_mov_b32_e32 v2, s10
+; SDAG-NEXT: v_mov_b32_e32 v3, s11
+; SDAG-NEXT: v_mov_b32_e32 v4, s12
+; SDAG-NEXT: v_mov_b32_e32 v5, s13
+; SDAG-NEXT: v_mov_b32_e32 v6, s14
+; SDAG-NEXT: v_mov_b32_e32 v7, s15
+; SDAG-NEXT: v_mov_b32_e32 v8, s16
+; SDAG-NEXT: v_mov_b32_e32 v9, s17
+; SDAG-NEXT: v_mov_b32_e32 v10, s18
+; SDAG-NEXT: v_mov_b32_e32 v11, s19
+; SDAG-NEXT: v_mov_b32_e32 v12, s20
+; SDAG-NEXT: v_mov_b32_e32 v13, s21
+; SDAG-NEXT: v_mov_b32_e32 v14, s22
+; SDAG-NEXT: v_mov_b32_e32 v15, s23
+; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[2:3]
+; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[0:1]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[8:15], v[16:23], v[0:3], v6, v5 op_sel:[1,1,0] op_sel_hi:[1,0,0]
+; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v22, v21 op_sel:[1,1,0] op_sel_hi:[1,0,0]
; SDAG-NEXT: s_nop 11
-; SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; SDAG-NEXT: global_store_dwordx4 v20, v[0:3], s[6:7]
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA_FP_literal__scaleB__FP_literal:
@@ -3059,32 +3059,32 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; AGPR-SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x0
; AGPR-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x40
; AGPR-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x50
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v5, 0.15915494
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v6, 1.0
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v4, 0
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, 0.15915494
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, 1.0
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, 0
; AGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v8, s8
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v9, s9
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v10, s10
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v11, s11
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v12, s12
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v13, s13
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v14, s14
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v15, s15
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s16
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s17
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s18
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s19
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s20
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s21
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, s22
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v23, s23
-; AGPR-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; AGPR-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v0, s8
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v1, s9
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v2, s10
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v3, s11
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v4, s12
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v5, s13
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v6, s14
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v7, s15
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v8, s16
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v9, s17
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v10, s18
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v11, s19
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v12, s20
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v13, s21
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v14, s22
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v15, s23
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[18:19], s[2:3]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[16:17], s[0:1]
; AGPR-SDAG-NEXT: s_nop 1
-; AGPR-SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[8:15], v[16:23], v[0:3], v6, v5 op_sel:[1,1,0] op_sel_hi:[1,0,0]
+; AGPR-SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v22, v21 op_sel:[1,1,0] op_sel_hi:[1,0,0]
; AGPR-SDAG-NEXT: s_nop 11
-; AGPR-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; AGPR-SDAG-NEXT: global_store_dwordx4 v20, v[0:3], s[6:7]
; AGPR-SDAG-NEXT: s_endpgm
;
; AGPR-GISEL-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA_FP_literal__scaleB__FP_literal:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll
index 749f9e5ae5ef2..202f9fc81f290 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll
@@ -7307,22 +7307,22 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4__vgprcd(<8 x i32>
; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x80
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[36:37]
-; SDAG-NEXT: v_mov_b32_e32 v18, s8
-; SDAG-NEXT: v_mov_b32_e32 v19, s9
-; SDAG-NEXT: v_mov_b32_e32 v20, s10
-; SDAG-NEXT: v_mov_b32_e32 v21, s11
-; SDAG-NEXT: v_mov_b32_e32 v22, s12
-; SDAG-NEXT: v_mov_b32_e32 v23, s13
-; SDAG-NEXT: v_mov_b32_e32 v24, s14
-; SDAG-NEXT: v_mov_b32_e32 v25, s15
-; SDAG-NEXT: v_mov_b32_e32 v26, s16
-; SDAG-NEXT: v_mov_b32_e32 v27, s17
-; SDAG-NEXT: v_mov_b32_e32 v28, s18
-; SDAG-NEXT: v_mov_b32_e32 v29, s19
-; SDAG-NEXT: v_mov_b32_e32 v30, s20
-; SDAG-NEXT: v_mov_b32_e32 v31, s21
-; SDAG-NEXT: v_mov_b32_e32 v32, s22
-; SDAG-NEXT: v_mov_b32_e32 v33, s23
+; SDAG-NEXT: v_mov_b32_e32 v16, s8
+; SDAG-NEXT: v_mov_b32_e32 v17, s9
+; SDAG-NEXT: v_mov_b32_e32 v18, s10
+; SDAG-NEXT: v_mov_b32_e32 v19, s11
+; SDAG-NEXT: v_mov_b32_e32 v20, s12
+; SDAG-NEXT: v_mov_b32_e32 v21, s13
+; SDAG-NEXT: v_mov_b32_e32 v22, s14
+; SDAG-NEXT: v_mov_b32_e32 v23, s15
+; SDAG-NEXT: v_mov_b32_e32 v24, s16
+; SDAG-NEXT: v_mov_b32_e32 v25, s17
+; SDAG-NEXT: v_mov_b32_e32 v26, s18
+; SDAG-NEXT: v_mov_b32_e32 v27, s19
+; SDAG-NEXT: v_mov_b32_e32 v28, s20
+; SDAG-NEXT: v_mov_b32_e32 v29, s21
+; SDAG-NEXT: v_mov_b32_e32 v30, s22
+; SDAG-NEXT: v_mov_b32_e32 v31, s23
; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[38:39]
; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[40:41]
; SDAG-NEXT: v_mov_b64_e32 v[6:7], s[42:43]
@@ -7330,10 +7330,10 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4__vgprcd(<8 x i32>
; SDAG-NEXT: v_mov_b64_e32 v[10:11], s[46:47]
; SDAG-NEXT: v_mov_b64_e32 v[12:13], s[48:49]
; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[50:51]
-; SDAG-NEXT: v_mov_b32_e32 v16, s0
-; SDAG-NEXT: v_mov_b32_e32 v17, s1
+; SDAG-NEXT: v_mov_b32_e32 v32, s0
+; SDAG-NEXT: v_mov_b32_e32 v33, s1
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[18:25], v[26:33], v[0:15], v16, v17 op_sel:[1,1,0] op_sel_hi:[1,0,0] blgp:2
+; SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[16:23], v[24:31], v[0:15], v32, v33 op_sel:[1,1,0] op_sel_hi:[1,0,0] blgp:2
; SDAG-NEXT: v_mov_b32_e32 v16, 0
; SDAG-NEXT: s_nop 15
; SDAG-NEXT: s_nop 2
@@ -7385,22 +7385,22 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4__vgprcd(<8 x i32>
; AGPR-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x80
; AGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[36:37]
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s8
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s9
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s10
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s11
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, s12
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v23, s13
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v24, s14
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v25, s15
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v26, s16
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v27, s17
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v28, s18
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v29, s19
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v30, s20
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v31, s21
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v32, s22
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v33, s23
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s8
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s9
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s10
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s11
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s12
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s13
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, s14
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v23, s15
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v24, s16
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v25, s17
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v26, s18
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v27, s19
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v28, s20
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v29, s21
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v30, s22
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v31, s23
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[38:39]
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[40:41]
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[42:43]
@@ -7408,10 +7408,10 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4__vgprcd(<8 x i32>
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[46:47]
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[12:13], s[48:49]
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[14:15], s[50:51]
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s0
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s1
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v32, s0
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v33, s1
; AGPR-SDAG-NEXT: s_nop 1
-; AGPR-SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[18:25], v[26:33], v[0:15], v16, v17 op_sel:[1,1,0] op_sel_hi:[1,0,0] blgp:2
+; AGPR-SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[16:23], v[24:31], v[0:15], v32, v33 op_sel:[1,1,0] op_sel_hi:[1,0,0] blgp:2
; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, 0
; AGPR-SDAG-NEXT: s_nop 15
; AGPR-SDAG-NEXT: s_nop 2
@@ -7465,26 +7465,26 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4__vgprcd___scaleA_
; SDAG: ; %bb.0:
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x0
; SDAG-NEXT: s_load_dwordx16 s[36:51], s[4:5], 0x40
-; SDAG-NEXT: v_mov_b32_e32 v16, -2
-; SDAG-NEXT: v_mov_b32_e32 v17, 0x41
+; SDAG-NEXT: v_mov_b32_e32 v32, -2
+; SDAG-NEXT: v_mov_b32_e32 v33, 0x41
; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x80
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v18, s8
-; SDAG-NEXT: v_mov_b32_e32 v19, s9
-; SDAG-NEXT: v_mov_b32_e32 v20, s10
-; SDAG-NEXT: v_mov_b32_e32 v21, s11
-; SDAG-NEXT: v_mov_b32_e32 v22, s12
-; SDAG-NEXT: v_mov_b32_e32 v23, s13
-; SDAG-NEXT: v_mov_b32_e32 v24, s14
-; SDAG-NEXT: v_mov_b32_e32 v25, s15
-; SDAG-NEXT: v_mov_b32_e32 v26, s16
-; SDAG-NEXT: v_mov_b32_e32 v27, s17
-; SDAG-NEXT: v_mov_b32_e32 v28, s18
-; SDAG-NEXT: v_mov_b32_e32 v29, s19
-; SDAG-NEXT: v_mov_b32_e32 v30, s20
-; SDAG-NEXT: v_mov_b32_e32 v31, s21
-; SDAG-NEXT: v_mov_b32_e32 v32, s22
-; SDAG-NEXT: v_mov_b32_e32 v33, s23
+; SDAG-NEXT: v_mov_b32_e32 v16, s8
+; SDAG-NEXT: v_mov_b32_e32 v17, s9
+; SDAG-NEXT: v_mov_b32_e32 v18, s10
+; SDAG-NEXT: v_mov_b32_e32 v19, s11
+; SDAG-NEXT: v_mov_b32_e32 v20, s12
+; SDAG-NEXT: v_mov_b32_e32 v21, s13
+; SDAG-NEXT: v_mov_b32_e32 v22, s14
+; SDAG-NEXT: v_mov_b32_e32 v23, s15
+; SDAG-NEXT: v_mov_b32_e32 v24, s16
+; SDAG-NEXT: v_mov_b32_e32 v25, s17
+; SDAG-NEXT: v_mov_b32_e32 v26, s18
+; SDAG-NEXT: v_mov_b32_e32 v27, s19
+; SDAG-NEXT: v_mov_b32_e32 v28, s20
+; SDAG-NEXT: v_mov_b32_e32 v29, s21
+; SDAG-NEXT: v_mov_b32_e32 v30, s22
+; SDAG-NEXT: v_mov_b32_e32 v31, s23
; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[36:37]
; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[38:39]
; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[40:41]
@@ -7494,7 +7494,7 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4__vgprcd___scaleA_
; SDAG-NEXT: v_mov_b64_e32 v[12:13], s[48:49]
; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[50:51]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[18:25], v[26:33], v[0:15], v17, v16 op_sel:[1,1,0] op_sel_hi:[1,0,0] blgp:2
+; SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[16:23], v[24:31], v[0:15], v33, v32 op_sel:[1,1,0] op_sel_hi:[1,0,0] blgp:2
; SDAG-NEXT: v_mov_b32_e32 v16, 0
; SDAG-NEXT: s_nop 15
; SDAG-NEXT: s_nop 2
@@ -7543,26 +7543,26 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4__vgprcd___scaleA_
; AGPR-SDAG: ; %bb.0:
; AGPR-SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x0
; AGPR-SDAG-NEXT: s_load_dwordx16 s[36:51], s[4:5], 0x40
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, -2
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, 0x41
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v32, -2
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v33, 0x41
; AGPR-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x80
; AGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s8
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s9
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s10
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s11
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, s12
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v23, s13
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v24, s14
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v25, s15
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v26, s16
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v27, s17
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v28, s18
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v29, s19
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v30, s20
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v31, s21
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v32, s22
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v33, s23
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s8
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s9
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s10
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s11
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s12
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s13
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, s14
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v23, s15
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v24, s16
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v25, s17
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v26, s18
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v27, s19
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v28, s20
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v29, s21
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v30, s22
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v31, s23
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[36:37]
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[38:39]
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[40:41]
@@ -7572,7 +7572,7 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4__vgprcd___scaleA_
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[12:13], s[48:49]
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[14:15], s[50:51]
; AGPR-SDAG-NEXT: s_nop 1
-; AGPR-SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[18:25], v[26:33], v[0:15], v17, v16 op_sel:[1,1,0] op_sel_hi:[1,0,0] blgp:2
+; AGPR-SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[16:23], v[24:31], v[0:15], v33, v32 op_sel:[1,1,0] op_sel_hi:[1,0,0] blgp:2
; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, 0
; AGPR-SDAG-NEXT: s_nop 15
; AGPR-SDAG-NEXT: s_nop 2
@@ -8245,41 +8245,42 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__vgprcd_nonma
; SDAG-NEXT: s_nop 1
; SDAG-NEXT: v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[32:39], v[40:47], v[16:31] blgp:2
; SDAG-NEXT: s_nop 14
-; SDAG-NEXT: v_mov_b32_e32 v18, s20
-; SDAG-NEXT: v_mov_b32_e32 v19, s21
-; SDAG-NEXT: v_mov_b32_e32 v20, s22
-; SDAG-NEXT: v_mov_b32_e32 v21, s23
-; SDAG-NEXT: v_mov_b64_e32 v[16:17], 48
-; SDAG-NEXT: global_store_dwordx4 v[16:17], v[18:21], off sc0 sc1
+; SDAG-NEXT: v_mov_b32_e32 v16, s20
+; SDAG-NEXT: v_mov_b32_e32 v17, s21
+; SDAG-NEXT: v_mov_b32_e32 v18, s22
+; SDAG-NEXT: v_mov_b32_e32 v19, s23
+; SDAG-NEXT: v_mov_b64_e32 v[20:21], 48
+; SDAG-NEXT: global_store_dwordx4 v[20:21], v[16:19], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v22, s18
-; SDAG-NEXT: v_mov_b32_e32 v23, s19
-; SDAG-NEXT: v_mov_b32_e32 v20, s16
-; SDAG-NEXT: v_mov_b32_e32 v21, s17
-; SDAG-NEXT: v_mov_b64_e32 v[18:19], 32
-; SDAG-NEXT: global_store_dwordx4 v[18:19], v[20:23], off sc0 sc1
+; SDAG-NEXT: v_mov_b64_e32 v[22:23], 32
+; SDAG-NEXT: v_mov_b64_e32 v[24:25], 16
+; SDAG-NEXT: v_mov_b32_e32 v16, s16
+; SDAG-NEXT: v_mov_b32_e32 v17, s17
+; SDAG-NEXT: v_mov_b32_e32 v18, s18
+; SDAG-NEXT: v_mov_b32_e32 v19, s19
+; SDAG-NEXT: global_store_dwordx4 v[22:23], v[16:19], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v24, s14
-; SDAG-NEXT: v_mov_b32_e32 v25, s15
-; SDAG-NEXT: v_mov_b32_e32 v22, s12
-; SDAG-NEXT: v_mov_b32_e32 v23, s13
-; SDAG-NEXT: v_mov_b64_e32 v[20:21], 16
-; SDAG-NEXT: global_store_dwordx4 v[20:21], v[22:25], off sc0 sc1
+; SDAG-NEXT: v_mov_b64_e32 v[26:27], 0
+; SDAG-NEXT: v_mov_b32_e32 v16, s12
+; SDAG-NEXT: v_mov_b32_e32 v17, s13
+; SDAG-NEXT: v_mov_b32_e32 v18, s14
+; SDAG-NEXT: v_mov_b32_e32 v19, s15
+; SDAG-NEXT: global_store_dwordx4 v[24:25], v[16:19], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v26, s10
-; SDAG-NEXT: v_mov_b32_e32 v27, s11
-; SDAG-NEXT: v_mov_b32_e32 v24, s8
-; SDAG-NEXT: v_mov_b32_e32 v25, s9
-; SDAG-NEXT: v_mov_b64_e32 v[22:23], 0
-; SDAG-NEXT: global_store_dwordx4 v[22:23], v[24:27], off sc0 sc1
+; SDAG-NEXT: s_nop 0
+; SDAG-NEXT: v_mov_b32_e32 v16, s8
+; SDAG-NEXT: v_mov_b32_e32 v17, s9
+; SDAG-NEXT: v_mov_b32_e32 v18, s10
+; SDAG-NEXT: v_mov_b32_e32 v19, s11
+; SDAG-NEXT: global_store_dwordx4 v[26:27], v[16:19], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v[18:19], v[8:11], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[22:23], v[8:11], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v[16:17], v[12:15], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[20:21], v[12:15], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v[22:23], v[0:3], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[26:27], v[0:3], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v[20:21], v[4:7], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[24:25], v[4:7], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_endpgm
;
@@ -8362,41 +8363,42 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__vgprcd_nonma
; AGPR-SDAG-NEXT: s_nop 1
; AGPR-SDAG-NEXT: v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[32:39], v[40:47], v[16:31] blgp:2
; AGPR-SDAG-NEXT: s_nop 14
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s20
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s21
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s22
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s23
-; AGPR-SDAG-NEXT: v_mov_b64_e32 v[16:17], 48
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[16:17], v[18:21], off sc0 sc1
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s20
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s21
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s22
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s23
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[20:21], 48
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[20:21], v[16:19], off sc0 sc1
; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, s18
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v23, s19
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s16
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s17
-; AGPR-SDAG-NEXT: v_mov_b64_e32 v[18:19], 32
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[18:19], v[20:23], off sc0 sc1
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[22:23], 32
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[24:25], 16
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s16
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s17
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s18
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s19
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[22:23], v[16:19], off sc0 sc1
; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v24, s14
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v25, s15
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, s12
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v23, s13
-; AGPR-SDAG-NEXT: v_mov_b64_e32 v[20:21], 16
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[20:21], v[22:25], off sc0 sc1
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[26:27], 0
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s12
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s13
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s14
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s15
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[24:25], v[16:19], off sc0 sc1
; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v26, s10
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v27, s11
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v24, s8
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v25, s9
-; AGPR-SDAG-NEXT: v_mov_b64_e32 v[22:23], 0
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[22:23], v[24:27], off sc0 sc1
+; AGPR-SDAG-NEXT: s_nop 0
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s8
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s9
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s10
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s11
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[26:27], v[16:19], off sc0 sc1
; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[18:19], v[8:11], off sc0 sc1
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[22:23], v[8:11], off sc0 sc1
; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[16:17], v[12:15], off sc0 sc1
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[20:21], v[12:15], off sc0 sc1
; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[22:23], v[0:3], off sc0 sc1
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[26:27], v[0:3], off sc0 sc1
; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[20:21], v[4:7], off sc0 sc1
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[24:25], v[4:7], off sc0 sc1
; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
; AGPR-SDAG-NEXT: s_endpgm
;
@@ -8455,26 +8457,26 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_25_42__vgprcd_non
; SDAG-LABEL: test_mfma_scale_f32_32x32x64_f8f6f4_25_42__vgprcd_nonmac:
; SDAG: ; %bb.0:
; SDAG-NEXT: s_load_dwordx16 s[12:27], s[4:5], 0x0
-; SDAG-NEXT: v_mov_b32_e32 v16, 42
-; SDAG-NEXT: v_mov_b32_e32 v17, 25
+; SDAG-NEXT: v_mov_b32_e32 v32, 42
+; SDAG-NEXT: v_mov_b32_e32 v33, 25
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v18, s12
-; SDAG-NEXT: v_mov_b32_e32 v19, s13
-; SDAG-NEXT: v_mov_b32_e32 v20, s14
-; SDAG-NEXT: v_mov_b32_e32 v21, s15
-; SDAG-NEXT: v_mov_b32_e32 v22, s16
-; SDAG-NEXT: v_mov_b32_e32 v23, s17
-; SDAG-NEXT: v_mov_b32_e32 v24, s18
-; SDAG-NEXT: v_mov_b32_e32 v25, s19
-; SDAG-NEXT: v_mov_b32_e32 v26, s20
-; SDAG-NEXT: v_mov_b32_e32 v27, s21
-; SDAG-NEXT: v_mov_b32_e32 v28, s22
-; SDAG-NEXT: v_mov_b32_e32 v29, s23
+; SDAG-NEXT: v_mov_b32_e32 v16, s12
+; SDAG-NEXT: v_mov_b32_e32 v17, s13
+; SDAG-NEXT: v_mov_b32_e32 v18, s14
+; SDAG-NEXT: v_mov_b32_e32 v19, s15
+; SDAG-NEXT: v_mov_b32_e32 v20, s16
+; SDAG-NEXT: v_mov_b32_e32 v21, s17
+; SDAG-NEXT: v_mov_b32_e32 v22, s18
+; SDAG-NEXT: v_mov_b32_e32 v23, s19
+; SDAG-NEXT: v_mov_b32_e32 v24, s20
+; SDAG-NEXT: v_mov_b32_e32 v25, s21
+; SDAG-NEXT: v_mov_b32_e32 v26, s22
+; SDAG-NEXT: v_mov_b32_e32 v27, s23
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x40
-; SDAG-NEXT: v_mov_b32_e32 v30, s24
-; SDAG-NEXT: v_mov_b32_e32 v31, s25
-; SDAG-NEXT: v_mov_b32_e32 v32, s26
-; SDAG-NEXT: v_mov_b32_e32 v33, s27
+; SDAG-NEXT: v_mov_b32_e32 v28, s24
+; SDAG-NEXT: v_mov_b32_e32 v29, s25
+; SDAG-NEXT: v_mov_b32_e32 v30, s26
+; SDAG-NEXT: v_mov_b32_e32 v31, s27
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
@@ -8485,42 +8487,43 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_25_42__vgprcd_non
; SDAG-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[18:25], v[26:33], v[0:15], v17, v16 op_sel_hi:[0,0,0] blgp:2
-; SDAG-NEXT: v_mov_b32_e32 v18, s20
-; SDAG-NEXT: v_mov_b32_e32 v19, s21
-; SDAG-NEXT: v_mov_b32_e32 v20, s22
-; SDAG-NEXT: v_mov_b32_e32 v21, s23
-; SDAG-NEXT: v_mov_b64_e32 v[16:17], 48
-; SDAG-NEXT: global_store_dwordx4 v[16:17], v[18:21], off sc0 sc1
+; SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[16:23], v[24:31], v[0:15], v33, v32 op_sel_hi:[0,0,0] blgp:2
+; SDAG-NEXT: v_mov_b32_e32 v16, s20
+; SDAG-NEXT: v_mov_b32_e32 v17, s21
+; SDAG-NEXT: v_mov_b32_e32 v18, s22
+; SDAG-NEXT: v_mov_b32_e32 v19, s23
+; SDAG-NEXT: v_mov_b64_e32 v[20:21], 48
+; SDAG-NEXT: global_store_dwordx4 v[20:21], v[16:19], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v22, s18
-; SDAG-NEXT: v_mov_b32_e32 v23, s19
-; SDAG-NEXT: v_mov_b32_e32 v20, s16
-; SDAG-NEXT: v_mov_b32_e32 v21, s17
-; SDAG-NEXT: v_mov_b64_e32 v[18:19], 32
-; SDAG-NEXT: global_store_dwordx4 v[18:19], v[20:23], off sc0 sc1
+; SDAG-NEXT: v_mov_b64_e32 v[22:23], 32
+; SDAG-NEXT: v_mov_b64_e32 v[24:25], 16
+; SDAG-NEXT: v_mov_b32_e32 v16, s16
+; SDAG-NEXT: v_mov_b32_e32 v17, s17
+; SDAG-NEXT: v_mov_b32_e32 v18, s18
+; SDAG-NEXT: v_mov_b32_e32 v19, s19
+; SDAG-NEXT: global_store_dwordx4 v[22:23], v[16:19], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v24, s14
-; SDAG-NEXT: v_mov_b32_e32 v25, s15
-; SDAG-NEXT: v_mov_b32_e32 v22, s12
-; SDAG-NEXT: v_mov_b32_e32 v23, s13
-; SDAG-NEXT: v_mov_b64_e32 v[20:21], 16
-; SDAG-NEXT: global_store_dwordx4 v[20:21], v[22:25], off sc0 sc1
+; SDAG-NEXT: v_mov_b64_e32 v[26:27], 0
+; SDAG-NEXT: v_mov_b32_e32 v16, s12
+; SDAG-NEXT: v_mov_b32_e32 v17, s13
+; SDAG-NEXT: v_mov_b32_e32 v18, s14
+; SDAG-NEXT: v_mov_b32_e32 v19, s15
+; SDAG-NEXT: global_store_dwordx4 v[24:25], v[16:19], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v26, s10
-; SDAG-NEXT: v_mov_b32_e32 v27, s11
-; SDAG-NEXT: v_mov_b32_e32 v24, s8
-; SDAG-NEXT: v_mov_b32_e32 v25, s9
-; SDAG-NEXT: v_mov_b64_e32 v[22:23], 0
-; SDAG-NEXT: global_store_dwordx4 v[22:23], v[24:27], off sc0 sc1
+; SDAG-NEXT: s_nop 0
+; SDAG-NEXT: v_mov_b32_e32 v16, s8
+; SDAG-NEXT: v_mov_b32_e32 v17, s9
+; SDAG-NEXT: v_mov_b32_e32 v18, s10
+; SDAG-NEXT: v_mov_b32_e32 v19, s11
+; SDAG-NEXT: global_store_dwordx4 v[26:27], v[16:19], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v[18:19], v[8:11], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[22:23], v[8:11], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v[16:17], v[12:15], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[20:21], v[12:15], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v[22:23], v[0:3], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[26:27], v[0:3], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v[20:21], v[4:7], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[24:25], v[4:7], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_endpgm
;
@@ -8582,26 +8585,26 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_25_42__vgprcd_non
; AGPR-SDAG-LABEL: test_mfma_scale_f32_32x32x64_f8f6f4_25_42__vgprcd_nonmac:
; AGPR-SDAG: ; %bb.0:
; AGPR-SDAG-NEXT: s_load_dwordx16 s[12:27], s[4:5], 0x0
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, 42
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, 25
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v32, 42
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v33, 25
; AGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s12
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s13
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s14
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s15
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, s16
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v23, s17
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v24, s18
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v25, s19
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v26, s20
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v27, s21
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v28, s22
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v29, s23
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s12
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s13
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s14
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s15
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s16
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s17
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, s18
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v23, s19
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v24, s20
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v25, s21
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v26, s22
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v27, s23
; AGPR-SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x40
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v30, s24
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v31, s25
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v32, s26
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v33, s27
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v28, s24
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v29, s25
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v30, s26
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v31, s27
; AGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
@@ -8612,42 +8615,43 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_25_42__vgprcd_non
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; AGPR-SDAG-NEXT: s_nop 1
-; AGPR-SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[18:25], v[26:33], v[0:15], v17, v16 op_sel_hi:[0,0,0] blgp:2
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s20
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s21
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s22
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s23
-; AGPR-SDAG-NEXT: v_mov_b64_e32 v[16:17], 48
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[16:17], v[18:21], off sc0 sc1
+; AGPR-SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[16:23], v[24:31], v[0:15], v33, v32 op_sel_hi:[0,0,0] blgp:2
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s20
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s21
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s22
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s23
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[20:21], 48
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[20:21], v[16:19], off sc0 sc1
; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, s18
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v23, s19
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s16
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s17
-; AGPR-SDAG-NEXT: v_mov_b64_e32 v[18:19], 32
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[18:19], v[20:23], off sc0 sc1
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[22:23], 32
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[24:25], 16
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s16
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s17
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s18
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s19
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[22:23], v[16:19], off sc0 sc1
; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v24, s14
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v25, s15
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, s12
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v23, s13
-; AGPR-SDAG-NEXT: v_mov_b64_e32 v[20:21], 16
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[20:21], v[22:25], off sc0 sc1
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[26:27], 0
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s12
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s13
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s14
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s15
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[24:25], v[16:19], off sc0 sc1
; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v26, s10
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v27, s11
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v24, s8
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v25, s9
-; AGPR-SDAG-NEXT: v_mov_b64_e32 v[22:23], 0
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[22:23], v[24:27], off sc0 sc1
+; AGPR-SDAG-NEXT: s_nop 0
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s8
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s9
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s10
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s11
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[26:27], v[16:19], off sc0 sc1
; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[18:19], v[8:11], off sc0 sc1
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[22:23], v[8:11], off sc0 sc1
; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[16:17], v[12:15], off sc0 sc1
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[20:21], v[12:15], off sc0 sc1
; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[22:23], v[0:3], off sc0 sc1
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[26:27], v[0:3], off sc0 sc1
; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
-; AGPR-SDAG-NEXT: global_store_dwordx4 v[20:21], v[4:7], off sc0 sc1
+; AGPR-SDAG-NEXT: global_store_dwordx4 v[24:25], v[4:7], off sc0 sc1
; AGPR-SDAG-NEXT: s_waitcnt vmcnt(0)
; AGPR-SDAG-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.smfmac.gfx950.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.smfmac.gfx950.ll
index 76b2d9ab4dc40..c16828d4d5444 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.smfmac.gfx950.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.smfmac.gfx950.ll
@@ -1386,26 +1386,26 @@ define amdgpu_kernel void @test_smfmac_i32_16x16x128_i8__vgpr(ptr addrspace(1) %
; SDAG-NEXT: s_load_dword s16, s[4:5], 0x64
; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: global_load_dwordx4 v[10:13], v0, s[6:7]
-; SDAG-NEXT: v_mov_b32_e32 v0, 0
-; SDAG-NEXT: v_mov_b32_e32 v14, s8
-; SDAG-NEXT: v_mov_b32_e32 v15, s9
-; SDAG-NEXT: v_mov_b32_e32 v16, s10
-; SDAG-NEXT: v_mov_b32_e32 v17, s11
-; SDAG-NEXT: v_mov_b32_e32 v2, s12
-; SDAG-NEXT: v_mov_b32_e32 v3, s13
-; SDAG-NEXT: v_mov_b32_e32 v4, s14
-; SDAG-NEXT: v_mov_b32_e32 v5, s15
-; SDAG-NEXT: v_mov_b32_e32 v6, s0
-; SDAG-NEXT: v_mov_b32_e32 v7, s1
-; SDAG-NEXT: v_mov_b32_e32 v8, s2
-; SDAG-NEXT: v_mov_b32_e32 v9, s3
-; SDAG-NEXT: v_mov_b32_e32 v1, s16
+; SDAG-NEXT: global_load_dwordx4 v[14:17], v0, s[6:7]
+; SDAG-NEXT: v_mov_b32_e32 v12, 0
+; SDAG-NEXT: v_mov_b32_e32 v8, s8
+; SDAG-NEXT: v_mov_b32_e32 v9, s9
+; SDAG-NEXT: v_mov_b32_e32 v10, s10
+; SDAG-NEXT: v_mov_b32_e32 v11, s11
+; SDAG-NEXT: v_mov_b32_e32 v0, s12
+; SDAG-NEXT: v_mov_b32_e32 v1, s13
+; SDAG-NEXT: v_mov_b32_e32 v2, s14
+; SDAG-NEXT: v_mov_b32_e32 v3, s15
+; SDAG-NEXT: v_mov_b32_e32 v4, s0
+; SDAG-NEXT: v_mov_b32_e32 v5, s1
+; SDAG-NEXT: v_mov_b32_e32 v6, s2
+; SDAG-NEXT: v_mov_b32_e32 v7, s3
+; SDAG-NEXT: v_mov_b32_e32 v13, s16
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
-; SDAG-NEXT: v_smfmac_i32_16x16x128_i8 v[10:13], v[14:17], v[2:9], v1 cbsz:1 abid:2
+; SDAG-NEXT: v_smfmac_i32_16x16x128_i8 v[14:17], v[8:11], v[0:7], v13 cbsz:1 abid:2
; SDAG-NEXT: s_nop 7
-; SDAG-NEXT: global_store_dwordx4 v0, v[10:13], s[6:7]
+; SDAG-NEXT: global_store_dwordx4 v12, v[14:17], s[6:7]
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: test_smfmac_i32_16x16x128_i8__vgpr:
@@ -1442,26 +1442,26 @@ define amdgpu_kernel void @test_smfmac_i32_16x16x128_i8__vgpr(ptr addrspace(1) %
; SDAG-VGPR-NEXT: s_load_dword s16, s[4:5], 0x64
; SDAG-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
; SDAG-VGPR-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-VGPR-NEXT: global_load_dwordx4 v[10:13], v0, s[6:7]
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v0, 0
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v14, s8
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v15, s9
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, s10
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v17, s11
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v2, s12
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v3, s13
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v4, s14
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v5, s15
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v6, s0
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v7, s1
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v8, s2
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v9, s3
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v1, s16
+; SDAG-VGPR-NEXT: global_load_dwordx4 v[14:17], v0, s[6:7]
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v12, 0
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v8, s8
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v9, s9
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v10, s10
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v11, s11
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v0, s12
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v1, s13
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v2, s14
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v3, s15
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v4, s0
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v5, s1
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v6, s2
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v7, s3
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v13, s16
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-VGPR-NEXT: s_nop 0
-; SDAG-VGPR-NEXT: v_smfmac_i32_16x16x128_i8 v[10:13], v[14:17], v[2:9], v1 cbsz:1 abid:2
+; SDAG-VGPR-NEXT: v_smfmac_i32_16x16x128_i8 v[14:17], v[8:11], v[0:7], v13 cbsz:1 abid:2
; SDAG-VGPR-NEXT: s_nop 7
-; SDAG-VGPR-NEXT: global_store_dwordx4 v0, v[10:13], s[6:7]
+; SDAG-VGPR-NEXT: global_store_dwordx4 v12, v[14:17], s[6:7]
; SDAG-VGPR-NEXT: s_endpgm
;
; GISEL-VGPR-LABEL: test_smfmac_i32_16x16x128_i8__vgpr:
@@ -1711,22 +1711,22 @@ define amdgpu_kernel void @test_smfmac_i32_32x32x64_i8__vgpr(ptr addrspace(1) %a
; SDAG-NEXT: global_load_dwordx4 v[8:11], v16, s[6:7] offset:32
; SDAG-NEXT: global_load_dwordx4 v[4:7], v16, s[6:7] offset:16
; SDAG-NEXT: global_load_dwordx4 v[0:3], v16, s[6:7]
-; SDAG-NEXT: v_mov_b32_e32 v26, s8
-; SDAG-NEXT: v_mov_b32_e32 v27, s9
-; SDAG-NEXT: v_mov_b32_e32 v28, s10
-; SDAG-NEXT: v_mov_b32_e32 v29, s11
-; SDAG-NEXT: v_mov_b32_e32 v18, s12
-; SDAG-NEXT: v_mov_b32_e32 v19, s13
-; SDAG-NEXT: v_mov_b32_e32 v20, s14
-; SDAG-NEXT: v_mov_b32_e32 v21, s15
-; SDAG-NEXT: v_mov_b32_e32 v22, s0
-; SDAG-NEXT: v_mov_b32_e32 v23, s1
-; SDAG-NEXT: v_mov_b32_e32 v24, s2
-; SDAG-NEXT: v_mov_b32_e32 v25, s3
-; SDAG-NEXT: v_mov_b32_e32 v16, s16
+; SDAG-NEXT: v_mov_b32_e32 v24, s8
+; SDAG-NEXT: v_mov_b32_e32 v25, s9
+; SDAG-NEXT: v_mov_b32_e32 v26, s10
+; SDAG-NEXT: v_mov_b32_e32 v27, s11
+; SDAG-NEXT: v_mov_b32_e32 v16, s12
+; SDAG-NEXT: v_mov_b32_e32 v17, s13
+; SDAG-NEXT: v_mov_b32_e32 v18, s14
+; SDAG-NEXT: v_mov_b32_e32 v19, s15
+; SDAG-NEXT: v_mov_b32_e32 v20, s0
+; SDAG-NEXT: v_mov_b32_e32 v21, s1
+; SDAG-NEXT: v_mov_b32_e32 v22, s2
+; SDAG-NEXT: v_mov_b32_e32 v23, s3
+; SDAG-NEXT: v_mov_b32_e32 v28, s16
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
-; SDAG-NEXT: v_smfmac_i32_32x32x64_i8 v[0:15], v[26:29], v[18:25], v16 cbsz:1 abid:2
+; SDAG-NEXT: v_smfmac_i32_32x32x64_i8 v[0:15], v[24:27], v[16:23], v28 cbsz:1 abid:2
; SDAG-NEXT: v_mov_b32_e32 v16, 0
; SDAG-NEXT: s_nop 10
; SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[6:7] offset:32
@@ -1779,22 +1779,22 @@ define amdgpu_kernel void @test_smfmac_i32_32x32x64_i8__vgpr(ptr addrspace(1) %a
; SDAG-VGPR-NEXT: global_load_dwordx4 v[8:11], v16, s[6:7] offset:32
; SDAG-VGPR-NEXT: global_load_dwordx4 v[4:7], v16, s[6:7] offset:16
; SDAG-VGPR-NEXT: global_load_dwordx4 v[0:3], v16, s[6:7]
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v26, s8
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v27, s9
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v28, s10
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v29, s11
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v18, s12
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v19, s13
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v20, s14
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v21, s15
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v22, s0
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v23, s1
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v24, s2
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v25, s3
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, s16
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v24, s8
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v25, s9
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v26, s10
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v27, s11
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, s12
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v17, s13
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v18, s14
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v19, s15
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v20, s0
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v21, s1
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v22, s2
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v23, s3
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v28, s16
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-VGPR-NEXT: s_nop 0
-; SDAG-VGPR-NEXT: v_smfmac_i32_32x32x64_i8 v[0:15], v[26:29], v[18:25], v16 cbsz:1 abid:2
+; SDAG-VGPR-NEXT: v_smfmac_i32_32x32x64_i8 v[0:15], v[24:27], v[16:23], v28 cbsz:1 abid:2
; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, 0
; SDAG-VGPR-NEXT: s_nop 10
; SDAG-VGPR-NEXT: global_store_dwordx4 v16, v[8:11], s[6:7] offset:32
@@ -2255,26 +2255,26 @@ define amdgpu_kernel void @test_smfmac_f32_16x16x128_bf8_bf8__vgpr(ptr addrspace
; SDAG-NEXT: s_load_dword s16, s[4:5], 0x64
; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: global_load_dwordx4 v[10:13], v0, s[6:7]
-; SDAG-NEXT: v_mov_b32_e32 v0, 0
-; SDAG-NEXT: v_mov_b32_e32 v14, s8
-; SDAG-NEXT: v_mov_b32_e32 v15, s9
-; SDAG-NEXT: v_mov_b32_e32 v16, s10
-; SDAG-NEXT: v_mov_b32_e32 v17, s11
-; SDAG-NEXT: v_mov_b32_e32 v2, s12
-; SDAG-NEXT: v_mov_b32_e32 v3, s13
-; SDAG-NEXT: v_mov_b32_e32 v4, s14
-; SDAG-NEXT: v_mov_b32_e32 v5, s15
-; SDAG-NEXT: v_mov_b32_e32 v6, s0
-; SDAG-NEXT: v_mov_b32_e32 v7, s1
-; SDAG-NEXT: v_mov_b32_e32 v8, s2
-; SDAG-NEXT: v_mov_b32_e32 v9, s3
-; SDAG-NEXT: v_mov_b32_e32 v1, s16
+; SDAG-NEXT: global_load_dwordx4 v[14:17], v0, s[6:7]
+; SDAG-NEXT: v_mov_b32_e32 v12, 0
+; SDAG-NEXT: v_mov_b32_e32 v8, s8
+; SDAG-NEXT: v_mov_b32_e32 v9, s9
+; SDAG-NEXT: v_mov_b32_e32 v10, s10
+; SDAG-NEXT: v_mov_b32_e32 v11, s11
+; SDAG-NEXT: v_mov_b32_e32 v0, s12
+; SDAG-NEXT: v_mov_b32_e32 v1, s13
+; SDAG-NEXT: v_mov_b32_e32 v2, s14
+; SDAG-NEXT: v_mov_b32_e32 v3, s15
+; SDAG-NEXT: v_mov_b32_e32 v4, s0
+; SDAG-NEXT: v_mov_b32_e32 v5, s1
+; SDAG-NEXT: v_mov_b32_e32 v6, s2
+; SDAG-NEXT: v_mov_b32_e32 v7, s3
+; SDAG-NEXT: v_mov_b32_e32 v13, s16
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
-; SDAG-NEXT: v_smfmac_f32_16x16x128_bf8_bf8 v[10:13], v[14:17], v[2:9], v1 cbsz:1 abid:2
+; SDAG-NEXT: v_smfmac_f32_16x16x128_bf8_bf8 v[14:17], v[8:11], v[0:7], v13 cbsz:1 abid:2
; SDAG-NEXT: s_nop 7
-; SDAG-NEXT: global_store_dwordx4 v0, v[10:13], s[6:7]
+; SDAG-NEXT: global_store_dwordx4 v12, v[14:17], s[6:7]
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: test_smfmac_f32_16x16x128_bf8_bf8__vgpr:
@@ -2311,26 +2311,26 @@ define amdgpu_kernel void @test_smfmac_f32_16x16x128_bf8_bf8__vgpr(ptr addrspace
; SDAG-VGPR-NEXT: s_load_dword s16, s[4:5], 0x64
; SDAG-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
; SDAG-VGPR-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-VGPR-NEXT: global_load_dwordx4 v[10:13], v0, s[6:7]
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v0, 0
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v14, s8
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v15, s9
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, s10
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v17, s11
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v2, s12
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v3, s13
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v4, s14
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v5, s15
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v6, s0
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v7, s1
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v8, s2
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v9, s3
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v1, s16
+; SDAG-VGPR-NEXT: global_load_dwordx4 v[14:17], v0, s[6:7]
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v12, 0
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v8, s8
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v9, s9
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v10, s10
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v11, s11
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v0, s12
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v1, s13
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v2, s14
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v3, s15
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v4, s0
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v5, s1
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v6, s2
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v7, s3
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v13, s16
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-VGPR-NEXT: s_nop 0
-; SDAG-VGPR-NEXT: v_smfmac_f32_16x16x128_bf8_bf8 v[10:13], v[14:17], v[2:9], v1 cbsz:1 abid:2
+; SDAG-VGPR-NEXT: v_smfmac_f32_16x16x128_bf8_bf8 v[14:17], v[8:11], v[0:7], v13 cbsz:1 abid:2
; SDAG-VGPR-NEXT: s_nop 7
-; SDAG-VGPR-NEXT: global_store_dwordx4 v0, v[10:13], s[6:7]
+; SDAG-VGPR-NEXT: global_store_dwordx4 v12, v[14:17], s[6:7]
; SDAG-VGPR-NEXT: s_endpgm
;
; GISEL-VGPR-LABEL: test_smfmac_f32_16x16x128_bf8_bf8__vgpr:
@@ -2576,26 +2576,26 @@ define amdgpu_kernel void @test_smfmac_f32_16x16x128_bf8_fp8__vgpr(ptr addrspace
; SDAG-NEXT: s_load_dword s16, s[4:5], 0x64
; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: global_load_dwordx4 v[10:13], v0, s[6:7]
-; SDAG-NEXT: v_mov_b32_e32 v0, 0
-; SDAG-NEXT: v_mov_b32_e32 v14, s8
-; SDAG-NEXT: v_mov_b32_e32 v15, s9
-; SDAG-NEXT: v_mov_b32_e32 v16, s10
-; SDAG-NEXT: v_mov_b32_e32 v17, s11
-; SDAG-NEXT: v_mov_b32_e32 v2, s12
-; SDAG-NEXT: v_mov_b32_e32 v3, s13
-; SDAG-NEXT: v_mov_b32_e32 v4, s14
-; SDAG-NEXT: v_mov_b32_e32 v5, s15
-; SDAG-NEXT: v_mov_b32_e32 v6, s0
-; SDAG-NEXT: v_mov_b32_e32 v7, s1
-; SDAG-NEXT: v_mov_b32_e32 v8, s2
-; SDAG-NEXT: v_mov_b32_e32 v9, s3
-; SDAG-NEXT: v_mov_b32_e32 v1, s16
+; SDAG-NEXT: global_load_dwordx4 v[14:17], v0, s[6:7]
+; SDAG-NEXT: v_mov_b32_e32 v12, 0
+; SDAG-NEXT: v_mov_b32_e32 v8, s8
+; SDAG-NEXT: v_mov_b32_e32 v9, s9
+; SDAG-NEXT: v_mov_b32_e32 v10, s10
+; SDAG-NEXT: v_mov_b32_e32 v11, s11
+; SDAG-NEXT: v_mov_b32_e32 v0, s12
+; SDAG-NEXT: v_mov_b32_e32 v1, s13
+; SDAG-NEXT: v_mov_b32_e32 v2, s14
+; SDAG-NEXT: v_mov_b32_e32 v3, s15
+; SDAG-NEXT: v_mov_b32_e32 v4, s0
+; SDAG-NEXT: v_mov_b32_e32 v5, s1
+; SDAG-NEXT: v_mov_b32_e32 v6, s2
+; SDAG-NEXT: v_mov_b32_e32 v7, s3
+; SDAG-NEXT: v_mov_b32_e32 v13, s16
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
-; SDAG-NEXT: v_smfmac_f32_16x16x128_bf8_fp8 v[10:13], v[14:17], v[2:9], v1 cbsz:1 abid:2
+; SDAG-NEXT: v_smfmac_f32_16x16x128_bf8_fp8 v[14:17], v[8:11], v[0:7], v13 cbsz:1 abid:2
; SDAG-NEXT: s_nop 7
-; SDAG-NEXT: global_store_dwordx4 v0, v[10:13], s[6:7]
+; SDAG-NEXT: global_store_dwordx4 v12, v[14:17], s[6:7]
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: test_smfmac_f32_16x16x128_bf8_fp8__vgpr:
@@ -2632,26 +2632,26 @@ define amdgpu_kernel void @test_smfmac_f32_16x16x128_bf8_fp8__vgpr(ptr addrspace
; SDAG-VGPR-NEXT: s_load_dword s16, s[4:5], 0x64
; SDAG-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
; SDAG-VGPR-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-VGPR-NEXT: global_load_dwordx4 v[10:13], v0, s[6:7]
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v0, 0
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v14, s8
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v15, s9
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, s10
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v17, s11
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v2, s12
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v3, s13
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v4, s14
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v5, s15
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v6, s0
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v7, s1
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v8, s2
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v9, s3
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v1, s16
+; SDAG-VGPR-NEXT: global_load_dwordx4 v[14:17], v0, s[6:7]
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v12, 0
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v8, s8
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v9, s9
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v10, s10
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v11, s11
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v0, s12
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v1, s13
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v2, s14
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v3, s15
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v4, s0
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v5, s1
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v6, s2
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v7, s3
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v13, s16
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-VGPR-NEXT: s_nop 0
-; SDAG-VGPR-NEXT: v_smfmac_f32_16x16x128_bf8_fp8 v[10:13], v[14:17], v[2:9], v1 cbsz:1 abid:2
+; SDAG-VGPR-NEXT: v_smfmac_f32_16x16x128_bf8_fp8 v[14:17], v[8:11], v[0:7], v13 cbsz:1 abid:2
; SDAG-VGPR-NEXT: s_nop 7
-; SDAG-VGPR-NEXT: global_store_dwordx4 v0, v[10:13], s[6:7]
+; SDAG-VGPR-NEXT: global_store_dwordx4 v12, v[14:17], s[6:7]
; SDAG-VGPR-NEXT: s_endpgm
;
; GISEL-VGPR-LABEL: test_smfmac_f32_16x16x128_bf8_fp8__vgpr:
@@ -2897,26 +2897,26 @@ define amdgpu_kernel void @test_smfmac_f32_16x16x128_fp8_bf8__vgpr(ptr addrspace
; SDAG-NEXT: s_load_dword s16, s[4:5], 0x64
; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: global_load_dwordx4 v[10:13], v0, s[6:7]
-; SDAG-NEXT: v_mov_b32_e32 v0, 0
-; SDAG-NEXT: v_mov_b32_e32 v14, s8
-; SDAG-NEXT: v_mov_b32_e32 v15, s9
-; SDAG-NEXT: v_mov_b32_e32 v16, s10
-; SDAG-NEXT: v_mov_b32_e32 v17, s11
-; SDAG-NEXT: v_mov_b32_e32 v2, s12
-; SDAG-NEXT: v_mov_b32_e32 v3, s13
-; SDAG-NEXT: v_mov_b32_e32 v4, s14
-; SDAG-NEXT: v_mov_b32_e32 v5, s15
-; SDAG-NEXT: v_mov_b32_e32 v6, s0
-; SDAG-NEXT: v_mov_b32_e32 v7, s1
-; SDAG-NEXT: v_mov_b32_e32 v8, s2
-; SDAG-NEXT: v_mov_b32_e32 v9, s3
-; SDAG-NEXT: v_mov_b32_e32 v1, s16
+; SDAG-NEXT: global_load_dwordx4 v[14:17], v0, s[6:7]
+; SDAG-NEXT: v_mov_b32_e32 v12, 0
+; SDAG-NEXT: v_mov_b32_e32 v8, s8
+; SDAG-NEXT: v_mov_b32_e32 v9, s9
+; SDAG-NEXT: v_mov_b32_e32 v10, s10
+; SDAG-NEXT: v_mov_b32_e32 v11, s11
+; SDAG-NEXT: v_mov_b32_e32 v0, s12
+; SDAG-NEXT: v_mov_b32_e32 v1, s13
+; SDAG-NEXT: v_mov_b32_e32 v2, s14
+; SDAG-NEXT: v_mov_b32_e32 v3, s15
+; SDAG-NEXT: v_mov_b32_e32 v4, s0
+; SDAG-NEXT: v_mov_b32_e32 v5, s1
+; SDAG-NEXT: v_mov_b32_e32 v6, s2
+; SDAG-NEXT: v_mov_b32_e32 v7, s3
+; SDAG-NEXT: v_mov_b32_e32 v13, s16
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
-; SDAG-NEXT: v_smfmac_f32_16x16x128_fp8_bf8 v[10:13], v[14:17], v[2:9], v1 cbsz:1 abid:2
+; SDAG-NEXT: v_smfmac_f32_16x16x128_fp8_bf8 v[14:17], v[8:11], v[0:7], v13 cbsz:1 abid:2
; SDAG-NEXT: s_nop 7
-; SDAG-NEXT: global_store_dwordx4 v0, v[10:13], s[6:7]
+; SDAG-NEXT: global_store_dwordx4 v12, v[14:17], s[6:7]
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: test_smfmac_f32_16x16x128_fp8_bf8__vgpr:
@@ -2953,26 +2953,26 @@ define amdgpu_kernel void @test_smfmac_f32_16x16x128_fp8_bf8__vgpr(ptr addrspace
; SDAG-VGPR-NEXT: s_load_dword s16, s[4:5], 0x64
; SDAG-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
; SDAG-VGPR-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-VGPR-NEXT: global_load_dwordx4 v[10:13], v0, s[6:7]
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v0, 0
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v14, s8
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v15, s9
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, s10
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v17, s11
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v2, s12
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v3, s13
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v4, s14
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v5, s15
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v6, s0
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v7, s1
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v8, s2
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v9, s3
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v1, s16
+; SDAG-VGPR-NEXT: global_load_dwordx4 v[14:17], v0, s[6:7]
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v12, 0
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v8, s8
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v9, s9
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v10, s10
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v11, s11
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v0, s12
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v1, s13
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v2, s14
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v3, s15
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v4, s0
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v5, s1
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v6, s2
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v7, s3
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v13, s16
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-VGPR-NEXT: s_nop 0
-; SDAG-VGPR-NEXT: v_smfmac_f32_16x16x128_fp8_bf8 v[10:13], v[14:17], v[2:9], v1 cbsz:1 abid:2
+; SDAG-VGPR-NEXT: v_smfmac_f32_16x16x128_fp8_bf8 v[14:17], v[8:11], v[0:7], v13 cbsz:1 abid:2
; SDAG-VGPR-NEXT: s_nop 7
-; SDAG-VGPR-NEXT: global_store_dwordx4 v0, v[10:13], s[6:7]
+; SDAG-VGPR-NEXT: global_store_dwordx4 v12, v[14:17], s[6:7]
; SDAG-VGPR-NEXT: s_endpgm
;
; GISEL-VGPR-LABEL: test_smfmac_f32_16x16x128_fp8_bf8__vgpr:
@@ -3218,26 +3218,26 @@ define amdgpu_kernel void @test_smfmac_f32_16x16x128_fp8_fp8__vgpr(ptr addrspace
; SDAG-NEXT: s_load_dword s16, s[4:5], 0x64
; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: global_load_dwordx4 v[10:13], v0, s[6:7]
-; SDAG-NEXT: v_mov_b32_e32 v0, 0
-; SDAG-NEXT: v_mov_b32_e32 v14, s8
-; SDAG-NEXT: v_mov_b32_e32 v15, s9
-; SDAG-NEXT: v_mov_b32_e32 v16, s10
-; SDAG-NEXT: v_mov_b32_e32 v17, s11
-; SDAG-NEXT: v_mov_b32_e32 v2, s12
-; SDAG-NEXT: v_mov_b32_e32 v3, s13
-; SDAG-NEXT: v_mov_b32_e32 v4, s14
-; SDAG-NEXT: v_mov_b32_e32 v5, s15
-; SDAG-NEXT: v_mov_b32_e32 v6, s0
-; SDAG-NEXT: v_mov_b32_e32 v7, s1
-; SDAG-NEXT: v_mov_b32_e32 v8, s2
-; SDAG-NEXT: v_mov_b32_e32 v9, s3
-; SDAG-NEXT: v_mov_b32_e32 v1, s16
+; SDAG-NEXT: global_load_dwordx4 v[14:17], v0, s[6:7]
+; SDAG-NEXT: v_mov_b32_e32 v12, 0
+; SDAG-NEXT: v_mov_b32_e32 v8, s8
+; SDAG-NEXT: v_mov_b32_e32 v9, s9
+; SDAG-NEXT: v_mov_b32_e32 v10, s10
+; SDAG-NEXT: v_mov_b32_e32 v11, s11
+; SDAG-NEXT: v_mov_b32_e32 v0, s12
+; SDAG-NEXT: v_mov_b32_e32 v1, s13
+; SDAG-NEXT: v_mov_b32_e32 v2, s14
+; SDAG-NEXT: v_mov_b32_e32 v3, s15
+; SDAG-NEXT: v_mov_b32_e32 v4, s0
+; SDAG-NEXT: v_mov_b32_e32 v5, s1
+; SDAG-NEXT: v_mov_b32_e32 v6, s2
+; SDAG-NEXT: v_mov_b32_e32 v7, s3
+; SDAG-NEXT: v_mov_b32_e32 v13, s16
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
-; SDAG-NEXT: v_smfmac_f32_16x16x128_fp8_fp8 v[10:13], v[14:17], v[2:9], v1 cbsz:1 abid:2
+; SDAG-NEXT: v_smfmac_f32_16x16x128_fp8_fp8 v[14:17], v[8:11], v[0:7], v13 cbsz:1 abid:2
; SDAG-NEXT: s_nop 7
-; SDAG-NEXT: global_store_dwordx4 v0, v[10:13], s[6:7]
+; SDAG-NEXT: global_store_dwordx4 v12, v[14:17], s[6:7]
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: test_smfmac_f32_16x16x128_fp8_fp8__vgpr:
@@ -3274,26 +3274,26 @@ define amdgpu_kernel void @test_smfmac_f32_16x16x128_fp8_fp8__vgpr(ptr addrspace
; SDAG-VGPR-NEXT: s_load_dword s16, s[4:5], 0x64
; SDAG-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
; SDAG-VGPR-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-VGPR-NEXT: global_load_dwordx4 v[10:13], v0, s[6:7]
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v0, 0
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v14, s8
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v15, s9
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, s10
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v17, s11
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v2, s12
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v3, s13
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v4, s14
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v5, s15
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v6, s0
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v7, s1
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v8, s2
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v9, s3
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v1, s16
+; SDAG-VGPR-NEXT: global_load_dwordx4 v[14:17], v0, s[6:7]
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v12, 0
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v8, s8
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v9, s9
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v10, s10
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v11, s11
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v0, s12
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v1, s13
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v2, s14
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v3, s15
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v4, s0
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v5, s1
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v6, s2
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v7, s3
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v13, s16
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-VGPR-NEXT: s_nop 0
-; SDAG-VGPR-NEXT: v_smfmac_f32_16x16x128_fp8_fp8 v[10:13], v[14:17], v[2:9], v1 cbsz:1 abid:2
+; SDAG-VGPR-NEXT: v_smfmac_f32_16x16x128_fp8_fp8 v[14:17], v[8:11], v[0:7], v13 cbsz:1 abid:2
; SDAG-VGPR-NEXT: s_nop 7
-; SDAG-VGPR-NEXT: global_store_dwordx4 v0, v[10:13], s[6:7]
+; SDAG-VGPR-NEXT: global_store_dwordx4 v12, v[14:17], s[6:7]
; SDAG-VGPR-NEXT: s_endpgm
;
; GISEL-VGPR-LABEL: test_smfmac_f32_16x16x128_fp8_fp8__vgpr:
@@ -3543,22 +3543,22 @@ define amdgpu_kernel void @test_smfmac_f32_32x32x64_bf8_bf8__vgpr(ptr addrspace(
; SDAG-NEXT: global_load_dwordx4 v[8:11], v16, s[6:7] offset:32
; SDAG-NEXT: global_load_dwordx4 v[4:7], v16, s[6:7] offset:16
; SDAG-NEXT: global_load_dwordx4 v[0:3], v16, s[6:7]
-; SDAG-NEXT: v_mov_b32_e32 v26, s8
-; SDAG-NEXT: v_mov_b32_e32 v27, s9
-; SDAG-NEXT: v_mov_b32_e32 v28, s10
-; SDAG-NEXT: v_mov_b32_e32 v29, s11
-; SDAG-NEXT: v_mov_b32_e32 v18, s12
-; SDAG-NEXT: v_mov_b32_e32 v19, s13
-; SDAG-NEXT: v_mov_b32_e32 v20, s14
-; SDAG-NEXT: v_mov_b32_e32 v21, s15
-; SDAG-NEXT: v_mov_b32_e32 v22, s0
-; SDAG-NEXT: v_mov_b32_e32 v23, s1
-; SDAG-NEXT: v_mov_b32_e32 v24, s2
-; SDAG-NEXT: v_mov_b32_e32 v25, s3
-; SDAG-NEXT: v_mov_b32_e32 v16, s16
+; SDAG-NEXT: v_mov_b32_e32 v24, s8
+; SDAG-NEXT: v_mov_b32_e32 v25, s9
+; SDAG-NEXT: v_mov_b32_e32 v26, s10
+; SDAG-NEXT: v_mov_b32_e32 v27, s11
+; SDAG-NEXT: v_mov_b32_e32 v16, s12
+; SDAG-NEXT: v_mov_b32_e32 v17, s13
+; SDAG-NEXT: v_mov_b32_e32 v18, s14
+; SDAG-NEXT: v_mov_b32_e32 v19, s15
+; SDAG-NEXT: v_mov_b32_e32 v20, s0
+; SDAG-NEXT: v_mov_b32_e32 v21, s1
+; SDAG-NEXT: v_mov_b32_e32 v22, s2
+; SDAG-NEXT: v_mov_b32_e32 v23, s3
+; SDAG-NEXT: v_mov_b32_e32 v28, s16
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
-; SDAG-NEXT: v_smfmac_f32_32x32x64_bf8_bf8 v[0:15], v[26:29], v[18:25], v16 cbsz:1 abid:2
+; SDAG-NEXT: v_smfmac_f32_32x32x64_bf8_bf8 v[0:15], v[24:27], v[16:23], v28 cbsz:1 abid:2
; SDAG-NEXT: v_mov_b32_e32 v16, 0
; SDAG-NEXT: s_nop 10
; SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[6:7] offset:32
@@ -3611,22 +3611,22 @@ define amdgpu_kernel void @test_smfmac_f32_32x32x64_bf8_bf8__vgpr(ptr addrspace(
; SDAG-VGPR-NEXT: global_load_dwordx4 v[8:11], v16, s[6:7] offset:32
; SDAG-VGPR-NEXT: global_load_dwordx4 v[4:7], v16, s[6:7] offset:16
; SDAG-VGPR-NEXT: global_load_dwordx4 v[0:3], v16, s[6:7]
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v26, s8
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v27, s9
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v28, s10
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v29, s11
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v18, s12
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v19, s13
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v20, s14
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v21, s15
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v22, s0
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v23, s1
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v24, s2
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v25, s3
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, s16
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v24, s8
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v25, s9
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v26, s10
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v27, s11
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, s12
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v17, s13
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v18, s14
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v19, s15
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v20, s0
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v21, s1
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v22, s2
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v23, s3
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v28, s16
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-VGPR-NEXT: s_nop 0
-; SDAG-VGPR-NEXT: v_smfmac_f32_32x32x64_bf8_bf8 v[0:15], v[26:29], v[18:25], v16 cbsz:1 abid:2
+; SDAG-VGPR-NEXT: v_smfmac_f32_32x32x64_bf8_bf8 v[0:15], v[24:27], v[16:23], v28 cbsz:1 abid:2
; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, 0
; SDAG-VGPR-NEXT: s_nop 10
; SDAG-VGPR-NEXT: global_store_dwordx4 v16, v[8:11], s[6:7] offset:32
@@ -4091,22 +4091,22 @@ define amdgpu_kernel void @test_smfmac_f32_32x32x64_bf8_fp8__vgpr(ptr addrspace(
; SDAG-NEXT: global_load_dwordx4 v[8:11], v16, s[6:7] offset:32
; SDAG-NEXT: global_load_dwordx4 v[4:7], v16, s[6:7] offset:16
; SDAG-NEXT: global_load_dwordx4 v[0:3], v16, s[6:7]
-; SDAG-NEXT: v_mov_b32_e32 v26, s8
-; SDAG-NEXT: v_mov_b32_e32 v27, s9
-; SDAG-NEXT: v_mov_b32_e32 v28, s10
-; SDAG-NEXT: v_mov_b32_e32 v29, s11
-; SDAG-NEXT: v_mov_b32_e32 v18, s12
-; SDAG-NEXT: v_mov_b32_e32 v19, s13
-; SDAG-NEXT: v_mov_b32_e32 v20, s14
-; SDAG-NEXT: v_mov_b32_e32 v21, s15
-; SDAG-NEXT: v_mov_b32_e32 v22, s0
-; SDAG-NEXT: v_mov_b32_e32 v23, s1
-; SDAG-NEXT: v_mov_b32_e32 v24, s2
-; SDAG-NEXT: v_mov_b32_e32 v25, s3
-; SDAG-NEXT: v_mov_b32_e32 v16, s16
+; SDAG-NEXT: v_mov_b32_e32 v24, s8
+; SDAG-NEXT: v_mov_b32_e32 v25, s9
+; SDAG-NEXT: v_mov_b32_e32 v26, s10
+; SDAG-NEXT: v_mov_b32_e32 v27, s11
+; SDAG-NEXT: v_mov_b32_e32 v16, s12
+; SDAG-NEXT: v_mov_b32_e32 v17, s13
+; SDAG-NEXT: v_mov_b32_e32 v18, s14
+; SDAG-NEXT: v_mov_b32_e32 v19, s15
+; SDAG-NEXT: v_mov_b32_e32 v20, s0
+; SDAG-NEXT: v_mov_b32_e32 v21, s1
+; SDAG-NEXT: v_mov_b32_e32 v22, s2
+; SDAG-NEXT: v_mov_b32_e32 v23, s3
+; SDAG-NEXT: v_mov_b32_e32 v28, s16
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
-; SDAG-NEXT: v_smfmac_f32_32x32x64_bf8_fp8 v[0:15], v[26:29], v[18:25], v16 cbsz:1 abid:2
+; SDAG-NEXT: v_smfmac_f32_32x32x64_bf8_fp8 v[0:15], v[24:27], v[16:23], v28 cbsz:1 abid:2
; SDAG-NEXT: v_mov_b32_e32 v16, 0
; SDAG-NEXT: s_nop 10
; SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[6:7] offset:32
@@ -4159,22 +4159,22 @@ define amdgpu_kernel void @test_smfmac_f32_32x32x64_bf8_fp8__vgpr(ptr addrspace(
; SDAG-VGPR-NEXT: global_load_dwordx4 v[8:11], v16, s[6:7] offset:32
; SDAG-VGPR-NEXT: global_load_dwordx4 v[4:7], v16, s[6:7] offset:16
; SDAG-VGPR-NEXT: global_load_dwordx4 v[0:3], v16, s[6:7]
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v26, s8
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v27, s9
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v28, s10
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v29, s11
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v18, s12
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v19, s13
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v20, s14
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v21, s15
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v22, s0
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v23, s1
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v24, s2
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v25, s3
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, s16
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v24, s8
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v25, s9
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v26, s10
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v27, s11
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, s12
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v17, s13
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v18, s14
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v19, s15
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v20, s0
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v21, s1
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v22, s2
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v23, s3
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v28, s16
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-VGPR-NEXT: s_nop 0
-; SDAG-VGPR-NEXT: v_smfmac_f32_32x32x64_bf8_fp8 v[0:15], v[26:29], v[18:25], v16 cbsz:1 abid:2
+; SDAG-VGPR-NEXT: v_smfmac_f32_32x32x64_bf8_fp8 v[0:15], v[24:27], v[16:23], v28 cbsz:1 abid:2
; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, 0
; SDAG-VGPR-NEXT: s_nop 10
; SDAG-VGPR-NEXT: global_store_dwordx4 v16, v[8:11], s[6:7] offset:32
@@ -4639,22 +4639,22 @@ define amdgpu_kernel void @test_smfmac_f32_32x32x64_fp8_bf8__vgpr(ptr addrspace(
; SDAG-NEXT: global_load_dwordx4 v[8:11], v16, s[6:7] offset:32
; SDAG-NEXT: global_load_dwordx4 v[4:7], v16, s[6:7] offset:16
; SDAG-NEXT: global_load_dwordx4 v[0:3], v16, s[6:7]
-; SDAG-NEXT: v_mov_b32_e32 v26, s8
-; SDAG-NEXT: v_mov_b32_e32 v27, s9
-; SDAG-NEXT: v_mov_b32_e32 v28, s10
-; SDAG-NEXT: v_mov_b32_e32 v29, s11
-; SDAG-NEXT: v_mov_b32_e32 v18, s12
-; SDAG-NEXT: v_mov_b32_e32 v19, s13
-; SDAG-NEXT: v_mov_b32_e32 v20, s14
-; SDAG-NEXT: v_mov_b32_e32 v21, s15
-; SDAG-NEXT: v_mov_b32_e32 v22, s0
-; SDAG-NEXT: v_mov_b32_e32 v23, s1
-; SDAG-NEXT: v_mov_b32_e32 v24, s2
-; SDAG-NEXT: v_mov_b32_e32 v25, s3
-; SDAG-NEXT: v_mov_b32_e32 v16, s16
+; SDAG-NEXT: v_mov_b32_e32 v24, s8
+; SDAG-NEXT: v_mov_b32_e32 v25, s9
+; SDAG-NEXT: v_mov_b32_e32 v26, s10
+; SDAG-NEXT: v_mov_b32_e32 v27, s11
+; SDAG-NEXT: v_mov_b32_e32 v16, s12
+; SDAG-NEXT: v_mov_b32_e32 v17, s13
+; SDAG-NEXT: v_mov_b32_e32 v18, s14
+; SDAG-NEXT: v_mov_b32_e32 v19, s15
+; SDAG-NEXT: v_mov_b32_e32 v20, s0
+; SDAG-NEXT: v_mov_b32_e32 v21, s1
+; SDAG-NEXT: v_mov_b32_e32 v22, s2
+; SDAG-NEXT: v_mov_b32_e32 v23, s3
+; SDAG-NEXT: v_mov_b32_e32 v28, s16
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
-; SDAG-NEXT: v_smfmac_f32_32x32x64_fp8_bf8 v[0:15], v[26:29], v[18:25], v16 cbsz:1 abid:2
+; SDAG-NEXT: v_smfmac_f32_32x32x64_fp8_bf8 v[0:15], v[24:27], v[16:23], v28 cbsz:1 abid:2
; SDAG-NEXT: v_mov_b32_e32 v16, 0
; SDAG-NEXT: s_nop 10
; SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[6:7] offset:32
@@ -4707,22 +4707,22 @@ define amdgpu_kernel void @test_smfmac_f32_32x32x64_fp8_bf8__vgpr(ptr addrspace(
; SDAG-VGPR-NEXT: global_load_dwordx4 v[8:11], v16, s[6:7] offset:32
; SDAG-VGPR-NEXT: global_load_dwordx4 v[4:7], v16, s[6:7] offset:16
; SDAG-VGPR-NEXT: global_load_dwordx4 v[0:3], v16, s[6:7]
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v26, s8
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v27, s9
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v28, s10
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v29, s11
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v18, s12
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v19, s13
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v20, s14
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v21, s15
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v22, s0
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v23, s1
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v24, s2
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v25, s3
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, s16
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v24, s8
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v25, s9
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v26, s10
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v27, s11
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, s12
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v17, s13
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v18, s14
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v19, s15
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v20, s0
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v21, s1
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v22, s2
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v23, s3
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v28, s16
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-VGPR-NEXT: s_nop 0
-; SDAG-VGPR-NEXT: v_smfmac_f32_32x32x64_fp8_bf8 v[0:15], v[26:29], v[18:25], v16 cbsz:1 abid:2
+; SDAG-VGPR-NEXT: v_smfmac_f32_32x32x64_fp8_bf8 v[0:15], v[24:27], v[16:23], v28 cbsz:1 abid:2
; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, 0
; SDAG-VGPR-NEXT: s_nop 10
; SDAG-VGPR-NEXT: global_store_dwordx4 v16, v[8:11], s[6:7] offset:32
@@ -5187,22 +5187,22 @@ define amdgpu_kernel void @test_smfmac_f32_32x32x64_fp8_fp8__vgpr(ptr addrspace(
; SDAG-NEXT: global_load_dwordx4 v[8:11], v16, s[6:7] offset:32
; SDAG-NEXT: global_load_dwordx4 v[4:7], v16, s[6:7] offset:16
; SDAG-NEXT: global_load_dwordx4 v[0:3], v16, s[6:7]
-; SDAG-NEXT: v_mov_b32_e32 v26, s8
-; SDAG-NEXT: v_mov_b32_e32 v27, s9
-; SDAG-NEXT: v_mov_b32_e32 v28, s10
-; SDAG-NEXT: v_mov_b32_e32 v29, s11
-; SDAG-NEXT: v_mov_b32_e32 v18, s12
-; SDAG-NEXT: v_mov_b32_e32 v19, s13
-; SDAG-NEXT: v_mov_b32_e32 v20, s14
-; SDAG-NEXT: v_mov_b32_e32 v21, s15
-; SDAG-NEXT: v_mov_b32_e32 v22, s0
-; SDAG-NEXT: v_mov_b32_e32 v23, s1
-; SDAG-NEXT: v_mov_b32_e32 v24, s2
-; SDAG-NEXT: v_mov_b32_e32 v25, s3
-; SDAG-NEXT: v_mov_b32_e32 v16, s16
+; SDAG-NEXT: v_mov_b32_e32 v24, s8
+; SDAG-NEXT: v_mov_b32_e32 v25, s9
+; SDAG-NEXT: v_mov_b32_e32 v26, s10
+; SDAG-NEXT: v_mov_b32_e32 v27, s11
+; SDAG-NEXT: v_mov_b32_e32 v16, s12
+; SDAG-NEXT: v_mov_b32_e32 v17, s13
+; SDAG-NEXT: v_mov_b32_e32 v18, s14
+; SDAG-NEXT: v_mov_b32_e32 v19, s15
+; SDAG-NEXT: v_mov_b32_e32 v20, s0
+; SDAG-NEXT: v_mov_b32_e32 v21, s1
+; SDAG-NEXT: v_mov_b32_e32 v22, s2
+; SDAG-NEXT: v_mov_b32_e32 v23, s3
+; SDAG-NEXT: v_mov_b32_e32 v28, s16
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
-; SDAG-NEXT: v_smfmac_f32_32x32x64_fp8_fp8 v[0:15], v[26:29], v[18:25], v16 cbsz:1 abid:2
+; SDAG-NEXT: v_smfmac_f32_32x32x64_fp8_fp8 v[0:15], v[24:27], v[16:23], v28 cbsz:1 abid:2
; SDAG-NEXT: v_mov_b32_e32 v16, 0
; SDAG-NEXT: s_nop 10
; SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[6:7] offset:32
@@ -5255,22 +5255,22 @@ define amdgpu_kernel void @test_smfmac_f32_32x32x64_fp8_fp8__vgpr(ptr addrspace(
; SDAG-VGPR-NEXT: global_load_dwordx4 v[8:11], v16, s[6:7] offset:32
; SDAG-VGPR-NEXT: global_load_dwordx4 v[4:7], v16, s[6:7] offset:16
; SDAG-VGPR-NEXT: global_load_dwordx4 v[0:3], v16, s[6:7]
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v26, s8
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v27, s9
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v28, s10
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v29, s11
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v18, s12
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v19, s13
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v20, s14
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v21, s15
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v22, s0
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v23, s1
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v24, s2
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v25, s3
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, s16
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v24, s8
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v25, s9
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v26, s10
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v27, s11
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, s12
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v17, s13
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v18, s14
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v19, s15
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v20, s0
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v21, s1
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v22, s2
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v23, s3
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v28, s16
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-VGPR-NEXT: s_nop 0
-; SDAG-VGPR-NEXT: v_smfmac_f32_32x32x64_fp8_fp8 v[0:15], v[26:29], v[18:25], v16 cbsz:1 abid:2
+; SDAG-VGPR-NEXT: v_smfmac_f32_32x32x64_fp8_fp8 v[0:15], v[24:27], v[16:23], v28 cbsz:1 abid:2
; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, 0
; SDAG-VGPR-NEXT: s_nop 10
; SDAG-VGPR-NEXT: global_store_dwordx4 v16, v[8:11], s[6:7] offset:32
diff --git a/llvm/test/CodeGen/AMDGPU/lo16-hi16-illegal-copy.mir b/llvm/test/CodeGen/AMDGPU/lo16-hi16-illegal-copy.mir
index b9a134df8238e..ba3df99adc909 100644
--- a/llvm/test/CodeGen/AMDGPU/lo16-hi16-illegal-copy.mir
+++ b/llvm/test/CodeGen/AMDGPU/lo16-hi16-illegal-copy.mir
@@ -1,7 +1,7 @@
-# RUN: not llc -mtriple=amdgpu8.02 -start-before postrapseudos -asm-verbose=0 -verify-machineinstrs %s -o - 2>&1 | FileCheck -check-prefixes=ERR,GFX8-ERR %s
-# RUN: not llc -mtriple=amdgpu8.02 -start-before postrapseudos -asm-verbose=0 -verify-machineinstrs %s -o - 2>&1 | FileCheck -check-prefix=GCN %s
-# RUN: not llc -mtriple=amdgpu9.00 -start-before postrapseudos -asm-verbose=0 -verify-machineinstrs %s -o - 2>&1 | FileCheck -check-prefix=ERR %s
-# RUN: not llc -mtriple=amdgpu9.00 -start-before postrapseudos -asm-verbose=0 -verify-machineinstrs %s -o - 2>&1 | FileCheck -check-prefixes=GCN,GFX9 %s
+# RUN: not llc -mtriple=amdgpu8.02 -start-before post-ra-pseudos -asm-verbose=0 -verify-machineinstrs %s -o - 2>&1 | FileCheck -check-prefixes=ERR,GFX8-ERR %s
+# RUN: not llc -mtriple=amdgpu8.02 -start-before post-ra-pseudos -asm-verbose=0 -verify-machineinstrs %s -o - 2>&1 | FileCheck -check-prefix=GCN %s
+# RUN: not llc -mtriple=amdgpu9.00 -start-before post-ra-pseudos -asm-verbose=0 -verify-machineinstrs %s -o - 2>&1 | FileCheck -check-prefix=ERR %s
+# RUN: not llc -mtriple=amdgpu9.00 -start-before post-ra-pseudos -asm-verbose=0 -verify-machineinstrs %s -o - 2>&1 | FileCheck -check-prefixes=GCN,GFX9 %s
# Note: GFX8 did not allow SDWA SGPR sources. Therefor no HI16 subregs can be used there.
diff --git a/llvm/test/CodeGen/AMDGPU/lo16-hi16-physreg-copy.mir b/llvm/test/CodeGen/AMDGPU/lo16-hi16-physreg-copy.mir
index 1601eaf8f0679..eda00f65c9ae7 100644
--- a/llvm/test/CodeGen/AMDGPU/lo16-hi16-physreg-copy.mir
+++ b/llvm/test/CodeGen/AMDGPU/lo16-hi16-physreg-copy.mir
@@ -1,6 +1,6 @@
-# RUN: llc -mtriple=amdgpu8.02 -start-before postrapseudos -asm-verbose=0 -verify-machineinstrs %s -o - | FileCheck -check-prefix=GCN %s
-# RUN: llc -mtriple=amdgpu9.00 -start-before postrapseudos -asm-verbose=0 -verify-machineinstrs %s -o - | FileCheck -check-prefix=GCN %s
-# RUN: llc -mtriple=amdgpu10.10 -start-before postrapseudos -asm-verbose=0 -verify-machineinstrs %s -o - | FileCheck -check-prefix=GCN %s
+# RUN: llc -mtriple=amdgpu8.02 -start-before post-ra-pseudos -asm-verbose=0 -verify-machineinstrs %s -o - | FileCheck -check-prefix=GCN %s
+# RUN: llc -mtriple=amdgpu9.00 -start-before post-ra-pseudos -asm-verbose=0 -verify-machineinstrs %s -o - | FileCheck -check-prefix=GCN %s
+# RUN: llc -mtriple=amdgpu10.10 -start-before post-ra-pseudos -asm-verbose=0 -verify-machineinstrs %s -o - | FileCheck -check-prefix=GCN %s
# GCN-LABEL: {{^}}lo_to_lo:
# GCN: v_mov_b32_sdwa v1, v0 dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0
diff --git a/llvm/test/CodeGen/AMDGPU/lo16-lo16-physreg-copy-agpr.mir b/llvm/test/CodeGen/AMDGPU/lo16-lo16-physreg-copy-agpr.mir
index 30b8b8f264bad..180b6c52309eb 100644
--- a/llvm/test/CodeGen/AMDGPU/lo16-lo16-physreg-copy-agpr.mir
+++ b/llvm/test/CodeGen/AMDGPU/lo16-lo16-physreg-copy-agpr.mir
@@ -1,4 +1,4 @@
-# RUN: llc -mtriple=amdgpu9.08 -start-before postrapseudos -asm-verbose=0 -verify-machineinstrs %s -o - | FileCheck -check-prefix=GCN %s
+# RUN: llc -mtriple=amdgpu9.08 -start-before post-ra-pseudos -asm-verbose=0 -verify-machineinstrs %s -o - | FileCheck -check-prefix=GCN %s
# GCN-LABEL: {{^}}lo_to_lo_agpr_to_agpr:
# GCN: v_accvgpr_read_b32 [[TMP:v[0-9]+]], a0
diff --git a/llvm/test/CodeGen/AMDGPU/lo16-lo16-physreg-copy-sgpr.mir b/llvm/test/CodeGen/AMDGPU/lo16-lo16-physreg-copy-sgpr.mir
index 87a29e75ed6b1..e19fefac74cbb 100644
--- a/llvm/test/CodeGen/AMDGPU/lo16-lo16-physreg-copy-sgpr.mir
+++ b/llvm/test/CodeGen/AMDGPU/lo16-lo16-physreg-copy-sgpr.mir
@@ -1,5 +1,5 @@
-# RUN: llc -mtriple=amdgpu9.00 -start-before postrapseudos -asm-verbose=0 -verify-machineinstrs %s -o - | FileCheck -check-prefix=GCN %s
-# RUN: llc -mtriple=amdgpu10.10 -start-before postrapseudos -asm-verbose=0 -verify-machineinstrs %s -o - | FileCheck -check-prefix=GCN %s
+# RUN: llc -mtriple=amdgpu9.00 -start-before post-ra-pseudos -asm-verbose=0 -verify-machineinstrs %s -o - | FileCheck -check-prefix=GCN %s
+# RUN: llc -mtriple=amdgpu10.10 -start-before post-ra-pseudos -asm-verbose=0 -verify-machineinstrs %s -o - | FileCheck -check-prefix=GCN %s
# Note: GFX8 did not allow SDWA SGPR sources. Therefor no HI16 subregs can be used there.
diff --git a/llvm/test/CodeGen/AMDGPU/lower-control-flow-live-variables-update.mir b/llvm/test/CodeGen/AMDGPU/lower-control-flow-live-variables-update.mir
index 32746ef5fce9e..c9d3b1a5e749f 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-control-flow-live-variables-update.mir
+++ b/llvm/test/CodeGen/AMDGPU/lower-control-flow-live-variables-update.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu7.00-amd-amdhsa -start-before=livevars -stop-after=twoaddressinstruction -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu7.00-amd-amdhsa -start-before=unreachable-mbb-elimination -stop-after=two-address-instruction -verify-machineinstrs -o - %s | FileCheck %s
# FIXME: update_mir_test_checks tries to incorrectly re-use a variable
# name used for a copy, so some of the check variable names were
diff --git a/llvm/test/CodeGen/AMDGPU/lower-control-flow-live-variables-update.xfail.mir b/llvm/test/CodeGen/AMDGPU/lower-control-flow-live-variables-update.xfail.mir
index 7a6fb07b598bb..c675879cf622f 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-control-flow-live-variables-update.xfail.mir
+++ b/llvm/test/CodeGen/AMDGPU/lower-control-flow-live-variables-update.xfail.mir
@@ -1,4 +1,4 @@
-# RUN: not --crash llc -mtriple=amdgpu7.00-amd-amdhsa -start-before=livevars -stop-after=twoaddressinstruction -verify-machineinstrs -o - %s 2>&1 | FileCheck %s
+# RUN: not --crash llc -mtriple=amdgpu7.00-amd-amdhsa -start-before=unreachable-mbb-elimination -stop-after=two-address-instruction -verify-each -o - %s 2>&1 | FileCheck %s
# CHECK: *** Bad machine code: LiveVariables: Block missing from AliveBlocks ***
# CHECK-NEXT: function: live_variables_update_block_split_split_def_before_si_end_cf_live_out
diff --git a/llvm/test/CodeGen/AMDGPU/machine-sink-cycle.mir b/llvm/test/CodeGen/AMDGPU/machine-sink-cycle.mir
index 9ae2440d21b25..e0f43eebff940 100644
--- a/llvm/test/CodeGen/AMDGPU/machine-sink-cycle.mir
+++ b/llvm/test/CodeGen/AMDGPU/machine-sink-cycle.mir
@@ -1,6 +1,6 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 3
-# RUN: llc -mtriple=amdgpu9.42-amd-amdhsa --sink-insts-to-avoid-spills=1 --stop-after=machine-sink -o - %s | FileCheck -check-prefixes=GFX9-SUNK %s
-# RUN: llc -mtriple=amdgpu10.31-amd-amdhsa --sink-insts-to-avoid-spills=1 -mattr=+wavefrontsize64 --stop-after=machine-sink -o - %s | FileCheck -check-prefixes=GFX10-SUNK %s
+# RUN: llc -mtriple=amdgpu9.42-amd-amdhsa --sink-insts-to-avoid-spills=1 --start-before=amdgpu-isel --stop-after=machine-sink -o - %s | FileCheck -check-prefixes=GFX9-SUNK %s
+# RUN: llc -mtriple=amdgpu10.31-amd-amdhsa --sink-insts-to-avoid-spills=1 -mattr=+wavefrontsize64 --start-before=amdgpu-isel --stop-after=machine-sink -o - %s | FileCheck -check-prefixes=GFX10-SUNK %s
---
name: test_sink_copy
diff --git a/llvm/test/CodeGen/AMDGPU/memcpy-crash-issue63986.ll b/llvm/test/CodeGen/AMDGPU/memcpy-crash-issue63986.ll
index d90c9771c3e4a..862b55d93179a 100644
--- a/llvm/test/CodeGen/AMDGPU/memcpy-crash-issue63986.ll
+++ b/llvm/test/CodeGen/AMDGPU/memcpy-crash-issue63986.ll
@@ -190,6 +190,7 @@ define void @issue63986_reduced_expanded(i64 %idxprom) {
; CHECK-NEXT: v_mov_b32_e32 v0, s4
; CHECK-NEXT: v_mov_b32_e32 v1, s5
; CHECK-NEXT: .LBB1_6: ; %loop-memcpy-residual
+; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: s_add_i32 s6, s8, 1
; CHECK-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[0:1]
; CHECK-NEXT: s_mov_b64 s[8:9], 1
@@ -205,6 +206,7 @@ define void @issue63986_reduced_expanded(i64 %idxprom) {
; CHECK-NEXT: s_and_b64 vcc, exec, 0
; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[2:5]
; CHECK-NEXT: .LBB1_9: ; %loop-memcpy-expansion2
+; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: s_mov_b64 vcc, vcc
; CHECK-NEXT: s_cbranch_vccz .LBB1_9
; CHECK-NEXT: ; %bb.10: ; %DummyReturnBlock
diff --git a/llvm/test/CodeGen/AMDGPU/memcpy_const_compare.ll b/llvm/test/CodeGen/AMDGPU/memcpy_const_compare.ll
index bb9011a3c43d7..b2373fd33fd54 100644
--- a/llvm/test/CodeGen/AMDGPU/memcpy_const_compare.ll
+++ b/llvm/test/CodeGen/AMDGPU/memcpy_const_compare.ll
@@ -40,26 +40,23 @@ define amdgpu_kernel void @_start() {
; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[2:5]
; CHECK-NEXT: s_cbranch_vccnz .LBB0_1
; CHECK-NEXT: ; %bb.2: ; %dynamic-memcpy-expansion-residual-cond
-; FIXME: Compare should be evaluated at compile time
; CHECK-NEXT: s_cmp_eq_u64 13, 0
; CHECK-NEXT: s_cbranch_scc1 .LBB0_5
; CHECK-NEXT: ; %bb.3: ; %dynamic-memcpy-expansion-residual-body.preheader
-; CHECK-NEXT: s_sub_u32 s4, 29, 13
-; CHECK-NEXT: s_subb_u32 s5, 0, 0
; CHECK-NEXT: s_getpc_b64 s[0:1]
; CHECK-NEXT: s_add_u32 s0, s0, src_array at gotpcrel32@lo+4
; CHECK-NEXT: s_addc_u32 s1, s1, src_array at gotpcrel32@hi+12
; CHECK-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: s_add_u32 s2, s0, s4
-; CHECK-NEXT: s_addc_u32 s3, s1, s5
+; CHECK-NEXT: s_add_u32 s2, s0, 16
+; CHECK-NEXT: s_addc_u32 s3, s1, 0
; CHECK-NEXT: s_getpc_b64 s[0:1]
; CHECK-NEXT: s_add_u32 s0, s0, dst_array at gotpcrel32@lo+4
; CHECK-NEXT: s_addc_u32 s1, s1, dst_array at gotpcrel32@hi+12
; CHECK-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: s_add_u32 s4, s0, s4
-; CHECK-NEXT: s_addc_u32 s5, s1, s5
+; CHECK-NEXT: s_add_u32 s4, s0, 16
+; CHECK-NEXT: s_addc_u32 s5, s1, 0
; CHECK-NEXT: s_mov_b64 s[0:1], 0
; CHECK-NEXT: .LBB0_4: ; %dynamic-memcpy-expansion-residual-body
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -79,6 +76,7 @@ define amdgpu_kernel void @_start() {
; CHECK-NEXT: s_cbranch_vccnz .LBB0_4
; CHECK-NEXT: .LBB0_5: ; %dynamic-memcpy-post-expansion
; CHECK-NEXT: s_endpgm
+; FIXME: Compare should be evaluated at compile time
%src_ptr = getelementptr inbounds [128 x i8], ptr @src_array, i64 0, i64 0
%dst_ptr = getelementptr inbounds [128 x i8], ptr @dst_array, i64 0, i64 0
call void @llvm.memcpy.p0.p0.i64(ptr %dst_ptr, ptr %src_ptr, i64 add (i64 sub (i64 16, i64 ptrtoint (ptr addrspacecast (ptr addrspace(4) null to ptr) to i64)), i64 13), i1 false)
diff --git a/llvm/test/CodeGen/AMDGPU/memmove-var-size.ll b/llvm/test/CodeGen/AMDGPU/memmove-var-size.ll
index eb8eb8b4f1cb2..71e50608d2ec5 100644
--- a/llvm/test/CodeGen/AMDGPU/memmove-var-size.ll
+++ b/llvm/test/CodeGen/AMDGPU/memmove-var-size.ll
@@ -1043,55 +1043,55 @@ define void @memmove_p1_p3(ptr addrspace(1) align 1 %dst, ptr addrspace(3) align
; CHECK-LABEL: memmove_p1_p3:
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_and_b32_e32 v7, -16, v3
-; CHECK-NEXT: v_mov_b32_e32 v8, v4
-; CHECK-NEXT: v_and_b32_e32 v5, 15, v3
-; CHECK-NEXT: v_mov_b32_e32 v6, 0
+; CHECK-NEXT: v_mov_b32_e32 v5, v3
+; CHECK-NEXT: v_mov_b32_e32 v7, 0
; CHECK-NEXT: s_mov_b64 s[4:5], 0
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[7:8]
+; CHECK-NEXT: v_and_b32_e32 v3, -16, v5
+; CHECK-NEXT: v_and_b32_e32 v6, 15, v5
+; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[3:4]
; CHECK-NEXT: s_cbranch_execz .LBB7_3
; CHECK-NEXT: ; %bb.1: ; %dynamic-memcpy-expansion-main-body.preheader
-; CHECK-NEXT: v_mov_b32_e32 v9, v2
+; CHECK-NEXT: v_mov_b32_e32 v8, v2
; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: .LBB7_2: ; %dynamic-memcpy-expansion-main-body
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: ds_read_b128 v[10:13], v9
-; CHECK-NEXT: v_add_co_u32 v14, vcc_lo, v0, s4
+; CHECK-NEXT: ds_read_b128 v[9:12], v8
+; CHECK-NEXT: v_add_co_u32 v13, vcc_lo, v0, s4
; CHECK-NEXT: s_add_u32 s4, s4, 16
-; CHECK-NEXT: v_add_co_ci_u32_e64 v15, null, s5, v1, vcc_lo
+; CHECK-NEXT: v_add_co_ci_u32_e64 v14, null, s5, v1, vcc_lo
; CHECK-NEXT: s_addc_u32 s5, s5, 0
-; CHECK-NEXT: v_add_nc_u32_e32 v9, 16, v9
-; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[7:8]
+; CHECK-NEXT: v_add_nc_u32_e32 v8, 16, v8
+; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[3:4]
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: global_store_dwordx4 v[14:15], v[10:13], off
+; CHECK-NEXT: global_store_dwordx4 v[13:14], v[9:12], off
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s7
; CHECK-NEXT: s_cbranch_execnz .LBB7_2
; CHECK-NEXT: .LBB7_3: ; %Flow9
; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s6
; CHECK-NEXT: s_mov_b64 s[4:5], 0
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[5:6]
+; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[6:7]
; CHECK-NEXT: s_cbranch_execz .LBB7_6
; CHECK-NEXT: ; %bb.4: ; %dynamic-memcpy-expansion-residual-body.preheader
-; CHECK-NEXT: v_and_b32_e32 v3, -16, v3
-; CHECK-NEXT: s_mov_b32 s7, 0
+; CHECK-NEXT: v_and_b32_e32 v5, -16, v5
; CHECK-NEXT: v_add_co_u32 v0, vcc_lo, v0, v3
-; CHECK-NEXT: v_add_nc_u32_e32 v2, v2, v3
; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc_lo
+; CHECK-NEXT: v_add_nc_u32_e32 v2, v2, v5
+; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: .LBB7_5: ; %dynamic-memcpy-expansion-residual-body
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: ds_read_u8 v7, v2
+; CHECK-NEXT: ds_read_u8 v5, v2
; CHECK-NEXT: v_add_co_u32 v3, vcc_lo, v0, s4
; CHECK-NEXT: s_add_u32 s4, s4, 1
; CHECK-NEXT: v_add_co_ci_u32_e64 v4, null, s5, v1, vcc_lo
; CHECK-NEXT: s_addc_u32 s5, s5, 0
; CHECK-NEXT: v_add_nc_u32_e32 v2, 1, v2
-; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[5:6]
+; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[6:7]
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: global_store_byte v[3:4], v7, off
+; CHECK-NEXT: global_store_byte v[3:4], v5, off
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s7
; CHECK-NEXT: s_cbranch_execnz .LBB7_5
; CHECK-NEXT: .LBB7_6: ; %Flow7
@@ -1252,60 +1252,60 @@ define void @memmove_p1_p5(ptr addrspace(1) align 1 %dst, ptr addrspace(5) align
; CHECK-LABEL: memmove_p1_p5:
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_and_b32_e32 v7, -16, v3
-; CHECK-NEXT: v_mov_b32_e32 v8, v4
-; CHECK-NEXT: v_and_b32_e32 v5, 15, v3
-; CHECK-NEXT: v_mov_b32_e32 v6, 0
+; CHECK-NEXT: v_mov_b32_e32 v5, v3
+; CHECK-NEXT: v_mov_b32_e32 v7, 0
; CHECK-NEXT: s_mov_b64 s[4:5], 0
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[7:8]
+; CHECK-NEXT: v_and_b32_e32 v3, -16, v5
+; CHECK-NEXT: v_and_b32_e32 v6, 15, v5
+; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[3:4]
; CHECK-NEXT: s_cbranch_execz .LBB9_3
; CHECK-NEXT: ; %bb.1: ; %dynamic-memcpy-expansion-main-body.preheader
-; CHECK-NEXT: v_mov_b32_e32 v9, v2
+; CHECK-NEXT: v_mov_b32_e32 v8, v2
; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: .p2align 6
; CHECK-NEXT: .LBB9_2: ; %dynamic-memcpy-expansion-main-body
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: s_clause 0x3
-; CHECK-NEXT: buffer_load_dword v10, v9, s[0:3], 0 offen
-; CHECK-NEXT: buffer_load_dword v11, v9, s[0:3], 0 offen offset:4
-; CHECK-NEXT: buffer_load_dword v12, v9, s[0:3], 0 offen offset:8
-; CHECK-NEXT: buffer_load_dword v13, v9, s[0:3], 0 offen offset:12
-; CHECK-NEXT: v_add_co_u32 v14, vcc_lo, v0, s4
+; CHECK-NEXT: buffer_load_dword v9, v8, s[0:3], 0 offen
+; CHECK-NEXT: buffer_load_dword v10, v8, s[0:3], 0 offen offset:4
+; CHECK-NEXT: buffer_load_dword v11, v8, s[0:3], 0 offen offset:8
+; CHECK-NEXT: buffer_load_dword v12, v8, s[0:3], 0 offen offset:12
+; CHECK-NEXT: v_add_co_u32 v13, vcc_lo, v0, s4
; CHECK-NEXT: s_add_u32 s4, s4, 16
-; CHECK-NEXT: v_add_co_ci_u32_e64 v15, null, s5, v1, vcc_lo
+; CHECK-NEXT: v_add_co_ci_u32_e64 v14, null, s5, v1, vcc_lo
; CHECK-NEXT: s_addc_u32 s5, s5, 0
-; CHECK-NEXT: v_add_nc_u32_e32 v9, 16, v9
-; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[7:8]
+; CHECK-NEXT: v_add_nc_u32_e32 v8, 16, v8
+; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[3:4]
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: global_store_dwordx4 v[14:15], v[10:13], off
+; CHECK-NEXT: global_store_dwordx4 v[13:14], v[9:12], off
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s7
; CHECK-NEXT: s_cbranch_execnz .LBB9_2
; CHECK-NEXT: .LBB9_3: ; %Flow9
; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s6
; CHECK-NEXT: s_mov_b64 s[4:5], 0
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[5:6]
+; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[6:7]
; CHECK-NEXT: s_cbranch_execz .LBB9_6
; CHECK-NEXT: ; %bb.4: ; %dynamic-memcpy-expansion-residual-body.preheader
-; CHECK-NEXT: v_and_b32_e32 v3, -16, v3
-; CHECK-NEXT: s_mov_b32 s7, 0
+; CHECK-NEXT: v_and_b32_e32 v5, -16, v5
; CHECK-NEXT: v_add_co_u32 v0, vcc_lo, v0, v3
-; CHECK-NEXT: v_add_nc_u32_e32 v2, v2, v3
; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc_lo
+; CHECK-NEXT: v_add_nc_u32_e32 v2, v2, v5
+; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: .LBB9_5: ; %dynamic-memcpy-expansion-residual-body
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: buffer_load_ubyte v7, v2, s[0:3], 0 offen
+; CHECK-NEXT: buffer_load_ubyte v5, v2, s[0:3], 0 offen
; CHECK-NEXT: v_add_co_u32 v3, vcc_lo, v0, s4
; CHECK-NEXT: s_add_u32 s4, s4, 1
; CHECK-NEXT: v_add_co_ci_u32_e64 v4, null, s5, v1, vcc_lo
; CHECK-NEXT: s_addc_u32 s5, s5, 0
; CHECK-NEXT: v_add_nc_u32_e32 v2, 1, v2
-; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[5:6]
+; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[6:7]
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: global_store_byte v[3:4], v7, off
+; CHECK-NEXT: global_store_byte v[3:4], v5, off
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s7
; CHECK-NEXT: s_cbranch_execnz .LBB9_5
; CHECK-NEXT: .LBB9_6: ; %Flow7
@@ -1465,50 +1465,50 @@ define void @memmove_p3_p1(ptr addrspace(3) align 1 %dst, ptr addrspace(1) align
; CHECK-LABEL: memmove_p3_p1:
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_and_b32_e32 v7, -16, v3
-; CHECK-NEXT: v_mov_b32_e32 v8, v4
-; CHECK-NEXT: v_and_b32_e32 v5, 15, v3
-; CHECK-NEXT: v_mov_b32_e32 v6, 0
+; CHECK-NEXT: v_mov_b32_e32 v5, v3
+; CHECK-NEXT: v_mov_b32_e32 v7, 0
; CHECK-NEXT: s_mov_b64 s[4:5], 0
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[7:8]
+; CHECK-NEXT: v_and_b32_e32 v3, -16, v5
+; CHECK-NEXT: v_and_b32_e32 v6, 15, v5
+; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[3:4]
; CHECK-NEXT: s_cbranch_execz .LBB11_3
; CHECK-NEXT: ; %bb.1: ; %dynamic-memcpy-expansion-main-body.preheader
-; CHECK-NEXT: v_mov_b32_e32 v9, v0
+; CHECK-NEXT: v_mov_b32_e32 v8, v0
; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: .LBB11_2: ; %dynamic-memcpy-expansion-main-body
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: v_add_co_u32 v10, vcc_lo, v1, s4
-; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, s5, v2, vcc_lo
+; CHECK-NEXT: v_add_co_u32 v9, vcc_lo, v1, s4
+; CHECK-NEXT: v_add_co_ci_u32_e64 v10, null, s5, v2, vcc_lo
; CHECK-NEXT: s_add_u32 s4, s4, 16
; CHECK-NEXT: s_addc_u32 s5, s5, 0
-; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[7:8]
-; CHECK-NEXT: global_load_dwordx4 v[10:13], v[10:11], off
+; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[3:4]
+; CHECK-NEXT: global_load_dwordx4 v[9:12], v[9:10], off
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: ds_write_b128 v9, v[10:13]
-; CHECK-NEXT: v_add_nc_u32_e32 v9, 16, v9
+; CHECK-NEXT: ds_write_b128 v8, v[9:12]
+; CHECK-NEXT: v_add_nc_u32_e32 v8, 16, v8
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s7
; CHECK-NEXT: s_cbranch_execnz .LBB11_2
; CHECK-NEXT: .LBB11_3: ; %Flow9
; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s6
; CHECK-NEXT: s_mov_b64 s[4:5], 0
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[5:6]
+; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[6:7]
; CHECK-NEXT: s_cbranch_execz .LBB11_6
; CHECK-NEXT: ; %bb.4: ; %dynamic-memcpy-expansion-residual-body.preheader
-; CHECK-NEXT: v_and_b32_e32 v3, -16, v3
-; CHECK-NEXT: s_mov_b32 s7, 0
+; CHECK-NEXT: v_and_b32_e32 v5, -16, v5
; CHECK-NEXT: v_add_co_u32 v1, vcc_lo, v1, v3
-; CHECK-NEXT: v_add_nc_u32_e32 v0, v0, v3
; CHECK-NEXT: v_add_co_ci_u32_e64 v2, null, v2, v4, vcc_lo
+; CHECK-NEXT: v_add_nc_u32_e32 v0, v0, v5
+; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: .LBB11_5: ; %dynamic-memcpy-expansion-residual-body
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: v_add_co_u32 v3, vcc_lo, v1, s4
; CHECK-NEXT: v_add_co_ci_u32_e64 v4, null, s5, v2, vcc_lo
; CHECK-NEXT: s_add_u32 s4, s4, 1
; CHECK-NEXT: s_addc_u32 s5, s5, 0
-; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[5:6]
+; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[6:7]
; CHECK-NEXT: global_load_ubyte v3, v[3:4], off
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_waitcnt vmcnt(0)
@@ -1656,50 +1656,50 @@ define void @memmove_p3_p4(ptr addrspace(3) align 1 %dst, ptr addrspace(4) align
; CHECK-LABEL: memmove_p3_p4:
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_and_b32_e32 v7, -16, v3
-; CHECK-NEXT: v_mov_b32_e32 v8, v4
-; CHECK-NEXT: v_and_b32_e32 v5, 15, v3
-; CHECK-NEXT: v_mov_b32_e32 v6, 0
+; CHECK-NEXT: v_mov_b32_e32 v5, v3
+; CHECK-NEXT: v_mov_b32_e32 v7, 0
; CHECK-NEXT: s_mov_b64 s[4:5], 0
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[7:8]
+; CHECK-NEXT: v_and_b32_e32 v3, -16, v5
+; CHECK-NEXT: v_and_b32_e32 v6, 15, v5
+; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[3:4]
; CHECK-NEXT: s_cbranch_execz .LBB13_3
; CHECK-NEXT: ; %bb.1: ; %dynamic-memcpy-expansion-main-body.preheader
-; CHECK-NEXT: v_mov_b32_e32 v9, v0
+; CHECK-NEXT: v_mov_b32_e32 v8, v0
; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: .LBB13_2: ; %dynamic-memcpy-expansion-main-body
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: v_add_co_u32 v10, vcc_lo, v1, s4
-; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, s5, v2, vcc_lo
+; CHECK-NEXT: v_add_co_u32 v9, vcc_lo, v1, s4
+; CHECK-NEXT: v_add_co_ci_u32_e64 v10, null, s5, v2, vcc_lo
; CHECK-NEXT: s_add_u32 s4, s4, 16
; CHECK-NEXT: s_addc_u32 s5, s5, 0
-; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[7:8]
-; CHECK-NEXT: global_load_dwordx4 v[10:13], v[10:11], off
+; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[3:4]
+; CHECK-NEXT: global_load_dwordx4 v[9:12], v[9:10], off
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: ds_write_b128 v9, v[10:13]
-; CHECK-NEXT: v_add_nc_u32_e32 v9, 16, v9
+; CHECK-NEXT: ds_write_b128 v8, v[9:12]
+; CHECK-NEXT: v_add_nc_u32_e32 v8, 16, v8
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s7
; CHECK-NEXT: s_cbranch_execnz .LBB13_2
; CHECK-NEXT: .LBB13_3: ; %Flow9
; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s6
; CHECK-NEXT: s_mov_b64 s[4:5], 0
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[5:6]
+; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[6:7]
; CHECK-NEXT: s_cbranch_execz .LBB13_6
; CHECK-NEXT: ; %bb.4: ; %dynamic-memcpy-expansion-residual-body.preheader
-; CHECK-NEXT: v_and_b32_e32 v3, -16, v3
-; CHECK-NEXT: s_mov_b32 s7, 0
+; CHECK-NEXT: v_and_b32_e32 v5, -16, v5
; CHECK-NEXT: v_add_co_u32 v1, vcc_lo, v1, v3
-; CHECK-NEXT: v_add_nc_u32_e32 v0, v0, v3
; CHECK-NEXT: v_add_co_ci_u32_e64 v2, null, v2, v4, vcc_lo
+; CHECK-NEXT: v_add_nc_u32_e32 v0, v0, v5
+; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: .LBB13_5: ; %dynamic-memcpy-expansion-residual-body
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: v_add_co_u32 v3, vcc_lo, v1, s4
; CHECK-NEXT: v_add_co_ci_u32_e64 v4, null, s5, v2, vcc_lo
; CHECK-NEXT: s_add_u32 s4, s4, 1
; CHECK-NEXT: s_addc_u32 s5, s5, 0
-; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[5:6]
+; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[6:7]
; CHECK-NEXT: global_load_ubyte v3, v[3:4], off
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_waitcnt vmcnt(0)
@@ -1936,32 +1936,32 @@ define void @memmove_p5_p1(ptr addrspace(5) align 1 %dst, ptr addrspace(1) align
; CHECK-LABEL: memmove_p5_p1:
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_and_b32_e32 v7, -16, v3
-; CHECK-NEXT: v_mov_b32_e32 v8, v4
-; CHECK-NEXT: v_and_b32_e32 v5, 15, v3
-; CHECK-NEXT: v_mov_b32_e32 v6, 0
+; CHECK-NEXT: v_mov_b32_e32 v5, v3
+; CHECK-NEXT: v_mov_b32_e32 v7, 0
; CHECK-NEXT: s_mov_b64 s[4:5], 0
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[7:8]
+; CHECK-NEXT: v_and_b32_e32 v3, -16, v5
+; CHECK-NEXT: v_and_b32_e32 v6, 15, v5
+; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[3:4]
; CHECK-NEXT: s_cbranch_execz .LBB16_3
; CHECK-NEXT: ; %bb.1: ; %dynamic-memcpy-expansion-main-body.preheader
-; CHECK-NEXT: v_mov_b32_e32 v9, v0
+; CHECK-NEXT: v_mov_b32_e32 v8, v0
; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: .p2align 6
; CHECK-NEXT: .LBB16_2: ; %dynamic-memcpy-expansion-main-body
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: v_add_co_u32 v10, vcc_lo, v1, s4
-; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, s5, v2, vcc_lo
+; CHECK-NEXT: v_add_co_u32 v9, vcc_lo, v1, s4
+; CHECK-NEXT: v_add_co_ci_u32_e64 v10, null, s5, v2, vcc_lo
; CHECK-NEXT: s_add_u32 s4, s4, 16
; CHECK-NEXT: s_addc_u32 s5, s5, 0
-; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[7:8]
-; CHECK-NEXT: global_load_dwordx4 v[10:13], v[10:11], off
+; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[3:4]
+; CHECK-NEXT: global_load_dwordx4 v[9:12], v[9:10], off
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: buffer_store_dword v13, v9, s[0:3], 0 offen offset:12
-; CHECK-NEXT: buffer_store_dword v12, v9, s[0:3], 0 offen offset:8
-; CHECK-NEXT: buffer_store_dword v11, v9, s[0:3], 0 offen offset:4
-; CHECK-NEXT: buffer_store_dword v10, v9, s[0:3], 0 offen
-; CHECK-NEXT: v_add_nc_u32_e32 v9, 16, v9
+; CHECK-NEXT: buffer_store_dword v12, v8, s[0:3], 0 offen offset:12
+; CHECK-NEXT: buffer_store_dword v11, v8, s[0:3], 0 offen offset:8
+; CHECK-NEXT: buffer_store_dword v10, v8, s[0:3], 0 offen offset:4
+; CHECK-NEXT: buffer_store_dword v9, v8, s[0:3], 0 offen
+; CHECK-NEXT: v_add_nc_u32_e32 v8, 16, v8
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s7
; CHECK-NEXT: s_cbranch_execnz .LBB16_2
@@ -1969,21 +1969,21 @@ define void @memmove_p5_p1(ptr addrspace(5) align 1 %dst, ptr addrspace(1) align
; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s6
; CHECK-NEXT: s_mov_b64 s[4:5], 0
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[5:6]
+; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[6:7]
; CHECK-NEXT: s_cbranch_execz .LBB16_6
; CHECK-NEXT: ; %bb.4: ; %dynamic-memcpy-expansion-residual-body.preheader
-; CHECK-NEXT: v_and_b32_e32 v3, -16, v3
-; CHECK-NEXT: s_mov_b32 s7, 0
+; CHECK-NEXT: v_and_b32_e32 v5, -16, v5
; CHECK-NEXT: v_add_co_u32 v1, vcc_lo, v1, v3
-; CHECK-NEXT: v_add_nc_u32_e32 v0, v0, v3
; CHECK-NEXT: v_add_co_ci_u32_e64 v2, null, v2, v4, vcc_lo
+; CHECK-NEXT: v_add_nc_u32_e32 v0, v0, v5
+; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: .LBB16_5: ; %dynamic-memcpy-expansion-residual-body
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: v_add_co_u32 v3, vcc_lo, v1, s4
; CHECK-NEXT: v_add_co_ci_u32_e64 v4, null, s5, v2, vcc_lo
; CHECK-NEXT: s_add_u32 s4, s4, 1
; CHECK-NEXT: s_addc_u32 s5, s5, 0
-; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[5:6]
+; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[6:7]
; CHECK-NEXT: global_load_ubyte v3, v[3:4], off
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_waitcnt vmcnt(0)
@@ -2068,32 +2068,32 @@ define void @memmove_p5_p4(ptr addrspace(5) align 1 %dst, ptr addrspace(4) align
; CHECK-LABEL: memmove_p5_p4:
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_and_b32_e32 v7, -16, v3
-; CHECK-NEXT: v_mov_b32_e32 v8, v4
-; CHECK-NEXT: v_and_b32_e32 v5, 15, v3
-; CHECK-NEXT: v_mov_b32_e32 v6, 0
+; CHECK-NEXT: v_mov_b32_e32 v5, v3
+; CHECK-NEXT: v_mov_b32_e32 v7, 0
; CHECK-NEXT: s_mov_b64 s[4:5], 0
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[7:8]
+; CHECK-NEXT: v_and_b32_e32 v3, -16, v5
+; CHECK-NEXT: v_and_b32_e32 v6, 15, v5
+; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[3:4]
; CHECK-NEXT: s_cbranch_execz .LBB18_3
; CHECK-NEXT: ; %bb.1: ; %dynamic-memcpy-expansion-main-body.preheader
-; CHECK-NEXT: v_mov_b32_e32 v9, v0
+; CHECK-NEXT: v_mov_b32_e32 v8, v0
; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: .p2align 6
; CHECK-NEXT: .LBB18_2: ; %dynamic-memcpy-expansion-main-body
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: v_add_co_u32 v10, vcc_lo, v1, s4
-; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, s5, v2, vcc_lo
+; CHECK-NEXT: v_add_co_u32 v9, vcc_lo, v1, s4
+; CHECK-NEXT: v_add_co_ci_u32_e64 v10, null, s5, v2, vcc_lo
; CHECK-NEXT: s_add_u32 s4, s4, 16
; CHECK-NEXT: s_addc_u32 s5, s5, 0
-; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[7:8]
-; CHECK-NEXT: global_load_dwordx4 v[10:13], v[10:11], off
+; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[3:4]
+; CHECK-NEXT: global_load_dwordx4 v[9:12], v[9:10], off
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: buffer_store_dword v13, v9, s[0:3], 0 offen offset:12
-; CHECK-NEXT: buffer_store_dword v12, v9, s[0:3], 0 offen offset:8
-; CHECK-NEXT: buffer_store_dword v11, v9, s[0:3], 0 offen offset:4
-; CHECK-NEXT: buffer_store_dword v10, v9, s[0:3], 0 offen
-; CHECK-NEXT: v_add_nc_u32_e32 v9, 16, v9
+; CHECK-NEXT: buffer_store_dword v12, v8, s[0:3], 0 offen offset:12
+; CHECK-NEXT: buffer_store_dword v11, v8, s[0:3], 0 offen offset:8
+; CHECK-NEXT: buffer_store_dword v10, v8, s[0:3], 0 offen offset:4
+; CHECK-NEXT: buffer_store_dword v9, v8, s[0:3], 0 offen
+; CHECK-NEXT: v_add_nc_u32_e32 v8, 16, v8
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s7
; CHECK-NEXT: s_cbranch_execnz .LBB18_2
@@ -2101,21 +2101,21 @@ define void @memmove_p5_p4(ptr addrspace(5) align 1 %dst, ptr addrspace(4) align
; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s6
; CHECK-NEXT: s_mov_b64 s[4:5], 0
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[5:6]
+; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[6:7]
; CHECK-NEXT: s_cbranch_execz .LBB18_6
; CHECK-NEXT: ; %bb.4: ; %dynamic-memcpy-expansion-residual-body.preheader
-; CHECK-NEXT: v_and_b32_e32 v3, -16, v3
-; CHECK-NEXT: s_mov_b32 s7, 0
+; CHECK-NEXT: v_and_b32_e32 v5, -16, v5
; CHECK-NEXT: v_add_co_u32 v1, vcc_lo, v1, v3
-; CHECK-NEXT: v_add_nc_u32_e32 v0, v0, v3
; CHECK-NEXT: v_add_co_ci_u32_e64 v2, null, v2, v4, vcc_lo
+; CHECK-NEXT: v_add_nc_u32_e32 v0, v0, v5
+; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: .LBB18_5: ; %dynamic-memcpy-expansion-residual-body
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: v_add_co_u32 v3, vcc_lo, v1, s4
; CHECK-NEXT: v_add_co_ci_u32_e64 v4, null, s5, v2, vcc_lo
; CHECK-NEXT: s_add_u32 s4, s4, 1
; CHECK-NEXT: s_addc_u32 s5, s5, 0
-; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[5:6]
+; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[6:7]
; CHECK-NEXT: global_load_ubyte v3, v[3:4], off
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/memset-param-combinations.ll b/llvm/test/CodeGen/AMDGPU/memset-param-combinations.ll
index 14f56571ca056..22c0ed2c87696 100644
--- a/llvm/test/CodeGen/AMDGPU/memset-param-combinations.ll
+++ b/llvm/test/CodeGen/AMDGPU/memset-param-combinations.ll
@@ -7,14 +7,12 @@ define void @memset_p0_varsize_align_4_varsetval(ptr addrspace(0) align 4 %dst,
; GFX942-SDAG-LABEL: memset_p0_varsize_align_4_varsetval:
; GFX942-SDAG: ; %bb.0: ; %entry
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v10, v3
-; GFX942-SDAG-NEXT: v_and_b32_e32 v12, -16, v10
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v13, v4
; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, v4
-; GFX942-SDAG-NEXT: v_and_b32_e32 v8, 15, v10
+; GFX942-SDAG-NEXT: v_and_b32_e32 v10, -16, v3
+; GFX942-SDAG-NEXT: v_and_b32_e32 v8, 15, v3
; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, 0
; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[12:13]
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-SDAG-NEXT: s_cbranch_execz .LBB0_3
; GFX942-SDAG-NEXT: ; %bb.1: ; %dynamic-memset-expansion-main-body.preheader
@@ -26,12 +24,12 @@ define void @memset_p0_varsize_align_4_varsetval(ptr addrspace(0) align 4 %dst,
; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
; GFX942-SDAG-NEXT: .LBB0_2: ; %dynamic-memset-expansion-main-body
; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[14:15], v[0:1], 0, s[4:5]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[12:13], v[0:1], 0, s[4:5]
; GFX942-SDAG-NEXT: s_add_u32 s4, s4, 16
; GFX942-SDAG-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[12:13]
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[10:11]
; GFX942-SDAG-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[14:15], v[4:7]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[12:13], v[4:7]
; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-SDAG-NEXT: s_cbranch_execnz .LBB0_2
; GFX942-SDAG-NEXT: .LBB0_3: ; %Flow4
@@ -41,7 +39,6 @@ define void @memset_p0_varsize_align_4_varsetval(ptr addrspace(0) align 4 %dst,
; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-SDAG-NEXT: s_cbranch_execz .LBB0_6
; GFX942-SDAG-NEXT: ; %bb.4: ; %dynamic-memset-expansion-residual-body.preheader
-; GFX942-SDAG-NEXT: v_and_b32_e32 v10, -16, v10
; GFX942-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, v[10:11]
; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
; GFX942-SDAG-NEXT: .LBB0_5: ; %dynamic-memset-expansion-residual-body
@@ -132,14 +129,12 @@ define void @memset_p1_varsize_align_4_varsetval(ptr addrspace(1) align 4 %dst,
; GFX942-SDAG-LABEL: memset_p1_varsize_align_4_varsetval:
; GFX942-SDAG: ; %bb.0: ; %entry
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v10, v3
-; GFX942-SDAG-NEXT: v_and_b32_e32 v12, -16, v10
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v13, v4
; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, v4
-; GFX942-SDAG-NEXT: v_and_b32_e32 v8, 15, v10
+; GFX942-SDAG-NEXT: v_and_b32_e32 v10, -16, v3
+; GFX942-SDAG-NEXT: v_and_b32_e32 v8, 15, v3
; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, 0
; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[12:13]
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-SDAG-NEXT: s_cbranch_execz .LBB1_3
; GFX942-SDAG-NEXT: ; %bb.1: ; %dynamic-memset-expansion-main-body.preheader
@@ -151,12 +146,12 @@ define void @memset_p1_varsize_align_4_varsetval(ptr addrspace(1) align 4 %dst,
; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
; GFX942-SDAG-NEXT: .LBB1_2: ; %dynamic-memset-expansion-main-body
; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[14:15], v[0:1], 0, s[4:5]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[12:13], v[0:1], 0, s[4:5]
; GFX942-SDAG-NEXT: s_add_u32 s4, s4, 16
; GFX942-SDAG-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[12:13]
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[10:11]
; GFX942-SDAG-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[14:15], v[4:7], off
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[12:13], v[4:7], off
; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-SDAG-NEXT: s_cbranch_execnz .LBB1_2
; GFX942-SDAG-NEXT: .LBB1_3: ; %Flow4
@@ -166,7 +161,6 @@ define void @memset_p1_varsize_align_4_varsetval(ptr addrspace(1) align 4 %dst,
; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-SDAG-NEXT: s_cbranch_execz .LBB1_6
; GFX942-SDAG-NEXT: ; %bb.4: ; %dynamic-memset-expansion-residual-body.preheader
-; GFX942-SDAG-NEXT: v_and_b32_e32 v10, -16, v10
; GFX942-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, v[10:11]
; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
; GFX942-SDAG-NEXT: .LBB1_5: ; %dynamic-memset-expansion-residual-body
@@ -1650,12 +1644,11 @@ define void @memset_p1_varsz_align_4_set40(ptr addrspace(1) align 4 %dst, i64 %s
; GFX942-SDAG-LABEL: memset_p1_varsz_align_4_set40:
; GFX942-SDAG: ; %bb.0: ; %entry
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-SDAG-NEXT: v_and_b32_e32 v10, -16, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, v3
; GFX942-SDAG-NEXT: v_and_b32_e32 v8, 15, v2
+; GFX942-SDAG-NEXT: v_and_b32_e32 v2, -16, v2
; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, 0
; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-SDAG-NEXT: s_cbranch_execz .LBB12_3
; GFX942-SDAG-NEXT: ; %bb.1: ; %dynamic-memset-expansion-main-body.preheader
@@ -1666,12 +1659,12 @@ define void @memset_p1_varsz_align_4_set40(ptr addrspace(1) align 4 %dst, i64 %s
; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
; GFX942-SDAG-NEXT: .LBB12_2: ; %dynamic-memset-expansion-main-body
; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[12:13], v[0:1], 0, s[4:5]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[10:11], v[0:1], 0, s[4:5]
; GFX942-SDAG-NEXT: s_add_u32 s4, s4, 16
; GFX942-SDAG-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[10:11]
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[2:3]
; GFX942-SDAG-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[12:13], v[4:7], off
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[10:11], v[4:7], off
; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-SDAG-NEXT: s_cbranch_execnz .LBB12_2
; GFX942-SDAG-NEXT: .LBB12_3: ; %Flow4
@@ -1681,7 +1674,6 @@ define void @memset_p1_varsz_align_4_set40(ptr addrspace(1) align 4 %dst, i64 %s
; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-SDAG-NEXT: s_cbranch_execz .LBB12_6
; GFX942-SDAG-NEXT: ; %bb.4: ; %dynamic-memset-expansion-residual-body.preheader
-; GFX942-SDAG-NEXT: v_and_b32_e32 v2, -16, v2
; GFX942-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, v[2:3]
; GFX942-SDAG-NEXT: v_mov_b32_e32 v2, 40
; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
@@ -1770,28 +1762,27 @@ define void @memset_p1_varsz_align_4_set0(ptr addrspace(1) align 4 %dst, i64 %si
; GFX942-SDAG-LABEL: memset_p1_varsz_align_4_set0:
; GFX942-SDAG: ; %bb.0: ; %entry
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-SDAG-NEXT: v_and_b32_e32 v6, -16, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v3
; GFX942-SDAG-NEXT: v_and_b32_e32 v4, 15, v2
+; GFX942-SDAG-NEXT: v_and_b32_e32 v2, -16, v2
; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, 0
; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-SDAG-NEXT: s_cbranch_execz .LBB13_3
; GFX942-SDAG-NEXT: ; %bb.1: ; %dynamic-memset-expansion-main-body.preheader
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v6, v5
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v5
; GFX942-SDAG-NEXT: v_mov_b32_e32 v8, v5
; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, v5
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v10, v5
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, v5
; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
; GFX942-SDAG-NEXT: .LBB13_2: ; %dynamic-memset-expansion-main-body
; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[12:13], v[0:1], 0, s[4:5]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[10:11], v[0:1], 0, s[4:5]
; GFX942-SDAG-NEXT: s_add_u32 s4, s4, 16
; GFX942-SDAG-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[6:7]
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[2:3]
; GFX942-SDAG-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[12:13], v[8:11], off
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[10:11], v[6:9], off
; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-SDAG-NEXT: s_cbranch_execnz .LBB13_2
; GFX942-SDAG-NEXT: .LBB13_3: ; %Flow4
@@ -1801,7 +1792,6 @@ define void @memset_p1_varsz_align_4_set0(ptr addrspace(1) align 4 %dst, i64 %si
; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-SDAG-NEXT: s_cbranch_execz .LBB13_6
; GFX942-SDAG-NEXT: ; %bb.4: ; %dynamic-memset-expansion-residual-body.preheader
-; GFX942-SDAG-NEXT: v_and_b32_e32 v2, -16, v2
; GFX942-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, v[2:3]
; GFX942-SDAG-NEXT: v_mov_b32_e32 v2, 0
; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
diff --git a/llvm/test/CodeGen/AMDGPU/mfma-no-register-aliasing.ll b/llvm/test/CodeGen/AMDGPU/mfma-no-register-aliasing.ll
index 07d04ea789a81..1cfaa0bba74e2 100644
--- a/llvm/test/CodeGen/AMDGPU/mfma-no-register-aliasing.ll
+++ b/llvm/test/CodeGen/AMDGPU/mfma-no-register-aliasing.ll
@@ -4,7 +4,7 @@
; RUN: llc -mtriple=amdgpu9.0a -early-live-intervals < %s | FileCheck -enable-var-scope --check-prefixes=GREEDY90A %s
; RUN: llc -mtriple=amdgpu9.42 < %s | FileCheck -enable-var-scope --check-prefixes=GREEDY942 %s
; RUN: llc -global-isel -mtriple=amdgpu9.0a < %s | FileCheck -enable-var-scope --check-prefixes=GREEDY90A-GISEL %s
-; RUN: llc -mtriple=amdgpu9.0a -sgpr-regalloc=fast -vgpr-regalloc=fast -wwm-regalloc=fast < %s | FileCheck -enable-var-scope --check-prefixes=FAST90A %s
+; RUN: llc -mtriple=amdgpu9.0a -sgpr-regalloc-npm=fast -vgpr-regalloc-npm=fast -wwm-regalloc-npm=fast < %s | FileCheck -enable-var-scope --check-prefixes=FAST90A %s
; This is better with 90a
diff --git a/llvm/test/CodeGen/AMDGPU/opt-exec-masking-pre-ra-update-liveness-wave32.mir b/llvm/test/CodeGen/AMDGPU/opt-exec-masking-pre-ra-update-liveness-wave32.mir
index 09d9e90e3d710..bbee2c4f107a1 100644
--- a/llvm/test/CodeGen/AMDGPU/opt-exec-masking-pre-ra-update-liveness-wave32.mir
+++ b/llvm/test/CodeGen/AMDGPU/opt-exec-masking-pre-ra-update-liveness-wave32.mir
@@ -1,6 +1,6 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
# REQUIRES: asserts
-# RUN: llc -mtriple=amdgpu10.10-amd-amdhsa -verify-regalloc -misched-only-block=999 -start-before=machine-scheduler -stop-after=greedy,0 -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu10.10-amd-amdhsa -verify-regalloc -misched-only-block=999 -start-before=machine-scheduler -stop-after=greedy,1 -o - %s | FileCheck %s
# This run line is a total hack to get the live intervals to make it
# to the verifier. This requires asserts to use
diff --git a/llvm/test/CodeGen/AMDGPU/opt-exec-masking-pre-ra-update-liveness.mir b/llvm/test/CodeGen/AMDGPU/opt-exec-masking-pre-ra-update-liveness.mir
index 4d32e22218cfc..3a416cd43fce6 100644
--- a/llvm/test/CodeGen/AMDGPU/opt-exec-masking-pre-ra-update-liveness.mir
+++ b/llvm/test/CodeGen/AMDGPU/opt-exec-masking-pre-ra-update-liveness.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -verify-regalloc -start-before=machine-scheduler -stop-after=greedy,0 -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -verify-regalloc -start-before=machine-scheduler -stop-after=greedy,1 -o - %s | FileCheck %s
# Make sure liveness is correctly updated when folding the cndmask and
# compare.
diff --git a/llvm/test/CodeGen/AMDGPU/optimize-exec-mask-pre-ra-alloc-failure.mir b/llvm/test/CodeGen/AMDGPU/optimize-exec-mask-pre-ra-alloc-failure.mir
index df83b74042ad2..91a9219548dc9 100644
--- a/llvm/test/CodeGen/AMDGPU/optimize-exec-mask-pre-ra-alloc-failure.mir
+++ b/llvm/test/CodeGen/AMDGPU/optimize-exec-mask-pre-ra-alloc-failure.mir
@@ -1,4 +1,4 @@
-# RUN: llc -mtriple=amdgpu10.30-amd-amdhsa -start-before=machine-scheduler -stop-after=virtregrewriter,0 -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu10.30-amd-amdhsa -start-before=machine-scheduler -stop-after=virt-reg-rewriter,1 -o - %s | FileCheck %s
---
# If optimize-exec-mask-pre-ra over approximates live intervals (not replicating splits)
diff --git a/llvm/test/CodeGen/AMDGPU/partial-regcopy-and-spill-missed-at-regalloc.ll b/llvm/test/CodeGen/AMDGPU/partial-regcopy-and-spill-missed-at-regalloc.ll
index 9431658b0e6b5..b9bd46bc0ae9f 100644
--- a/llvm/test/CodeGen/AMDGPU/partial-regcopy-and-spill-missed-at-regalloc.ll
+++ b/llvm/test/CodeGen/AMDGPU/partial-regcopy-and-spill-missed-at-regalloc.ll
@@ -1,7 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-;RUN: llc -mtriple=amdgpu9.08-amd-amdhsa --stop-after=greedy,2 < %s | FileCheck -check-prefix=REGALLOC-GFX908 %s
+;RUN: llc -mtriple=amdgpu9.08-amd-amdhsa --stop-after=greedy,3 < %s | FileCheck -check-prefix=REGALLOC-GFX908 %s
;RUN: llc -mtriple=amdgpu9.08-amd-amdhsa --stop-after=prolog-epilog < %s | FileCheck -check-prefix=PEI-GFX908 %s
-;RUN: llc -mtriple=amdgpu9.0a-amd-amdhsa -amdgpu-mfma-vgpr-form=0 --stop-after=greedy,2 < %s | FileCheck -check-prefix=REGALLOC-GFX90A %s
+;RUN: llc -mtriple=amdgpu9.0a-amd-amdhsa -amdgpu-mfma-vgpr-form=0 --stop-after=greedy,3 < %s | FileCheck -check-prefix=REGALLOC-GFX90A %s
;RUN: llc -mtriple=amdgpu9.0a-amd-amdhsa -amdgpu-mfma-vgpr-form=0 --stop-after=prolog-epilog < %s | FileCheck -check-prefix=PEI-GFX90A %s
; Partial reg copy and spill missed during regalloc handled later at frame lowering.
diff --git a/llvm/test/CodeGen/AMDGPU/pr51516.mir b/llvm/test/CodeGen/AMDGPU/pr51516.mir
index 033656354f2c1..404e4cdc13340 100644
--- a/llvm/test/CodeGen/AMDGPU/pr51516.mir
+++ b/llvm/test/CodeGen/AMDGPU/pr51516.mir
@@ -1,6 +1,6 @@
# NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-# RUN: llc -mtriple=amdgpu9.00--amdhsa -amdgpu-disable-unclustered-high-rp-reschedule -verify-misched -start-before=machine-scheduler -stop-after=virtregrewriter,2 -o - %s | FileCheck -check-prefix=GCN %s
-# RUN: llc -mtriple=amdgpu9.00--amdhsa -amdgpu-disable-unclustered-high-rp-reschedule -amdgpu-use-amdgpu-trackers=1 -verify-misched -start-before=machine-scheduler -stop-after=virtregrewriter,2 -o - %s | FileCheck -check-prefix=GCN-GCNTRACKER %s
+# RUN: llc -mtriple=amdgpu9.00--amdhsa -amdgpu-disable-unclustered-high-rp-reschedule -verify-misched -start-before=machine-scheduler -stop-after=virt-reg-rewriter,3 -o - %s | FileCheck -check-prefix=GCN %s
+# RUN: llc -mtriple=amdgpu9.00--amdhsa -amdgpu-disable-unclustered-high-rp-reschedule -amdgpu-use-amdgpu-trackers=1 -verify-misched -start-before=machine-scheduler -stop-after=virt-reg-rewriter,3 -o - %s | FileCheck -check-prefix=GCN-GCNTRACKER %s
# Check that %3 was not rematerialized before the last store since its operand %1
# is killed by that store.
diff --git a/llvm/test/CodeGen/AMDGPU/ran-out-of-registers-error-all-regs-reserved.ll b/llvm/test/CodeGen/AMDGPU/ran-out-of-registers-error-all-regs-reserved.ll
index 0825688cb03c0..c6624a1dbf93e 100644
--- a/llvm/test/CodeGen/AMDGPU/ran-out-of-registers-error-all-regs-reserved.ll
+++ b/llvm/test/CodeGen/AMDGPU/ran-out-of-registers-error-all-regs-reserved.ll
@@ -1,6 +1,6 @@
-; RUN: not llc -mtriple=amdgpu9.08-amd-amdhsa -vgpr-regalloc=greedy -filetype=null %s 2>&1 | FileCheck -implicit-check-not=error %s
-; RUN: not llc -mtriple=amdgpu9.08-amd-amdhsa -vgpr-regalloc=basic -filetype=null %s 2>&1 | FileCheck -implicit-check-not=error %s
-; RUN: not llc -mtriple=amdgpu9.08-amd-amdhsa -vgpr-regalloc=fast -filetype=null %s 2>&1 | FileCheck -implicit-check-not=error %s
+; RUN: not llc -mtriple=amdgpu9.08-amd-amdhsa -vgpr-regalloc-npm=greedy -filetype=null %s 2>&1 | FileCheck -implicit-check-not=error %s
+; RUN: not llc -enable-new-pm=0 -mtriple=amdgpu9.08-amd-amdhsa -vgpr-regalloc=basic -filetype=null %s 2>&1 | FileCheck -implicit-check-not=error %s
+; RUN: not llc -mtriple=amdgpu9.08-amd-amdhsa -vgpr-regalloc-npm=fast -filetype=null %s 2>&1 | FileCheck -implicit-check-not=error %s
declare <32 x i32> @llvm.amdgcn.mfma.i32.32x32x4i8(i32, i32, <32 x i32>, i32 immarg, i32 immarg, i32 immarg)
diff --git a/llvm/test/CodeGen/AMDGPU/ran-out-of-registers-errors.ll b/llvm/test/CodeGen/AMDGPU/ran-out-of-registers-errors.ll
index 11b5363aa7ca1..dc2f3fb6bd447 100644
--- a/llvm/test/CodeGen/AMDGPU/ran-out-of-registers-errors.ll
+++ b/llvm/test/CodeGen/AMDGPU/ran-out-of-registers-errors.ll
@@ -1,9 +1,9 @@
-; RUN: not llc -mtriple=amdgpu9.08-amd-amdhsa -stress-regalloc=1 -vgpr-regalloc=greedy -filetype=null %s 2>&1 | FileCheck -check-prefixes=CHECK,GREEDY -implicit-check-not=error %s
-; RUN: not llc -mtriple=amdgpu9.08-amd-amdhsa -stress-regalloc=1 -vgpr-regalloc=basic -filetype=null %s 2>&1 | FileCheck -implicit-check-not=error -check-prefixes=CHECK,BASIC %s
-; RUN: not llc -mtriple=amdgpu9.08-amd-amdhsa -stress-regalloc=1 -vgpr-regalloc=fast -filetype=null %s 2>&1 | FileCheck -implicit-check-not=error -check-prefixes=CHECK,FAST %s
+; RUN: not llc -mtriple=amdgpu9.08-amd-amdhsa -stress-regalloc=1 -vgpr-regalloc-npm=greedy -filetype=null %s 2>&1 | FileCheck -check-prefixes=CHECK,GREEDY -implicit-check-not=error %s
+; RUN: not llc -enable-new-pm=0 -mtriple=amdgpu9.08-amd-amdhsa -stress-regalloc=1 -vgpr-regalloc=basic -filetype=null %s 2>&1 | FileCheck -implicit-check-not=error -check-prefixes=CHECK,BASIC %s
+; RUN: not llc -mtriple=amdgpu9.08-amd-amdhsa -stress-regalloc=1 -vgpr-regalloc-npm=fast -filetype=null %s 2>&1 | FileCheck -implicit-check-not=error -check-prefixes=CHECK,FAST %s
; RUN: opt -passes=debugify -o %t.bc %s
-; RUN: not llc -mtriple=amdgpu9.08-amd-amdhsa -stress-regalloc=1 -vgpr-regalloc=greedy -filetype=null %t.bc 2>&1 | FileCheck -implicit-check-not=error -check-prefixes=DBGINFO-CHECK,DBGINFO-GREEDY %s
-; RUN: not llc -mtriple=amdgpu9.08-amd-amdhsa -stress-regalloc=1 -vgpr-regalloc=basic -filetype=null %t.bc 2>&1 | FileCheck -implicit-check-not=error -check-prefixes=DBGINFO-CHECK,DBGINFO-BASIC %s
+; RUN: not llc -mtriple=amdgpu9.08-amd-amdhsa -stress-regalloc=1 -vgpr-regalloc-npm=greedy -filetype=null %t.bc 2>&1 | FileCheck -implicit-check-not=error -check-prefixes=DBGINFO-CHECK,DBGINFO-GREEDY %s
+; RUN: not llc -enable-new-pm=0 -mtriple=amdgpu9.08-amd-amdhsa -stress-regalloc=1 -vgpr-regalloc=basic -filetype=null %t.bc 2>&1 | FileCheck -implicit-check-not=error -check-prefixes=DBGINFO-CHECK,DBGINFO-BASIC %s
; FIXME: Asserts when using -O2 + -vgpr-regalloc=fast
; RUN: not llc -mtriple=amdgpu9.08-amd-amdhsa -stress-regalloc=1 -O0 -filetype=null %t.bc 2>&1 | FileCheck -implicit-check-not=error -check-prefixes=DBGINFO-CHECK,DBGINFO-FAST %s
diff --git a/llvm/test/CodeGen/AMDGPU/ran-out-of-sgprs-allocation-failure.mir b/llvm/test/CodeGen/AMDGPU/ran-out-of-sgprs-allocation-failure.mir
index 7ecbecec82898..878bc6db21569 100644
--- a/llvm/test/CodeGen/AMDGPU/ran-out-of-sgprs-allocation-failure.mir
+++ b/llvm/test/CodeGen/AMDGPU/ran-out-of-sgprs-allocation-failure.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu9.0a-amd-amdhsa -verify-regalloc -start-before=greedy,0 -stop-after=virtregrewriter,0 -greedy-regclass-priority-trumps-globalness=1 -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.0a-amd-amdhsa -verify-regalloc -start-before=greedy,1 -stop-after=virt-reg-rewriter,1 -greedy-regclass-priority-trumps-globalness=1 -o - %s | FileCheck %s
# The allocation would previously fail due to poor ordering based on
# register class. The super wide tuples should be allocated first so
diff --git a/llvm/test/CodeGen/AMDGPU/recursive-resource-usage-mcexpr.ll b/llvm/test/CodeGen/AMDGPU/recursive-resource-usage-mcexpr.ll
index b26e2784fab17..329b3d4d2ceab 100644
--- a/llvm/test/CodeGen/AMDGPU/recursive-resource-usage-mcexpr.ll
+++ b/llvm/test/CodeGen/AMDGPU/recursive-resource-usage-mcexpr.ll
@@ -2,27 +2,19 @@
; Recursion: foo -> bar -> baz -> qux -> foo
-; CHECK-LABEL: {{^}}qux
-; CHECK: .set .Lqux.num_vgpr, max(71, .Lfoo.num_vgpr)
-; CHECK: .set .Lqux.num_agpr, max(0, .Lfoo.num_agpr)
-; CHECK: .set .Lqux.numbered_sgpr, max(46, .Lfoo.numbered_sgpr)
-; CHECK: .set .Lqux.private_seg_size, 16
-; CHECK: .set .Lqux.uses_vcc, or(1, .Lfoo.uses_vcc)
-; CHECK: .set .Lqux.uses_flat_scratch, or(0, .Lfoo.uses_flat_scratch)
-; CHECK: .set .Lqux.has_dyn_sized_stack, or(0, .Lfoo.has_dyn_sized_stack)
-; CHECK: .set .Lqux.has_recursion, or(1, .Lfoo.has_recursion)
-; CHECK: .set .Lqux.has_indirect_call, or(0, .Lfoo.has_indirect_call)
-
-; CHECK-LABEL: {{^}}baz
-; CHECK: .set .Lbaz.num_vgpr, max(61, .Lqux.num_vgpr)
-; CHECK: .set .Lbaz.num_agpr, max(0, .Lqux.num_agpr)
-; CHECK: .set .Lbaz.numbered_sgpr, max(51, .Lqux.numbered_sgpr)
-; CHECK: .set .Lbaz.private_seg_size, 16+max(.Lqux.private_seg_size)
-; CHECK: .set .Lbaz.uses_vcc, or(1, .Lqux.uses_vcc)
-; CHECK: .set .Lbaz.uses_flat_scratch, or(0, .Lqux.uses_flat_scratch)
-; CHECK: .set .Lbaz.has_dyn_sized_stack, or(0, .Lqux.has_dyn_sized_stack)
-; CHECK: .set .Lbaz.has_recursion, or(1, .Lqux.has_recursion)
-; CHECK: .set .Lbaz.has_indirect_call, or(0, .Lqux.has_indirect_call)
+; The new pass manager emits an SCC in reverse order compared to the legacy
+; pass manager: the entry (foo) is emitted first and refers to its successors
+; symbolically, while the last-emitted member (qux) is the fully-resolved base.
+; CHECK-LABEL: {{^}}foo
+; CHECK: .set .Lfoo.num_vgpr, max(46, .Lbar.num_vgpr)
+; CHECK: .set .Lfoo.num_agpr, max(0, .Lbar.num_agpr)
+; CHECK: .set .Lfoo.numbered_sgpr, max(71, .Lbar.numbered_sgpr)
+; CHECK: .set .Lfoo.private_seg_size, 16+max(.Lbar.private_seg_size)
+; CHECK: .set .Lfoo.uses_vcc, or(1, .Lbar.uses_vcc)
+; CHECK: .set .Lfoo.uses_flat_scratch, or(0, .Lbar.uses_flat_scratch)
+; CHECK: .set .Lfoo.has_dyn_sized_stack, or(0, .Lbar.has_dyn_sized_stack)
+; CHECK: .set .Lfoo.has_recursion, or(1, .Lbar.has_recursion)
+; CHECK: .set .Lfoo.has_indirect_call, or(0, .Lbar.has_indirect_call)
; CHECK-LABEL: {{^}}bar
; CHECK: .set .Lbar.num_vgpr, max(51, .Lbaz.num_vgpr)
@@ -35,16 +27,27 @@
; CHECK: .set .Lbar.has_recursion, or(1, .Lbaz.has_recursion)
; CHECK: .set .Lbar.has_indirect_call, or(0, .Lbaz.has_indirect_call)
-; CHECK-LABEL: {{^}}foo
-; CHECK: .set .Lfoo.num_vgpr, max(46, 71)
-; CHECK: .set .Lfoo.num_agpr, max(0, 0)
-; CHECK: .set .Lfoo.numbered_sgpr, max(71, 61)
-; CHECK: .set .Lfoo.private_seg_size, 16
-; CHECK: .set .Lfoo.uses_vcc, 1
-; CHECK: .set .Lfoo.uses_flat_scratch, 0
-; CHECK: .set .Lfoo.has_dyn_sized_stack, 0
-; CHECK: .set .Lfoo.has_recursion, 1
-; CHECK: .set .Lfoo.has_indirect_call, 0
+; CHECK-LABEL: {{^}}baz
+; CHECK: .set .Lbaz.num_vgpr, max(61, .Lqux.num_vgpr)
+; CHECK: .set .Lbaz.num_agpr, max(0, .Lqux.num_agpr)
+; CHECK: .set .Lbaz.numbered_sgpr, max(51, .Lqux.numbered_sgpr)
+; CHECK: .set .Lbaz.private_seg_size, 16+max(.Lqux.private_seg_size)
+; CHECK: .set .Lbaz.uses_vcc, or(1, .Lqux.uses_vcc)
+; CHECK: .set .Lbaz.uses_flat_scratch, or(0, .Lqux.uses_flat_scratch)
+; CHECK: .set .Lbaz.has_dyn_sized_stack, or(0, .Lqux.has_dyn_sized_stack)
+; CHECK: .set .Lbaz.has_recursion, or(1, .Lqux.has_recursion)
+; CHECK: .set .Lbaz.has_indirect_call, or(0, .Lqux.has_indirect_call)
+
+; CHECK-LABEL: {{^}}qux
+; CHECK: .set .Lqux.num_vgpr, max(71, 61)
+; CHECK: .set .Lqux.num_agpr, max(0, 0)
+; CHECK: .set .Lqux.numbered_sgpr, max(71, 71)
+; CHECK: .set .Lqux.private_seg_size, 16
+; CHECK: .set .Lqux.uses_vcc, 1
+; CHECK: .set .Lqux.uses_flat_scratch, 0
+; CHECK: .set .Lqux.has_dyn_sized_stack, 0
+; CHECK: .set .Lqux.has_recursion, 1
+; CHECK: .set .Lqux.has_indirect_call, 0
define void @foo() {
entry:
@@ -95,32 +98,21 @@ define amdgpu_kernel void @usefoo() {
; Recursion: A -> B -> C -> A && C -> D -> C
-; CHECK-LABEL: {{^}}D
-; CHECK: .set .LD.num_vgpr, max(71, .LC.num_vgpr)
-; CHECK: .set .LD.num_agpr, max(0, .LC.num_agpr)
-; CHECK: .set .LD.numbered_sgpr, max(71, .LC.numbered_sgpr)
-; CHECK: .set .LD.private_seg_size, 16+max(.LC.private_seg_size)
-; CHECK: .set .LD.uses_vcc, or(1, .LC.uses_vcc)
-; CHECK: .set .LD.uses_flat_scratch, or(0, .LC.uses_flat_scratch)
-; CHECK: .set .LD.has_dyn_sized_stack, or(0, .LC.has_dyn_sized_stack)
-; CHECK: .set .LD.has_recursion, or(1, .LC.has_recursion)
-; CHECK: .set .LD.has_indirect_call, or(0, .LC.has_indirect_call)
-
-; CHECK-LABEL: {{^}}C
-; CHECK: .set .LC.num_vgpr, max(42, .LA.num_vgpr, 71)
-; CHECK: .set .LC.num_agpr, max(0, .LA.num_agpr, 0)
-; CHECK: .set .LC.numbered_sgpr, max(71, .LA.numbered_sgpr, 71)
-; CHECK: .set .LC.private_seg_size, 16+max(.LA.private_seg_size)
-; CHECK: .set .LC.uses_vcc, or(1, .LA.uses_vcc)
-; CHECK: .set .LC.uses_flat_scratch, or(0, .LA.uses_flat_scratch)
-; CHECK: .set .LC.has_dyn_sized_stack, or(0, .LA.has_dyn_sized_stack)
-; CHECK: .set .LC.has_recursion, or(1, .LA.has_recursion)
-; CHECK: .set .LC.has_indirect_call, or(0, .LA.has_indirect_call)
+; CHECK-LABEL: {{^}}A
+; CHECK: .set .LA.num_vgpr, max(41, .LB.num_vgpr)
+; CHECK: .set .LA.num_agpr, max(0, .LB.num_agpr)
+; CHECK: .set .LA.numbered_sgpr, max(51, .LB.numbered_sgpr)
+; CHECK: .set .LA.private_seg_size, 16+max(.LB.private_seg_size)
+; CHECK: .set .LA.uses_vcc, or(1, .LB.uses_vcc)
+; CHECK: .set .LA.uses_flat_scratch, or(0, .LB.uses_flat_scratch)
+; CHECK: .set .LA.has_dyn_sized_stack, or(0, .LB.has_dyn_sized_stack)
+; CHECK: .set .LA.has_recursion, or(1, .LB.has_recursion)
+; CHECK: .set .LA.has_indirect_call, or(0, .LB.has_indirect_call)
; CHECK-LABEL: {{^}}B
-; CHECK: .set .LB.num_vgpr, max(42, .LC.num_vgpr)
+; CHECK: .set .LB.num_vgpr, max(41, .LC.num_vgpr)
; CHECK: .set .LB.num_agpr, max(0, .LC.num_agpr)
-; CHECK: .set .LB.numbered_sgpr, max(71, .LC.numbered_sgpr)
+; CHECK: .set .LB.numbered_sgpr, max(34, .LC.numbered_sgpr)
; CHECK: .set .LB.private_seg_size, 16+max(.LC.private_seg_size)
; CHECK: .set .LB.uses_vcc, or(1, .LC.uses_vcc)
; CHECK: .set .LB.uses_flat_scratch, or(0, .LC.uses_flat_scratch)
@@ -128,16 +120,27 @@ define amdgpu_kernel void @usefoo() {
; CHECK: .set .LB.has_recursion, or(1, .LC.has_recursion)
; CHECK: .set .LB.has_indirect_call, or(0, .LC.has_indirect_call)
-; CHECK-LABEL: {{^}}A
-; CHECK: .set .LA.num_vgpr, max(42, 71)
-; CHECK: .set .LA.num_agpr, max(0, 0)
-; CHECK: .set .LA.numbered_sgpr, max(71, 71)
-; CHECK: .set .LA.private_seg_size, 16
-; CHECK: .set .LA.uses_vcc, 1
-; CHECK: .set .LA.uses_flat_scratch, 0
-; CHECK: .set .LA.has_dyn_sized_stack, 0
-; CHECK: .set .LA.has_recursion, 1
-; CHECK: .set .LA.has_indirect_call, 0
+; CHECK-LABEL: {{^}}C
+; CHECK: .set .LC.num_vgpr, max(43, 41, .LD.num_vgpr)
+; CHECK: .set .LC.num_agpr, max(0, 0, .LD.num_agpr)
+; CHECK: .set .LC.numbered_sgpr, max(55, 51, .LD.numbered_sgpr)
+; CHECK: .set .LC.private_seg_size, 16+max(.LD.private_seg_size)
+; CHECK: .set .LC.uses_vcc, or(1, .LD.uses_vcc)
+; CHECK: .set .LC.uses_flat_scratch, or(0, .LD.uses_flat_scratch)
+; CHECK: .set .LC.has_dyn_sized_stack, or(0, .LD.has_dyn_sized_stack)
+; CHECK: .set .LC.has_recursion, or(1, .LD.has_recursion)
+; CHECK: .set .LC.has_indirect_call, or(0, .LD.has_indirect_call)
+
+; CHECK-LABEL: {{^}}D
+; CHECK: .set .LD.num_vgpr, max(71, 43)
+; CHECK: .set .LD.num_agpr, max(0, 0)
+; CHECK: .set .LD.numbered_sgpr, max(71, 55)
+; CHECK: .set .LD.private_seg_size, 16
+; CHECK: .set .LD.uses_vcc, 1
+; CHECK: .set .LD.uses_flat_scratch, 0
+; CHECK: .set .LD.has_dyn_sized_stack, 0
+; CHECK: .set .LD.has_recursion, 1
+; CHECK: .set .LD.has_indirect_call, 0
define void @A() {
call void @B()
diff --git a/llvm/test/CodeGen/AMDGPU/regalloc-failure-overlapping-insert-assert.mir b/llvm/test/CodeGen/AMDGPU/regalloc-failure-overlapping-insert-assert.mir
index 4f702f1356588..7dc4b5336b193 100644
--- a/llvm/test/CodeGen/AMDGPU/regalloc-failure-overlapping-insert-assert.mir
+++ b/llvm/test/CodeGen/AMDGPU/regalloc-failure-overlapping-insert-assert.mir
@@ -1,4 +1,4 @@
-# RUN: not llc -mtriple=amdgpu9.08 -verify-machineinstrs -start-before=greedy,1 -stop-after=virtregrewriter,2 %s -filetype=null 2>&1 | FileCheck -check-prefix=ERR %s
+# RUN: not llc -mtriple=amdgpu9.08 -verify-machineinstrs -start-before=greedy,2 -stop-after=virt-reg-rewriter,3 %s -filetype=null 2>&1 | FileCheck -check-prefix=ERR %s
# Make sure there's no machine verifier error after failure.
diff --git a/llvm/test/CodeGen/AMDGPU/regalloc-fast-dont-drop-subreg-index-issue61134.mir b/llvm/test/CodeGen/AMDGPU/regalloc-fast-dont-drop-subreg-index-issue61134.mir
index 27e0747fdd232..3f7b79275930a 100644
--- a/llvm/test/CodeGen/AMDGPU/regalloc-fast-dont-drop-subreg-index-issue61134.mir
+++ b/llvm/test/CodeGen/AMDGPU/regalloc-fast-dont-drop-subreg-index-issue61134.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -sgpr-regalloc=fast -start-before=regallocfast,0 -stop-before=greedy -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -sgpr-regalloc-npm=fast -start-before=regallocfast,1 -stop-before=greedy -o - %s | FileCheck %s
# RegAllocFast was incorrectly dropping subregister indexes on
# unassigned virtual registers. Make sure they are passed through
diff --git a/llvm/test/CodeGen/AMDGPU/regalloc-introduces-copy-sgpr-to-agpr.mir b/llvm/test/CodeGen/AMDGPU/regalloc-introduces-copy-sgpr-to-agpr.mir
index 578a1f8e261e6..f0349ddff548a 100644
--- a/llvm/test/CodeGen/AMDGPU/regalloc-introduces-copy-sgpr-to-agpr.mir
+++ b/llvm/test/CodeGen/AMDGPU/regalloc-introduces-copy-sgpr-to-agpr.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu9.08-amd-amdhsa -start-before=greedy,0 -stop-after=postrapseudos -o - %s | FileCheck -check-prefix=GFX908 %s
+# RUN: llc -mtriple=amdgpu9.08-amd-amdhsa -start-before=greedy,1 -stop-after=post-ra-pseudos -o - %s | FileCheck -check-prefix=GFX908 %s
# This testcase has a long sequence of sgpr to vgpr copies with a lot of vgpr
# pressure, encouraging the allocator to displace some into AGPRs. This
diff --git a/llvm/test/CodeGen/AMDGPU/regalloc-sgpr128-partial-def.mir b/llvm/test/CodeGen/AMDGPU/regalloc-sgpr128-partial-def.mir
index 16dd33326213b..45e20180058e2 100644
--- a/llvm/test/CodeGen/AMDGPU/regalloc-sgpr128-partial-def.mir
+++ b/llvm/test/CodeGen/AMDGPU/regalloc-sgpr128-partial-def.mir
@@ -1,4 +1,4 @@
-# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -verify-machineinstrs --stress-regalloc=2 -start-before=greedy -stop-after=virtregrewriter \
+# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -verify-machineinstrs --stress-regalloc=2 -start-before=greedy -stop-after=virt-reg-rewriter \
# RUN: %s -o - | FileCheck %s
# Test that S_MOV_B64 rematerialization is shrunk to S_MOV_B32 when only a
diff --git a/llvm/test/CodeGen/AMDGPU/regalloc-undef-copy-fold.mir b/llvm/test/CodeGen/AMDGPU/regalloc-undef-copy-fold.mir
index f9ad9ecf51a6f..4f7380565ab89 100644
--- a/llvm/test/CodeGen/AMDGPU/regalloc-undef-copy-fold.mir
+++ b/llvm/test/CodeGen/AMDGPU/regalloc-undef-copy-fold.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
-# RUN: llc -mtriple=amdgpu9.08 -verify-machineinstrs --start-before=greedy,2 --stop-after=greedy,2 %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.08 -verify-machineinstrs --start-before=greedy,3 --stop-after=greedy,3 %s -o - | FileCheck %s
# Make sure there's no machine verifier error
diff --git a/llvm/test/CodeGen/AMDGPU/register-killed-error-after-alloc-failure0.mir b/llvm/test/CodeGen/AMDGPU/register-killed-error-after-alloc-failure0.mir
index fb1866c379e5c..c92df2e1baed6 100644
--- a/llvm/test/CodeGen/AMDGPU/register-killed-error-after-alloc-failure0.mir
+++ b/llvm/test/CodeGen/AMDGPU/register-killed-error-after-alloc-failure0.mir
@@ -1,5 +1,5 @@
-# RUN: not llc -mtriple=amdgpu9.00-amd-amdhsa -vgpr-regalloc=basic -sgpr-regalloc=basic -start-before=regallocbasic,0 -stop-after=virtregrewriter,2 -verify-regalloc -o - %s 2> %t.basic.err | FileCheck -check-prefix=BASIC %s
-# RUN: not llc -mtriple=amdgpu9.00-amd-amdhsa -start-before=greedy,0 -stop-after=virtregrewriter,2 -verify-regalloc -o - %s 2> %t.greedy.err | FileCheck -check-prefix=GREEDY %s
+# RUN: not llc -enable-new-pm=0 -mtriple=amdgpu9.00-amd-amdhsa -vgpr-regalloc=basic -sgpr-regalloc=basic -start-before=regallocbasic,0 -stop-after=virtregrewriter,2 -verify-regalloc -o - %s 2> %t.basic.err | FileCheck -check-prefix=BASIC %s
+# RUN: not llc -mtriple=amdgpu9.00-amd-amdhsa -start-before=greedy,1 -stop-after=virt-reg-rewriter,3 -verify-regalloc -o - %s 2> %t.greedy.err | FileCheck -check-prefix=GREEDY %s
# RUN: FileCheck -check-prefix=ERR -implicit-check-not=error %s < %t.basic.err
# RUN: FileCheck -check-prefix=ERR -implicit-check-not=error %s < %t.greedy.err
diff --git a/llvm/test/CodeGen/AMDGPU/remaining-virtual-register-operands.mir b/llvm/test/CodeGen/AMDGPU/remaining-virtual-register-operands.mir
index 3476342177d5e..4bc178f3c5bdb 100644
--- a/llvm/test/CodeGen/AMDGPU/remaining-virtual-register-operands.mir
+++ b/llvm/test/CodeGen/AMDGPU/remaining-virtual-register-operands.mir
@@ -1,4 +1,4 @@
-# RUN: not llc -mtriple=amdgpu9.08-amd-amdhsa -verify-machineinstrs -start-before=greedy,2 -filetype=null %s 2>&1 | FileCheck %s
+# RUN: not llc -mtriple=amdgpu9.08-amd-amdhsa -verify-machineinstrs -start-before=greedy,3 -filetype=null %s 2>&1 | FileCheck %s
# This testcase fails register allocation at the same time it performs
# virtual register splitting (by introducing VGPR to AGPR copies). We
diff --git a/llvm/test/CodeGen/AMDGPU/remat-dead-subreg.mir b/llvm/test/CodeGen/AMDGPU/remat-dead-subreg.mir
index 29373f9df5d30..ba4c6e9ebb5c5 100644
--- a/llvm/test/CodeGen/AMDGPU/remat-dead-subreg.mir
+++ b/llvm/test/CodeGen/AMDGPU/remat-dead-subreg.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu6.00 -start-before=greedy -stop-after=virtregrewriter -stress-regalloc=3 -verify-machineinstrs -o - %s | FileCheck -check-prefix=GCN %s
+# RUN: llc -mtriple=amdgpu6.00 -start-before=greedy -stop-after=virt-reg-rewriter -stress-regalloc=3 -verify-machineinstrs -o - %s | FileCheck -check-prefix=GCN %s
# Check that subreg use is live at the point of materialization, not just the main range.
# Do not rematerialize if used subreg is dead at a new index.
diff --git a/llvm/test/CodeGen/AMDGPU/remat-smrd.mir b/llvm/test/CodeGen/AMDGPU/remat-smrd.mir
index efd9893c604fd..61d57e829d98b 100644
--- a/llvm/test/CodeGen/AMDGPU/remat-smrd.mir
+++ b/llvm/test/CodeGen/AMDGPU/remat-smrd.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -verify-machineinstrs --stress-regalloc=2 -start-before=greedy -stop-after=virtregrewriter -o - %s | FileCheck -check-prefix=GCN %s
+# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -verify-machineinstrs --stress-regalloc=2 -start-before=greedy -stop-after=virt-reg-rewriter -o - %s | FileCheck -check-prefix=GCN %s
# Case that should really rematerialize
---
diff --git a/llvm/test/CodeGen/AMDGPU/remat-sop.mir b/llvm/test/CodeGen/AMDGPU/remat-sop.mir
index 4cf0f7a2722bd..cedd70580d5ca 100644
--- a/llvm/test/CodeGen/AMDGPU/remat-sop.mir
+++ b/llvm/test/CodeGen/AMDGPU/remat-sop.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -verify-machineinstrs --stress-regalloc=2 -start-before=greedy -stop-after=virtregrewriter -o - %s | FileCheck -check-prefix=GCN %s
+# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -verify-machineinstrs --stress-regalloc=2 -start-before=greedy -stop-after=virt-reg-rewriter -o - %s | FileCheck -check-prefix=GCN %s
---
name: test_remat_s_mov_b32
diff --git a/llvm/test/CodeGen/AMDGPU/remat-vop.mir b/llvm/test/CodeGen/AMDGPU/remat-vop.mir
index 418b6e68a24e9..7720ee85ef6c9 100644
--- a/llvm/test/CodeGen/AMDGPU/remat-vop.mir
+++ b/llvm/test/CodeGen/AMDGPU/remat-vop.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu9.0a-amd-amdhsa -verify-regalloc --stress-regalloc=2 -start-before=greedy,0 -stop-after=virtregrewriter,1 -o - %s | FileCheck -check-prefix=GCN %s
+# RUN: llc -mtriple=amdgpu9.0a-amd-amdhsa -verify-regalloc --stress-regalloc=2 -start-before=greedy,1 -stop-after=virt-reg-rewriter,2 -o - %s | FileCheck -check-prefix=GCN %s
---
name: test_remat_v_mov_b32_e32
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-partial-reg-uses-dbg.mir b/llvm/test/CodeGen/AMDGPU/rewrite-partial-reg-uses-dbg.mir
index 8eb8050dbd9f2..8d54bb894eb97 100644
--- a/llvm/test/CodeGen/AMDGPU/rewrite-partial-reg-uses-dbg.mir
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-partial-reg-uses-dbg.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 2
-# RUN: llc -mtriple=amdgpu7.00-amd-amdhsa -amdgpu-enable-rewrite-partial-reg-uses=true -verify-machineinstrs -start-before=rename-independent-subregs -stop-after=rewrite-partial-reg-uses %s -o - | FileCheck -check-prefix=CHECK %s
+# RUN: llc -mtriple=amdgpu7.00-amd-amdhsa -amdgpu-enable-rewrite-partial-reg-uses=true -verify-machineinstrs -start-before=rename-independent-subregs -stop-after=amdgpu-rewrite-partial-reg-uses %s -o - | FileCheck -check-prefix=CHECK %s
# RUN: llc -mtriple=amdgpu7.00-amd-amdhsa -passes="rename-independent-subregs,amdgpu-rewrite-partial-reg-uses" %s -o - | FileCheck -check-prefix=CHECK %s
--- |
define void @test_vreg_96_w64() !dbg !5 {
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-partial-reg-uses-gen.mir b/llvm/test/CodeGen/AMDGPU/rewrite-partial-reg-uses-gen.mir
index 8c6e4c400b44a..78951bae47ee5 100644
--- a/llvm/test/CodeGen/AMDGPU/rewrite-partial-reg-uses-gen.mir
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-partial-reg-uses-gen.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu7.00-amd-amdhsa -amdgpu-enable-rewrite-partial-reg-uses=true -verify-machineinstrs -start-before=rename-independent-subregs -stop-after=rewrite-partial-reg-uses %s -o - | FileCheck -check-prefix=CHECK %s
+# RUN: llc -mtriple=amdgpu7.00-amd-amdhsa -amdgpu-enable-rewrite-partial-reg-uses=true -verify-machineinstrs -start-before=rename-independent-subregs -stop-after=amdgpu-rewrite-partial-reg-uses %s -o - | FileCheck -check-prefix=CHECK %s
# RUN: llc -mtriple=amdgpu7.00-amd-amdhsa -passes="rename-independent-subregs,amdgpu-rewrite-partial-reg-uses" %s -o - | FileCheck -check-prefix=CHECK %s
---
name: test_subregs_composition_vreg_1024
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-partial-reg-uses.mir b/llvm/test/CodeGen/AMDGPU/rewrite-partial-reg-uses.mir
index 67c5797c872fa..f8df0253fe5ab 100644
--- a/llvm/test/CodeGen/AMDGPU/rewrite-partial-reg-uses.mir
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-partial-reg-uses.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu7.00-amd-amdhsa -amdgpu-enable-rewrite-partial-reg-uses=true -verify-machineinstrs -start-before=rename-independent-subregs -stop-after=rewrite-partial-reg-uses %s -o - | FileCheck -check-prefix=CHECK %s
+# RUN: llc -mtriple=amdgpu7.00-amd-amdhsa -amdgpu-enable-rewrite-partial-reg-uses=true -verify-machineinstrs -start-before=rename-independent-subregs -stop-after=amdgpu-rewrite-partial-reg-uses %s -o - | FileCheck -check-prefix=CHECK %s
# RUN: llc -mtriple=amdgpu7.00-amd-amdhsa -passes="rename-independent-subregs,amdgpu-rewrite-partial-reg-uses" %s -o - | FileCheck -check-prefix=CHECK %s
---
name: test_subregs_composition_vreg_1024
diff --git a/llvm/test/CodeGen/AMDGPU/schedule-regpressure-ilp-metric-spills.mir b/llvm/test/CodeGen/AMDGPU/schedule-regpressure-ilp-metric-spills.mir
index 6d9f8b6b3c2c5..37112359130bb 100644
--- a/llvm/test/CodeGen/AMDGPU/schedule-regpressure-ilp-metric-spills.mir
+++ b/llvm/test/CodeGen/AMDGPU/schedule-regpressure-ilp-metric-spills.mir
@@ -1,5 +1,5 @@
-# RUN: llc -mtriple=amdgpu11.00 -start-before=machine-scheduler -stop-after=greedy,2 -verify-machineinstrs -o - %s | FileCheck -check-prefix=GCN %s
-# RUN: llc -mtriple=amdgpu11.00 -start-before=machine-scheduler -stop-after=greedy,2 -amdgpu-use-amdgpu-trackers=1 -verify-machineinstrs -o - %s | FileCheck -check-prefix=GCN-GCNTRACKER %s
+# RUN: llc -mtriple=amdgpu11.00 -start-before=machine-scheduler -stop-after=greedy,3 -verify-machineinstrs -o - %s | FileCheck -check-prefix=GCN %s
+# RUN: llc -mtriple=amdgpu11.00 -start-before=machine-scheduler -stop-after=greedy,3 -amdgpu-use-amdgpu-trackers=1 -verify-machineinstrs -o - %s | FileCheck -check-prefix=GCN-GCNTRACKER %s
--- |
define amdgpu_kernel void @no_sched_metric_due_to_spills() #0 {
diff --git a/llvm/test/CodeGen/AMDGPU/schedule-regpressure-no-unclustered-regions.mir b/llvm/test/CodeGen/AMDGPU/schedule-regpressure-no-unclustered-regions.mir
index 8b70b5c320eb2..7e7615f052cc1 100644
--- a/llvm/test/CodeGen/AMDGPU/schedule-regpressure-no-unclustered-regions.mir
+++ b/llvm/test/CodeGen/AMDGPU/schedule-regpressure-no-unclustered-regions.mir
@@ -1,5 +1,5 @@
# REQUIRES: asserts
-# RUN: llc -mtriple=amdgpu11.00 -start-before=machine-scheduler -stop-after=greedy,2 -stress-regalloc=4 -debug-only=machine-scheduler %s -o - 2>&1 | FileCheck %s
+# RUN: llc -mtriple=amdgpu11.00 -start-before=machine-scheduler -stop-after=greedy,3 -stress-regalloc=4 -debug-only=machine-scheduler %s -o - 2>&1 | FileCheck %s
--- |
define amdgpu_kernel void @no_sched_metric_due_to_spills() #0 {
diff --git a/llvm/test/CodeGen/AMDGPU/sgpr-regalloc-flags.ll b/llvm/test/CodeGen/AMDGPU/sgpr-regalloc-flags.ll
index d47611c3fec7e..c179a7b3951bd 100644
--- a/llvm/test/CodeGen/AMDGPU/sgpr-regalloc-flags.ll
+++ b/llvm/test/CodeGen/AMDGPU/sgpr-regalloc-flags.ll
@@ -1,16 +1,16 @@
; REQUIRES: asserts
-; RUN: llc -verify-machineinstrs=0 -mtriple=amdgpu7.00-amd-amdhsa -debug-pass=Structure -filetype=null %s 2>&1 | FileCheck -check-prefix=DEFAULT %s
-; RUN: llc -verify-machineinstrs=0 -sgpr-regalloc=greedy -wwm-regalloc=greedy -vgpr-regalloc=greedy -mtriple=amdgpu7.00-amd-amdhsa -debug-pass=Structure -filetype=null %s 2>&1 | FileCheck -check-prefix=DEFAULT %s
+; RUN: llc -enable-new-pm=0 -verify-machineinstrs=0 -mtriple=amdgpu7.00-amd-amdhsa -debug-pass=Structure -filetype=null %s 2>&1 | FileCheck -check-prefix=DEFAULT %s
+; RUN: llc -enable-new-pm=0 -verify-machineinstrs=0 -sgpr-regalloc=greedy -wwm-regalloc=greedy -vgpr-regalloc=greedy -mtriple=amdgpu7.00-amd-amdhsa -debug-pass=Structure -filetype=null %s 2>&1 | FileCheck -check-prefix=DEFAULT %s
-; RUN: llc -verify-machineinstrs=0 -O0 -mtriple=amdgpu7.00-amd-amdhsa -debug-pass=Structure -filetype=null %s 2>&1 | FileCheck -check-prefix=O0 %s
+; RUN: llc -enable-new-pm=0 -verify-machineinstrs=0 -O0 -mtriple=amdgpu7.00-amd-amdhsa -debug-pass=Structure -filetype=null %s 2>&1 | FileCheck -check-prefix=O0 %s
-; RUN: llc -verify-machineinstrs=0 -wwm-regalloc=basic -vgpr-regalloc=basic -mtriple=amdgpu7.00-amd-amdhsa -debug-pass=Structure -filetype=null %s 2>&1 | FileCheck -check-prefix=DEFAULT-BASIC %s
-; RUN: llc -verify-machineinstrs=0 -sgpr-regalloc=basic -mtriple=amdgpu7.00-amd-amdhsa -debug-pass=Structure -filetype=null %s 2>&1 | FileCheck -check-prefix=BASIC-DEFAULT %s
-; RUN: llc -verify-machineinstrs=0 -sgpr-regalloc=basic -wwm-regalloc=basic -vgpr-regalloc=basic -mtriple=amdgpu7.00-amd-amdhsa -debug-pass=Structure -filetype=null %s 2>&1 | FileCheck -check-prefix=BASIC-BASIC %s
+; RUN: llc -enable-new-pm=0 -verify-machineinstrs=0 -wwm-regalloc=basic -vgpr-regalloc=basic -mtriple=amdgpu7.00-amd-amdhsa -debug-pass=Structure -filetype=null %s 2>&1 | FileCheck -check-prefix=DEFAULT-BASIC %s
+; RUN: llc -enable-new-pm=0 -verify-machineinstrs=0 -sgpr-regalloc=basic -mtriple=amdgpu7.00-amd-amdhsa -debug-pass=Structure -filetype=null %s 2>&1 | FileCheck -check-prefix=BASIC-DEFAULT %s
+; RUN: llc -enable-new-pm=0 -verify-machineinstrs=0 -sgpr-regalloc=basic -wwm-regalloc=basic -vgpr-regalloc=basic -mtriple=amdgpu7.00-amd-amdhsa -debug-pass=Structure -filetype=null %s 2>&1 | FileCheck -check-prefix=BASIC-BASIC %s
-; RUN: not llc -verify-machineinstrs=0 -regalloc=basic -mtriple=amdgpu7.00-amd-amdhsa -debug-pass=Structure -filetype=null %s 2>&1 | FileCheck -check-prefix=REGALLOC %s
-; RUN: not llc -verify-machineinstrs=0 -regalloc=fast -O0 -mtriple=amdgpu7.00-amd-amdhsa -debug-pass=Structure -filetype=null %s 2>&1 | FileCheck -check-prefix=REGALLOC %s
+; RUN: not llc -enable-new-pm=0 -verify-machineinstrs=0 -regalloc=basic -mtriple=amdgpu7.00-amd-amdhsa -debug-pass=Structure -filetype=null %s 2>&1 | FileCheck -check-prefix=REGALLOC %s
+; RUN: not llc -enable-new-pm=0 -verify-machineinstrs=0 -regalloc=fast -O0 -mtriple=amdgpu7.00-amd-amdhsa -debug-pass=Structure -filetype=null %s 2>&1 | FileCheck -check-prefix=REGALLOC %s
; REGALLOC: -regalloc not supported with amdgcn. Use -sgpr-regalloc, -wwm-regalloc, and -vgpr-regalloc
diff --git a/llvm/test/CodeGen/AMDGPU/sgpr-spill-update-only-slot-indexes.ll b/llvm/test/CodeGen/AMDGPU/sgpr-spill-update-only-slot-indexes.ll
index 7d1ba484442ff..e4d14180dfe82 100644
--- a/llvm/test/CodeGen/AMDGPU/sgpr-spill-update-only-slot-indexes.ll
+++ b/llvm/test/CodeGen/AMDGPU/sgpr-spill-update-only-slot-indexes.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=amdgpu9.00 -sgpr-regalloc=fast -vgpr-regalloc=fast -wwm-regalloc=fast < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -mtriple=amdgpu9.00 -sgpr-regalloc-npm=fast -vgpr-regalloc-npm=fast -wwm-regalloc-npm=fast < %s | FileCheck -check-prefix=GCN %s
; Make sure there's no verifier error from improperly updated
; SlotIndexes if regalloc fast is manually used.
diff --git a/llvm/test/CodeGen/AMDGPU/shift-select.ll b/llvm/test/CodeGen/AMDGPU/shift-select.ll
index 69786d0a16786..dbc262a37f8e8 100644
--- a/llvm/test/CodeGen/AMDGPU/shift-select.ll
+++ b/llvm/test/CodeGen/AMDGPU/shift-select.ll
@@ -1,7 +1,8 @@
-; RUN: llc -mtriple=amdgpu6.00 -stop-after=instruction-select < %s | FileCheck -check-prefixes=GCN,GFX6 %s
-; RUN: llc -mtriple=amdgpu8.03 -stop-after=instruction-select < %s | FileCheck -check-prefixes=GCN,GFX8PLUS %s
-; RUN: llc -mtriple=amdgpu10.10 -stop-after=instruction-select < %s | FileCheck -check-prefixes=GCN,GFX8PLUS %s
-; RUN: llc -mtriple=amdgpu11.00 -amdgpu-enable-vopd=0 -stop-after=instruction-select < %s | FileCheck -check-prefixes=GCN,GFX8PLUS %s
+; FIXME: use NPM when GIsel passes have been ported.
+; RUN: llc -enable-new-pm=0 -mtriple=amdgpu6.00 -stop-after=instruction-select < %s | FileCheck -check-prefixes=GCN,GFX6 %s
+; RUN: llc -enable-new-pm=0 -mtriple=amdgpu8.03 -stop-after=instruction-select < %s | FileCheck -check-prefixes=GCN,GFX8PLUS %s
+; RUN: llc -enable-new-pm=0 -mtriple=amdgpu10.10 -stop-after=instruction-select < %s | FileCheck -check-prefixes=GCN,GFX8PLUS %s
+; RUN: llc -enable-new-pm=0 -mtriple=amdgpu11.00 -amdgpu-enable-vopd=0 -stop-after=instruction-select < %s | FileCheck -check-prefixes=GCN,GFX8PLUS %s
; GCN-LABEL: name: s_shl_i32
; GCN: S_LSHL_B32
diff --git a/llvm/test/CodeGen/AMDGPU/si-lower-sgpr-spills-cycle-header.mir b/llvm/test/CodeGen/AMDGPU/si-lower-sgpr-spills-cycle-header.mir
index 469a10d47fc1e..20dfa7537f4df 100644
--- a/llvm/test/CodeGen/AMDGPU/si-lower-sgpr-spills-cycle-header.mir
+++ b/llvm/test/CodeGen/AMDGPU/si-lower-sgpr-spills-cycle-header.mir
@@ -1,7 +1,7 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
# RUN: llc -mtriple=amdgpu7.00-amd-amdhsa -verify-machineinstrs -run-pass=si-lower-sgpr-spills -o - %s | FileCheck -check-prefix=SGPR-SPILL %s
-# RUN: llc -mtriple=amdgpu7.00-amd-amdhsa -O0 -start-before=si-lower-sgpr-spills -stop-after=regallocfast,1 -verify-machineinstrs %s -o - | FileCheck -check-prefix=WWM-REGALLOC %s
+# RUN: llc -mtriple=amdgpu7.00-amd-amdhsa -O0 -start-before=si-lower-sgpr-spills -stop-after=regallocfast,2 -verify-machineinstrs %s -o - | FileCheck -check-prefix=WWM-REGALLOC %s
# When SGPR spills to a virtual VGPR lane occur in both a loop header and the latch,
diff --git a/llvm/test/CodeGen/AMDGPU/si-lower-sgpr-spills-multi-entry-cycle.mir b/llvm/test/CodeGen/AMDGPU/si-lower-sgpr-spills-multi-entry-cycle.mir
index 63939ff5d69ae..e544210ed5815 100644
--- a/llvm/test/CodeGen/AMDGPU/si-lower-sgpr-spills-multi-entry-cycle.mir
+++ b/llvm/test/CodeGen/AMDGPU/si-lower-sgpr-spills-multi-entry-cycle.mir
@@ -1,6 +1,6 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
# RUN: llc -mtriple=amdgpu7.00-amd-amdhsa -verify-machineinstrs -run-pass=si-lower-sgpr-spills -o - %s | FileCheck -check-prefix=SGPR-SPILL %s
-# RUN: llc -mtriple=amdgpu7.00-amd-amdhsa -O0 -start-before=si-lower-sgpr-spills -stop-after=regallocfast,1 -verify-machineinstrs %s -o - | FileCheck -check-prefix=WWM-REGALLOC %s
+# RUN: llc -mtriple=amdgpu7.00-amd-amdhsa -O0 -start-before=si-lower-sgpr-spills -stop-after=regallocfast,2 -verify-machineinstrs %s -o - | FileCheck -check-prefix=WWM-REGALLOC %s
# Ensure that for a multi-entry cycle si-lower-sgpr-spills inserts
# IMPLICIT_DEF into the NCD of the cycle's entries.
diff --git a/llvm/test/CodeGen/AMDGPU/si-lower-sgpr-spills-vgpr-lanes-usage.mir b/llvm/test/CodeGen/AMDGPU/si-lower-sgpr-spills-vgpr-lanes-usage.mir
index 570469d0955ce..d156bbc8a2919 100644
--- a/llvm/test/CodeGen/AMDGPU/si-lower-sgpr-spills-vgpr-lanes-usage.mir
+++ b/llvm/test/CodeGen/AMDGPU/si-lower-sgpr-spills-vgpr-lanes-usage.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
-# RUN: llc -mtriple=amdgpu9.08-amd-amdhsa -verify-machineinstrs -stress-regalloc=3 -start-before=greedy,0 -stop-after=si-lower-sgpr-spills -o - %s | FileCheck -check-prefix=SGPR_SPILLED %s
+# RUN: llc -mtriple=amdgpu9.08-amd-amdhsa -verify-machineinstrs -stress-regalloc=3 -start-before=greedy,1 -stop-after=si-lower-sgpr-spills -o - %s | FileCheck -check-prefix=SGPR_SPILLED %s
# INFO : The test starts from the sgpr-regalloc pipeline.
# INFO : Now, StackSlotColoring pass comes just after sgpr-regalloc pipeline.
diff --git a/llvm/test/CodeGen/AMDGPU/snippet-copy-bundle-regression.mir b/llvm/test/CodeGen/AMDGPU/snippet-copy-bundle-regression.mir
index 3c99fab308ad0..50789cffb4cfa 100644
--- a/llvm/test/CodeGen/AMDGPU/snippet-copy-bundle-regression.mir
+++ b/llvm/test/CodeGen/AMDGPU/snippet-copy-bundle-regression.mir
@@ -1,6 +1,6 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 2
# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa \
-# RUN: -start-before=greedy,0 -stop-after=virtregrewriter,1 -verify-regalloc -o - %s | FileCheck %s
+# RUN: -start-before=greedy,1 -stop-after=virt-reg-rewriter,2 -verify-regalloc -o - %s | FileCheck %s
# This hit an assert when identifying snippet copy bundles from
# running off the end of the block due to using
diff --git a/llvm/test/CodeGen/AMDGPU/spill-empty-live-interval.mir b/llvm/test/CodeGen/AMDGPU/spill-empty-live-interval.mir
index 75b68900a10a7..2b7f1af29e41e 100644
--- a/llvm/test/CodeGen/AMDGPU/spill-empty-live-interval.mir
+++ b/llvm/test/CodeGen/AMDGPU/spill-empty-live-interval.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu7.00-amd-amdhsa -verify-machineinstrs -amdgpu-enable-rewrite-partial-reg-uses=false -amdgpu-dce-in-ra=0 -stress-regalloc=1 -start-before=register-coalescer -stop-after=greedy,1 -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu7.00-amd-amdhsa -verify-machineinstrs -amdgpu-enable-rewrite-partial-reg-uses=false -amdgpu-dce-in-ra=0 -stress-regalloc=1 -start-before=register-coalescer -stop-after=greedy,2 -o - %s | FileCheck %s
# https://bugs.llvm.org/show_bug.cgi?id=33620
---
diff --git a/llvm/test/CodeGen/AMDGPU/spill-scavenge-offset.ll b/llvm/test/CodeGen/AMDGPU/spill-scavenge-offset.ll
index 5c829d7ec88ef..e8122bf96a01d 100644
--- a/llvm/test/CodeGen/AMDGPU/spill-scavenge-offset.ll
+++ b/llvm/test/CodeGen/AMDGPU/spill-scavenge-offset.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=amdgpu6.01 -enable-misched=0 -post-RA-scheduler=0 -amdgpu-spill-sgpr-to-vgpr=0 < %s | FileCheck -check-prefixes=CHECK,GFX6 %s
-; RUN: llc -sgpr-regalloc=basic -vgpr-regalloc=basic -mtriple=amdgpu8.02 -enable-misched=0 -post-RA-scheduler=0 -amdgpu-spill-sgpr-to-vgpr=0 < %s | FileCheck --check-prefix=CHECK %s
+; RUN: llc -enable-new-pm=0 -sgpr-regalloc=basic -vgpr-regalloc=basic -mtriple=amdgpu8.02 -enable-misched=0 -post-RA-scheduler=0 -amdgpu-spill-sgpr-to-vgpr=0 < %s | FileCheck --check-prefix=CHECK %s
; RUN: llc -mtriple=amdgpu9.00 --amdgpu-xnack=false -mattr=+enable-flat-scratch -enable-misched=0 -post-RA-scheduler=0 -amdgpu-spill-sgpr-to-vgpr=0 < %s | FileCheck -check-prefixes=CHECK,GFX9-FLATSCR,FLATSCR %s
; RUN: llc -mtriple=amdgpu10.30 -enable-misched=0 -post-RA-scheduler=0 -amdgpu-spill-sgpr-to-vgpr=0 -mattr=+enable-flat-scratch < %s | FileCheck -check-prefixes=CHECK,GFX10-FLATSCR,FLATSCR %s
;
diff --git a/llvm/test/CodeGen/AMDGPU/spill-vector-superclass.ll b/llvm/test/CodeGen/AMDGPU/spill-vector-superclass.ll
index 29c5092609a82..ae3bbd6e3eded 100644
--- a/llvm/test/CodeGen/AMDGPU/spill-vector-superclass.ll
+++ b/llvm/test/CodeGen/AMDGPU/spill-vector-superclass.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -mtriple=amdgpu9.08-amd-amdhsa -stop-after=greedy,1 -o - %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -mtriple=amdgpu9.08-amd-amdhsa -stop-after=greedy,2 -o - %s | FileCheck -check-prefix=GCN %s
; Convert AV spills into VGPR spills by introducing appropriate copies in between.
define amdgpu_kernel void @test_spill_av_class(<4 x i32> %arg) #0 {
diff --git a/llvm/test/CodeGen/AMDGPU/splitkit-do-not-undo-subclass-split-with-remat.mir b/llvm/test/CodeGen/AMDGPU/splitkit-do-not-undo-subclass-split-with-remat.mir
index 70d6975dad8f3..c01688ae02c52 100644
--- a/llvm/test/CodeGen/AMDGPU/splitkit-do-not-undo-subclass-split-with-remat.mir
+++ b/llvm/test/CodeGen/AMDGPU/splitkit-do-not-undo-subclass-split-with-remat.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
-# RUN: llc -mtriple=amdgpu9.0a-amd-amdhsa -verify-regalloc -start-before=greedy,2 -stop-before=virtregrewriter,2 -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.0a-amd-amdhsa -verify-regalloc -start-before=greedy,3 -stop-before=virt-reg-rewriter,3 -o - %s | FileCheck %s
# FIXME: Assert if run to end
# The V_MFMA_F32_32X32X1F32_vgprcd_e64 as written requires 66 VGPRs
diff --git a/llvm/test/CodeGen/AMDGPU/swdev502267-use-after-free-last-chance-recoloring-alloc-succeeds.mir b/llvm/test/CodeGen/AMDGPU/swdev502267-use-after-free-last-chance-recoloring-alloc-succeeds.mir
index 2ef6908d3701d..57b65681557fa 100644
--- a/llvm/test/CodeGen/AMDGPU/swdev502267-use-after-free-last-chance-recoloring-alloc-succeeds.mir
+++ b/llvm/test/CodeGen/AMDGPU/swdev502267-use-after-free-last-chance-recoloring-alloc-succeeds.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
-# RUN: llc -mtriple=amdgpu9.42-amd-amdhsa -stress-regalloc=4 -verify-regalloc -start-before=greedy,2 -stop-after=virtregrewriter,2 -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.42-amd-amdhsa -stress-regalloc=4 -verify-regalloc -start-before=greedy,3 -stop-after=virt-reg-rewriter,3 -o - %s | FileCheck %s
# This testcase hit a situation where greedy would hit a use after
# free during last chance recoloring. This case successfully allocates
diff --git a/llvm/test/CodeGen/AMDGPU/threeaddr-wmma.mir b/llvm/test/CodeGen/AMDGPU/threeaddr-wmma.mir
index 695619300e298..af244ac580aa4 100644
--- a/llvm/test/CodeGen/AMDGPU/threeaddr-wmma.mir
+++ b/llvm/test/CodeGen/AMDGPU/threeaddr-wmma.mir
@@ -1,4 +1,4 @@
-# RUN: llc -mtriple=amdgpu11.00 %s -start-after postrapseudos -verify-machineinstrs -o - | FileCheck -check-prefix=GCN %s
+# RUN: llc -mtriple=amdgpu11.00 %s -start-after post-ra-pseudos -verify-machineinstrs -o - | FileCheck -check-prefix=GCN %s
# GCN-LABEL: test_V_WMMA_F32_16X16X16_F16_threeaddr_w32:
# GCN: v_wmma_f32_16x16x16_f16 v[34:41], v[0:7], v[8:15], v[16:23]
diff --git a/llvm/test/CodeGen/AMDGPU/true16-ra-pre-gfx11-regression-test.mir b/llvm/test/CodeGen/AMDGPU/true16-ra-pre-gfx11-regression-test.mir
index 90cbbaef3ab2e..d97415df573e3 100644
--- a/llvm/test/CodeGen/AMDGPU/true16-ra-pre-gfx11-regression-test.mir
+++ b/llvm/test/CodeGen/AMDGPU/true16-ra-pre-gfx11-regression-test.mir
@@ -1,4 +1,4 @@
-# RUN: llc -mtriple=amdgpu10.10 -start-before=greedy,0 -stop-after=virtregrewriter,2 -verify-machineinstrs -o - %s | FileCheck --check-prefixes=GCN %s
+# RUN: llc -mtriple=amdgpu10.10 -start-before=greedy,1 -stop-after=virt-reg-rewriter,3 -verify-machineinstrs -o - %s | FileCheck --check-prefixes=GCN %s
--- |
define amdgpu_ps void @e32() #0 {
diff --git a/llvm/test/CodeGen/AMDGPU/vgpr-agpr-limit-gfx90a.ll b/llvm/test/CodeGen/AMDGPU/vgpr-agpr-limit-gfx90a.ll
index 12ffc853188e7..6138ba6df8771 100644
--- a/llvm/test/CodeGen/AMDGPU/vgpr-agpr-limit-gfx90a.ll
+++ b/llvm/test/CodeGen/AMDGPU/vgpr-agpr-limit-gfx90a.ll
@@ -1,6 +1,6 @@
; -enable-misched=false makes the register usage more predictable
; -regalloc=fast just makes the test run faster
-; RUN: llc -mtriple=amdgpu9.0a -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc=fast -vgpr-regalloc=fast -wwm-regalloc=fast < %s | FileCheck %s --check-prefixes=GCN,GFX90A
+; RUN: llc -mtriple=amdgpu9.0a -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc-npm=fast -vgpr-regalloc-npm=fast -wwm-regalloc-npm=fast < %s | FileCheck %s --check-prefixes=GCN,GFX90A
define internal void @use256vgprs() {
%v0 = call i32 asm sideeffect "; def $0", "=v"()
diff --git a/llvm/test/CodeGen/AMDGPU/vgpr-limit-gfx1250.ll b/llvm/test/CodeGen/AMDGPU/vgpr-limit-gfx1250.ll
index 8c690f6579d89..416e89554ee89 100644
--- a/llvm/test/CodeGen/AMDGPU/vgpr-limit-gfx1250.ll
+++ b/llvm/test/CodeGen/AMDGPU/vgpr-limit-gfx1250.ll
@@ -1,6 +1,6 @@
; -enable-misched=false makes the register usage more predictable
; -regalloc=fast just makes the test run faster
-; RUN: llc -mtriple=amdgpu12.50 -O0 -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc=fast -vgpr-regalloc=fast -wwm-regalloc=fast < %s | FileCheck %s --check-prefixes=GCN,GFX1250
+; RUN: llc -mtriple=amdgpu12.50 -O0 -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc-npm=fast -vgpr-regalloc-npm=fast -wwm-regalloc-npm=fast < %s | FileCheck %s --check-prefixes=GCN,GFX1250
define internal void @use256vgprs_asm() {
%v0 = call i32 asm sideeffect "; def $0", "=v"()
diff --git a/llvm/test/CodeGen/AMDGPU/virtregrewrite-undef-identity-copy.mir b/llvm/test/CodeGen/AMDGPU/virtregrewrite-undef-identity-copy.mir
index 6d57b9dae6e32..ace8bd0e2a554 100644
--- a/llvm/test/CodeGen/AMDGPU/virtregrewrite-undef-identity-copy.mir
+++ b/llvm/test/CodeGen/AMDGPU/virtregrewrite-undef-identity-copy.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu7.00-amd-amdhsa -start-before=greedy,0 -stop-after=virtregrewriter,1 -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu7.00-amd-amdhsa -start-before=greedy,1 -stop-after=virt-reg-rewriter,2 -verify-machineinstrs -o - %s | FileCheck %s
# The undef copy of %4 is allocated to $vgpr3, and the identity copy
# was deleted, and $vgpr3 was considered undef. The code to replace
diff --git a/llvm/test/CodeGen/AMDGPU/vopc-remat.mir b/llvm/test/CodeGen/AMDGPU/vopc-remat.mir
index d15ed36b1b1fe..4e6a57ad8056a 100644
--- a/llvm/test/CodeGen/AMDGPU/vopc-remat.mir
+++ b/llvm/test/CodeGen/AMDGPU/vopc-remat.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu9.42-amd-amdhsa --stress-regalloc=2 -start-before=greedy -stop-after=virtregrewriter -o - %s | FileCheck -check-prefix=GCN %s
+# RUN: llc -mtriple=amdgpu9.42-amd-amdhsa --stress-regalloc=2 -start-before=greedy -stop-after=virt-reg-rewriter -o - %s | FileCheck -check-prefix=GCN %s
# Test that V_CMP_*_e64 instructions are rematerialized instead of spilled.
diff --git a/llvm/test/CodeGen/AMDGPU/wave32.ll b/llvm/test/CodeGen/AMDGPU/wave32.ll
index ba191cd41be04..0a18c6c6c9ad0 100644
--- a/llvm/test/CodeGen/AMDGPU/wave32.ll
+++ b/llvm/test/CodeGen/AMDGPU/wave32.ll
@@ -2310,11 +2310,12 @@ define amdgpu_kernel void @test_branch_true() #2 {
; GFX1032-NEXT: s_cbranch_execnz .LBB45_2
; GFX1032-NEXT: ; %bb.1: ; %for.body.lr.ph
; GFX1032-NEXT: s_branch .LBB45_3
-; GFX1032-NEXT: .LBB45_2: ; %Flow
+; GFX1032-NEXT: .LBB45_2: ; %Flow1
; GFX1032-NEXT: s_branch .LBB45_5
; GFX1032-NEXT: .LBB45_3: ; %for.body
-; GFX1032-NEXT: s_mov_b32 vcc_lo, 0
-; GFX1032-NEXT: ; %bb.4: ; %for.end.loopexit
+; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_cbranch_scc1 .LBB45_3
+; GFX1032-NEXT: ; %bb.4: ; %Flow
; GFX1032-NEXT: s_branch .LBB45_2
; GFX1032-NEXT: .LBB45_5: ; %for.end
; GFX1032-NEXT: s_endpgm
@@ -2325,11 +2326,12 @@ define amdgpu_kernel void @test_branch_true() #2 {
; GFX1064-NEXT: s_cbranch_execnz .LBB45_2
; GFX1064-NEXT: ; %bb.1: ; %for.body.lr.ph
; GFX1064-NEXT: s_branch .LBB45_3
-; GFX1064-NEXT: .LBB45_2: ; %Flow
+; GFX1064-NEXT: .LBB45_2: ; %Flow1
; GFX1064-NEXT: s_branch .LBB45_5
; GFX1064-NEXT: .LBB45_3: ; %for.body
-; GFX1064-NEXT: s_mov_b64 vcc, 0
-; GFX1064-NEXT: ; %bb.4: ; %for.end.loopexit
+; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_cbranch_scc1 .LBB45_3
+; GFX1064-NEXT: ; %bb.4: ; %Flow
; GFX1064-NEXT: s_branch .LBB45_2
; GFX1064-NEXT: .LBB45_5: ; %for.end
; GFX1064-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/wwm-spill-superclass-pseudo.mir b/llvm/test/CodeGen/AMDGPU/wwm-spill-superclass-pseudo.mir
index ce8aebe7458ed..3c445919d7a26 100644
--- a/llvm/test/CodeGen/AMDGPU/wwm-spill-superclass-pseudo.mir
+++ b/llvm/test/CodeGen/AMDGPU/wwm-spill-superclass-pseudo.mir
@@ -1,5 +1,5 @@
-# RUN: llc -mtriple=amdgpu9.08-amd-amdhsa -start-before=si-lower-sgpr-spills -stop-after=greedy,1 -verify-machineinstrs --stress-regalloc=2 -o - %s | FileCheck -check-prefix GCN-REGALLOC %s
-# RUN: llc -mtriple=amdgpu9.08-amd-amdhsa -start-before=si-lower-sgpr-spills -stop-after=virtregrewriter,1 -verify-machineinstrs --stress-regalloc=2 -o - %s | FileCheck -check-prefix GCN-REWRITER %s
+# RUN: llc -mtriple=amdgpu9.08-amd-amdhsa -start-before=si-lower-sgpr-spills -stop-after=greedy,2 -verify-machineinstrs --stress-regalloc=2 -o - %s | FileCheck -check-prefix GCN-REGALLOC %s
+# RUN: llc -mtriple=amdgpu9.08-amd-amdhsa -start-before=si-lower-sgpr-spills -stop-after=virt-reg-rewriter,2 -verify-machineinstrs --stress-regalloc=2 -o - %s | FileCheck -check-prefix GCN-REWRITER %s
name: test_wwm_reg_superclass_spill
tracksRegLiveness: true
diff --git a/llvm/test/DebugInfo/AMDGPU/debug-loc-copy.ll b/llvm/test/DebugInfo/AMDGPU/debug-loc-copy.ll
index f81a57f12173b..d35b727edb3ef 100644
--- a/llvm/test/DebugInfo/AMDGPU/debug-loc-copy.ll
+++ b/llvm/test/DebugInfo/AMDGPU/debug-loc-copy.ll
@@ -24,7 +24,6 @@ define void @_Z12lane_pc_testj() #0 !dbg !9 {
; GCN-NEXT: .cfi_undefined 39
; GCN-NEXT: .cfi_undefined 40
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: ; %bb.1: ; %lab
; GCN-NEXT: .Ltmp0:
; GCN-NEXT: .loc 0 12 1 prologue_end ; t.cpp:12:1
; GCN-NEXT: s_mov_b64 s[4:5], 0
@@ -34,13 +33,14 @@ define void @_Z12lane_pc_testj() #0 !dbg !9 {
; GCN-NEXT: s_cmp_lg_u32 s8, s6
; GCN-NEXT: s_cselect_b32 s5, s7, s5
; GCN-NEXT: s_cselect_b32 s4, s8, s4
-; GCN-NEXT: v_mov_b32_e32 v2, 0
+; GCN-NEXT: ; %bb.1: ; %lab
; GCN-NEXT: .loc 0 13 1 ; t.cpp:13:1
+; GCN-NEXT: v_mov_b32_e32 v2, 0
; GCN-NEXT: v_mov_b32_e32 v0, s4
; GCN-NEXT: v_mov_b32_e32 v1, s5
; GCN-NEXT: flat_store_dword v[0:1], v2
-; GCN-NEXT: v_mov_b32_e32 v2, 1
; GCN-NEXT: .loc 0 14 1 ; t.cpp:14:1
+; GCN-NEXT: v_mov_b32_e32 v2, 1
; GCN-NEXT: v_mov_b32_e32 v0, s4
; GCN-NEXT: v_mov_b32_e32 v1, s5
; GCN-NEXT: flat_store_dword v[0:1], v2
diff --git a/llvm/tools/llc/NewPMDriver.cpp b/llvm/tools/llc/NewPMDriver.cpp
index 97a7a8a2a6ba8..ac74619acfc20 100644
--- a/llvm/tools/llc/NewPMDriver.cpp
+++ b/llvm/tools/llc/NewPMDriver.cpp
@@ -88,11 +88,12 @@ bool LLCDiagnosticHandler::handleDiagnostics(const DiagnosticInfo &DI) {
static llvm::ExitOnError ExitOnErr;
int llvm::compileModuleWithNewPM(
- StringRef Arg0, std::unique_ptr<Module> M, std::unique_ptr<MIRParser> MIR,
- std::unique_ptr<TargetMachine> Target, std::unique_ptr<ToolOutputFile> Out,
- std::unique_ptr<ToolOutputFile> DwoOut, LLVMContext &Context,
+ StringRef Arg0, std::unique_ptr<Module> &M, std::unique_ptr<MIRParser> &MIR,
+ std::unique_ptr<TargetMachine> &Target,
+ std::unique_ptr<ToolOutputFile> &Out,
+ std::unique_ptr<ToolOutputFile> &DwoOut, LLVMContext &Context,
const TargetLibraryInfoImpl &TLII, VerifierKind VK, StringRef PassPipeline,
- CodeGenFileType FileType) {
+ CodeGenFileType FileType, CGPassBuilderOption &Opt) {
if (!PassPipeline.empty() && TargetPassConfig::hasLimitedCodeGenPipeline()) {
WithColor::error(errs(), Arg0)
@@ -110,8 +111,6 @@ int llvm::compileModuleWithNewPM(
OS = BOS.get();
}
- // Fetch options from TargetPassConfig
- CGPassBuilderOption Opt = getCGPassBuilderOption();
Opt.DisableVerify = VK != VerifierKind::InputOutput;
Opt.DebugPM = DebugPM;
Opt.RegAlloc = RegAlloc;
diff --git a/llvm/tools/llc/NewPMDriver.h b/llvm/tools/llc/NewPMDriver.h
index 0dbd46797dabc..40e046f5316f3 100644
--- a/llvm/tools/llc/NewPMDriver.h
+++ b/llvm/tools/llc/NewPMDriver.h
@@ -30,6 +30,7 @@ class TargetMachine;
class ToolOutputFile;
class LLVMContext;
class MIRParser;
+class CGPassBuilderOption;
enum class VerifierKind { None, InputOutput, EachPass };
@@ -37,14 +38,15 @@ struct LLCDiagnosticHandler : public DiagnosticHandler {
bool handleDiagnostics(const DiagnosticInfo &DI) override;
};
-int compileModuleWithNewPM(StringRef Arg0, std::unique_ptr<Module> M,
- std::unique_ptr<MIRParser> MIR,
- std::unique_ptr<TargetMachine> Target,
- std::unique_ptr<ToolOutputFile> Out,
- std::unique_ptr<ToolOutputFile> DwoOut,
+int compileModuleWithNewPM(StringRef Arg0, std::unique_ptr<Module> &M,
+ std::unique_ptr<MIRParser> &MIR,
+ std::unique_ptr<TargetMachine> &Target,
+ std::unique_ptr<ToolOutputFile> &Out,
+ std::unique_ptr<ToolOutputFile> &DwoOut,
LLVMContext &Context,
const TargetLibraryInfoImpl &TLII, VerifierKind VK,
- StringRef PassPipeline, CodeGenFileType FileType);
+ StringRef PassPipeline, CodeGenFileType FileType,
+ CGPassBuilderOption &Opt);
} // namespace llvm
#endif
diff --git a/llvm/tools/llc/llc.cpp b/llvm/tools/llc/llc.cpp
index dcd54c18401d0..d20e2f24249bc 100644
--- a/llvm/tools/llc/llc.cpp
+++ b/llvm/tools/llc/llc.cpp
@@ -495,6 +495,11 @@ static bool addPass(PassManagerBase &PM, const char *argv0, StringRef PassName,
return false;
}
+bool targetSupportsNPMBackend(const Triple &T, const CGPassBuilderOption &Opt) {
+ return T.isAMDGCN() &&
+ Opt.EnableGlobalISelOption != cl::boolOrDefault::BOU_TRUE;
+}
+
static int compileModule(char **argv, SmallVectorImpl<PassPlugin> &PluginList,
LLVMContext &Context, std::string &OutputFilename) {
// Load the module to be compiled...
@@ -749,11 +754,23 @@ static int compileModule(char **argv, SmallVectorImpl<PassPlugin> &PluginList,
else if (VerifyEach)
VK = VerifierKind::EachPass;
- if (EnableNewPassManager || !PassPipeline.empty()) {
- return compileModuleWithNewPM(argv[0], std::move(M), std::move(MIR),
- std::move(Target), std::move(Out),
- std::move(DwoOut), Context, TLII, VK,
- PassPipeline, codegen::getFileType());
+ // Fetch options from TargetPassConfig.
+ CGPassBuilderOption Opt = getCGPassBuilderOption();
+ bool UseNPM = false;
+ if (EnableNewPassManager.getNumOccurrences()) {
+ UseNPM = EnableNewPassManager;
+ } else if (!PassPipeline.empty()) {
+ UseNPM = true;
+ } else if (RunPass.getNumOccurrences())
+ UseNPM = false;
+ else if (targetSupportsNPMBackend(TheTriple, Opt)) {
+ UseNPM = true;
+ }
+
+ if (UseNPM) {
+ return compileModuleWithNewPM(argv[0], M, MIR, Target, Out, DwoOut, Context,
+ TLII, VK, PassPipeline,
+ codegen::getFileType(), Opt);
}
// Build up all of the passes that we want to do to the module.
More information about the llvm-branch-commits
mailing list