[llvm-branch-commits] [llvm] [LLC][AMDGPU] start using new pass manager for backend codegen (PR #210251)
Vikram Hegde via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Thu Jul 16 23:53:21 PDT 2026
https://github.com/vikramRH updated https://github.com/llvm/llvm-project/pull/210251
>From 0cc72933f8d474d33be7959c7d7536d13d6bf857 Mon Sep 17 00:00:00 2001
From: vikhegde <vikram.hegde at amd.com>
Date: Thu, 16 Jul 2026 17:15:53 +0530
Subject: [PATCH] [LLC][AMDGPU] start using new pass manager for backend
codegen
---
...ee-registers-assertion-after-ra-failure.ll | 2 +-
.../AMDGPU/alloc-aligned-tuples-gfx1250.mir | 2 +-
.../AMDGPU/alloc-aligned-tuples-gfx908.mir | 2 +-
.../AMDGPU/alloc-aligned-tuples-gfx90a.mir | 2 +-
...-amdgpu-flat-work-group-size-vgpr-limit.ll | 20 +-
.../AMDGPU/av_spill_cross_bb_usage.mir | 2 +-
.../bad-agpr-vgpr-regalloc-priority.mir | 2 +-
.../AMDGPU/bb-prolog-spill-during-regalloc.ll | 2 +-
.../block-should-not-be-in-alive-blocks.mir | 2 +-
.../AMDGPU/call-alias-register-usage-agpr.ll | 36 +-
.../AMDGPU/call-alias-register-usage0.ll | 18 +-
.../AMDGPU/call-alias-register-usage1.ll | 22 +-
.../AMDGPU/call-alias-register-usage2.ll | 22 +-
.../AMDGPU/call-alias-register-usage3.ll | 22 +-
llvm/test/CodeGen/AMDGPU/call-constexpr.ll | 388 -----------
.../CodeGen/AMDGPU/convergence-laneops.ll | 4 +-
llvm/test/CodeGen/AMDGPU/dead_bundle.mir | 2 +-
.../dummy-regalloc-priority-advisor.mir | 4 +-
...gpr-vgpr-count-propagation-direct-chain.ll | 42 +-
.../dvgpr-vgpr-count-propagation-indirect.ll | 26 +-
.../AMDGPU/dvgpr-vgpr-count-propagation.ll | 50 +-
.../AMDGPU/extend-wwm-virt-reg-liveness.mir | 2 +-
.../CodeGen/AMDGPU/fastregalloc-sgpr-only.mir | 2 +-
llvm/test/CodeGen/AMDGPU/fmamk_fmaak-t16.mir | 2 +-
.../CodeGen/AMDGPU/fold-restore-undef-use.mir | 2 +-
.../CodeGen/AMDGPU/function-resource-usage.ll | 92 +--
.../greedy-alloc-fail-sgpr1024-spill.mir | 2 +-
.../greedy-instruction-split-subrange.mir | 2 +-
.../greedy-remark-crash-unassigned-reg.mir | 2 +-
.../AMDGPU/illegal-eviction-assert.mir | 2 +-
.../AMDGPU/indirect-call-known-callees.ll | 73 +-
.../CodeGen/AMDGPU/inflate-av-remat-imm.mir | 2 +-
...class-vgpr-mfma-to-agpr-negative-tests.mir | 2 +-
...class-vgpr-mfma-to-av-with-load-source.mir | 2 +-
.../inflate-reg-class-vgpr-mfma-to-av.mir | 2 +-
...-reg-class-snippet-copy-use-after-free.mir | 2 +-
...nfloop-subrange-spill-inspect-subrange.mir | 2 +-
.../CodeGen/AMDGPU/infloop-subrange-spill.mir | 2 +-
llvm/test/CodeGen/AMDGPU/issue48473.mir | 2 +-
...sue98474-assigned-physreg-interference.mir | 2 +-
...egrewriter-live-out-undef-subregisters.mir | 2 +-
.../AMDGPU/large-avgpr-assign-last.mir | 2 +-
llvm/test/CodeGen/AMDGPU/llc-pipeline.ll | 10 +-
.../CodeGen/AMDGPU/machine-sink-cycle.mir | 4 +-
.../CodeGen/AMDGPU/memcpy-crash-issue63986.ll | 2 +
.../CodeGen/AMDGPU/memcpy_const_compare.ll | 12 +-
llvm/test/CodeGen/AMDGPU/memmove-var-size.ll | 230 +++----
.../AMDGPU/memset-param-combinations.ll | 633 ++++++++++++------
.../AMDGPU/mfma-no-register-aliasing.ll | 2 +-
...-masking-pre-ra-update-liveness-wave32.mir | 2 +-
...pt-exec-masking-pre-ra-update-liveness.mir | 2 +-
...ptimize-exec-mask-pre-ra-alloc-failure.mir | 2 +-
...al-regcopy-and-spill-missed-at-regalloc.ll | 4 +-
llvm/test/CodeGen/AMDGPU/pr51516.mir | 4 +-
...ut-of-registers-error-all-regs-reserved.ll | 6 +-
.../AMDGPU/ran-out-of-registers-errors.ll | 10 +-
.../ran-out-of-sgprs-allocation-failure.mir | 2 +-
.../AMDGPU/recursive-resource-usage-mcexpr.ll | 131 ++--
...lloc-failure-overlapping-insert-assert.mir | 2 +-
...fast-dont-drop-subreg-index-issue61134.mir | 2 +-
.../regalloc-introduces-copy-sgpr-to-agpr.mir | 2 +-
.../AMDGPU/regalloc-undef-copy-fold.mir | 2 +-
...ster-killed-error-after-alloc-failure0.mir | 4 +-
.../remaining-virtual-register-operands.mir | 2 +-
llvm/test/CodeGen/AMDGPU/remat-vop.mir | 2 +-
...schedule-regpressure-ilp-metric-spills.mir | 4 +-
...ule-regpressure-no-unclustered-regions.mir | 2 +-
.../CodeGen/AMDGPU/sgpr-regalloc-flags.ll | 16 +-
.../sgpr-spill-update-only-slot-indexes.ll | 2 +-
.../si-lower-sgpr-spills-cycle-header.mir | 2 +-
...si-lower-sgpr-spills-multi-entry-cycle.mir | 2 +-
.../si-lower-sgpr-spills-vgpr-lanes-usage.mir | 2 +-
.../AMDGPU/snippet-copy-bundle-regression.mir | 2 +-
.../AMDGPU/spill-empty-live-interval.mir | 2 +-
.../CodeGen/AMDGPU/spill-scavenge-offset.ll | 2 +-
.../CodeGen/AMDGPU/spill-vector-superclass.ll | 2 +-
...-do-not-undo-subclass-split-with-remat.mir | 2 +-
...-last-chance-recoloring-alloc-succeeds.mir | 2 +-
.../true16-ra-pre-gfx11-regression-test.mir | 2 +-
.../CodeGen/AMDGPU/vgpr-agpr-limit-gfx90a.ll | 2 +-
.../test/CodeGen/AMDGPU/vgpr-limit-gfx1250.ll | 2 +-
.../virtregrewrite-undef-identity-copy.mir | 2 +-
llvm/test/CodeGen/AMDGPU/wave32.ll | 14 +-
.../AMDGPU/wwm-spill-superclass-pseudo.mir | 4 +-
llvm/test/DebugInfo/AMDGPU/debug-loc-copy.ll | 6 +-
llvm/tools/llc/NewPMDriver.cpp | 32 +-
llvm/tools/llc/NewPMDriver.h | 21 +-
llvm/tools/llc/llc.cpp | 32 +-
88 files changed, 979 insertions(+), 1127 deletions(-)
delete mode 100644 llvm/test/CodeGen/AMDGPU/call-constexpr.ll
diff --git a/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers-assertion-after-ra-failure.ll b/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers-assertion-after-ra-failure.ll
index d35f04c64ac89..f3b0a315d6166 100644
--- a/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers-assertion-after-ra-failure.ll
+++ b/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers-assertion-after-ra-failure.ll
@@ -1,5 +1,5 @@
; RUN: not llc -mtriple=amdgpu9.0a-amd-amdhsa -filetype=null %s 2>&1 | FileCheck -check-prefix=ERR %s
-; RUN: not llc -mtriple=amdgpu9.0a-amd-amdhsa -vgpr-regalloc=basic -filetype=null %s 2>&1 | FileCheck -check-prefix=ERR %s
+; RUN: not llc -enable-new-pm=0 -mtriple=amdgpu9.0a-amd-amdhsa -vgpr-regalloc=basic -filetype=null %s 2>&1 | FileCheck -check-prefix=ERR %s
; ERR: error: <unknown>:0:0: no registers from class available to allocate in function 'no_free_vgprs_at_agpr_to_agpr_copy'
diff --git a/llvm/test/CodeGen/AMDGPU/alloc-aligned-tuples-gfx1250.mir b/llvm/test/CodeGen/AMDGPU/alloc-aligned-tuples-gfx1250.mir
index 9b4e364063f36..3eb221620587f 100644
--- a/llvm/test/CodeGen/AMDGPU/alloc-aligned-tuples-gfx1250.mir
+++ b/llvm/test/CodeGen/AMDGPU/alloc-aligned-tuples-gfx1250.mir
@@ -1,4 +1,4 @@
-# RUN: llc -mtriple=amdgpu12.50 -start-before=greedy,0 -stop-after=virtregrewriter,2 -verify-machineinstrs -o - %s | FileCheck --check-prefixes=GCN,GFX1250 %s
+# RUN: llc -mtriple=amdgpu12.50 -start-before=greedy,1 -stop-after=virt-reg-rewriter,3 -verify-machineinstrs -o - %s | FileCheck --check-prefixes=GCN,GFX1250 %s
# Using the unaligned vector tuples are OK as long as they aren't used
# in a real instruction.
diff --git a/llvm/test/CodeGen/AMDGPU/alloc-aligned-tuples-gfx908.mir b/llvm/test/CodeGen/AMDGPU/alloc-aligned-tuples-gfx908.mir
index cb042fa6bc261..76c807aaeb0da 100644
--- a/llvm/test/CodeGen/AMDGPU/alloc-aligned-tuples-gfx908.mir
+++ b/llvm/test/CodeGen/AMDGPU/alloc-aligned-tuples-gfx908.mir
@@ -1,4 +1,4 @@
-# RUN: llc -mtriple=amdgpu9.08 -start-before=greedy,0 -stop-after=virtregrewriter,2 -verify-machineinstrs -o - %s | FileCheck --check-prefixes=GCN,GFX908 %s
+# RUN: llc -mtriple=amdgpu9.08 -start-before=greedy,1 -stop-after=virt-reg-rewriter,3 -verify-machineinstrs -o - %s | FileCheck --check-prefixes=GCN,GFX908 %s
---
# GCN-LABEL: name: alloc_vgpr_64
diff --git a/llvm/test/CodeGen/AMDGPU/alloc-aligned-tuples-gfx90a.mir b/llvm/test/CodeGen/AMDGPU/alloc-aligned-tuples-gfx90a.mir
index c6593c0dbeb45..6bebdb59a1548 100644
--- a/llvm/test/CodeGen/AMDGPU/alloc-aligned-tuples-gfx90a.mir
+++ b/llvm/test/CodeGen/AMDGPU/alloc-aligned-tuples-gfx90a.mir
@@ -1,4 +1,4 @@
-# RUN: llc -mtriple=amdgpu9.0a -start-before=greedy,0 -stop-after=virtregrewriter,2 -verify-machineinstrs -o - %s | FileCheck --check-prefixes=GCN,GFX90A %s
+# RUN: llc -mtriple=amdgpu9.0a -start-before=greedy,1 -stop-after=virt-reg-rewriter,3 -verify-machineinstrs -o - %s | FileCheck --check-prefixes=GCN,GFX90A %s
# RUN: llc -enable-new-pm -mtriple=amdgpu9.0a -passes='greedy<sgpr>,virt-reg-rewriter<no-clear-vregs>,stack-slot-coloring,si-lower-sgpr-spills,si-pre-allocate-wwm-regs,greedy<wwm>,si-lower-wwm-copies,virt-reg-rewriter<no-clear-vregs>,amdgpu-reserve-wwm-regs,greedy<vgpr>,amdgpu-nsa-reassign,virt-reg-rewriter' -o - %s | FileCheck --check-prefixes=GCN,GFX90A %s
# Using the unaligned vector tuples are OK as long as they aren't used
diff --git a/llvm/test/CodeGen/AMDGPU/attr-amdgpu-flat-work-group-size-vgpr-limit.ll b/llvm/test/CodeGen/AMDGPU/attr-amdgpu-flat-work-group-size-vgpr-limit.ll
index 412933f78660b..332f237d8d43e 100644
--- a/llvm/test/CodeGen/AMDGPU/attr-amdgpu-flat-work-group-size-vgpr-limit.ll
+++ b/llvm/test/CodeGen/AMDGPU/attr-amdgpu-flat-work-group-size-vgpr-limit.ll
@@ -1,15 +1,15 @@
; -enable-misched=false makes the register usage more predictable
; -regalloc=fast just makes the test run faster
-; RUN: llc -mtriple=amdgpu9.00 -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc=fast -vgpr-regalloc=fast -wwm-regalloc=fast < %s | FileCheck %s --check-prefixes=GCN,GFX9
-; RUN: llc -mtriple=amdgpu9.0a -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc=fast -vgpr-regalloc=fast -wwm-regalloc=fast < %s | FileCheck %s --check-prefixes=GCN,GFX90A
-; RUN: llc -mtriple=amdgpu10.10 -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc=fast -vgpr-regalloc=fast -wwm-regalloc=fast < %s | FileCheck %s --check-prefixes=GCN,GFX10WGP-WAVE32
-; RUN: llc -mtriple=amdgpu10.10 -mattr=+wavefrontsize64 -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc=fast -vgpr-regalloc=fast -wwm-regalloc=fast < %s | FileCheck %s --check-prefixes=GCN,GFX10WGP-WAVE64
-; RUN: llc -mtriple=amdgpu10.10 -mattr=+cumode -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc=fast -vgpr-regalloc=fast -wwm-regalloc=fast < %s | FileCheck %s --check-prefixes=GCN,GFX10CU-WAVE32
-; RUN: llc -mtriple=amdgpu10.10 -mattr=+cumode,+wavefrontsize64 -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc=fast -vgpr-regalloc=fast -wwm-regalloc=fast < %s | FileCheck %s --check-prefixes=GCN,GFX10CU-WAVE64
-; RUN: llc -mtriple=amdgpu11.00 -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc=fast -vgpr-regalloc=fast -wwm-regalloc=fast < %s | FileCheck %s --check-prefixes=GCN,GFX11WGP-WAVE32
-; RUN: llc -mtriple=amdgpu11.00 -mattr=+wavefrontsize64 -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc=fast -vgpr-regalloc=fast -wwm-regalloc=fast < %s | FileCheck %s --check-prefixes=GCN,GFX11WGP-WAVE64
-; RUN: llc -mtriple=amdgpu11.00 -mattr=+cumode -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc=fast -vgpr-regalloc=fast -wwm-regalloc=fast < %s | FileCheck %s --check-prefixes=GCN,GFX11CU-WAVE32
-; RUN: llc -mtriple=amdgpu11.00 -mattr=+cumode,+wavefrontsize64 -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc=fast -vgpr-regalloc=fast -wwm-regalloc=fast < %s | FileCheck %s --check-prefixes=GCN,GFX11CU-WAVE64
+; RUN: llc -mtriple=amdgpu9.00 -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc-npm=fast -vgpr-regalloc-npm=fast -wwm-regalloc-npm=fast < %s | FileCheck %s --check-prefixes=GCN,GFX9
+; RUN: llc -mtriple=amdgpu9.0a -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc-npm=fast -vgpr-regalloc-npm=fast -wwm-regalloc-npm=fast < %s | FileCheck %s --check-prefixes=GCN,GFX90A
+; RUN: llc -mtriple=amdgpu10.10 -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc-npm=fast -vgpr-regalloc-npm=fast -wwm-regalloc-npm=fast < %s | FileCheck %s --check-prefixes=GCN,GFX10WGP-WAVE32
+; RUN: llc -mtriple=amdgpu10.10 -mattr=+wavefrontsize64 -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc-npm=fast -vgpr-regalloc-npm=fast -wwm-regalloc-npm=fast < %s | FileCheck %s --check-prefixes=GCN,GFX10WGP-WAVE64
+; RUN: llc -mtriple=amdgpu10.10 -mattr=+cumode -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc-npm=fast -vgpr-regalloc-npm=fast -wwm-regalloc-npm=fast < %s | FileCheck %s --check-prefixes=GCN,GFX10CU-WAVE32
+; RUN: llc -mtriple=amdgpu10.10 -mattr=+cumode,+wavefrontsize64 -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc-npm=fast -vgpr-regalloc-npm=fast -wwm-regalloc-npm=fast < %s | FileCheck %s --check-prefixes=GCN,GFX10CU-WAVE64
+; RUN: llc -mtriple=amdgpu11.00 -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc-npm=fast -vgpr-regalloc-npm=fast -wwm-regalloc-npm=fast < %s | FileCheck %s --check-prefixes=GCN,GFX11WGP-WAVE32
+; RUN: llc -mtriple=amdgpu11.00 -mattr=+wavefrontsize64 -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc-npm=fast -vgpr-regalloc-npm=fast -wwm-regalloc-npm=fast < %s | FileCheck %s --check-prefixes=GCN,GFX11WGP-WAVE64
+; RUN: llc -mtriple=amdgpu11.00 -mattr=+cumode -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc-npm=fast -vgpr-regalloc-npm=fast -wwm-regalloc-npm=fast < %s | FileCheck %s --check-prefixes=GCN,GFX11CU-WAVE32
+; RUN: llc -mtriple=amdgpu11.00 -mattr=+cumode,+wavefrontsize64 -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc-npm=fast -vgpr-regalloc-npm=fast -wwm-regalloc-npm=fast < %s | FileCheck %s --check-prefixes=GCN,GFX11CU-WAVE64
define internal void @use256vgprs() {
%v0 = call i32 asm sideeffect "; def $0", "=v"()
diff --git a/llvm/test/CodeGen/AMDGPU/av_spill_cross_bb_usage.mir b/llvm/test/CodeGen/AMDGPU/av_spill_cross_bb_usage.mir
index 11b009307ce1e..6eb7181d99c53 100644
--- a/llvm/test/CodeGen/AMDGPU/av_spill_cross_bb_usage.mir
+++ b/llvm/test/CodeGen/AMDGPU/av_spill_cross_bb_usage.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu9.08-amd-amdhsa -start-before=greedy,1 -stop-after=prolog-epilog -verify-machineinstrs -verify-regalloc -o - %s | FileCheck --check-prefixes=GCN %s
+# RUN: llc -mtriple=amdgpu9.08-amd-amdhsa -start-before=greedy,2 -stop-after=prolog-epilog -verify-machineinstrs -verify-regalloc -o - %s | FileCheck --check-prefixes=GCN %s
# The VGPR pair spilled and restored around the callsite is used in the next basic block.
#
diff --git a/llvm/test/CodeGen/AMDGPU/bad-agpr-vgpr-regalloc-priority.mir b/llvm/test/CodeGen/AMDGPU/bad-agpr-vgpr-regalloc-priority.mir
index a2763a1a1ccfd..7d0ced376f1cb 100644
--- a/llvm/test/CodeGen/AMDGPU/bad-agpr-vgpr-regalloc-priority.mir
+++ b/llvm/test/CodeGen/AMDGPU/bad-agpr-vgpr-regalloc-priority.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
-# RUN: llc -mtriple=amdgpu9.08-amd-amdhsa -start-before=greedy,2 -stop-after=virtregrewriter,2 -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.08-amd-amdhsa -start-before=greedy,3 -stop-after=virt-reg-rewriter,3 -o - %s | FileCheck %s
---
name: bad_ra
diff --git a/llvm/test/CodeGen/AMDGPU/bb-prolog-spill-during-regalloc.ll b/llvm/test/CodeGen/AMDGPU/bb-prolog-spill-during-regalloc.ll
index 7694e30a612af..71134ed6b67fb 100644
--- a/llvm/test/CodeGen/AMDGPU/bb-prolog-spill-during-regalloc.ll
+++ b/llvm/test/CodeGen/AMDGPU/bb-prolog-spill-during-regalloc.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 3
-; RUN: llc -mtriple=amdgpu9.06-amd-amdhsa -O0 --stop-after=regallocfast,2 -o - %s | FileCheck -check-prefix=REGALLOC %s
+; RUN: llc -mtriple=amdgpu9.06-amd-amdhsa -O0 --stop-after=regallocfast,3 -o - %s | FileCheck -check-prefix=REGALLOC %s
; Test to check if the bb prolog spills are inserted correctly during regalloc.
define i32 @prolog_spill(i32 %arg0, i32 %arg1, i32 %arg2) {
diff --git a/llvm/test/CodeGen/AMDGPU/block-should-not-be-in-alive-blocks.mir b/llvm/test/CodeGen/AMDGPU/block-should-not-be-in-alive-blocks.mir
index 7ecb1cd7a5343..e5e436c8fc97a 100644
--- a/llvm/test/CodeGen/AMDGPU/block-should-not-be-in-alive-blocks.mir
+++ b/llvm/test/CodeGen/AMDGPU/block-should-not-be-in-alive-blocks.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu10.31-amd-amdhsa -verify-machineinstrs -start-after=unreachable-mbb-elimination -stop-after=phi-node-elimination -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu10.31-amd-amdhsa -verify-machineinstrs -start-after=unreachable-mbb-elimination -stop-after=si-lower-control-flow -o - %s | FileCheck %s
# FIXME: Should be able to just use run-pass, but need to keep
# LiveVariables live after for the verifier. Also -start-before
diff --git a/llvm/test/CodeGen/AMDGPU/call-alias-register-usage-agpr.ll b/llvm/test/CodeGen/AMDGPU/call-alias-register-usage-agpr.ll
index 4ee6585c0a84a..b39b6cc4712de 100644
--- a/llvm/test/CodeGen/AMDGPU/call-alias-register-usage-agpr.ll
+++ b/llvm/test/CodeGen/AMDGPU/call-alias-register-usage-agpr.ll
@@ -1,16 +1,25 @@
-; RUN: llc -O0 -mtriple=amdgpu9.08-amd-amdhsa < %s | FileCheck -check-prefix=ALL %s
+; RUN: llc -O0 -mtriple=amdgpu9.08-amd-amdhsa < %s | FileCheck -check-prefix=ALL,GFX908 %s
; RUN: llc -O0 -mtriple=amdgpu9.0a-amd-amdhsa < %s | FileCheck -check-prefixes=ALL,GFX90A %s
-; CallGraphAnalysis, which CodeGenSCC order depends on, does not look
-; through aliases. If GlobalOpt is never run, we do not see direct
-; calls,
-
@alias = hidden alias void (), ptr @aliasee_default
-; ALL-LABEL: {{^}}kernel:
-; ALL: .amdhsa_next_free_vgpr max(totalnumvgprs(.Lkernel.num_agpr, .Lkernel.num_vgpr), 1, 0)
-; ALL-NEXT: .amdhsa_next_free_sgpr max(.Lkernel.numbered_sgpr+extrasgprs(.Lkernel.uses_vcc, .Lkernel.uses_flat_scratch, 1), 1, 0)-extrasgprs(.Lkernel.uses_vcc, .Lkernel.uses_flat_scratch, 1)
-; GFX90A-NEXT: .amdhsa_accum_offset (((((alignto(max(1, .Lkernel.num_vgpr), 4)/4)-1)&~65536)&63)+1)*4
+define internal void @aliasee_default() #1 {
+bb:
+ call void asm sideeffect "; clobber a26 ", "~{a26}"()
+ ret void
+}
+; ALL: .set .Laliasee_default.num_vgpr, 0
+; ALL-NEXT: .set .Laliasee_default.num_agpr, 27
+; ALL-NEXT: .set .Laliasee_default.numbered_sgpr, 32
+
+; GFX908-LABEL: {{^}}kernel:
+; GFX908: .amdhsa_next_free_vgpr 41
+; GFX908-NEXT: .amdhsa_next_free_sgpr 33
+
+; GFX90A-LABEL: {{^}}kernel:
+; GFX90A: .amdhsa_next_free_vgpr 71
+; GFX90A-NEXT: .amdhsa_next_free_sgpr 33
+; GFX90A-NEXT: .amdhsa_accum_offset 44
; ALL: .set .Lkernel.num_vgpr, max(41, .Laliasee_default.num_vgpr)
; ALL-NEXT: .set .Lkernel.num_agpr, max(0, .Laliasee_default.num_agpr)
@@ -21,15 +30,6 @@ bb:
ret void
}
-define internal void @aliasee_default() #1 {
-bb:
- call void asm sideeffect "; clobber a26 ", "~{a26}"()
- ret void
-}
-; ALL: .set .Laliasee_default.num_vgpr, 0
-; ALL-NEXT: .set .Laliasee_default.num_agpr, 27
-; ALL-NEXT: .set .Laliasee_default.numbered_sgpr, 32
-
attributes #0 = { noinline norecurse nounwind optnone }
attributes #1 = { noinline norecurse nounwind readnone willreturn }
attributes #2 = { nounwind readnone willreturn }
diff --git a/llvm/test/CodeGen/AMDGPU/call-alias-register-usage0.ll b/llvm/test/CodeGen/AMDGPU/call-alias-register-usage0.ll
index 8e74dcdd9490f..296507e2c2bbf 100644
--- a/llvm/test/CodeGen/AMDGPU/call-alias-register-usage0.ll
+++ b/llvm/test/CodeGen/AMDGPU/call-alias-register-usage0.ll
@@ -6,6 +6,15 @@
@alias0 = hidden alias void (), ptr @aliasee_default_vgpr64_sgpr102
+; CHECK: .set .Laliasee_default_vgpr64_sgpr102.num_vgpr, 53
+; CHECK-NEXT: .set .Laliasee_default_vgpr64_sgpr102.num_agpr, 0
+; CHECK-NEXT: .set .Laliasee_default_vgpr64_sgpr102.numbered_sgpr, 32
+define internal void @aliasee_default_vgpr64_sgpr102() #1 {
+bb:
+ call void asm sideeffect "; clobber v52 ", "~{v52}"()
+ ret void
+}
+
; CHECK-LABEL: {{^}}kernel0:
; CHECK: .set .Lkernel0.num_vgpr, max(41, .Laliasee_default_vgpr64_sgpr102.num_vgpr)
; CHECK-NEXT: .set .Lkernel0.num_agpr, max(0, .Laliasee_default_vgpr64_sgpr102.num_agpr)
@@ -16,15 +25,6 @@ bb:
ret void
}
-; CHECK: .set .Laliasee_default_vgpr64_sgpr102.num_vgpr, 53
-; CHECK-NEXT: .set .Laliasee_default_vgpr64_sgpr102.num_agpr, 0
-; CHECK-NEXT: .set .Laliasee_default_vgpr64_sgpr102.numbered_sgpr, 32
-define internal void @aliasee_default_vgpr64_sgpr102() #1 {
-bb:
- call void asm sideeffect "; clobber v52 ", "~{v52}"()
- ret void
-}
-
attributes #0 = { noinline norecurse nounwind optnone }
attributes #1 = { noinline norecurse nounwind readnone willreturn }
attributes #2 = { nounwind readnone willreturn }
diff --git a/llvm/test/CodeGen/AMDGPU/call-alias-register-usage1.ll b/llvm/test/CodeGen/AMDGPU/call-alias-register-usage1.ll
index 05412b343ecb7..81ee5b96a297f 100644
--- a/llvm/test/CodeGen/AMDGPU/call-alias-register-usage1.ll
+++ b/llvm/test/CodeGen/AMDGPU/call-alias-register-usage1.ll
@@ -8,9 +8,18 @@
; The parent kernel has a higher VGPR usage than the possible callees.
+; CHECK: .set .Laliasee_vgpr32_sgpr76.num_vgpr, 27
+; CHECK-NEXT: .set .Laliasee_vgpr32_sgpr76.num_agpr, 0
+; CHECK-NEXT: .set .Laliasee_vgpr32_sgpr76.numbered_sgpr, 32
+define internal void @aliasee_vgpr32_sgpr76() #1 {
+bb:
+ call void asm sideeffect "; clobber v26 ", "~{v26}"()
+ ret void
+}
+
; CHECK-LABEL: {{^}}kernel1:
-; CHECK: .amdhsa_next_free_vgpr max(totalnumvgprs(.Lkernel1.num_agpr, .Lkernel1.num_vgpr), 1, 0)
-; CHECK-NEXT: .amdhsa_next_free_sgpr max(.Lkernel1.numbered_sgpr+extrasgprs(.Lkernel1.uses_vcc, .Lkernel1.uses_flat_scratch, 1), 1, 0)-extrasgprs(.Lkernel1.uses_vcc, .Lkernel1.uses_flat_scratch, 1)
+; CHECK: .amdhsa_next_free_vgpr 42
+; CHECK-NEXT: .amdhsa_next_free_sgpr 33
; CHECK: .set .Lkernel1.num_vgpr, max(42, .Laliasee_vgpr32_sgpr76.num_vgpr)
; CHECK-NEXT: .set .Lkernel1.num_agpr, max(0, .Laliasee_vgpr32_sgpr76.num_agpr)
@@ -22,15 +31,6 @@ bb:
ret void
}
-; CHECK: .set .Laliasee_vgpr32_sgpr76.num_vgpr, 27
-; CHECK-NEXT: .set .Laliasee_vgpr32_sgpr76.num_agpr, 0
-; CHECK-NEXT: .set .Laliasee_vgpr32_sgpr76.numbered_sgpr, 32
-define internal void @aliasee_vgpr32_sgpr76() #1 {
-bb:
- call void asm sideeffect "; clobber v26 ", "~{v26}"()
- ret void
-}
-
attributes #0 = { noinline norecurse nounwind optnone }
attributes #1 = { noinline norecurse nounwind readnone willreturn "amdgpu-waves-per-eu"="8,10" }
attributes #2 = { nounwind readnone willreturn }
diff --git a/llvm/test/CodeGen/AMDGPU/call-alias-register-usage2.ll b/llvm/test/CodeGen/AMDGPU/call-alias-register-usage2.ll
index 5735ef21b084d..afcac9cb4327d 100644
--- a/llvm/test/CodeGen/AMDGPU/call-alias-register-usage2.ll
+++ b/llvm/test/CodeGen/AMDGPU/call-alias-register-usage2.ll
@@ -6,9 +6,18 @@
@alias2 = hidden alias void (), ptr @aliasee_vgpr64_sgpr102
+; CHECK: .set .Laliasee_vgpr64_sgpr102.num_vgpr, 53
+; CHECK-NEXT: .set .Laliasee_vgpr64_sgpr102.num_agpr, 0
+; CHECK-NEXT: .set .Laliasee_vgpr64_sgpr102.numbered_sgpr, 32
+define internal void @aliasee_vgpr64_sgpr102() #1 {
+bb:
+ call void asm sideeffect "; clobber v52 ", "~{v52}"()
+ ret void
+}
+
; CHECK-LABEL: {{^}}kernel2:
-; CHECK: .amdhsa_next_free_vgpr max(totalnumvgprs(.Lkernel2.num_agpr, .Lkernel2.num_vgpr), 1, 0)
-; CHECK-NEXT: .amdhsa_next_free_sgpr max(.Lkernel2.numbered_sgpr+extrasgprs(.Lkernel2.uses_vcc, .Lkernel2.uses_flat_scratch, 1), 1, 0)-extrasgprs(.Lkernel2.uses_vcc, .Lkernel2.uses_flat_scratch, 1)
+; CHECK: .amdhsa_next_free_vgpr 53
+; CHECK-NEXT: .amdhsa_next_free_sgpr 33
; CHECK: .set .Lkernel2.num_vgpr, max(41, .Laliasee_vgpr64_sgpr102.num_vgpr)
; CHECK-NEXT: .set .Lkernel2.num_agpr, max(0, .Laliasee_vgpr64_sgpr102.num_agpr)
@@ -19,15 +28,6 @@ bb:
ret void
}
-; CHECK: .set .Laliasee_vgpr64_sgpr102.num_vgpr, 53
-; CHECK-NEXT: .set .Laliasee_vgpr64_sgpr102.num_agpr, 0
-; CHECK-NEXT: .set .Laliasee_vgpr64_sgpr102.numbered_sgpr, 32
-define internal void @aliasee_vgpr64_sgpr102() #1 {
-bb:
- call void asm sideeffect "; clobber v52 ", "~{v52}"()
- ret void
-}
-
attributes #0 = { noinline norecurse nounwind optnone }
attributes #1 = { noinline norecurse nounwind readnone willreturn "amdgpu-waves-per-eu"="4,10" }
attributes #2 = { nounwind readnone willreturn }
diff --git a/llvm/test/CodeGen/AMDGPU/call-alias-register-usage3.ll b/llvm/test/CodeGen/AMDGPU/call-alias-register-usage3.ll
index 7e6be8593e3c5..4c094d75ecd70 100644
--- a/llvm/test/CodeGen/AMDGPU/call-alias-register-usage3.ll
+++ b/llvm/test/CodeGen/AMDGPU/call-alias-register-usage3.ll
@@ -6,9 +6,18 @@
@alias3 = hidden alias void (), ptr @aliasee_vgpr256_sgpr102
+; CHECK: .set .Laliasee_vgpr256_sgpr102.num_vgpr, 253
+; CHECK-NEXT: .set .Laliasee_vgpr256_sgpr102.num_agpr, 0
+; CHECK-NEXT: .set .Laliasee_vgpr256_sgpr102.numbered_sgpr, 33
+define internal void @aliasee_vgpr256_sgpr102() #1 {
+bb:
+ call void asm sideeffect "; clobber v252 ", "~{v252}"()
+ ret void
+}
+
; CHECK-LABEL: {{^}}kernel3:
-; CHECK: .amdhsa_next_free_vgpr max(totalnumvgprs(.Lkernel3.num_agpr, .Lkernel3.num_vgpr), 1, 0)
-; CHECK-NEXT: .amdhsa_next_free_sgpr max(.Lkernel3.numbered_sgpr+extrasgprs(.Lkernel3.uses_vcc, .Lkernel3.uses_flat_scratch, 1), 1, 0)-extrasgprs(.Lkernel3.uses_vcc, .Lkernel3.uses_flat_scratch, 1)
+; CHECK: .amdhsa_next_free_vgpr 253
+; CHECK-NEXT: .amdhsa_next_free_sgpr 33
; CHECK: .set .Lkernel3.num_vgpr, max(41, .Laliasee_vgpr256_sgpr102.num_vgpr)
; CHECK-NEXT: .set .Lkernel3.num_agpr, max(0, .Laliasee_vgpr256_sgpr102.num_agpr)
@@ -19,15 +28,6 @@ bb:
ret void
}
-; CHECK: .set .Laliasee_vgpr256_sgpr102.num_vgpr, 253
-; CHECK-NEXT: .set .Laliasee_vgpr256_sgpr102.num_agpr, 0
-; CHECK-NEXT: .set .Laliasee_vgpr256_sgpr102.numbered_sgpr, 33
-define internal void @aliasee_vgpr256_sgpr102() #1 {
-bb:
- call void asm sideeffect "; clobber v252 ", "~{v252}"()
- ret void
-}
-
attributes #0 = { noinline norecurse nounwind optnone }
attributes #1 = { noinline norecurse nounwind readnone willreturn "amdgpu-flat-work-group-size"="1,256" "amdgpu-waves-per-eu"="1,1" }
attributes #2 = { nounwind readnone willreturn }
diff --git a/llvm/test/CodeGen/AMDGPU/call-constexpr.ll b/llvm/test/CodeGen/AMDGPU/call-constexpr.ll
deleted file mode 100644
index 53867b9031f6b..0000000000000
--- a/llvm/test/CodeGen/AMDGPU/call-constexpr.ll
+++ /dev/null
@@ -1,388 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
-; RUN: llc -global-isel=0 -mtriple=amdgpu7.00-amd-amdhsa < %s | FileCheck -check-prefixes=GCN,SDAG %s
-; RUN: llc -global-isel=1 -mtriple=amdgpu7.00-amd-amdhsa < %s | FileCheck -check-prefixes=GCN,GISEL %s
-
-define amdgpu_kernel void @test_bitcast_return_type_noinline() #0 {
-; SDAG-LABEL: test_bitcast_return_type_noinline:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: s_add_i32 s12, s12, s17
-; SDAG-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; SDAG-NEXT: s_add_u32 s0, s0, s17
-; SDAG-NEXT: v_lshlrev_b32_e32 v1, 10, v1
-; SDAG-NEXT: s_addc_u32 s1, s1, 0
-; SDAG-NEXT: v_lshlrev_b32_e32 v2, 20, v2
-; SDAG-NEXT: v_or_b32_e32 v0, v0, v1
-; SDAG-NEXT: s_mov_b32 flat_scratch_lo, s13
-; SDAG-NEXT: s_mov_b32 s13, s15
-; SDAG-NEXT: s_mov_b32 s12, s14
-; SDAG-NEXT: s_getpc_b64 s[18:19]
-; SDAG-NEXT: s_add_u32 s18, s18, ret_i32_noinline at rel32@lo+4
-; SDAG-NEXT: s_addc_u32 s19, s19, ret_i32_noinline at rel32@hi+12
-; SDAG-NEXT: v_or_b32_e32 v31, v0, v2
-; SDAG-NEXT: s_mov_b32 s14, s16
-; SDAG-NEXT: s_mov_b32 s32, 0
-; SDAG-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; SDAG-NEXT: v_add_f32_e32 v0, 1.0, v0
-; SDAG-NEXT: flat_store_dword v[0:1], v0
-; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: test_bitcast_return_type_noinline:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: s_add_i32 s12, s12, s17
-; GISEL-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; GISEL-NEXT: s_add_u32 s0, s0, s17
-; GISEL-NEXT: v_lshlrev_b32_e32 v1, 10, v1
-; GISEL-NEXT: s_addc_u32 s1, s1, 0
-; GISEL-NEXT: v_or_b32_e32 v0, v0, v1
-; GISEL-NEXT: v_lshlrev_b32_e32 v1, 20, v2
-; GISEL-NEXT: s_mov_b32 flat_scratch_lo, s13
-; GISEL-NEXT: s_mov_b32 s13, s15
-; GISEL-NEXT: s_mov_b32 s12, s14
-; GISEL-NEXT: v_or_b32_e32 v31, v0, v1
-; GISEL-NEXT: s_getpc_b64 s[18:19]
-; GISEL-NEXT: s_add_u32 s18, s18, ret_i32_noinline at rel32@lo+4
-; GISEL-NEXT: s_addc_u32 s19, s19, ret_i32_noinline at rel32@hi+12
-; GISEL-NEXT: s_mov_b32 s14, s16
-; GISEL-NEXT: s_mov_b32 s32, 0
-; GISEL-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GISEL-NEXT: v_add_f32_e32 v0, 1.0, v0
-; GISEL-NEXT: flat_store_dword v[0:1], v0
-; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: s_endpgm
- %val = call float @ret_i32_noinline()
- %op = fadd float %val, 1.0
- store volatile float %op, ptr addrspace(1) poison
- ret void
-}
-
-define amdgpu_kernel void @test_bitcast_return_type_alwaysinline() #0 {
-; SDAG-LABEL: test_bitcast_return_type_alwaysinline:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: s_add_i32 s12, s12, s17
-; SDAG-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; SDAG-NEXT: s_add_u32 s0, s0, s17
-; SDAG-NEXT: v_lshlrev_b32_e32 v1, 10, v1
-; SDAG-NEXT: s_addc_u32 s1, s1, 0
-; SDAG-NEXT: v_lshlrev_b32_e32 v2, 20, v2
-; SDAG-NEXT: v_or_b32_e32 v0, v0, v1
-; SDAG-NEXT: s_mov_b32 flat_scratch_lo, s13
-; SDAG-NEXT: s_mov_b32 s13, s15
-; SDAG-NEXT: s_mov_b32 s12, s14
-; SDAG-NEXT: s_getpc_b64 s[18:19]
-; SDAG-NEXT: s_add_u32 s18, s18, ret_i32_alwaysinline at rel32@lo+4
-; SDAG-NEXT: s_addc_u32 s19, s19, ret_i32_alwaysinline at rel32@hi+12
-; SDAG-NEXT: v_or_b32_e32 v31, v0, v2
-; SDAG-NEXT: s_mov_b32 s14, s16
-; SDAG-NEXT: s_mov_b32 s32, 0
-; SDAG-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; SDAG-NEXT: v_add_f32_e32 v0, 1.0, v0
-; SDAG-NEXT: flat_store_dword v[0:1], v0
-; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: test_bitcast_return_type_alwaysinline:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: s_add_i32 s12, s12, s17
-; GISEL-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; GISEL-NEXT: s_add_u32 s0, s0, s17
-; GISEL-NEXT: v_lshlrev_b32_e32 v1, 10, v1
-; GISEL-NEXT: s_addc_u32 s1, s1, 0
-; GISEL-NEXT: v_or_b32_e32 v0, v0, v1
-; GISEL-NEXT: v_lshlrev_b32_e32 v1, 20, v2
-; GISEL-NEXT: s_mov_b32 flat_scratch_lo, s13
-; GISEL-NEXT: s_mov_b32 s13, s15
-; GISEL-NEXT: s_mov_b32 s12, s14
-; GISEL-NEXT: v_or_b32_e32 v31, v0, v1
-; GISEL-NEXT: s_getpc_b64 s[18:19]
-; GISEL-NEXT: s_add_u32 s18, s18, ret_i32_alwaysinline at rel32@lo+4
-; GISEL-NEXT: s_addc_u32 s19, s19, ret_i32_alwaysinline at rel32@hi+12
-; GISEL-NEXT: s_mov_b32 s14, s16
-; GISEL-NEXT: s_mov_b32 s32, 0
-; GISEL-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GISEL-NEXT: v_add_f32_e32 v0, 1.0, v0
-; GISEL-NEXT: flat_store_dword v[0:1], v0
-; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: s_endpgm
- %val = call float @ret_i32_alwaysinline()
- %op = fadd float %val, 1.0
- store volatile float %op, ptr addrspace(1) poison
- ret void
-}
-
-define amdgpu_kernel void @test_bitcast_argument_type() #0 {
-; SDAG-LABEL: test_bitcast_argument_type:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: s_add_i32 s12, s12, s17
-; SDAG-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; SDAG-NEXT: s_add_u32 s0, s0, s17
-; SDAG-NEXT: v_lshlrev_b32_e32 v1, 10, v1
-; SDAG-NEXT: s_addc_u32 s1, s1, 0
-; SDAG-NEXT: v_lshlrev_b32_e32 v2, 20, v2
-; SDAG-NEXT: v_or_b32_e32 v0, v0, v1
-; SDAG-NEXT: s_mov_b32 flat_scratch_lo, s13
-; SDAG-NEXT: s_mov_b32 s13, s15
-; SDAG-NEXT: s_mov_b32 s12, s14
-; SDAG-NEXT: s_getpc_b64 s[18:19]
-; SDAG-NEXT: s_add_u32 s18, s18, ident_i32 at rel32@lo+4
-; SDAG-NEXT: s_addc_u32 s19, s19, ident_i32 at rel32@hi+12
-; SDAG-NEXT: v_or_b32_e32 v31, v0, v2
-; SDAG-NEXT: s_mov_b32 s14, s16
-; SDAG-NEXT: v_mov_b32_e32 v0, 2.0
-; SDAG-NEXT: s_mov_b32 s32, 0
-; SDAG-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; SDAG-NEXT: v_add_i32_e32 v0, vcc, 1, v0
-; SDAG-NEXT: flat_store_dword v[0:1], v0
-; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: test_bitcast_argument_type:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: s_add_i32 s12, s12, s17
-; GISEL-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; GISEL-NEXT: s_add_u32 s0, s0, s17
-; GISEL-NEXT: v_lshlrev_b32_e32 v1, 10, v1
-; GISEL-NEXT: s_addc_u32 s1, s1, 0
-; GISEL-NEXT: v_or_b32_e32 v0, v0, v1
-; GISEL-NEXT: v_lshlrev_b32_e32 v1, 20, v2
-; GISEL-NEXT: s_mov_b32 flat_scratch_lo, s13
-; GISEL-NEXT: s_mov_b32 s13, s15
-; GISEL-NEXT: s_mov_b32 s12, s14
-; GISEL-NEXT: v_or_b32_e32 v31, v0, v1
-; GISEL-NEXT: s_getpc_b64 s[18:19]
-; GISEL-NEXT: s_add_u32 s18, s18, ident_i32 at rel32@lo+4
-; GISEL-NEXT: s_addc_u32 s19, s19, ident_i32 at rel32@hi+12
-; GISEL-NEXT: v_mov_b32_e32 v0, 2.0
-; GISEL-NEXT: s_mov_b32 s14, s16
-; GISEL-NEXT: s_mov_b32 s32, 0
-; GISEL-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, 1, v0
-; GISEL-NEXT: flat_store_dword v[0:1], v0
-; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: s_endpgm
- %val = call i32 @ident_i32(float 2.0)
- %op = add i32 %val, 1
- store volatile i32 %op, ptr addrspace(1) poison
- ret void
-}
-
-define amdgpu_kernel void @test_bitcast_argument_and_return_types() #0 {
-; SDAG-LABEL: test_bitcast_argument_and_return_types:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: s_add_i32 s12, s12, s17
-; SDAG-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; SDAG-NEXT: s_add_u32 s0, s0, s17
-; SDAG-NEXT: v_lshlrev_b32_e32 v1, 10, v1
-; SDAG-NEXT: s_addc_u32 s1, s1, 0
-; SDAG-NEXT: v_lshlrev_b32_e32 v2, 20, v2
-; SDAG-NEXT: v_or_b32_e32 v0, v0, v1
-; SDAG-NEXT: s_mov_b32 flat_scratch_lo, s13
-; SDAG-NEXT: s_mov_b32 s13, s15
-; SDAG-NEXT: s_mov_b32 s12, s14
-; SDAG-NEXT: s_getpc_b64 s[18:19]
-; SDAG-NEXT: s_add_u32 s18, s18, ident_i32 at rel32@lo+4
-; SDAG-NEXT: s_addc_u32 s19, s19, ident_i32 at rel32@hi+12
-; SDAG-NEXT: v_or_b32_e32 v31, v0, v2
-; SDAG-NEXT: s_mov_b32 s14, s16
-; SDAG-NEXT: v_mov_b32_e32 v0, 2.0
-; SDAG-NEXT: s_mov_b32 s32, 0
-; SDAG-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; SDAG-NEXT: v_add_f32_e32 v0, 1.0, v0
-; SDAG-NEXT: flat_store_dword v[0:1], v0
-; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: test_bitcast_argument_and_return_types:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: s_add_i32 s12, s12, s17
-; GISEL-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; GISEL-NEXT: s_add_u32 s0, s0, s17
-; GISEL-NEXT: v_lshlrev_b32_e32 v1, 10, v1
-; GISEL-NEXT: s_addc_u32 s1, s1, 0
-; GISEL-NEXT: v_or_b32_e32 v0, v0, v1
-; GISEL-NEXT: v_lshlrev_b32_e32 v1, 20, v2
-; GISEL-NEXT: s_mov_b32 flat_scratch_lo, s13
-; GISEL-NEXT: s_mov_b32 s13, s15
-; GISEL-NEXT: s_mov_b32 s12, s14
-; GISEL-NEXT: v_or_b32_e32 v31, v0, v1
-; GISEL-NEXT: s_getpc_b64 s[18:19]
-; GISEL-NEXT: s_add_u32 s18, s18, ident_i32 at rel32@lo+4
-; GISEL-NEXT: s_addc_u32 s19, s19, ident_i32 at rel32@hi+12
-; GISEL-NEXT: v_mov_b32_e32 v0, 2.0
-; GISEL-NEXT: s_mov_b32 s14, s16
-; GISEL-NEXT: s_mov_b32 s32, 0
-; GISEL-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GISEL-NEXT: v_add_f32_e32 v0, 1.0, v0
-; GISEL-NEXT: flat_store_dword v[0:1], v0
-; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: s_endpgm
- %val = call float @ident_i32(float 2.0)
- %op = fadd float %val, 1.0
- store volatile float %op, ptr addrspace(1) poison
- ret void
-}
-
-define hidden i32 @use_workitem_id_x(i32 %arg0) #3 {
-; GCN-LABEL: use_workitem_id_x:
-; GCN: ; %bb.0:
-; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v1, 0x3ff, v31
-; GCN-NEXT: v_add_i32_e32 v0, vcc, v1, v0
-; GCN-NEXT: s_setpc_b64 s[30:31]
- %id = call i32 @llvm.amdgcn.workitem.id.x()
- %op = add i32 %id, %arg0
- ret i32 %op
-}
-
-define amdgpu_kernel void @test_bitcast_use_workitem_id_x() #3 {
-; SDAG-LABEL: test_bitcast_use_workitem_id_x:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: s_add_i32 s12, s12, s17
-; SDAG-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; SDAG-NEXT: s_add_u32 s0, s0, s17
-; SDAG-NEXT: s_addc_u32 s1, s1, 0
-; SDAG-NEXT: s_mov_b32 flat_scratch_lo, s13
-; SDAG-NEXT: s_mov_b32 s13, s15
-; SDAG-NEXT: s_mov_b32 s12, s14
-; SDAG-NEXT: s_getpc_b64 s[18:19]
-; SDAG-NEXT: s_add_u32 s18, s18, use_workitem_id_x at rel32@lo+4
-; SDAG-NEXT: s_addc_u32 s19, s19, use_workitem_id_x at rel32@hi+12
-; SDAG-NEXT: s_mov_b32 s14, s16
-; SDAG-NEXT: v_mov_b32_e32 v31, v0
-; SDAG-NEXT: v_mov_b32_e32 v0, 9
-; SDAG-NEXT: s_mov_b32 s32, 0
-; SDAG-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; SDAG-NEXT: v_add_f32_e32 v0, 1.0, v0
-; SDAG-NEXT: flat_store_dword v[0:1], v0
-; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: test_bitcast_use_workitem_id_x:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: s_add_i32 s12, s12, s17
-; GISEL-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; GISEL-NEXT: s_add_u32 s0, s0, s17
-; GISEL-NEXT: s_addc_u32 s1, s1, 0
-; GISEL-NEXT: s_mov_b32 flat_scratch_lo, s13
-; GISEL-NEXT: v_mov_b32_e32 v31, v0
-; GISEL-NEXT: s_mov_b32 s13, s15
-; GISEL-NEXT: s_mov_b32 s12, s14
-; GISEL-NEXT: s_getpc_b64 s[18:19]
-; GISEL-NEXT: s_add_u32 s18, s18, use_workitem_id_x at rel32@lo+4
-; GISEL-NEXT: s_addc_u32 s19, s19, use_workitem_id_x at rel32@hi+12
-; GISEL-NEXT: v_mov_b32_e32 v0, 9
-; GISEL-NEXT: s_mov_b32 s14, s16
-; GISEL-NEXT: s_mov_b32 s32, 0
-; GISEL-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GISEL-NEXT: v_add_f32_e32 v0, 1.0, v0
-; GISEL-NEXT: flat_store_dword v[0:1], v0
-; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: s_endpgm
- %val = call float @use_workitem_id_x(i32 9)
- %op = fadd float %val, 1.0
- store volatile float %op, ptr addrspace(1) poison
- ret void
-}
-
- at _ZTIi = external global ptr
-declare i32 @__gxx_personality_v0(...)
-define amdgpu_kernel void @test_invoke() #0 personality ptr @__gxx_personality_v0 {
-; SDAG-LABEL: test_invoke:
-; SDAG: ; %bb.0:
-; SDAG-NEXT: s_add_i32 s12, s12, s17
-; SDAG-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; SDAG-NEXT: s_add_u32 s0, s0, s17
-; SDAG-NEXT: v_lshlrev_b32_e32 v1, 10, v1
-; SDAG-NEXT: s_addc_u32 s1, s1, 0
-; SDAG-NEXT: v_lshlrev_b32_e32 v2, 20, v2
-; SDAG-NEXT: v_or_b32_e32 v0, v0, v1
-; SDAG-NEXT: s_mov_b32 flat_scratch_lo, s13
-; SDAG-NEXT: s_mov_b32 s13, s15
-; SDAG-NEXT: s_mov_b32 s12, s14
-; SDAG-NEXT: s_getpc_b64 s[18:19]
-; SDAG-NEXT: s_add_u32 s18, s18, ident_i32 at rel32@lo+4
-; SDAG-NEXT: s_addc_u32 s19, s19, ident_i32 at rel32@hi+12
-; SDAG-NEXT: v_or_b32_e32 v31, v0, v2
-; SDAG-NEXT: s_mov_b32 s14, s16
-; SDAG-NEXT: v_mov_b32_e32 v0, 2.0
-; SDAG-NEXT: s_mov_b32 s32, 0
-; SDAG-NEXT: ; implicit-def: $sgpr15
-; SDAG-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; SDAG-NEXT: v_add_f32_e32 v0, 1.0, v0
-; SDAG-NEXT: flat_store_dword v[0:1], v0
-; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: s_endpgm
-;
-; GISEL-LABEL: test_invoke:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: s_add_i32 s12, s12, s17
-; GISEL-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; GISEL-NEXT: s_add_u32 s0, s0, s17
-; GISEL-NEXT: v_lshlrev_b32_e32 v1, 10, v1
-; GISEL-NEXT: s_addc_u32 s1, s1, 0
-; GISEL-NEXT: v_or_b32_e32 v0, v0, v1
-; GISEL-NEXT: v_lshlrev_b32_e32 v1, 20, v2
-; GISEL-NEXT: s_mov_b32 flat_scratch_lo, s13
-; GISEL-NEXT: s_mov_b32 s13, s15
-; GISEL-NEXT: s_mov_b32 s12, s14
-; GISEL-NEXT: v_or_b32_e32 v31, v0, v1
-; GISEL-NEXT: s_getpc_b64 s[18:19]
-; GISEL-NEXT: s_add_u32 s18, s18, ident_i32 at rel32@lo+4
-; GISEL-NEXT: s_addc_u32 s19, s19, ident_i32 at rel32@hi+12
-; GISEL-NEXT: v_mov_b32_e32 v0, 2.0
-; GISEL-NEXT: s_mov_b32 s14, s16
-; GISEL-NEXT: s_mov_b32 s32, 0
-; GISEL-NEXT: ; implicit-def: $sgpr15
-; GISEL-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GISEL-NEXT: v_add_f32_e32 v0, 1.0, v0
-; GISEL-NEXT: flat_store_dword v[0:1], v0
-; GISEL-NEXT: s_waitcnt vmcnt(0)
-; GISEL-NEXT: s_endpgm
- %val = invoke float @ident_i32(float 2.0)
- to label %continue unwind label %broken
-
-broken:
- landingpad { ptr, i32 } catch ptr @_ZTIi
- ret void
-
-continue:
- %op = fadd float %val, 1.0
- store volatile float %op, ptr addrspace(1) poison
- ret void
-}
-
-; Callees appears last in source file to test that we still lower their
-; arguments before we lower any calls to them.
-
-define hidden i32 @ret_i32_noinline() #0 {
-; GCN-LABEL: ret_i32_noinline:
-; GCN: ; %bb.0:
-; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v0, 4
-; GCN-NEXT: s_setpc_b64 s[30:31]
- ret i32 4
-}
-
-define hidden i32 @ret_i32_alwaysinline() #1 {
-; GCN-LABEL: ret_i32_alwaysinline:
-; GCN: ; %bb.0:
-; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v0, 4
-; GCN-NEXT: s_setpc_b64 s[30:31]
- ret i32 4
-}
-
-define hidden i32 @ident_i32(i32 %i) #0 {
-; GCN-LABEL: ident_i32:
-; GCN: ; %bb.0:
-; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: s_setpc_b64 s[30:31]
- ret i32 %i
-}
-
-declare i32 @llvm.amdgcn.workitem.id.x() #2
-
-attributes #0 = { nounwind noinline }
-attributes #1 = { alwaysinline nounwind }
-attributes #2 = { nounwind readnone speculatable }
-attributes #3 = { nounwind noinline "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" }
diff --git a/llvm/test/CodeGen/AMDGPU/convergence-laneops.ll b/llvm/test/CodeGen/AMDGPU/convergence-laneops.ll
index 64a6edcc72c93..3353f200629dc 100644
--- a/llvm/test/CodeGen/AMDGPU/convergence-laneops.ll
+++ b/llvm/test/CodeGen/AMDGPU/convergence-laneops.ll
@@ -1,5 +1,5 @@
-; RUN: llc -stop-after=amdgpu-isel -mtriple=amdgpu11.00-- -verify-machineinstrs -o - %s | FileCheck --check-prefixes=CHECK,ISEL %s
-; RUN: not --crash llc -mtriple=amdgpu11.00--amdhsa -verify-machineinstrs < %s 2>&1 | FileCheck --check-prefix=CHECK-ERROR %s
+; RUN: llc -stop-after=amdgpu-isel -mtriple=amdgpu11.00-- -verify-each -o - %s | FileCheck --check-prefixes=CHECK,ISEL %s
+; RUN: not --crash llc -mtriple=amdgpu11.00--amdhsa -verify-each < %s 2>&1 | FileCheck --check-prefix=CHECK-ERROR %s
; FIXME: Merge these tests with existing lane op tests (llvm.amdgcn.readlane.ll, llvm.amdgcn.writelane.ll ...) once the crash is fixed.
diff --git a/llvm/test/CodeGen/AMDGPU/dead_bundle.mir b/llvm/test/CodeGen/AMDGPU/dead_bundle.mir
index 77d4a437d167d..b761deb2f4cf7 100644
--- a/llvm/test/CodeGen/AMDGPU/dead_bundle.mir
+++ b/llvm/test/CodeGen/AMDGPU/dead_bundle.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu11.00--amdpal -verify-machineinstrs=1 -start-before=greedy,0 -stop-after=virtregrewriter,0 -stress-regalloc=5 %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgpu11.00--amdpal -verify-machineinstrs=1 -start-before=greedy,1 -stop-after=virt-reg-rewriter,1 -stress-regalloc=5 %s -o - | FileCheck %s
# This test checks that dead bundles are handled correctly.
---
diff --git a/llvm/test/CodeGen/AMDGPU/dummy-regalloc-priority-advisor.mir b/llvm/test/CodeGen/AMDGPU/dummy-regalloc-priority-advisor.mir
index 3f034d709b102..f32ebfaeb4a78 100644
--- a/llvm/test/CodeGen/AMDGPU/dummy-regalloc-priority-advisor.mir
+++ b/llvm/test/CodeGen/AMDGPU/dummy-regalloc-priority-advisor.mir
@@ -1,6 +1,6 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
-# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -start-before=greedy,2 -stress-regalloc=4 -stop-after=virtregrewriter,2 -regalloc-enable-priority-advisor=default -o - %s | FileCheck -check-prefixes=CHECK,DEFAULT %s
-# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -start-before=greedy,2 -stress-regalloc=4 -stop-after=virtregrewriter,2 -regalloc-enable-priority-advisor=dummy -o - %s | FileCheck -check-prefixes=CHECK,DUMMY %s
+# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -start-before=greedy,3 -stress-regalloc=4 -stop-after=virt-reg-rewriter,3 -regalloc-enable-priority-advisor=default -o - %s | FileCheck -check-prefixes=CHECK,DEFAULT %s
+# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -start-before=greedy,3 -stress-regalloc=4 -stop-after=virt-reg-rewriter,3 -regalloc-enable-priority-advisor=dummy -o - %s | FileCheck -check-prefixes=CHECK,DUMMY %s
# Check that the regalloc-enable-priority-advisor=dummy option works
# and the result is different from the default. Ordinarily %1 would be
diff --git a/llvm/test/CodeGen/AMDGPU/dvgpr-vgpr-count-propagation-direct-chain.ll b/llvm/test/CodeGen/AMDGPU/dvgpr-vgpr-count-propagation-direct-chain.ll
index 88683131078cf..08e2cb9332ede 100644
--- a/llvm/test/CodeGen/AMDGPU/dvgpr-vgpr-count-propagation-direct-chain.ll
+++ b/llvm/test/CodeGen/AMDGPU/dvgpr-vgpr-count-propagation-direct-chain.ll
@@ -10,27 +10,27 @@
; func.2 and func.4 have DVGPRs disabled
-; DVGPR: .set .Lgfx_func_a.num_vgpr, 40
-; DVGPR: .set .Lamdgpu_cs_main.num_vgpr, max(42, .Lgfx_func_a.num_vgpr)
-; DVGPR: .set .Lfunc.0.num_vgpr, max(11, .Lgfx_func_a.num_vgpr)
-; DVGPR: .set .Lfunc.1.num_vgpr, 82
-; DVGPR: .set .Lfunc.2.num_vgpr, max(11, amdgpu.max_num_vgpr)
-; DVGPR: .set .Lfunc.3.num_vgpr, 11
-; DVGPR: .set .Lfunc.4.num_vgpr, max(82, amdgpu.max_num_vgpr)
-; DVGPR: .set .Lretry_vgpr_alloc.num_vgpr, 11
-; DVGPR: .set .Lfirst_retry_wrapper.num_vgpr, 11
-; DVGPR: .set amdgpu.max_num_vgpr, 82
-
-; NODVGPR: .set .Lgfx_func_a.num_vgpr, 40
-; NODVGPR: .set .Lamdgpu_cs_main.num_vgpr, max(42, amdgpu.max_num_vgpr)
-; NODVGPR: .set .Lfunc.0.num_vgpr, max(11, amdgpu.max_num_vgpr)
-; NODVGPR: .set .Lfunc.1.num_vgpr, max(82, amdgpu.max_num_vgpr)
-; NODVGPR: .set .Lfunc.2.num_vgpr, max(11, amdgpu.max_num_vgpr)
-; NODVGPR: .set .Lfunc.3.num_vgpr, max(11, amdgpu.max_num_vgpr)
-; NODVGPR: .set .Lfunc.4.num_vgpr, max(82, amdgpu.max_num_vgpr)
-; NODVGPR: .set .Lretry_vgpr_alloc.num_vgpr, max(11, amdgpu.max_num_vgpr)
-; NODVGPR: .set .Lfirst_retry_wrapper.num_vgpr, max(11, amdgpu.max_num_vgpr)
-; NODVGPR: .set amdgpu.max_num_vgpr, 82
+; DVGPR-DAG: .set .Lgfx_func_a.num_vgpr, 40
+; DVGPR-DAG: .set .Lamdgpu_cs_main.num_vgpr, max(42, .Lgfx_func_a.num_vgpr)
+; DVGPR-DAG: .set .Lfunc.0.num_vgpr, max(11, .Lgfx_func_a.num_vgpr)
+; DVGPR-DAG: .set .Lfunc.1.num_vgpr, 82
+; DVGPR-DAG: .set .Lfunc.2.num_vgpr, max(11, amdgpu.max_num_vgpr)
+; DVGPR-DAG: .set .Lfunc.3.num_vgpr, 11
+; DVGPR-DAG: .set .Lfunc.4.num_vgpr, max(82, amdgpu.max_num_vgpr)
+; DVGPR-DAG: .set .Lretry_vgpr_alloc.num_vgpr, 11
+; DVGPR-DAG: .set .Lfirst_retry_wrapper.num_vgpr, 11
+; DVGPR-DAG: .set amdgpu.max_num_vgpr, 82
+
+; NODVGPR-DAG: .set .Lgfx_func_a.num_vgpr, 40
+; NODVGPR-DAG: .set .Lamdgpu_cs_main.num_vgpr, max(42, amdgpu.max_num_vgpr)
+; NODVGPR-DAG: .set .Lfunc.0.num_vgpr, max(11, amdgpu.max_num_vgpr)
+; NODVGPR-DAG: .set .Lfunc.1.num_vgpr, max(82, amdgpu.max_num_vgpr)
+; NODVGPR-DAG: .set .Lfunc.2.num_vgpr, max(11, amdgpu.max_num_vgpr)
+; NODVGPR-DAG: .set .Lfunc.3.num_vgpr, max(11, amdgpu.max_num_vgpr)
+; NODVGPR-DAG: .set .Lfunc.4.num_vgpr, max(82, amdgpu.max_num_vgpr)
+; NODVGPR-DAG: .set .Lretry_vgpr_alloc.num_vgpr, max(11, amdgpu.max_num_vgpr)
+; NODVGPR-DAG: .set .Lfirst_retry_wrapper.num_vgpr, max(11, amdgpu.max_num_vgpr)
+; NODVGPR-DAG: .set amdgpu.max_num_vgpr, 82
; DVGPR: - .hardware_stages:
; DVGPR: .vgpr_count: 0x2a
diff --git a/llvm/test/CodeGen/AMDGPU/dvgpr-vgpr-count-propagation-indirect.ll b/llvm/test/CodeGen/AMDGPU/dvgpr-vgpr-count-propagation-indirect.ll
index 21028afc34003..b9f27d01205eb 100644
--- a/llvm/test/CodeGen/AMDGPU/dvgpr-vgpr-count-propagation-indirect.ll
+++ b/llvm/test/CodeGen/AMDGPU/dvgpr-vgpr-count-propagation-indirect.ll
@@ -6,19 +6,19 @@
; their `num_vgpr` count set to the max of their local count and the module-wide
; max VGPR count of all non-chain functions.
-; DVGPR: .set .Lgfx_func_a.num_vgpr, 40
-; DVGPR: .set .Lgfx_func_b.num_vgpr, 80
-; DVGPR: .set .Lfunc_with_indirect_call.num_vgpr, max(11, amdgpu.max_num_vgpr)
-; DVGPR: .set .Lfunc_direct_only.num_vgpr, max(11, .Lgfx_func_a.num_vgpr)
-; DVGPR: .set .Lfunc_chain_only.num_vgpr, 11
-; DVGPR: .set amdgpu.max_num_vgpr, 80
-
-; NODVGPR: .set .Lgfx_func_a.num_vgpr, 40
-; NODVGPR: .set .Lgfx_func_b.num_vgpr, 80
-; NODVGPR: .set .Lfunc_with_indirect_call.num_vgpr, max(11, amdgpu.max_num_vgpr)
-; NODVGPR: .set .Lfunc_direct_only.num_vgpr, max(11, amdgpu.max_num_vgpr)
-; NODVGPR: .set .Lfunc_chain_only.num_vgpr, max(11, amdgpu.max_num_vgpr)
-; NODVGPR: .set amdgpu.max_num_vgpr, 80
+; DVGPR-DAG: .set .Lgfx_func_a.num_vgpr, 40
+; DVGPR-DAG: .set .Lgfx_func_b.num_vgpr, 80
+; DVGPR-DAG: .set .Lfunc_with_indirect_call.num_vgpr, max(11, amdgpu.max_num_vgpr)
+; DVGPR-DAG: .set .Lfunc_direct_only.num_vgpr, max(11, .Lgfx_func_a.num_vgpr)
+; DVGPR-DAG: .set .Lfunc_chain_only.num_vgpr, 11
+; DVGPR-DAG: .set amdgpu.max_num_vgpr, 80
+
+; NODVGPR-DAG: .set .Lgfx_func_a.num_vgpr, 40
+; NODVGPR-DAG: .set .Lgfx_func_b.num_vgpr, 80
+; NODVGPR-DAG: .set .Lfunc_with_indirect_call.num_vgpr, max(11, amdgpu.max_num_vgpr)
+; NODVGPR-DAG: .set .Lfunc_direct_only.num_vgpr, max(11, amdgpu.max_num_vgpr)
+; NODVGPR-DAG: .set .Lfunc_chain_only.num_vgpr, max(11, amdgpu.max_num_vgpr)
+; NODVGPR-DAG: .set amdgpu.max_num_vgpr, 80
define amdgpu_gfx void @gfx_func_a() #0 {
call void asm sideeffect "", "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7},~{v8},~{v9},~{v10},~{v11},~{v12},~{v13},~{v14},~{v15},~{v16},~{v17},~{v18},~{v19},~{v20},~{v21},~{v22},~{v23},~{v24},~{v25},~{v26},~{v27},~{v28},~{v29},~{v30},~{v31},~{v32},~{v33},~{v34},~{v35},~{v36},~{v37},~{v38},~{v39}"()
diff --git a/llvm/test/CodeGen/AMDGPU/dvgpr-vgpr-count-propagation.ll b/llvm/test/CodeGen/AMDGPU/dvgpr-vgpr-count-propagation.ll
index 22ea185019cf9..43f107611f4b8 100644
--- a/llvm/test/CodeGen/AMDGPU/dvgpr-vgpr-count-propagation.ll
+++ b/llvm/test/CodeGen/AMDGPU/dvgpr-vgpr-count-propagation.ll
@@ -11,31 +11,31 @@
; non-chain functions. With DVGPRs disabled, it represents module-wide max VGPR
; count.
-; DVGPR: .set .Lgfx_func_a.num_vgpr, 40
-; DVGPR: .set .Lgfx_func_b2.num_vgpr, 80
-; DVGPR: .set .Lgfx_func_b.num_vgpr, max(61, .Lgfx_func_b2.num_vgpr)
-; DVGPR: .set .Lamdgpu_cs_main.num_vgpr, max(42, .Lgfx_func_a.num_vgpr)
-; DVGPR: .set .Lfunc.0.num_vgpr, 13
-; DVGPR: .set .Lfunc.1.num_vgpr, max(14, .Lgfx_func_a.num_vgpr, .Lgfx_func_b.num_vgpr)
-; DVGPR: .set .Lfunc.2.num_vgpr, max(16, .Lgfx_func_a.num_vgpr)
-; DVGPR: .set .Lfunc.3.num_vgpr, max(15, .Lgfx_func_b.num_vgpr)
-; DVGPR: .set .Lfunc.4.num_vgpr, max(100, .Lgfx_func_b.num_vgpr)
-; DVGPR: .set .Lretry_vgpr_alloc.num_vgpr, 11
-; DVGPR: .set .Lfirst_retry_wrapper.num_vgpr, 11
-; DVGPR: .set amdgpu.max_num_vgpr, 80
-
-; NODVGPR: .set .Lgfx_func_a.num_vgpr, 40
-; NODVGPR: .set .Lgfx_func_b2.num_vgpr, 80
-; NODVGPR: .set .Lgfx_func_b.num_vgpr, max(61, .Lgfx_func_b2.num_vgpr)
-; NODVGPR: .set .Lamdgpu_cs_main.num_vgpr, max(42, amdgpu.max_num_vgpr)
-; NODVGPR: .set .Lfunc.0.num_vgpr, max(13, amdgpu.max_num_vgpr)
-; NODVGPR: .set .Lfunc.1.num_vgpr, max(14, amdgpu.max_num_vgpr)
-; NODVGPR: .set .Lfunc.2.num_vgpr, max(16, amdgpu.max_num_vgpr)
-; NODVGPR: .set .Lfunc.3.num_vgpr, max(15, amdgpu.max_num_vgpr)
-; NODVGPR: .set .Lfunc.4.num_vgpr, max(100, amdgpu.max_num_vgpr)
-; NODVGPR: .set .Lretry_vgpr_alloc.num_vgpr, max(11, amdgpu.max_num_vgpr)
-; NODVGPR: .set .Lfirst_retry_wrapper.num_vgpr, max(11, amdgpu.max_num_vgpr)
-; NODVGPR: .set amdgpu.max_num_vgpr, 100
+; DVGPR-DAG: .set .Lgfx_func_a.num_vgpr, 40
+; DVGPR-DAG: .set .Lgfx_func_b2.num_vgpr, 80
+; DVGPR-DAG: .set .Lgfx_func_b.num_vgpr, max(61, .Lgfx_func_b2.num_vgpr)
+; DVGPR-DAG: .set .Lamdgpu_cs_main.num_vgpr, max(42, .Lgfx_func_a.num_vgpr)
+; DVGPR-DAG: .set .Lfunc.0.num_vgpr, 13
+; DVGPR-DAG: .set .Lfunc.1.num_vgpr, max(14, .Lgfx_func_a.num_vgpr, .Lgfx_func_b.num_vgpr)
+; DVGPR-DAG: .set .Lfunc.2.num_vgpr, max(16, .Lgfx_func_a.num_vgpr)
+; DVGPR-DAG: .set .Lfunc.3.num_vgpr, max(15, .Lgfx_func_b.num_vgpr)
+; DVGPR-DAG: .set .Lfunc.4.num_vgpr, max(100, .Lgfx_func_b.num_vgpr)
+; DVGPR-DAG: .set .Lretry_vgpr_alloc.num_vgpr, 11
+; DVGPR-DAG: .set .Lfirst_retry_wrapper.num_vgpr, 11
+; DVGPR-DAG: .set amdgpu.max_num_vgpr, 80
+
+; NODVGPR-DAG: .set .Lgfx_func_a.num_vgpr, 40
+; NODVGPR-DAG: .set .Lgfx_func_b2.num_vgpr, 80
+; NODVGPR-DAG: .set .Lgfx_func_b.num_vgpr, max(61, .Lgfx_func_b2.num_vgpr)
+; NODVGPR-DAG: .set .Lamdgpu_cs_main.num_vgpr, max(42, amdgpu.max_num_vgpr)
+; NODVGPR-DAG: .set .Lfunc.0.num_vgpr, max(13, amdgpu.max_num_vgpr)
+; NODVGPR-DAG: .set .Lfunc.1.num_vgpr, max(14, amdgpu.max_num_vgpr)
+; NODVGPR-DAG: .set .Lfunc.2.num_vgpr, max(16, amdgpu.max_num_vgpr)
+; NODVGPR-DAG: .set .Lfunc.3.num_vgpr, max(15, amdgpu.max_num_vgpr)
+; NODVGPR-DAG: .set .Lfunc.4.num_vgpr, max(100, amdgpu.max_num_vgpr)
+; NODVGPR-DAG: .set .Lretry_vgpr_alloc.num_vgpr, max(11, amdgpu.max_num_vgpr)
+; NODVGPR-DAG: .set .Lfirst_retry_wrapper.num_vgpr, max(11, amdgpu.max_num_vgpr)
+; NODVGPR-DAG: .set amdgpu.max_num_vgpr, 100
; DVGPR: - .hardware_stages:
; DVGPR: .vgpr_count: 0x2a
diff --git a/llvm/test/CodeGen/AMDGPU/extend-wwm-virt-reg-liveness.mir b/llvm/test/CodeGen/AMDGPU/extend-wwm-virt-reg-liveness.mir
index 3433534698dd4..0b7ad9256c71d 100644
--- a/llvm/test/CodeGen/AMDGPU/extend-wwm-virt-reg-liveness.mir
+++ b/llvm/test/CodeGen/AMDGPU/extend-wwm-virt-reg-liveness.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu9.06-amd-amdhsa -start-before=si-lower-sgpr-spills -stop-after=virtregrewriter,1 -verify-machineinstrs %s -o - | FileCheck -check-prefix=GCN %s
+# RUN: llc -mtriple=amdgpu9.06-amd-amdhsa -start-before=si-lower-sgpr-spills -stop-after=virt-reg-rewriter,2 -verify-machineinstrs %s -o - | FileCheck -check-prefix=GCN %s
# Tests to check the conservative lieness extension for the wwm registers during SGPR spill lowering.
diff --git a/llvm/test/CodeGen/AMDGPU/fastregalloc-sgpr-only.mir b/llvm/test/CodeGen/AMDGPU/fastregalloc-sgpr-only.mir
index 22908c4834b63..305ec1ff8708c 100644
--- a/llvm/test/CodeGen/AMDGPU/fastregalloc-sgpr-only.mir
+++ b/llvm/test/CodeGen/AMDGPU/fastregalloc-sgpr-only.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -O0 -mtriple=amdgpu9.00-amd-amdhsa -start-before=regallocfast,0 --stop-after=regallocfast,0 -o - %s | FileCheck -check-prefix=GCN %s
+# RUN: llc -O0 -mtriple=amdgpu9.00-amd-amdhsa -start-before=regallocfast,1 --stop-after=regallocfast,1 -o - %s | FileCheck -check-prefix=GCN %s
---
name: copy_vgpr_allocation
diff --git a/llvm/test/CodeGen/AMDGPU/fmamk_fmaak-t16.mir b/llvm/test/CodeGen/AMDGPU/fmamk_fmaak-t16.mir
index 24bfc46380e6a..d6f0e129782f6 100644
--- a/llvm/test/CodeGen/AMDGPU/fmamk_fmaak-t16.mir
+++ b/llvm/test/CodeGen/AMDGPU/fmamk_fmaak-t16.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
-# RUN: llc -mtriple=amdgpu11.00 -mattr=+real-true16 -start-before=greedy,0 -stop-after=virtregrewriter,2 -o - %s | FileCheck -check-prefix=GFX11 %s
+# RUN: llc -mtriple=amdgpu11.00 -mattr=+real-true16 -start-before=greedy,1 -stop-after=virt-reg-rewriter,3 -o - %s | FileCheck -check-prefix=GFX11 %s
---
name: v_fmamk_f16
diff --git a/llvm/test/CodeGen/AMDGPU/fold-restore-undef-use.mir b/llvm/test/CodeGen/AMDGPU/fold-restore-undef-use.mir
index 9297b44695c68..06725af984963 100644
--- a/llvm/test/CodeGen/AMDGPU/fold-restore-undef-use.mir
+++ b/llvm/test/CodeGen/AMDGPU/fold-restore-undef-use.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu9.0a-amd-amdhsa -stress-regalloc=4 -verify-regalloc -start-before=greedy,0 -stop-after=virtregrewriter,0 %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.0a-amd-amdhsa -stress-regalloc=4 -verify-regalloc -start-before=greedy,1 -stop-after=virt-reg-rewriter,1 %s -o - | FileCheck %s
# RUN: llc -enable-new-pm -mtriple=amdgpu9.0a-amd-amdhsa -stress-regalloc=4 -verify-regalloc -passes="greedy<sgpr>,virt-reg-rewriter<no-clear-vregs>" %s -o - | FileCheck %s
diff --git a/llvm/test/CodeGen/AMDGPU/function-resource-usage.ll b/llvm/test/CodeGen/AMDGPU/function-resource-usage.ll
index 6db3a35aaae72..f721cca014554 100644
--- a/llvm/test/CodeGen/AMDGPU/function-resource-usage.ll
+++ b/llvm/test/CodeGen/AMDGPU/function-resource-usage.ll
@@ -482,29 +482,29 @@ define amdgpu_kernel void @usage_direct_recursion(i32 %n) #0 {
ret void
}
-; GCN-LABEL: {{^}}multi_stage_recurse2:
-; GCN: .set .Lmulti_stage_recurse2.num_vgpr, max(43, .Lmulti_stage_recurse1.num_vgpr)
-; GCN: .set .Lmulti_stage_recurse2.num_agpr, max(0, .Lmulti_stage_recurse1.num_agpr)
-; GCN: .set .Lmulti_stage_recurse2.numbered_sgpr, max(34, .Lmulti_stage_recurse1.numbered_sgpr)
-; GCN: .set .Lmulti_stage_recurse2.private_seg_size, 16+max(.Lmulti_stage_recurse1.private_seg_size)
-; GCN: .set .Lmulti_stage_recurse2.uses_vcc, or(1, .Lmulti_stage_recurse1.uses_vcc)
-; GCN: .set .Lmulti_stage_recurse2.uses_flat_scratch, or(0, .Lmulti_stage_recurse1.uses_flat_scratch)
-; GCN: .set .Lmulti_stage_recurse2.has_dyn_sized_stack, or(0, .Lmulti_stage_recurse1.has_dyn_sized_stack)
-; GCN: .set .Lmulti_stage_recurse2.has_recursion, or(1, .Lmulti_stage_recurse1.has_recursion)
-; GCN: .set .Lmulti_stage_recurse2.has_indirect_call, or(0, .Lmulti_stage_recurse1.has_indirect_call)
-; GCN: TotalNumSgprs: .Lmulti_stage_recurse2.numbered_sgpr+extrasgprs(.Lmulti_stage_recurse2.uses_vcc, .Lmulti_stage_recurse2.uses_flat_scratch, 1)
-; GCN: NumVgprs: max(43, .Lmulti_stage_recurse1.num_vgpr)
-; GCN: ScratchSize: 16+max(.Lmulti_stage_recurse1.private_seg_size)
; GCN-LABEL: {{^}}multi_stage_recurse1:
-; GCN: .set .Lmulti_stage_recurse1.num_vgpr, max(48, 43)
-; GCN: .set .Lmulti_stage_recurse1.num_agpr, max(0, 0)
-; GCN: .set .Lmulti_stage_recurse1.numbered_sgpr, max(34, 34)
-; GCN: .set .Lmulti_stage_recurse1.private_seg_size, 16
-; GCN: .set .Lmulti_stage_recurse1.uses_vcc, 1
-; GCN: .set .Lmulti_stage_recurse1.uses_flat_scratch, 0
-; GCN: .set .Lmulti_stage_recurse1.has_dyn_sized_stack, 0
-; GCN: .set .Lmulti_stage_recurse1.has_recursion, 1
-; GCN: .set .Lmulti_stage_recurse1.has_indirect_call, 0
+; GCN: .set .Lmulti_stage_recurse1.num_vgpr, max(48, .Lmulti_stage_recurse2.num_vgpr)
+; GCN: .set .Lmulti_stage_recurse1.num_agpr, max(0, .Lmulti_stage_recurse2.num_agpr)
+; GCN: .set .Lmulti_stage_recurse1.numbered_sgpr, max(34, .Lmulti_stage_recurse2.numbered_sgpr)
+; GCN: .set .Lmulti_stage_recurse1.private_seg_size, 16+max(.Lmulti_stage_recurse2.private_seg_size)
+; GCN: .set .Lmulti_stage_recurse1.uses_vcc, or(1, .Lmulti_stage_recurse2.uses_vcc)
+; GCN: .set .Lmulti_stage_recurse1.uses_flat_scratch, or(0, .Lmulti_stage_recurse2.uses_flat_scratch)
+; GCN: .set .Lmulti_stage_recurse1.has_dyn_sized_stack, or(0, .Lmulti_stage_recurse2.has_dyn_sized_stack)
+; GCN: .set .Lmulti_stage_recurse1.has_recursion, or(1, .Lmulti_stage_recurse2.has_recursion)
+; GCN: .set .Lmulti_stage_recurse1.has_indirect_call, or(0, .Lmulti_stage_recurse2.has_indirect_call)
+; GCN: TotalNumSgprs: .Lmulti_stage_recurse1.numbered_sgpr+extrasgprs(.Lmulti_stage_recurse1.uses_vcc, .Lmulti_stage_recurse1.uses_flat_scratch, 1)
+; GCN: NumVgprs: max(48, .Lmulti_stage_recurse2.num_vgpr)
+; GCN: ScratchSize: 16+max(.Lmulti_stage_recurse2.private_seg_size)
+; GCN-LABEL: {{^}}multi_stage_recurse2:
+; GCN: .set .Lmulti_stage_recurse2.num_vgpr, max(43, 48)
+; GCN: .set .Lmulti_stage_recurse2.num_agpr, max(0, 0)
+; GCN: .set .Lmulti_stage_recurse2.numbered_sgpr, max(34, 34)
+; GCN: .set .Lmulti_stage_recurse2.private_seg_size, 16
+; GCN: .set .Lmulti_stage_recurse2.uses_vcc, 1
+; GCN: .set .Lmulti_stage_recurse2.uses_flat_scratch, 0
+; GCN: .set .Lmulti_stage_recurse2.has_dyn_sized_stack, 0
+; GCN: .set .Lmulti_stage_recurse2.has_recursion, 1
+; GCN: .set .Lmulti_stage_recurse2.has_indirect_call, 0
; GCN: TotalNumSgprs: 38
; GCN: NumVgprs: 48
; GCN: ScratchSize: 16
@@ -531,35 +531,35 @@ define void @multi_stage_recurse2(i32 %val) #2 {
; GCN: .set .Lusage_multi_stage_recurse.has_indirect_call, or(0, .Lmulti_stage_recurse1.has_indirect_call)
; GCN: TotalNumSgprs: 40
; GCN: NumVgprs: 48
-; GCN: ScratchSize: 16
+; GCN: ScratchSize: 32
define amdgpu_kernel void @usage_multi_stage_recurse(i32 %n) #0 {
call void @multi_stage_recurse1(i32 %n)
ret void
}
-; GCN-LABEL: {{^}}multi_stage_recurse_noattr2:
-; GCN: .set .Lmulti_stage_recurse_noattr2.num_vgpr, max(41, .Lmulti_stage_recurse_noattr1.num_vgpr)
-; GCN: .set .Lmulti_stage_recurse_noattr2.num_agpr, max(0, .Lmulti_stage_recurse_noattr1.num_agpr)
-; GCN: .set .Lmulti_stage_recurse_noattr2.numbered_sgpr, max(54, .Lmulti_stage_recurse_noattr1.numbered_sgpr)
-; GCN: .set .Lmulti_stage_recurse_noattr2.private_seg_size, 16+max(.Lmulti_stage_recurse_noattr1.private_seg_size)
-; GCN: .set .Lmulti_stage_recurse_noattr2.uses_vcc, or(1, .Lmulti_stage_recurse_noattr1.uses_vcc)
-; GCN: .set .Lmulti_stage_recurse_noattr2.uses_flat_scratch, or(0, .Lmulti_stage_recurse_noattr1.uses_flat_scratch)
-; GCN: .set .Lmulti_stage_recurse_noattr2.has_dyn_sized_stack, or(0, .Lmulti_stage_recurse_noattr1.has_dyn_sized_stack)
-; GCN: .set .Lmulti_stage_recurse_noattr2.has_recursion, or(0, .Lmulti_stage_recurse_noattr1.has_recursion)
-; GCN: .set .Lmulti_stage_recurse_noattr2.has_indirect_call, or(0, .Lmulti_stage_recurse_noattr1.has_indirect_call)
-; GCN: TotalNumSgprs: .Lmulti_stage_recurse_noattr2.numbered_sgpr+extrasgprs(.Lmulti_stage_recurse_noattr2.uses_vcc, .Lmulti_stage_recurse_noattr2.uses_flat_scratch, 1)
-; GCN: NumVgprs: max(41, .Lmulti_stage_recurse_noattr1.num_vgpr)
-; GCN: ScratchSize: 16+max(.Lmulti_stage_recurse_noattr1.private_seg_size)
; GCN-LABEL: {{^}}multi_stage_recurse_noattr1:
-; GCN: .set .Lmulti_stage_recurse_noattr1.num_vgpr, max(41, 41)
-; GCN: .set .Lmulti_stage_recurse_noattr1.num_agpr, max(0, 0)
-; GCN: .set .Lmulti_stage_recurse_noattr1.numbered_sgpr, max(57, 54)
-; GCN: .set .Lmulti_stage_recurse_noattr1.private_seg_size, 16
-; GCN: .set .Lmulti_stage_recurse_noattr1.uses_vcc, 1
-; GCN: .set .Lmulti_stage_recurse_noattr1.uses_flat_scratch, 0
-; GCN: .set .Lmulti_stage_recurse_noattr1.has_dyn_sized_stack, 0
-; GCN: .set .Lmulti_stage_recurse_noattr1.has_recursion, 0
-; GCN: .set .Lmulti_stage_recurse_noattr1.has_indirect_call, 0
+; GCN: .set .Lmulti_stage_recurse_noattr1.num_vgpr, max(41, .Lmulti_stage_recurse_noattr2.num_vgpr)
+; GCN: .set .Lmulti_stage_recurse_noattr1.num_agpr, max(0, .Lmulti_stage_recurse_noattr2.num_agpr)
+; GCN: .set .Lmulti_stage_recurse_noattr1.numbered_sgpr, max(57, .Lmulti_stage_recurse_noattr2.numbered_sgpr)
+; GCN: .set .Lmulti_stage_recurse_noattr1.private_seg_size, 16+max(.Lmulti_stage_recurse_noattr2.private_seg_size)
+; GCN: .set .Lmulti_stage_recurse_noattr1.uses_vcc, or(1, .Lmulti_stage_recurse_noattr2.uses_vcc)
+; GCN: .set .Lmulti_stage_recurse_noattr1.uses_flat_scratch, or(0, .Lmulti_stage_recurse_noattr2.uses_flat_scratch)
+; GCN: .set .Lmulti_stage_recurse_noattr1.has_dyn_sized_stack, or(0, .Lmulti_stage_recurse_noattr2.has_dyn_sized_stack)
+; GCN: .set .Lmulti_stage_recurse_noattr1.has_recursion, or(0, .Lmulti_stage_recurse_noattr2.has_recursion)
+; GCN: .set .Lmulti_stage_recurse_noattr1.has_indirect_call, or(0, .Lmulti_stage_recurse_noattr2.has_indirect_call)
+; GCN: TotalNumSgprs: .Lmulti_stage_recurse_noattr1.numbered_sgpr+extrasgprs(.Lmulti_stage_recurse_noattr1.uses_vcc, .Lmulti_stage_recurse_noattr1.uses_flat_scratch, 1)
+; GCN: NumVgprs: max(41, .Lmulti_stage_recurse_noattr2.num_vgpr)
+; GCN: ScratchSize: 16+max(.Lmulti_stage_recurse_noattr2.private_seg_size)
+; GCN-LABEL: {{^}}multi_stage_recurse_noattr2:
+; GCN: .set .Lmulti_stage_recurse_noattr2.num_vgpr, max(41, 41)
+; GCN: .set .Lmulti_stage_recurse_noattr2.num_agpr, max(0, 0)
+; GCN: .set .Lmulti_stage_recurse_noattr2.numbered_sgpr, max(54, 57)
+; GCN: .set .Lmulti_stage_recurse_noattr2.private_seg_size, 16
+; GCN: .set .Lmulti_stage_recurse_noattr2.uses_vcc, 1
+; GCN: .set .Lmulti_stage_recurse_noattr2.uses_flat_scratch, 0
+; GCN: .set .Lmulti_stage_recurse_noattr2.has_dyn_sized_stack, 0
+; GCN: .set .Lmulti_stage_recurse_noattr2.has_recursion, 0
+; GCN: .set .Lmulti_stage_recurse_noattr2.has_indirect_call, 0
; GCN: TotalNumSgprs: 61
; GCN: NumVgprs: 41
; GCN: ScratchSize: 16
@@ -586,7 +586,7 @@ define void @multi_stage_recurse_noattr2(i32 %val) #0 {
; GCN: .set .Lusage_multi_stage_recurse_noattrs.has_indirect_call, or(0, .Lmulti_stage_recurse_noattr1.has_indirect_call)
; GCN: TotalNumSgprs: 63
; GCN: NumVgprs: 41
-; GCN: ScratchSize: 16
+; GCN: ScratchSize: 32
define amdgpu_kernel void @usage_multi_stage_recurse_noattrs(i32 %n) #0 {
call void @multi_stage_recurse_noattr1(i32 %n)
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/greedy-alloc-fail-sgpr1024-spill.mir b/llvm/test/CodeGen/AMDGPU/greedy-alloc-fail-sgpr1024-spill.mir
index 94c22b1aa8664..20ec57429619b 100644
--- a/llvm/test/CodeGen/AMDGPU/greedy-alloc-fail-sgpr1024-spill.mir
+++ b/llvm/test/CodeGen/AMDGPU/greedy-alloc-fail-sgpr1024-spill.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu9.08-amd-amdhsa -start-before=greedy,0 -stop-after=virtregrewriter,0 -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.08-amd-amdhsa -start-before=greedy,1 -stop-after=virt-reg-rewriter,1 -o - %s | FileCheck %s
# This testcase used to fail due to introducing a spill of an SGPR
# 1024 for every subregister use inside the loop. With overlapping
diff --git a/llvm/test/CodeGen/AMDGPU/greedy-instruction-split-subrange.mir b/llvm/test/CodeGen/AMDGPU/greedy-instruction-split-subrange.mir
index a3523221c7637..dceb62e0907c9 100644
--- a/llvm/test/CodeGen/AMDGPU/greedy-instruction-split-subrange.mir
+++ b/llvm/test/CodeGen/AMDGPU/greedy-instruction-split-subrange.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -verify-regalloc -stress-regalloc=3 -start-before=greedy,1 -stop-before=virtregrewriter,1 -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -verify-regalloc -stress-regalloc=3 -start-before=greedy,2 -stop-before=virt-reg-rewriter,2 -o - %s | FileCheck %s
---
name: split_instruction_subranges
alignment: 1
diff --git a/llvm/test/CodeGen/AMDGPU/greedy-remark-crash-unassigned-reg.mir b/llvm/test/CodeGen/AMDGPU/greedy-remark-crash-unassigned-reg.mir
index 71e99a6589882..1f3aa4030d623 100644
--- a/llvm/test/CodeGen/AMDGPU/greedy-remark-crash-unassigned-reg.mir
+++ b/llvm/test/CodeGen/AMDGPU/greedy-remark-crash-unassigned-reg.mir
@@ -1,5 +1,5 @@
# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa \
-# RUN: -start-before=greedy,0 -stop-after=virtregrewriter,0 -pass-remarks='.*' -pass-remarks-output=%t.yaml -filetype=null %s
+# RUN: -start-before=greedy,1 -stop-after=virt-reg-rewriter,1 -pass-remarks='.*' -pass-remarks-output=%t.yaml -filetype=null %s
# RUN: FileCheck %s < %t.yaml
# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa \
diff --git a/llvm/test/CodeGen/AMDGPU/illegal-eviction-assert.mir b/llvm/test/CodeGen/AMDGPU/illegal-eviction-assert.mir
index 7666aa35a4214..cdad480b3aaaa 100644
--- a/llvm/test/CodeGen/AMDGPU/illegal-eviction-assert.mir
+++ b/llvm/test/CodeGen/AMDGPU/illegal-eviction-assert.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: not llc -mtriple=amdgpu9.00 -start-before=greedy,0 -stop-after=virtregrewriter,2 -o - 2>%t.err %s | FileCheck %s
+# RUN: not llc -mtriple=amdgpu9.00 -start-before=greedy,1 -stop-after=virt-reg-rewriter,3 -o - 2>%t.err %s | FileCheck %s
# RUN: FileCheck -check-prefix=ERR %s < %t.err
# This testcase cannot be compiled. An attempted eviction legality
diff --git a/llvm/test/CodeGen/AMDGPU/indirect-call-known-callees.ll b/llvm/test/CodeGen/AMDGPU/indirect-call-known-callees.ll
index 4166f5a2ef3b2..2e54c5a68453b 100644
--- a/llvm/test/CodeGen/AMDGPU/indirect-call-known-callees.ll
+++ b/llvm/test/CodeGen/AMDGPU/indirect-call-known-callees.ll
@@ -8,6 +8,42 @@
; FIXME: Passing real values for workitem ID, and 0s that can be undef
+define void @snork() {
+; GFX9-LABEL: snork:
+; GFX9: ; %bb.0: ; %bb
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: snork:
+; GFX12: ; %bb.0: ; %bb
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+bb:
+ ret void
+}
+
+define void @wobble() {
+; GFX9-LABEL: wobble:
+; GFX9: ; %bb.0: ; %bb
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: wobble:
+; GFX12: ; %bb.0: ; %bb
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
+bb:
+ ret void
+}
+
define amdgpu_kernel void @indirect_call_known_no_special_inputs() {
; GFX9-LABEL: indirect_call_known_no_special_inputs:
; GFX9: ; %bb.0: ; %bb
@@ -71,46 +107,9 @@ define amdgpu_kernel void @indirect_call_known_no_special_inputs() {
; GFX12-NEXT: s_mov_b64 s[6:7], s[2:3]
; GFX12-NEXT: s_swappc_b64 s[30:31], s[12:13]
; GFX12-NEXT: s_endpgm
-
bb:
%cond = load i1, ptr addrspace(4) null
%tmp = select i1 %cond, ptr @wobble, ptr @snork
call void %tmp(ptr poison, i32 poison, ptr poison)
ret void
}
-
-define void @wobble() {
-; GFX9-LABEL: wobble:
-; GFX9: ; %bb.0: ; %bb
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-LABEL: wobble:
-; GFX12: ; %bb.0: ; %bb
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_setpc_b64 s[30:31]
-bb:
- ret void
-}
-
-define void @snork() {
-; GFX9-LABEL: snork:
-; GFX9: ; %bb.0: ; %bb
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-LABEL: snork:
-; GFX12: ; %bb.0: ; %bb
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_setpc_b64 s[30:31]
-bb:
- ret void
-}
diff --git a/llvm/test/CodeGen/AMDGPU/inflate-av-remat-imm.mir b/llvm/test/CodeGen/AMDGPU/inflate-av-remat-imm.mir
index 498a5cea9c3f2..6bca79c818280 100644
--- a/llvm/test/CodeGen/AMDGPU/inflate-av-remat-imm.mir
+++ b/llvm/test/CodeGen/AMDGPU/inflate-av-remat-imm.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
-# RUN: llc -mtriple=amdgpu9.0a-amd-amdhsa -simplify-mir -start-before=greedy,2 -stress-regalloc=3 -stop-after=postrapseudos -o - -verify-regalloc %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.0a-amd-amdhsa -simplify-mir -start-before=greedy,3 -stress-regalloc=3 -stop-after=post-ra-pseudos -o - -verify-regalloc %s | FileCheck %s
# Compare results of using V_MOV_B32 vs. AV_MOV_B32_IMM_PSEUDO during
# allocation.
diff --git a/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-agpr-negative-tests.mir b/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-agpr-negative-tests.mir
index 462714a57e30e..5e0e62b367261 100644
--- a/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-agpr-negative-tests.mir
+++ b/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-agpr-negative-tests.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
-# RUN: llc -mtriple=amdgpu9.42-amd-amdhsa -start-before=greedy,2 -stop-after=virtregrewriter,2 -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.42-amd-amdhsa -start-before=greedy,3 -stop-after=virt-reg-rewriter,3 -o - %s | FileCheck %s
--- |
diff --git a/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-av-with-load-source.mir b/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-av-with-load-source.mir
index 5bc25d5994ff6..c35ccca162e2b 100644
--- a/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-av-with-load-source.mir
+++ b/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-av-with-load-source.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
-# RUN: llc -mtriple=amdgpu9.42-amd-amdhsa -start-before=greedy,2 -stop-after=virtregrewriter,2 -verify-regalloc -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.42-amd-amdhsa -start-before=greedy,3 -stop-after=virt-reg-rewriter,3 -verify-regalloc -verify-machineinstrs -o - %s | FileCheck %s
# There aren't enough VGPRs for %0 to stay in a VGPR. %0 should be
# split and inflated to AV_512. The VGPR version of the instruction
diff --git a/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-av.mir b/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-av.mir
index 3cb539d3a423a..7cf0ee7fcb0ed 100644
--- a/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-av.mir
+++ b/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-av.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
-# RUN: llc -mtriple=amdgpu9.42-amd-amdhsa -start-before=greedy,2 -stop-after=virtregrewriter,2 -verify-regalloc -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.42-amd-amdhsa -start-before=greedy,3 -stop-after=virt-reg-rewriter,3 -verify-regalloc -verify-machineinstrs -o - %s | FileCheck %s
# There aren't enough VGPRs for %0 to stay in a VGPR. %0 should be
# split and inflated to AV_512. The VGPR version of the instruction
diff --git a/llvm/test/CodeGen/AMDGPU/inflated-reg-class-snippet-copy-use-after-free.mir b/llvm/test/CodeGen/AMDGPU/inflated-reg-class-snippet-copy-use-after-free.mir
index 3ed98899d6019..97cef18bd6254 100644
--- a/llvm/test/CodeGen/AMDGPU/inflated-reg-class-snippet-copy-use-after-free.mir
+++ b/llvm/test/CodeGen/AMDGPU/inflated-reg-class-snippet-copy-use-after-free.mir
@@ -1,4 +1,4 @@
-# RUN: not llc -mtriple=amdgpu9.0a-amd-amdhsa -simplify-mir -start-before=greedy,2 -stress-regalloc=4 -stop-before=virtregrewriter,2 -o - -verify-regalloc %s 2> %t.err | FileCheck %s
+# RUN: not llc -mtriple=amdgpu9.0a-amd-amdhsa -simplify-mir -start-before=greedy,3 -stress-regalloc=4 -stop-before=virt-reg-rewriter,3 -o - -verify-regalloc %s 2> %t.err | FileCheck %s
# RUN: FileCheck -check-prefix=ERR %s < %t.err
# To allocate the vreg_512_align2, the allocation will attempt to
diff --git a/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill-inspect-subrange.mir b/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill-inspect-subrange.mir
index 63ee4473a56e5..befc33e18aa94 100644
--- a/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill-inspect-subrange.mir
+++ b/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill-inspect-subrange.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 4
-# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -verify-regalloc -start-before=greedy,0 -stop-after=virtregrewriter,0 -simplify-mir -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -verify-regalloc -start-before=greedy,1 -stop-after=virt-reg-rewriter,1 -simplify-mir -o - %s | FileCheck %s
--- |
diff --git a/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill.mir b/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill.mir
index a8419a4bfc749..e6f29a8c11afc 100644
--- a/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill.mir
+++ b/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 3
-# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -start-before=greedy,0 -stop-after=virtregrewriter,0 -simplify-mir -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -start-before=greedy,1 -stop-after=virt-reg-rewriter,1 -simplify-mir -o - %s | FileCheck %s
--- |
define void @main() #0 {
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/issue48473.mir b/llvm/test/CodeGen/AMDGPU/issue48473.mir
index 7a8fde7bef6b5..097a11ee88493 100644
--- a/llvm/test/CodeGen/AMDGPU/issue48473.mir
+++ b/llvm/test/CodeGen/AMDGPU/issue48473.mir
@@ -1,4 +1,4 @@
-# RUN: not llc -mtriple=amdgpu9.08-amd-amdhsa -start-before=greedy,0 -stop-after=virtregrewriter,1 -verify-machineinstrs -o - 2> %t.err %s | FileCheck %s
+# RUN: not llc -mtriple=amdgpu9.08-amd-amdhsa -start-before=greedy,1 -stop-after=virt-reg-rewriter,2 -verify-machineinstrs -o - 2> %t.err %s | FileCheck %s
# RUN: FileCheck -check-prefix=ERR %s < %t.err
# ERR: error: register allocation failed: maximum depth for recoloring reached. Use -fexhaustive-register-search to skip cutoffs
diff --git a/llvm/test/CodeGen/AMDGPU/issue98474-assigned-physreg-interference.mir b/llvm/test/CodeGen/AMDGPU/issue98474-assigned-physreg-interference.mir
index f8d7635d8a6c1..a078979e8b33b 100644
--- a/llvm/test/CodeGen/AMDGPU/issue98474-assigned-physreg-interference.mir
+++ b/llvm/test/CodeGen/AMDGPU/issue98474-assigned-physreg-interference.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
-# RUN: llc -mtriple=amdgpu9.00-amd-amdpal -start-before=greedy,2 -stop-after=tailduplication -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.00-amd-amdpal -start-before=greedy,3 -stop-after=tailduplication -verify-machineinstrs -o - %s | FileCheck %s
---
name: undef_subreg_def_live_out_tailduplicate_vreg96_undef_sub1_sub2_assigned_physreg_interference
diff --git a/llvm/test/CodeGen/AMDGPU/issue98474-virtregrewriter-live-out-undef-subregisters.mir b/llvm/test/CodeGen/AMDGPU/issue98474-virtregrewriter-live-out-undef-subregisters.mir
index 886c8e6494be9..5e5aa5782ef82 100644
--- a/llvm/test/CodeGen/AMDGPU/issue98474-virtregrewriter-live-out-undef-subregisters.mir
+++ b/llvm/test/CodeGen/AMDGPU/issue98474-virtregrewriter-live-out-undef-subregisters.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
-# RUN: llc -mtriple=amdgpu9.00-amd-amdpal -start-before=greedy,2 -stop-after=tailduplication -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.00-amd-amdpal -start-before=greedy,3 -stop-after=tailduplication -verify-machineinstrs -o - %s | FileCheck %s
# The partial def of %0 introduces a live out undef def of %0.sub1
# into bb.3. We need to maintain this liveness with an explicit def of
diff --git a/llvm/test/CodeGen/AMDGPU/large-avgpr-assign-last.mir b/llvm/test/CodeGen/AMDGPU/large-avgpr-assign-last.mir
index fcb8461963dbb..854677b970e61 100644
--- a/llvm/test/CodeGen/AMDGPU/large-avgpr-assign-last.mir
+++ b/llvm/test/CodeGen/AMDGPU/large-avgpr-assign-last.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
-# RUN: llc -mtriple=amdgpu9.50-amd-amdhsa -verify-regalloc -greedy-regclass-priority-trumps-globalness=1 -start-after=machine-scheduler -stop-after=virtregrewriter,2 -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.50-amd-amdhsa -verify-regalloc -greedy-regclass-priority-trumps-globalness=1 -start-after=machine-scheduler -stop-after=virt-reg-rewriter,3 -o - %s | FileCheck %s
--- |
define void @temp_vgpr_to_agpr_should_not_undo_split_with_remat() #0 {
diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
index d10df2ed1d11f..4d87cfea36ab2 100644
--- a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
+++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
@@ -1,14 +1,14 @@
; UNSUPPORTED: expensive_checks
-; RUN: llc -O0 -mtriple=amdgpu7.00--amdhsa -disable-verify -debug-pass=Structure < %s 2>&1 \
+; RUN: llc -O0 -enable-new-pm=0 -mtriple=amdgpu7.00--amdhsa -disable-verify -debug-pass=Structure < %s 2>&1 \
; RUN: | FileCheck -match-full-lines -strict-whitespace -check-prefix=GCN-O0 %s
-; RUN: llc -O1 -mtriple=amdgpu7.00--amdhsa -disable-verify -debug-pass=Structure < %s 2>&1 \
+; RUN: llc -O1 -enable-new-pm=0 -mtriple=amdgpu7.00--amdhsa -disable-verify -debug-pass=Structure < %s 2>&1 \
; RUN: | FileCheck -match-full-lines -strict-whitespace -check-prefix=GCN-O1 %s
-; RUN: llc -O1 -mtriple=amdgpu7.00--amdhsa -disable-verify -amdgpu-scalar-ir-passes -amdgpu-sdwa-peephole \
+; RUN: llc -O1 -enable-new-pm=0 -mtriple=amdgpu7.00--amdhsa -disable-verify -amdgpu-scalar-ir-passes -amdgpu-sdwa-peephole \
; RUN: -amdgpu-load-store-vectorizer -amdgpu-enable-pre-ra-optimizations -amdgpu-loop-prefetch -debug-pass=Structure < %s 2>&1 \
; RUN: | FileCheck -match-full-lines -strict-whitespace -check-prefix=GCN-O1-OPTS %s
-; RUN: llc -O2 -mtriple=amdgpu7.00--amdhsa -disable-verify -debug-pass=Structure < %s 2>&1 \
+; RUN: llc -O2 -enable-new-pm=0 -mtriple=amdgpu7.00--amdhsa -disable-verify -debug-pass=Structure < %s 2>&1 \
; RUN: | FileCheck -match-full-lines -strict-whitespace -check-prefix=GCN-O2 %s
-; RUN: llc -O3 -mtriple=amdgpu7.00--amdhsa -disable-verify -debug-pass=Structure < %s 2>&1 \
+; RUN: llc -O3 -enable-new-pm=0 -mtriple=amdgpu7.00--amdhsa -disable-verify -debug-pass=Structure < %s 2>&1 \
; RUN: | FileCheck -match-full-lines -strict-whitespace -check-prefix=GCN-O3 %s
; REQUIRES: asserts
diff --git a/llvm/test/CodeGen/AMDGPU/machine-sink-cycle.mir b/llvm/test/CodeGen/AMDGPU/machine-sink-cycle.mir
index f228e579763f1..ed517470da28a 100644
--- a/llvm/test/CodeGen/AMDGPU/machine-sink-cycle.mir
+++ b/llvm/test/CodeGen/AMDGPU/machine-sink-cycle.mir
@@ -1,6 +1,6 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 3
-# RUN: llc -mtriple=amdgpu9.42-amd-amdhsa --sink-insts-to-avoid-spills=1 --stop-after=machine-sink -o - %s | FileCheck -check-prefixes=GFX9-SUNK %s
-# RUN: llc -mtriple=amdgpu10.31-amd-amdhsa --sink-insts-to-avoid-spills=1 -mattr=+wavefrontsize64 --stop-after=machine-sink -o - %s | FileCheck -check-prefixes=GFX10-SUNK %s
+# RUN: llc -mtriple=amdgpu9.42-amd-amdhsa --sink-insts-to-avoid-spills=1 --start-before=amdgpu-isel --stop-after=machine-sink -o - %s | FileCheck -check-prefixes=GFX9-SUNK %s
+# RUN: llc -mtriple=amdgpu10.31-amd-amdhsa --sink-insts-to-avoid-spills=1 -mattr=+wavefrontsize64 --start-before=amdgpu-isel --stop-after=machine-sink -o - %s | FileCheck -check-prefixes=GFX10-SUNK %s
---
name: test_sink_copy
diff --git a/llvm/test/CodeGen/AMDGPU/memcpy-crash-issue63986.ll b/llvm/test/CodeGen/AMDGPU/memcpy-crash-issue63986.ll
index d90c9771c3e4a..862b55d93179a 100644
--- a/llvm/test/CodeGen/AMDGPU/memcpy-crash-issue63986.ll
+++ b/llvm/test/CodeGen/AMDGPU/memcpy-crash-issue63986.ll
@@ -190,6 +190,7 @@ define void @issue63986_reduced_expanded(i64 %idxprom) {
; CHECK-NEXT: v_mov_b32_e32 v0, s4
; CHECK-NEXT: v_mov_b32_e32 v1, s5
; CHECK-NEXT: .LBB1_6: ; %loop-memcpy-residual
+; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: s_add_i32 s6, s8, 1
; CHECK-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[0:1]
; CHECK-NEXT: s_mov_b64 s[8:9], 1
@@ -205,6 +206,7 @@ define void @issue63986_reduced_expanded(i64 %idxprom) {
; CHECK-NEXT: s_and_b64 vcc, exec, 0
; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[2:5]
; CHECK-NEXT: .LBB1_9: ; %loop-memcpy-expansion2
+; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: s_mov_b64 vcc, vcc
; CHECK-NEXT: s_cbranch_vccz .LBB1_9
; CHECK-NEXT: ; %bb.10: ; %DummyReturnBlock
diff --git a/llvm/test/CodeGen/AMDGPU/memcpy_const_compare.ll b/llvm/test/CodeGen/AMDGPU/memcpy_const_compare.ll
index bb9011a3c43d7..b2373fd33fd54 100644
--- a/llvm/test/CodeGen/AMDGPU/memcpy_const_compare.ll
+++ b/llvm/test/CodeGen/AMDGPU/memcpy_const_compare.ll
@@ -40,26 +40,23 @@ define amdgpu_kernel void @_start() {
; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[2:5]
; CHECK-NEXT: s_cbranch_vccnz .LBB0_1
; CHECK-NEXT: ; %bb.2: ; %dynamic-memcpy-expansion-residual-cond
-; FIXME: Compare should be evaluated at compile time
; CHECK-NEXT: s_cmp_eq_u64 13, 0
; CHECK-NEXT: s_cbranch_scc1 .LBB0_5
; CHECK-NEXT: ; %bb.3: ; %dynamic-memcpy-expansion-residual-body.preheader
-; CHECK-NEXT: s_sub_u32 s4, 29, 13
-; CHECK-NEXT: s_subb_u32 s5, 0, 0
; CHECK-NEXT: s_getpc_b64 s[0:1]
; CHECK-NEXT: s_add_u32 s0, s0, src_array at gotpcrel32@lo+4
; CHECK-NEXT: s_addc_u32 s1, s1, src_array at gotpcrel32@hi+12
; CHECK-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: s_add_u32 s2, s0, s4
-; CHECK-NEXT: s_addc_u32 s3, s1, s5
+; CHECK-NEXT: s_add_u32 s2, s0, 16
+; CHECK-NEXT: s_addc_u32 s3, s1, 0
; CHECK-NEXT: s_getpc_b64 s[0:1]
; CHECK-NEXT: s_add_u32 s0, s0, dst_array at gotpcrel32@lo+4
; CHECK-NEXT: s_addc_u32 s1, s1, dst_array at gotpcrel32@hi+12
; CHECK-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: s_add_u32 s4, s0, s4
-; CHECK-NEXT: s_addc_u32 s5, s1, s5
+; CHECK-NEXT: s_add_u32 s4, s0, 16
+; CHECK-NEXT: s_addc_u32 s5, s1, 0
; CHECK-NEXT: s_mov_b64 s[0:1], 0
; CHECK-NEXT: .LBB0_4: ; %dynamic-memcpy-expansion-residual-body
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -79,6 +76,7 @@ define amdgpu_kernel void @_start() {
; CHECK-NEXT: s_cbranch_vccnz .LBB0_4
; CHECK-NEXT: .LBB0_5: ; %dynamic-memcpy-post-expansion
; CHECK-NEXT: s_endpgm
+; FIXME: Compare should be evaluated at compile time
%src_ptr = getelementptr inbounds [128 x i8], ptr @src_array, i64 0, i64 0
%dst_ptr = getelementptr inbounds [128 x i8], ptr @dst_array, i64 0, i64 0
call void @llvm.memcpy.p0.p0.i64(ptr %dst_ptr, ptr %src_ptr, i64 add (i64 sub (i64 16, i64 ptrtoint (ptr addrspacecast (ptr addrspace(4) null to ptr) to i64)), i64 13), i1 false)
diff --git a/llvm/test/CodeGen/AMDGPU/memmove-var-size.ll b/llvm/test/CodeGen/AMDGPU/memmove-var-size.ll
index eb8eb8b4f1cb2..71e50608d2ec5 100644
--- a/llvm/test/CodeGen/AMDGPU/memmove-var-size.ll
+++ b/llvm/test/CodeGen/AMDGPU/memmove-var-size.ll
@@ -1043,55 +1043,55 @@ define void @memmove_p1_p3(ptr addrspace(1) align 1 %dst, ptr addrspace(3) align
; CHECK-LABEL: memmove_p1_p3:
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_and_b32_e32 v7, -16, v3
-; CHECK-NEXT: v_mov_b32_e32 v8, v4
-; CHECK-NEXT: v_and_b32_e32 v5, 15, v3
-; CHECK-NEXT: v_mov_b32_e32 v6, 0
+; CHECK-NEXT: v_mov_b32_e32 v5, v3
+; CHECK-NEXT: v_mov_b32_e32 v7, 0
; CHECK-NEXT: s_mov_b64 s[4:5], 0
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[7:8]
+; CHECK-NEXT: v_and_b32_e32 v3, -16, v5
+; CHECK-NEXT: v_and_b32_e32 v6, 15, v5
+; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[3:4]
; CHECK-NEXT: s_cbranch_execz .LBB7_3
; CHECK-NEXT: ; %bb.1: ; %dynamic-memcpy-expansion-main-body.preheader
-; CHECK-NEXT: v_mov_b32_e32 v9, v2
+; CHECK-NEXT: v_mov_b32_e32 v8, v2
; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: .LBB7_2: ; %dynamic-memcpy-expansion-main-body
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: ds_read_b128 v[10:13], v9
-; CHECK-NEXT: v_add_co_u32 v14, vcc_lo, v0, s4
+; CHECK-NEXT: ds_read_b128 v[9:12], v8
+; CHECK-NEXT: v_add_co_u32 v13, vcc_lo, v0, s4
; CHECK-NEXT: s_add_u32 s4, s4, 16
-; CHECK-NEXT: v_add_co_ci_u32_e64 v15, null, s5, v1, vcc_lo
+; CHECK-NEXT: v_add_co_ci_u32_e64 v14, null, s5, v1, vcc_lo
; CHECK-NEXT: s_addc_u32 s5, s5, 0
-; CHECK-NEXT: v_add_nc_u32_e32 v9, 16, v9
-; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[7:8]
+; CHECK-NEXT: v_add_nc_u32_e32 v8, 16, v8
+; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[3:4]
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: global_store_dwordx4 v[14:15], v[10:13], off
+; CHECK-NEXT: global_store_dwordx4 v[13:14], v[9:12], off
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s7
; CHECK-NEXT: s_cbranch_execnz .LBB7_2
; CHECK-NEXT: .LBB7_3: ; %Flow9
; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s6
; CHECK-NEXT: s_mov_b64 s[4:5], 0
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[5:6]
+; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[6:7]
; CHECK-NEXT: s_cbranch_execz .LBB7_6
; CHECK-NEXT: ; %bb.4: ; %dynamic-memcpy-expansion-residual-body.preheader
-; CHECK-NEXT: v_and_b32_e32 v3, -16, v3
-; CHECK-NEXT: s_mov_b32 s7, 0
+; CHECK-NEXT: v_and_b32_e32 v5, -16, v5
; CHECK-NEXT: v_add_co_u32 v0, vcc_lo, v0, v3
-; CHECK-NEXT: v_add_nc_u32_e32 v2, v2, v3
; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc_lo
+; CHECK-NEXT: v_add_nc_u32_e32 v2, v2, v5
+; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: .LBB7_5: ; %dynamic-memcpy-expansion-residual-body
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: ds_read_u8 v7, v2
+; CHECK-NEXT: ds_read_u8 v5, v2
; CHECK-NEXT: v_add_co_u32 v3, vcc_lo, v0, s4
; CHECK-NEXT: s_add_u32 s4, s4, 1
; CHECK-NEXT: v_add_co_ci_u32_e64 v4, null, s5, v1, vcc_lo
; CHECK-NEXT: s_addc_u32 s5, s5, 0
; CHECK-NEXT: v_add_nc_u32_e32 v2, 1, v2
-; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[5:6]
+; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[6:7]
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: global_store_byte v[3:4], v7, off
+; CHECK-NEXT: global_store_byte v[3:4], v5, off
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s7
; CHECK-NEXT: s_cbranch_execnz .LBB7_5
; CHECK-NEXT: .LBB7_6: ; %Flow7
@@ -1252,60 +1252,60 @@ define void @memmove_p1_p5(ptr addrspace(1) align 1 %dst, ptr addrspace(5) align
; CHECK-LABEL: memmove_p1_p5:
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_and_b32_e32 v7, -16, v3
-; CHECK-NEXT: v_mov_b32_e32 v8, v4
-; CHECK-NEXT: v_and_b32_e32 v5, 15, v3
-; CHECK-NEXT: v_mov_b32_e32 v6, 0
+; CHECK-NEXT: v_mov_b32_e32 v5, v3
+; CHECK-NEXT: v_mov_b32_e32 v7, 0
; CHECK-NEXT: s_mov_b64 s[4:5], 0
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[7:8]
+; CHECK-NEXT: v_and_b32_e32 v3, -16, v5
+; CHECK-NEXT: v_and_b32_e32 v6, 15, v5
+; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[3:4]
; CHECK-NEXT: s_cbranch_execz .LBB9_3
; CHECK-NEXT: ; %bb.1: ; %dynamic-memcpy-expansion-main-body.preheader
-; CHECK-NEXT: v_mov_b32_e32 v9, v2
+; CHECK-NEXT: v_mov_b32_e32 v8, v2
; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: .p2align 6
; CHECK-NEXT: .LBB9_2: ; %dynamic-memcpy-expansion-main-body
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: s_clause 0x3
-; CHECK-NEXT: buffer_load_dword v10, v9, s[0:3], 0 offen
-; CHECK-NEXT: buffer_load_dword v11, v9, s[0:3], 0 offen offset:4
-; CHECK-NEXT: buffer_load_dword v12, v9, s[0:3], 0 offen offset:8
-; CHECK-NEXT: buffer_load_dword v13, v9, s[0:3], 0 offen offset:12
-; CHECK-NEXT: v_add_co_u32 v14, vcc_lo, v0, s4
+; CHECK-NEXT: buffer_load_dword v9, v8, s[0:3], 0 offen
+; CHECK-NEXT: buffer_load_dword v10, v8, s[0:3], 0 offen offset:4
+; CHECK-NEXT: buffer_load_dword v11, v8, s[0:3], 0 offen offset:8
+; CHECK-NEXT: buffer_load_dword v12, v8, s[0:3], 0 offen offset:12
+; CHECK-NEXT: v_add_co_u32 v13, vcc_lo, v0, s4
; CHECK-NEXT: s_add_u32 s4, s4, 16
-; CHECK-NEXT: v_add_co_ci_u32_e64 v15, null, s5, v1, vcc_lo
+; CHECK-NEXT: v_add_co_ci_u32_e64 v14, null, s5, v1, vcc_lo
; CHECK-NEXT: s_addc_u32 s5, s5, 0
-; CHECK-NEXT: v_add_nc_u32_e32 v9, 16, v9
-; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[7:8]
+; CHECK-NEXT: v_add_nc_u32_e32 v8, 16, v8
+; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[3:4]
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: global_store_dwordx4 v[14:15], v[10:13], off
+; CHECK-NEXT: global_store_dwordx4 v[13:14], v[9:12], off
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s7
; CHECK-NEXT: s_cbranch_execnz .LBB9_2
; CHECK-NEXT: .LBB9_3: ; %Flow9
; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s6
; CHECK-NEXT: s_mov_b64 s[4:5], 0
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[5:6]
+; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[6:7]
; CHECK-NEXT: s_cbranch_execz .LBB9_6
; CHECK-NEXT: ; %bb.4: ; %dynamic-memcpy-expansion-residual-body.preheader
-; CHECK-NEXT: v_and_b32_e32 v3, -16, v3
-; CHECK-NEXT: s_mov_b32 s7, 0
+; CHECK-NEXT: v_and_b32_e32 v5, -16, v5
; CHECK-NEXT: v_add_co_u32 v0, vcc_lo, v0, v3
-; CHECK-NEXT: v_add_nc_u32_e32 v2, v2, v3
; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc_lo
+; CHECK-NEXT: v_add_nc_u32_e32 v2, v2, v5
+; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: .LBB9_5: ; %dynamic-memcpy-expansion-residual-body
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: buffer_load_ubyte v7, v2, s[0:3], 0 offen
+; CHECK-NEXT: buffer_load_ubyte v5, v2, s[0:3], 0 offen
; CHECK-NEXT: v_add_co_u32 v3, vcc_lo, v0, s4
; CHECK-NEXT: s_add_u32 s4, s4, 1
; CHECK-NEXT: v_add_co_ci_u32_e64 v4, null, s5, v1, vcc_lo
; CHECK-NEXT: s_addc_u32 s5, s5, 0
; CHECK-NEXT: v_add_nc_u32_e32 v2, 1, v2
-; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[5:6]
+; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[6:7]
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: global_store_byte v[3:4], v7, off
+; CHECK-NEXT: global_store_byte v[3:4], v5, off
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s7
; CHECK-NEXT: s_cbranch_execnz .LBB9_5
; CHECK-NEXT: .LBB9_6: ; %Flow7
@@ -1465,50 +1465,50 @@ define void @memmove_p3_p1(ptr addrspace(3) align 1 %dst, ptr addrspace(1) align
; CHECK-LABEL: memmove_p3_p1:
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_and_b32_e32 v7, -16, v3
-; CHECK-NEXT: v_mov_b32_e32 v8, v4
-; CHECK-NEXT: v_and_b32_e32 v5, 15, v3
-; CHECK-NEXT: v_mov_b32_e32 v6, 0
+; CHECK-NEXT: v_mov_b32_e32 v5, v3
+; CHECK-NEXT: v_mov_b32_e32 v7, 0
; CHECK-NEXT: s_mov_b64 s[4:5], 0
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[7:8]
+; CHECK-NEXT: v_and_b32_e32 v3, -16, v5
+; CHECK-NEXT: v_and_b32_e32 v6, 15, v5
+; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[3:4]
; CHECK-NEXT: s_cbranch_execz .LBB11_3
; CHECK-NEXT: ; %bb.1: ; %dynamic-memcpy-expansion-main-body.preheader
-; CHECK-NEXT: v_mov_b32_e32 v9, v0
+; CHECK-NEXT: v_mov_b32_e32 v8, v0
; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: .LBB11_2: ; %dynamic-memcpy-expansion-main-body
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: v_add_co_u32 v10, vcc_lo, v1, s4
-; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, s5, v2, vcc_lo
+; CHECK-NEXT: v_add_co_u32 v9, vcc_lo, v1, s4
+; CHECK-NEXT: v_add_co_ci_u32_e64 v10, null, s5, v2, vcc_lo
; CHECK-NEXT: s_add_u32 s4, s4, 16
; CHECK-NEXT: s_addc_u32 s5, s5, 0
-; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[7:8]
-; CHECK-NEXT: global_load_dwordx4 v[10:13], v[10:11], off
+; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[3:4]
+; CHECK-NEXT: global_load_dwordx4 v[9:12], v[9:10], off
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: ds_write_b128 v9, v[10:13]
-; CHECK-NEXT: v_add_nc_u32_e32 v9, 16, v9
+; CHECK-NEXT: ds_write_b128 v8, v[9:12]
+; CHECK-NEXT: v_add_nc_u32_e32 v8, 16, v8
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s7
; CHECK-NEXT: s_cbranch_execnz .LBB11_2
; CHECK-NEXT: .LBB11_3: ; %Flow9
; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s6
; CHECK-NEXT: s_mov_b64 s[4:5], 0
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[5:6]
+; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[6:7]
; CHECK-NEXT: s_cbranch_execz .LBB11_6
; CHECK-NEXT: ; %bb.4: ; %dynamic-memcpy-expansion-residual-body.preheader
-; CHECK-NEXT: v_and_b32_e32 v3, -16, v3
-; CHECK-NEXT: s_mov_b32 s7, 0
+; CHECK-NEXT: v_and_b32_e32 v5, -16, v5
; CHECK-NEXT: v_add_co_u32 v1, vcc_lo, v1, v3
-; CHECK-NEXT: v_add_nc_u32_e32 v0, v0, v3
; CHECK-NEXT: v_add_co_ci_u32_e64 v2, null, v2, v4, vcc_lo
+; CHECK-NEXT: v_add_nc_u32_e32 v0, v0, v5
+; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: .LBB11_5: ; %dynamic-memcpy-expansion-residual-body
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: v_add_co_u32 v3, vcc_lo, v1, s4
; CHECK-NEXT: v_add_co_ci_u32_e64 v4, null, s5, v2, vcc_lo
; CHECK-NEXT: s_add_u32 s4, s4, 1
; CHECK-NEXT: s_addc_u32 s5, s5, 0
-; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[5:6]
+; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[6:7]
; CHECK-NEXT: global_load_ubyte v3, v[3:4], off
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_waitcnt vmcnt(0)
@@ -1656,50 +1656,50 @@ define void @memmove_p3_p4(ptr addrspace(3) align 1 %dst, ptr addrspace(4) align
; CHECK-LABEL: memmove_p3_p4:
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_and_b32_e32 v7, -16, v3
-; CHECK-NEXT: v_mov_b32_e32 v8, v4
-; CHECK-NEXT: v_and_b32_e32 v5, 15, v3
-; CHECK-NEXT: v_mov_b32_e32 v6, 0
+; CHECK-NEXT: v_mov_b32_e32 v5, v3
+; CHECK-NEXT: v_mov_b32_e32 v7, 0
; CHECK-NEXT: s_mov_b64 s[4:5], 0
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[7:8]
+; CHECK-NEXT: v_and_b32_e32 v3, -16, v5
+; CHECK-NEXT: v_and_b32_e32 v6, 15, v5
+; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[3:4]
; CHECK-NEXT: s_cbranch_execz .LBB13_3
; CHECK-NEXT: ; %bb.1: ; %dynamic-memcpy-expansion-main-body.preheader
-; CHECK-NEXT: v_mov_b32_e32 v9, v0
+; CHECK-NEXT: v_mov_b32_e32 v8, v0
; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: .LBB13_2: ; %dynamic-memcpy-expansion-main-body
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: v_add_co_u32 v10, vcc_lo, v1, s4
-; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, s5, v2, vcc_lo
+; CHECK-NEXT: v_add_co_u32 v9, vcc_lo, v1, s4
+; CHECK-NEXT: v_add_co_ci_u32_e64 v10, null, s5, v2, vcc_lo
; CHECK-NEXT: s_add_u32 s4, s4, 16
; CHECK-NEXT: s_addc_u32 s5, s5, 0
-; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[7:8]
-; CHECK-NEXT: global_load_dwordx4 v[10:13], v[10:11], off
+; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[3:4]
+; CHECK-NEXT: global_load_dwordx4 v[9:12], v[9:10], off
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: ds_write_b128 v9, v[10:13]
-; CHECK-NEXT: v_add_nc_u32_e32 v9, 16, v9
+; CHECK-NEXT: ds_write_b128 v8, v[9:12]
+; CHECK-NEXT: v_add_nc_u32_e32 v8, 16, v8
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s7
; CHECK-NEXT: s_cbranch_execnz .LBB13_2
; CHECK-NEXT: .LBB13_3: ; %Flow9
; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s6
; CHECK-NEXT: s_mov_b64 s[4:5], 0
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[5:6]
+; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[6:7]
; CHECK-NEXT: s_cbranch_execz .LBB13_6
; CHECK-NEXT: ; %bb.4: ; %dynamic-memcpy-expansion-residual-body.preheader
-; CHECK-NEXT: v_and_b32_e32 v3, -16, v3
-; CHECK-NEXT: s_mov_b32 s7, 0
+; CHECK-NEXT: v_and_b32_e32 v5, -16, v5
; CHECK-NEXT: v_add_co_u32 v1, vcc_lo, v1, v3
-; CHECK-NEXT: v_add_nc_u32_e32 v0, v0, v3
; CHECK-NEXT: v_add_co_ci_u32_e64 v2, null, v2, v4, vcc_lo
+; CHECK-NEXT: v_add_nc_u32_e32 v0, v0, v5
+; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: .LBB13_5: ; %dynamic-memcpy-expansion-residual-body
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: v_add_co_u32 v3, vcc_lo, v1, s4
; CHECK-NEXT: v_add_co_ci_u32_e64 v4, null, s5, v2, vcc_lo
; CHECK-NEXT: s_add_u32 s4, s4, 1
; CHECK-NEXT: s_addc_u32 s5, s5, 0
-; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[5:6]
+; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[6:7]
; CHECK-NEXT: global_load_ubyte v3, v[3:4], off
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_waitcnt vmcnt(0)
@@ -1936,32 +1936,32 @@ define void @memmove_p5_p1(ptr addrspace(5) align 1 %dst, ptr addrspace(1) align
; CHECK-LABEL: memmove_p5_p1:
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_and_b32_e32 v7, -16, v3
-; CHECK-NEXT: v_mov_b32_e32 v8, v4
-; CHECK-NEXT: v_and_b32_e32 v5, 15, v3
-; CHECK-NEXT: v_mov_b32_e32 v6, 0
+; CHECK-NEXT: v_mov_b32_e32 v5, v3
+; CHECK-NEXT: v_mov_b32_e32 v7, 0
; CHECK-NEXT: s_mov_b64 s[4:5], 0
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[7:8]
+; CHECK-NEXT: v_and_b32_e32 v3, -16, v5
+; CHECK-NEXT: v_and_b32_e32 v6, 15, v5
+; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[3:4]
; CHECK-NEXT: s_cbranch_execz .LBB16_3
; CHECK-NEXT: ; %bb.1: ; %dynamic-memcpy-expansion-main-body.preheader
-; CHECK-NEXT: v_mov_b32_e32 v9, v0
+; CHECK-NEXT: v_mov_b32_e32 v8, v0
; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: .p2align 6
; CHECK-NEXT: .LBB16_2: ; %dynamic-memcpy-expansion-main-body
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: v_add_co_u32 v10, vcc_lo, v1, s4
-; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, s5, v2, vcc_lo
+; CHECK-NEXT: v_add_co_u32 v9, vcc_lo, v1, s4
+; CHECK-NEXT: v_add_co_ci_u32_e64 v10, null, s5, v2, vcc_lo
; CHECK-NEXT: s_add_u32 s4, s4, 16
; CHECK-NEXT: s_addc_u32 s5, s5, 0
-; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[7:8]
-; CHECK-NEXT: global_load_dwordx4 v[10:13], v[10:11], off
+; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[3:4]
+; CHECK-NEXT: global_load_dwordx4 v[9:12], v[9:10], off
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: buffer_store_dword v13, v9, s[0:3], 0 offen offset:12
-; CHECK-NEXT: buffer_store_dword v12, v9, s[0:3], 0 offen offset:8
-; CHECK-NEXT: buffer_store_dword v11, v9, s[0:3], 0 offen offset:4
-; CHECK-NEXT: buffer_store_dword v10, v9, s[0:3], 0 offen
-; CHECK-NEXT: v_add_nc_u32_e32 v9, 16, v9
+; CHECK-NEXT: buffer_store_dword v12, v8, s[0:3], 0 offen offset:12
+; CHECK-NEXT: buffer_store_dword v11, v8, s[0:3], 0 offen offset:8
+; CHECK-NEXT: buffer_store_dword v10, v8, s[0:3], 0 offen offset:4
+; CHECK-NEXT: buffer_store_dword v9, v8, s[0:3], 0 offen
+; CHECK-NEXT: v_add_nc_u32_e32 v8, 16, v8
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s7
; CHECK-NEXT: s_cbranch_execnz .LBB16_2
@@ -1969,21 +1969,21 @@ define void @memmove_p5_p1(ptr addrspace(5) align 1 %dst, ptr addrspace(1) align
; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s6
; CHECK-NEXT: s_mov_b64 s[4:5], 0
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[5:6]
+; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[6:7]
; CHECK-NEXT: s_cbranch_execz .LBB16_6
; CHECK-NEXT: ; %bb.4: ; %dynamic-memcpy-expansion-residual-body.preheader
-; CHECK-NEXT: v_and_b32_e32 v3, -16, v3
-; CHECK-NEXT: s_mov_b32 s7, 0
+; CHECK-NEXT: v_and_b32_e32 v5, -16, v5
; CHECK-NEXT: v_add_co_u32 v1, vcc_lo, v1, v3
-; CHECK-NEXT: v_add_nc_u32_e32 v0, v0, v3
; CHECK-NEXT: v_add_co_ci_u32_e64 v2, null, v2, v4, vcc_lo
+; CHECK-NEXT: v_add_nc_u32_e32 v0, v0, v5
+; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: .LBB16_5: ; %dynamic-memcpy-expansion-residual-body
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: v_add_co_u32 v3, vcc_lo, v1, s4
; CHECK-NEXT: v_add_co_ci_u32_e64 v4, null, s5, v2, vcc_lo
; CHECK-NEXT: s_add_u32 s4, s4, 1
; CHECK-NEXT: s_addc_u32 s5, s5, 0
-; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[5:6]
+; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[6:7]
; CHECK-NEXT: global_load_ubyte v3, v[3:4], off
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_waitcnt vmcnt(0)
@@ -2068,32 +2068,32 @@ define void @memmove_p5_p4(ptr addrspace(5) align 1 %dst, ptr addrspace(4) align
; CHECK-LABEL: memmove_p5_p4:
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_and_b32_e32 v7, -16, v3
-; CHECK-NEXT: v_mov_b32_e32 v8, v4
-; CHECK-NEXT: v_and_b32_e32 v5, 15, v3
-; CHECK-NEXT: v_mov_b32_e32 v6, 0
+; CHECK-NEXT: v_mov_b32_e32 v5, v3
+; CHECK-NEXT: v_mov_b32_e32 v7, 0
; CHECK-NEXT: s_mov_b64 s[4:5], 0
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[7:8]
+; CHECK-NEXT: v_and_b32_e32 v3, -16, v5
+; CHECK-NEXT: v_and_b32_e32 v6, 15, v5
+; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[3:4]
; CHECK-NEXT: s_cbranch_execz .LBB18_3
; CHECK-NEXT: ; %bb.1: ; %dynamic-memcpy-expansion-main-body.preheader
-; CHECK-NEXT: v_mov_b32_e32 v9, v0
+; CHECK-NEXT: v_mov_b32_e32 v8, v0
; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: .p2align 6
; CHECK-NEXT: .LBB18_2: ; %dynamic-memcpy-expansion-main-body
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: v_add_co_u32 v10, vcc_lo, v1, s4
-; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, s5, v2, vcc_lo
+; CHECK-NEXT: v_add_co_u32 v9, vcc_lo, v1, s4
+; CHECK-NEXT: v_add_co_ci_u32_e64 v10, null, s5, v2, vcc_lo
; CHECK-NEXT: s_add_u32 s4, s4, 16
; CHECK-NEXT: s_addc_u32 s5, s5, 0
-; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[7:8]
-; CHECK-NEXT: global_load_dwordx4 v[10:13], v[10:11], off
+; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[3:4]
+; CHECK-NEXT: global_load_dwordx4 v[9:12], v[9:10], off
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: buffer_store_dword v13, v9, s[0:3], 0 offen offset:12
-; CHECK-NEXT: buffer_store_dword v12, v9, s[0:3], 0 offen offset:8
-; CHECK-NEXT: buffer_store_dword v11, v9, s[0:3], 0 offen offset:4
-; CHECK-NEXT: buffer_store_dword v10, v9, s[0:3], 0 offen
-; CHECK-NEXT: v_add_nc_u32_e32 v9, 16, v9
+; CHECK-NEXT: buffer_store_dword v12, v8, s[0:3], 0 offen offset:12
+; CHECK-NEXT: buffer_store_dword v11, v8, s[0:3], 0 offen offset:8
+; CHECK-NEXT: buffer_store_dword v10, v8, s[0:3], 0 offen offset:4
+; CHECK-NEXT: buffer_store_dword v9, v8, s[0:3], 0 offen
+; CHECK-NEXT: v_add_nc_u32_e32 v8, 16, v8
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s7
; CHECK-NEXT: s_cbranch_execnz .LBB18_2
@@ -2101,21 +2101,21 @@ define void @memmove_p5_p4(ptr addrspace(5) align 1 %dst, ptr addrspace(4) align
; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s6
; CHECK-NEXT: s_mov_b64 s[4:5], 0
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[5:6]
+; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[6:7]
; CHECK-NEXT: s_cbranch_execz .LBB18_6
; CHECK-NEXT: ; %bb.4: ; %dynamic-memcpy-expansion-residual-body.preheader
-; CHECK-NEXT: v_and_b32_e32 v3, -16, v3
-; CHECK-NEXT: s_mov_b32 s7, 0
+; CHECK-NEXT: v_and_b32_e32 v5, -16, v5
; CHECK-NEXT: v_add_co_u32 v1, vcc_lo, v1, v3
-; CHECK-NEXT: v_add_nc_u32_e32 v0, v0, v3
; CHECK-NEXT: v_add_co_ci_u32_e64 v2, null, v2, v4, vcc_lo
+; CHECK-NEXT: v_add_nc_u32_e32 v0, v0, v5
+; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: .LBB18_5: ; %dynamic-memcpy-expansion-residual-body
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: v_add_co_u32 v3, vcc_lo, v1, s4
; CHECK-NEXT: v_add_co_ci_u32_e64 v4, null, s5, v2, vcc_lo
; CHECK-NEXT: s_add_u32 s4, s4, 1
; CHECK-NEXT: s_addc_u32 s5, s5, 0
-; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[5:6]
+; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[6:7]
; CHECK-NEXT: global_load_ubyte v3, v[3:4], off
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/memset-param-combinations.ll b/llvm/test/CodeGen/AMDGPU/memset-param-combinations.ll
index 1d0565d150ac3..2a951e9602913 100644
--- a/llvm/test/CodeGen/AMDGPU/memset-param-combinations.ll
+++ b/llvm/test/CodeGen/AMDGPU/memset-param-combinations.ll
@@ -4,120 +4,224 @@
define void @memset_p0_varsize_align_4_varsetval(ptr addrspace(0) align 4 %dst, i8 %setval, i64 %size) #0 {
-; GFX942-LABEL: memset_p0_varsize_align_4_varsetval:
-; GFX942: ; %bb.0: ; %entry
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v10, v3
-; GFX942-NEXT: v_and_b32_e32 v12, -16, v10
-; GFX942-NEXT: v_mov_b32_e32 v13, v4
-; GFX942-NEXT: v_mov_b32_e32 v11, v4
-; GFX942-NEXT: v_and_b32_e32 v8, 15, v10
-; GFX942-NEXT: v_mov_b32_e32 v9, 0
-; GFX942-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[12:13]
-; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
-; GFX942-NEXT: s_cbranch_execz .LBB0_3
-; GFX942-NEXT: ; %bb.1: ; %dynamic-memset-expansion-main-body.preheader
-; GFX942-NEXT: s_mov_b32 s4, 0x4040404
-; GFX942-NEXT: v_perm_b32 v4, v2, v2, s4
-; GFX942-NEXT: v_mov_b32_e32 v5, v4
-; GFX942-NEXT: v_mov_b32_e32 v6, v4
-; GFX942-NEXT: v_mov_b32_e32 v7, v4
-; GFX942-NEXT: s_mov_b64 s[4:5], 0
-; GFX942-NEXT: .LBB0_2: ; %dynamic-memset-expansion-main-body
-; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshl_add_u64 v[14:15], v[0:1], 0, s[4:5]
-; GFX942-NEXT: s_add_u32 s4, s4, 16
-; GFX942-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[12:13]
-; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: flat_store_dwordx4 v[14:15], v[4:7]
-; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX942-NEXT: s_cbranch_execnz .LBB0_2
-; GFX942-NEXT: .LBB0_3: ; %Flow4
-; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[8:9]
-; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
-; GFX942-NEXT: s_cbranch_execz .LBB0_6
-; GFX942-NEXT: ; %bb.4: ; %dynamic-memset-expansion-residual-body.preheader
-; GFX942-NEXT: v_and_b32_e32 v10, -16, v10
-; GFX942-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, v[10:11]
-; GFX942-NEXT: s_mov_b64 s[4:5], 0
-; GFX942-NEXT: .LBB0_5: ; %dynamic-memset-expansion-residual-body
-; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[4:5]
-; GFX942-NEXT: s_add_u32 s4, s4, 1
-; GFX942-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[8:9]
-; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: flat_store_byte v[4:5], v2
-; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX942-NEXT: s_cbranch_execnz .LBB0_5
-; GFX942-NEXT: .LBB0_6: ; %Flow2
-; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
+; GFX942-SDAG-LABEL: memset_p0_varsize_align_4_varsetval:
+; GFX942-SDAG: ; %bb.0: ; %entry
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, v4
+; GFX942-SDAG-NEXT: v_and_b32_e32 v10, -16, v3
+; GFX942-SDAG-NEXT: v_and_b32_e32 v8, 15, v3
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, 0
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
+; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-SDAG-NEXT: s_cbranch_execz .LBB0_3
+; GFX942-SDAG-NEXT: ; %bb.1: ; %dynamic-memset-expansion-main-body.preheader
+; GFX942-SDAG-NEXT: s_mov_b32 s4, 0x4040404
+; GFX942-SDAG-NEXT: v_perm_b32 v4, v2, v2, s4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v6, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v4
+; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-SDAG-NEXT: .LBB0_2: ; %dynamic-memset-expansion-main-body
+; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[12:13], v[0:1], 0, s[4:5]
+; GFX942-SDAG-NEXT: s_add_u32 s4, s4, 16
+; GFX942-SDAG-NEXT: s_addc_u32 s5, s5, 0
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[10:11]
+; GFX942-SDAG-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[12:13], v[4:7]
+; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX942-SDAG-NEXT: s_cbranch_execnz .LBB0_2
+; GFX942-SDAG-NEXT: .LBB0_3: ; %Flow4
+; GFX942-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[8:9]
+; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-SDAG-NEXT: s_cbranch_execz .LBB0_6
+; GFX942-SDAG-NEXT: ; %bb.4: ; %dynamic-memset-expansion-residual-body.preheader
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, v[10:11]
+; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-SDAG-NEXT: .LBB0_5: ; %dynamic-memset-expansion-residual-body
+; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[4:5]
+; GFX942-SDAG-NEXT: s_add_u32 s4, s4, 1
+; GFX942-SDAG-NEXT: s_addc_u32 s5, s5, 0
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[8:9]
+; GFX942-SDAG-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX942-SDAG-NEXT: flat_store_byte v[4:5], v2
+; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX942-SDAG-NEXT: s_cbranch_execnz .LBB0_5
+; GFX942-SDAG-NEXT: .LBB0_6: ; %Flow2
+; GFX942-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-GISEL-LABEL: memset_p0_varsize_align_4_varsetval:
+; GFX942-GISEL: ; %bb.0: ; %entry
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v10, v3
+; GFX942-GISEL-NEXT: v_and_b32_e32 v12, -16, v10
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v13, v4
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v11, v4
+; GFX942-GISEL-NEXT: v_and_b32_e32 v8, 15, v10
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v9, 0
+; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[12:13]
+; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-GISEL-NEXT: s_cbranch_execz .LBB0_3
+; GFX942-GISEL-NEXT: ; %bb.1: ; %dynamic-memset-expansion-main-body.preheader
+; GFX942-GISEL-NEXT: s_mov_b32 s4, 0x4040404
+; GFX942-GISEL-NEXT: v_perm_b32 v4, v2, v2, s4
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, v4
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v6, v4
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v7, v4
+; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-GISEL-NEXT: .LBB0_2: ; %dynamic-memset-expansion-main-body
+; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-GISEL-NEXT: v_lshl_add_u64 v[14:15], v[0:1], 0, s[4:5]
+; GFX942-GISEL-NEXT: s_add_u32 s4, s4, 16
+; GFX942-GISEL-NEXT: s_addc_u32 s5, s5, 0
+; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[12:13]
+; GFX942-GISEL-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX942-GISEL-NEXT: flat_store_dwordx4 v[14:15], v[4:7]
+; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX942-GISEL-NEXT: s_cbranch_execnz .LBB0_2
+; GFX942-GISEL-NEXT: .LBB0_3: ; %Flow4
+; GFX942-GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[8:9]
+; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-GISEL-NEXT: s_cbranch_execz .LBB0_6
+; GFX942-GISEL-NEXT: ; %bb.4: ; %dynamic-memset-expansion-residual-body.preheader
+; GFX942-GISEL-NEXT: v_and_b32_e32 v10, -16, v10
+; GFX942-GISEL-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, v[10:11]
+; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-GISEL-NEXT: .LBB0_5: ; %dynamic-memset-expansion-residual-body
+; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-GISEL-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[4:5]
+; GFX942-GISEL-NEXT: s_add_u32 s4, s4, 1
+; GFX942-GISEL-NEXT: s_addc_u32 s5, s5, 0
+; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[8:9]
+; GFX942-GISEL-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX942-GISEL-NEXT: flat_store_byte v[4:5], v2
+; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX942-GISEL-NEXT: s_cbranch_execnz .LBB0_5
+; GFX942-GISEL-NEXT: .LBB0_6: ; %Flow2
+; GFX942-GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
tail call void @llvm.memset.p0.i64(ptr addrspace(0) noundef nonnull align 4 %dst, i8 %setval, i64 %size, i1 false)
ret void
}
define void @memset_p1_varsize_align_4_varsetval(ptr addrspace(1) align 4 %dst, i8 %setval, i64 %size) #0 {
-; GFX942-LABEL: memset_p1_varsize_align_4_varsetval:
-; GFX942: ; %bb.0: ; %entry
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v10, v3
-; GFX942-NEXT: v_and_b32_e32 v12, -16, v10
-; GFX942-NEXT: v_mov_b32_e32 v13, v4
-; GFX942-NEXT: v_mov_b32_e32 v11, v4
-; GFX942-NEXT: v_and_b32_e32 v8, 15, v10
-; GFX942-NEXT: v_mov_b32_e32 v9, 0
-; GFX942-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[12:13]
-; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
-; GFX942-NEXT: s_cbranch_execz .LBB1_3
-; GFX942-NEXT: ; %bb.1: ; %dynamic-memset-expansion-main-body.preheader
-; GFX942-NEXT: s_mov_b32 s4, 0x4040404
-; GFX942-NEXT: v_perm_b32 v4, v2, v2, s4
-; GFX942-NEXT: v_mov_b32_e32 v5, v4
-; GFX942-NEXT: v_mov_b32_e32 v6, v4
-; GFX942-NEXT: v_mov_b32_e32 v7, v4
-; GFX942-NEXT: s_mov_b64 s[4:5], 0
-; GFX942-NEXT: .LBB1_2: ; %dynamic-memset-expansion-main-body
-; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshl_add_u64 v[14:15], v[0:1], 0, s[4:5]
-; GFX942-NEXT: s_add_u32 s4, s4, 16
-; GFX942-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[12:13]
-; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: global_store_dwordx4 v[14:15], v[4:7], off
-; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX942-NEXT: s_cbranch_execnz .LBB1_2
-; GFX942-NEXT: .LBB1_3: ; %Flow4
-; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[8:9]
-; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
-; GFX942-NEXT: s_cbranch_execz .LBB1_6
-; GFX942-NEXT: ; %bb.4: ; %dynamic-memset-expansion-residual-body.preheader
-; GFX942-NEXT: v_and_b32_e32 v10, -16, v10
-; GFX942-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, v[10:11]
-; GFX942-NEXT: s_mov_b64 s[4:5], 0
-; GFX942-NEXT: .LBB1_5: ; %dynamic-memset-expansion-residual-body
-; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[4:5]
-; GFX942-NEXT: s_add_u32 s4, s4, 1
-; GFX942-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[8:9]
-; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: global_store_byte v[4:5], v2, off
-; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX942-NEXT: s_cbranch_execnz .LBB1_5
-; GFX942-NEXT: .LBB1_6: ; %Flow2
-; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
+; GFX942-SDAG-LABEL: memset_p1_varsize_align_4_varsetval:
+; GFX942-SDAG: ; %bb.0: ; %entry
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, v4
+; GFX942-SDAG-NEXT: v_and_b32_e32 v10, -16, v3
+; GFX942-SDAG-NEXT: v_and_b32_e32 v8, 15, v3
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, 0
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
+; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-SDAG-NEXT: s_cbranch_execz .LBB1_3
+; GFX942-SDAG-NEXT: ; %bb.1: ; %dynamic-memset-expansion-main-body.preheader
+; GFX942-SDAG-NEXT: s_mov_b32 s4, 0x4040404
+; GFX942-SDAG-NEXT: v_perm_b32 v4, v2, v2, s4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v6, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v4
+; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-SDAG-NEXT: .LBB1_2: ; %dynamic-memset-expansion-main-body
+; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[12:13], v[0:1], 0, s[4:5]
+; GFX942-SDAG-NEXT: s_add_u32 s4, s4, 16
+; GFX942-SDAG-NEXT: s_addc_u32 s5, s5, 0
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[10:11]
+; GFX942-SDAG-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[12:13], v[4:7], off
+; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX942-SDAG-NEXT: s_cbranch_execnz .LBB1_2
+; GFX942-SDAG-NEXT: .LBB1_3: ; %Flow4
+; GFX942-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[8:9]
+; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-SDAG-NEXT: s_cbranch_execz .LBB1_6
+; GFX942-SDAG-NEXT: ; %bb.4: ; %dynamic-memset-expansion-residual-body.preheader
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, v[10:11]
+; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-SDAG-NEXT: .LBB1_5: ; %dynamic-memset-expansion-residual-body
+; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[4:5]
+; GFX942-SDAG-NEXT: s_add_u32 s4, s4, 1
+; GFX942-SDAG-NEXT: s_addc_u32 s5, s5, 0
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[8:9]
+; GFX942-SDAG-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX942-SDAG-NEXT: global_store_byte v[4:5], v2, off
+; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX942-SDAG-NEXT: s_cbranch_execnz .LBB1_5
+; GFX942-SDAG-NEXT: .LBB1_6: ; %Flow2
+; GFX942-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-GISEL-LABEL: memset_p1_varsize_align_4_varsetval:
+; GFX942-GISEL: ; %bb.0: ; %entry
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v10, v3
+; GFX942-GISEL-NEXT: v_and_b32_e32 v12, -16, v10
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v13, v4
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v11, v4
+; GFX942-GISEL-NEXT: v_and_b32_e32 v8, 15, v10
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v9, 0
+; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[12:13]
+; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-GISEL-NEXT: s_cbranch_execz .LBB1_3
+; GFX942-GISEL-NEXT: ; %bb.1: ; %dynamic-memset-expansion-main-body.preheader
+; GFX942-GISEL-NEXT: s_mov_b32 s4, 0x4040404
+; GFX942-GISEL-NEXT: v_perm_b32 v4, v2, v2, s4
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, v4
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v6, v4
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v7, v4
+; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-GISEL-NEXT: .LBB1_2: ; %dynamic-memset-expansion-main-body
+; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-GISEL-NEXT: v_lshl_add_u64 v[14:15], v[0:1], 0, s[4:5]
+; GFX942-GISEL-NEXT: s_add_u32 s4, s4, 16
+; GFX942-GISEL-NEXT: s_addc_u32 s5, s5, 0
+; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[12:13]
+; GFX942-GISEL-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[14:15], v[4:7], off
+; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX942-GISEL-NEXT: s_cbranch_execnz .LBB1_2
+; GFX942-GISEL-NEXT: .LBB1_3: ; %Flow4
+; GFX942-GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[8:9]
+; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-GISEL-NEXT: s_cbranch_execz .LBB1_6
+; GFX942-GISEL-NEXT: ; %bb.4: ; %dynamic-memset-expansion-residual-body.preheader
+; GFX942-GISEL-NEXT: v_and_b32_e32 v10, -16, v10
+; GFX942-GISEL-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, v[10:11]
+; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-GISEL-NEXT: .LBB1_5: ; %dynamic-memset-expansion-residual-body
+; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-GISEL-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[4:5]
+; GFX942-GISEL-NEXT: s_add_u32 s4, s4, 1
+; GFX942-GISEL-NEXT: s_addc_u32 s5, s5, 0
+; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[8:9]
+; GFX942-GISEL-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX942-GISEL-NEXT: global_store_byte v[4:5], v2, off
+; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX942-GISEL-NEXT: s_cbranch_execnz .LBB1_5
+; GFX942-GISEL-NEXT: .LBB1_6: ; %Flow2
+; GFX942-GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
tail call void @llvm.memset.p1.i64(ptr addrspace(1) noundef nonnull align 4 %dst, i8 %setval, i64 %size, i1 false)
ret void
@@ -982,116 +1086,218 @@ entry:
}
define void @memset_p1_varsz_align_4_set40(ptr addrspace(1) align 4 %dst, i64 %size) #0 {
-; GFX942-LABEL: memset_p1_varsz_align_4_set40:
-; GFX942: ; %bb.0: ; %entry
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_and_b32_e32 v10, -16, v2
-; GFX942-NEXT: v_mov_b32_e32 v11, v3
-; GFX942-NEXT: v_and_b32_e32 v8, 15, v2
-; GFX942-NEXT: v_mov_b32_e32 v9, 0
-; GFX942-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
-; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
-; GFX942-NEXT: s_cbranch_execz .LBB12_3
-; GFX942-NEXT: ; %bb.1: ; %dynamic-memset-expansion-main-body.preheader
-; GFX942-NEXT: v_mov_b32_e32 v4, 0x28282828
-; GFX942-NEXT: v_mov_b32_e32 v5, v4
-; GFX942-NEXT: v_mov_b32_e32 v6, v4
-; GFX942-NEXT: v_mov_b32_e32 v7, v4
-; GFX942-NEXT: s_mov_b64 s[4:5], 0
-; GFX942-NEXT: .LBB12_2: ; %dynamic-memset-expansion-main-body
-; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshl_add_u64 v[12:13], v[0:1], 0, s[4:5]
-; GFX942-NEXT: s_add_u32 s4, s4, 16
-; GFX942-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[10:11]
-; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: global_store_dwordx4 v[12:13], v[4:7], off
-; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX942-NEXT: s_cbranch_execnz .LBB12_2
-; GFX942-NEXT: .LBB12_3: ; %Flow4
-; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[8:9]
-; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
-; GFX942-NEXT: s_cbranch_execz .LBB12_6
-; GFX942-NEXT: ; %bb.4: ; %dynamic-memset-expansion-residual-body.preheader
-; GFX942-NEXT: v_and_b32_e32 v2, -16, v2
-; GFX942-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, v[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v2, 40
-; GFX942-NEXT: s_mov_b64 s[4:5], 0
-; GFX942-NEXT: .LBB12_5: ; %dynamic-memset-expansion-residual-body
-; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[4:5]
-; GFX942-NEXT: s_add_u32 s4, s4, 1
-; GFX942-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[8:9]
-; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: global_store_byte v[4:5], v2, off
-; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX942-NEXT: s_cbranch_execnz .LBB12_5
-; GFX942-NEXT: .LBB12_6: ; %Flow2
-; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
+; GFX942-SDAG-LABEL: memset_p1_varsz_align_4_set40:
+; GFX942-SDAG: ; %bb.0: ; %entry
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: v_and_b32_e32 v8, 15, v2
+; GFX942-SDAG-NEXT: v_and_b32_e32 v2, -16, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, 0
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
+; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-SDAG-NEXT: s_cbranch_execz .LBB12_3
+; GFX942-SDAG-NEXT: ; %bb.1: ; %dynamic-memset-expansion-main-body.preheader
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v4, 0x28282828
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v6, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v4
+; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-SDAG-NEXT: .LBB12_2: ; %dynamic-memset-expansion-main-body
+; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[10:11], v[0:1], 0, s[4:5]
+; GFX942-SDAG-NEXT: s_add_u32 s4, s4, 16
+; GFX942-SDAG-NEXT: s_addc_u32 s5, s5, 0
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[2:3]
+; GFX942-SDAG-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[10:11], v[4:7], off
+; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX942-SDAG-NEXT: s_cbranch_execnz .LBB12_2
+; GFX942-SDAG-NEXT: .LBB12_3: ; %Flow4
+; GFX942-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[8:9]
+; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-SDAG-NEXT: s_cbranch_execz .LBB12_6
+; GFX942-SDAG-NEXT: ; %bb.4: ; %dynamic-memset-expansion-residual-body.preheader
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, v[2:3]
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v2, 40
+; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-SDAG-NEXT: .LBB12_5: ; %dynamic-memset-expansion-residual-body
+; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[4:5]
+; GFX942-SDAG-NEXT: s_add_u32 s4, s4, 1
+; GFX942-SDAG-NEXT: s_addc_u32 s5, s5, 0
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[8:9]
+; GFX942-SDAG-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX942-SDAG-NEXT: global_store_byte v[4:5], v2, off
+; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX942-SDAG-NEXT: s_cbranch_execnz .LBB12_5
+; GFX942-SDAG-NEXT: .LBB12_6: ; %Flow2
+; GFX942-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-GISEL-LABEL: memset_p1_varsz_align_4_set40:
+; GFX942-GISEL: ; %bb.0: ; %entry
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_and_b32_e32 v10, -16, v2
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v11, v3
+; GFX942-GISEL-NEXT: v_and_b32_e32 v8, 15, v2
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v9, 0
+; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
+; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-GISEL-NEXT: s_cbranch_execz .LBB12_3
+; GFX942-GISEL-NEXT: ; %bb.1: ; %dynamic-memset-expansion-main-body.preheader
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, 0x28282828
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, v4
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v6, v4
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v7, v4
+; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-GISEL-NEXT: .LBB12_2: ; %dynamic-memset-expansion-main-body
+; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-GISEL-NEXT: v_lshl_add_u64 v[12:13], v[0:1], 0, s[4:5]
+; GFX942-GISEL-NEXT: s_add_u32 s4, s4, 16
+; GFX942-GISEL-NEXT: s_addc_u32 s5, s5, 0
+; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[10:11]
+; GFX942-GISEL-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[12:13], v[4:7], off
+; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX942-GISEL-NEXT: s_cbranch_execnz .LBB12_2
+; GFX942-GISEL-NEXT: .LBB12_3: ; %Flow4
+; GFX942-GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[8:9]
+; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-GISEL-NEXT: s_cbranch_execz .LBB12_6
+; GFX942-GISEL-NEXT: ; %bb.4: ; %dynamic-memset-expansion-residual-body.preheader
+; GFX942-GISEL-NEXT: v_and_b32_e32 v2, -16, v2
+; GFX942-GISEL-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, v[2:3]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v2, 40
+; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-GISEL-NEXT: .LBB12_5: ; %dynamic-memset-expansion-residual-body
+; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-GISEL-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[4:5]
+; GFX942-GISEL-NEXT: s_add_u32 s4, s4, 1
+; GFX942-GISEL-NEXT: s_addc_u32 s5, s5, 0
+; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[8:9]
+; GFX942-GISEL-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX942-GISEL-NEXT: global_store_byte v[4:5], v2, off
+; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX942-GISEL-NEXT: s_cbranch_execnz .LBB12_5
+; GFX942-GISEL-NEXT: .LBB12_6: ; %Flow2
+; GFX942-GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
tail call void @llvm.memset.p1.i64(ptr addrspace(1) noundef nonnull align 4 %dst, i8 40, i64 %size, i1 false)
ret void
}
define void @memset_p1_varsz_align_4_set0(ptr addrspace(1) align 4 %dst, i64 %size) #0 {
-; GFX942-LABEL: memset_p1_varsz_align_4_set0:
-; GFX942: ; %bb.0: ; %entry
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_and_b32_e32 v6, -16, v2
-; GFX942-NEXT: v_mov_b32_e32 v7, v3
-; GFX942-NEXT: v_and_b32_e32 v4, 15, v2
-; GFX942-NEXT: v_mov_b32_e32 v5, 0
-; GFX942-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
-; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
-; GFX942-NEXT: s_cbranch_execz .LBB13_3
-; GFX942-NEXT: ; %bb.1: ; %dynamic-memset-expansion-main-body.preheader
-; GFX942-NEXT: v_mov_b32_e32 v8, v5
-; GFX942-NEXT: v_mov_b32_e32 v9, v5
-; GFX942-NEXT: v_mov_b32_e32 v10, v5
-; GFX942-NEXT: v_mov_b32_e32 v11, v5
-; GFX942-NEXT: s_mov_b64 s[4:5], 0
-; GFX942-NEXT: .LBB13_2: ; %dynamic-memset-expansion-main-body
-; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshl_add_u64 v[12:13], v[0:1], 0, s[4:5]
-; GFX942-NEXT: s_add_u32 s4, s4, 16
-; GFX942-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[6:7]
-; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: global_store_dwordx4 v[12:13], v[8:11], off
-; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX942-NEXT: s_cbranch_execnz .LBB13_2
-; GFX942-NEXT: .LBB13_3: ; %Flow4
-; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
-; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
-; GFX942-NEXT: s_cbranch_execz .LBB13_6
-; GFX942-NEXT: ; %bb.4: ; %dynamic-memset-expansion-residual-body.preheader
-; GFX942-NEXT: v_and_b32_e32 v2, -16, v2
-; GFX942-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, v[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v2, 0
-; GFX942-NEXT: s_mov_b64 s[4:5], 0
-; GFX942-NEXT: .LBB13_5: ; %dynamic-memset-expansion-residual-body
-; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshl_add_u64 v[6:7], v[0:1], 0, s[4:5]
-; GFX942-NEXT: s_add_u32 s4, s4, 1
-; GFX942-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[4:5]
-; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: global_store_byte v[6:7], v2, off
-; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX942-NEXT: s_cbranch_execnz .LBB13_5
-; GFX942-NEXT: .LBB13_6: ; %Flow2
-; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
+; GFX942-SDAG-LABEL: memset_p1_varsz_align_4_set0:
+; GFX942-SDAG: ; %bb.0: ; %entry
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: v_and_b32_e32 v4, 15, v2
+; GFX942-SDAG-NEXT: v_and_b32_e32 v2, -16, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, 0
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
+; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-SDAG-NEXT: s_cbranch_execz .LBB13_3
+; GFX942-SDAG-NEXT: ; %bb.1: ; %dynamic-memset-expansion-main-body.preheader
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v6, v5
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v5
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v8, v5
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, v5
+; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-SDAG-NEXT: .LBB13_2: ; %dynamic-memset-expansion-main-body
+; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[10:11], v[0:1], 0, s[4:5]
+; GFX942-SDAG-NEXT: s_add_u32 s4, s4, 16
+; GFX942-SDAG-NEXT: s_addc_u32 s5, s5, 0
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[2:3]
+; GFX942-SDAG-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[10:11], v[6:9], off
+; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX942-SDAG-NEXT: s_cbranch_execnz .LBB13_2
+; GFX942-SDAG-NEXT: .LBB13_3: ; %Flow4
+; GFX942-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
+; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-SDAG-NEXT: s_cbranch_execz .LBB13_6
+; GFX942-SDAG-NEXT: ; %bb.4: ; %dynamic-memset-expansion-residual-body.preheader
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, v[2:3]
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-SDAG-NEXT: .LBB13_5: ; %dynamic-memset-expansion-residual-body
+; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[6:7], v[0:1], 0, s[4:5]
+; GFX942-SDAG-NEXT: s_add_u32 s4, s4, 1
+; GFX942-SDAG-NEXT: s_addc_u32 s5, s5, 0
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[4:5]
+; GFX942-SDAG-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX942-SDAG-NEXT: global_store_byte v[6:7], v2, off
+; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX942-SDAG-NEXT: s_cbranch_execnz .LBB13_5
+; GFX942-SDAG-NEXT: .LBB13_6: ; %Flow2
+; GFX942-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-GISEL-LABEL: memset_p1_varsz_align_4_set0:
+; GFX942-GISEL: ; %bb.0: ; %entry
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_and_b32_e32 v6, -16, v2
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v7, v3
+; GFX942-GISEL-NEXT: v_and_b32_e32 v4, 15, v2
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, 0
+; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
+; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-GISEL-NEXT: s_cbranch_execz .LBB13_3
+; GFX942-GISEL-NEXT: ; %bb.1: ; %dynamic-memset-expansion-main-body.preheader
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v8, v5
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v9, v5
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v10, v5
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v11, v5
+; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-GISEL-NEXT: .LBB13_2: ; %dynamic-memset-expansion-main-body
+; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-GISEL-NEXT: v_lshl_add_u64 v[12:13], v[0:1], 0, s[4:5]
+; GFX942-GISEL-NEXT: s_add_u32 s4, s4, 16
+; GFX942-GISEL-NEXT: s_addc_u32 s5, s5, 0
+; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[6:7]
+; GFX942-GISEL-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[12:13], v[8:11], off
+; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX942-GISEL-NEXT: s_cbranch_execnz .LBB13_2
+; GFX942-GISEL-NEXT: .LBB13_3: ; %Flow4
+; GFX942-GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
+; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-GISEL-NEXT: s_cbranch_execz .LBB13_6
+; GFX942-GISEL-NEXT: ; %bb.4: ; %dynamic-memset-expansion-residual-body.preheader
+; GFX942-GISEL-NEXT: v_and_b32_e32 v2, -16, v2
+; GFX942-GISEL-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, v[2:3]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-GISEL-NEXT: .LBB13_5: ; %dynamic-memset-expansion-residual-body
+; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-GISEL-NEXT: v_lshl_add_u64 v[6:7], v[0:1], 0, s[4:5]
+; GFX942-GISEL-NEXT: s_add_u32 s4, s4, 1
+; GFX942-GISEL-NEXT: s_addc_u32 s5, s5, 0
+; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[4:5]
+; GFX942-GISEL-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX942-GISEL-NEXT: global_store_byte v[6:7], v2, off
+; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX942-GISEL-NEXT: s_cbranch_execnz .LBB13_5
+; GFX942-GISEL-NEXT: .LBB13_6: ; %Flow2
+; GFX942-GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
tail call void @llvm.memset.p1.i64(ptr addrspace(1) noundef nonnull align 4 %dst, i8 0, i64 %size, i1 false)
ret void
@@ -1104,6 +1310,3 @@ declare void @llvm.memset.p5.i64(ptr addrspace(5) noalias nocapture writeonly, i
attributes #0 = { nounwind }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GFX942-GISEL: {{.*}}
-; GFX942-SDAG: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/mfma-no-register-aliasing.ll b/llvm/test/CodeGen/AMDGPU/mfma-no-register-aliasing.ll
index 07d04ea789a81..1cfaa0bba74e2 100644
--- a/llvm/test/CodeGen/AMDGPU/mfma-no-register-aliasing.ll
+++ b/llvm/test/CodeGen/AMDGPU/mfma-no-register-aliasing.ll
@@ -4,7 +4,7 @@
; RUN: llc -mtriple=amdgpu9.0a -early-live-intervals < %s | FileCheck -enable-var-scope --check-prefixes=GREEDY90A %s
; RUN: llc -mtriple=amdgpu9.42 < %s | FileCheck -enable-var-scope --check-prefixes=GREEDY942 %s
; RUN: llc -global-isel -mtriple=amdgpu9.0a < %s | FileCheck -enable-var-scope --check-prefixes=GREEDY90A-GISEL %s
-; RUN: llc -mtriple=amdgpu9.0a -sgpr-regalloc=fast -vgpr-regalloc=fast -wwm-regalloc=fast < %s | FileCheck -enable-var-scope --check-prefixes=FAST90A %s
+; RUN: llc -mtriple=amdgpu9.0a -sgpr-regalloc-npm=fast -vgpr-regalloc-npm=fast -wwm-regalloc-npm=fast < %s | FileCheck -enable-var-scope --check-prefixes=FAST90A %s
; This is better with 90a
diff --git a/llvm/test/CodeGen/AMDGPU/opt-exec-masking-pre-ra-update-liveness-wave32.mir b/llvm/test/CodeGen/AMDGPU/opt-exec-masking-pre-ra-update-liveness-wave32.mir
index 09d9e90e3d710..bbee2c4f107a1 100644
--- a/llvm/test/CodeGen/AMDGPU/opt-exec-masking-pre-ra-update-liveness-wave32.mir
+++ b/llvm/test/CodeGen/AMDGPU/opt-exec-masking-pre-ra-update-liveness-wave32.mir
@@ -1,6 +1,6 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
# REQUIRES: asserts
-# RUN: llc -mtriple=amdgpu10.10-amd-amdhsa -verify-regalloc -misched-only-block=999 -start-before=machine-scheduler -stop-after=greedy,0 -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu10.10-amd-amdhsa -verify-regalloc -misched-only-block=999 -start-before=machine-scheduler -stop-after=greedy,1 -o - %s | FileCheck %s
# This run line is a total hack to get the live intervals to make it
# to the verifier. This requires asserts to use
diff --git a/llvm/test/CodeGen/AMDGPU/opt-exec-masking-pre-ra-update-liveness.mir b/llvm/test/CodeGen/AMDGPU/opt-exec-masking-pre-ra-update-liveness.mir
index 4d32e22218cfc..3a416cd43fce6 100644
--- a/llvm/test/CodeGen/AMDGPU/opt-exec-masking-pre-ra-update-liveness.mir
+++ b/llvm/test/CodeGen/AMDGPU/opt-exec-masking-pre-ra-update-liveness.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -verify-regalloc -start-before=machine-scheduler -stop-after=greedy,0 -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -verify-regalloc -start-before=machine-scheduler -stop-after=greedy,1 -o - %s | FileCheck %s
# Make sure liveness is correctly updated when folding the cndmask and
# compare.
diff --git a/llvm/test/CodeGen/AMDGPU/optimize-exec-mask-pre-ra-alloc-failure.mir b/llvm/test/CodeGen/AMDGPU/optimize-exec-mask-pre-ra-alloc-failure.mir
index df83b74042ad2..91a9219548dc9 100644
--- a/llvm/test/CodeGen/AMDGPU/optimize-exec-mask-pre-ra-alloc-failure.mir
+++ b/llvm/test/CodeGen/AMDGPU/optimize-exec-mask-pre-ra-alloc-failure.mir
@@ -1,4 +1,4 @@
-# RUN: llc -mtriple=amdgpu10.30-amd-amdhsa -start-before=machine-scheduler -stop-after=virtregrewriter,0 -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu10.30-amd-amdhsa -start-before=machine-scheduler -stop-after=virt-reg-rewriter,1 -o - %s | FileCheck %s
---
# If optimize-exec-mask-pre-ra over approximates live intervals (not replicating splits)
diff --git a/llvm/test/CodeGen/AMDGPU/partial-regcopy-and-spill-missed-at-regalloc.ll b/llvm/test/CodeGen/AMDGPU/partial-regcopy-and-spill-missed-at-regalloc.ll
index 9431658b0e6b5..b9bd46bc0ae9f 100644
--- a/llvm/test/CodeGen/AMDGPU/partial-regcopy-and-spill-missed-at-regalloc.ll
+++ b/llvm/test/CodeGen/AMDGPU/partial-regcopy-and-spill-missed-at-regalloc.ll
@@ -1,7 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-;RUN: llc -mtriple=amdgpu9.08-amd-amdhsa --stop-after=greedy,2 < %s | FileCheck -check-prefix=REGALLOC-GFX908 %s
+;RUN: llc -mtriple=amdgpu9.08-amd-amdhsa --stop-after=greedy,3 < %s | FileCheck -check-prefix=REGALLOC-GFX908 %s
;RUN: llc -mtriple=amdgpu9.08-amd-amdhsa --stop-after=prolog-epilog < %s | FileCheck -check-prefix=PEI-GFX908 %s
-;RUN: llc -mtriple=amdgpu9.0a-amd-amdhsa -amdgpu-mfma-vgpr-form=0 --stop-after=greedy,2 < %s | FileCheck -check-prefix=REGALLOC-GFX90A %s
+;RUN: llc -mtriple=amdgpu9.0a-amd-amdhsa -amdgpu-mfma-vgpr-form=0 --stop-after=greedy,3 < %s | FileCheck -check-prefix=REGALLOC-GFX90A %s
;RUN: llc -mtriple=amdgpu9.0a-amd-amdhsa -amdgpu-mfma-vgpr-form=0 --stop-after=prolog-epilog < %s | FileCheck -check-prefix=PEI-GFX90A %s
; Partial reg copy and spill missed during regalloc handled later at frame lowering.
diff --git a/llvm/test/CodeGen/AMDGPU/pr51516.mir b/llvm/test/CodeGen/AMDGPU/pr51516.mir
index 033656354f2c1..404e4cdc13340 100644
--- a/llvm/test/CodeGen/AMDGPU/pr51516.mir
+++ b/llvm/test/CodeGen/AMDGPU/pr51516.mir
@@ -1,6 +1,6 @@
# NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-# RUN: llc -mtriple=amdgpu9.00--amdhsa -amdgpu-disable-unclustered-high-rp-reschedule -verify-misched -start-before=machine-scheduler -stop-after=virtregrewriter,2 -o - %s | FileCheck -check-prefix=GCN %s
-# RUN: llc -mtriple=amdgpu9.00--amdhsa -amdgpu-disable-unclustered-high-rp-reschedule -amdgpu-use-amdgpu-trackers=1 -verify-misched -start-before=machine-scheduler -stop-after=virtregrewriter,2 -o - %s | FileCheck -check-prefix=GCN-GCNTRACKER %s
+# RUN: llc -mtriple=amdgpu9.00--amdhsa -amdgpu-disable-unclustered-high-rp-reschedule -verify-misched -start-before=machine-scheduler -stop-after=virt-reg-rewriter,3 -o - %s | FileCheck -check-prefix=GCN %s
+# RUN: llc -mtriple=amdgpu9.00--amdhsa -amdgpu-disable-unclustered-high-rp-reschedule -amdgpu-use-amdgpu-trackers=1 -verify-misched -start-before=machine-scheduler -stop-after=virt-reg-rewriter,3 -o - %s | FileCheck -check-prefix=GCN-GCNTRACKER %s
# Check that %3 was not rematerialized before the last store since its operand %1
# is killed by that store.
diff --git a/llvm/test/CodeGen/AMDGPU/ran-out-of-registers-error-all-regs-reserved.ll b/llvm/test/CodeGen/AMDGPU/ran-out-of-registers-error-all-regs-reserved.ll
index 0825688cb03c0..c6624a1dbf93e 100644
--- a/llvm/test/CodeGen/AMDGPU/ran-out-of-registers-error-all-regs-reserved.ll
+++ b/llvm/test/CodeGen/AMDGPU/ran-out-of-registers-error-all-regs-reserved.ll
@@ -1,6 +1,6 @@
-; RUN: not llc -mtriple=amdgpu9.08-amd-amdhsa -vgpr-regalloc=greedy -filetype=null %s 2>&1 | FileCheck -implicit-check-not=error %s
-; RUN: not llc -mtriple=amdgpu9.08-amd-amdhsa -vgpr-regalloc=basic -filetype=null %s 2>&1 | FileCheck -implicit-check-not=error %s
-; RUN: not llc -mtriple=amdgpu9.08-amd-amdhsa -vgpr-regalloc=fast -filetype=null %s 2>&1 | FileCheck -implicit-check-not=error %s
+; RUN: not llc -mtriple=amdgpu9.08-amd-amdhsa -vgpr-regalloc-npm=greedy -filetype=null %s 2>&1 | FileCheck -implicit-check-not=error %s
+; RUN: not llc -enable-new-pm=0 -mtriple=amdgpu9.08-amd-amdhsa -vgpr-regalloc=basic -filetype=null %s 2>&1 | FileCheck -implicit-check-not=error %s
+; RUN: not llc -mtriple=amdgpu9.08-amd-amdhsa -vgpr-regalloc-npm=fast -filetype=null %s 2>&1 | FileCheck -implicit-check-not=error %s
declare <32 x i32> @llvm.amdgcn.mfma.i32.32x32x4i8(i32, i32, <32 x i32>, i32 immarg, i32 immarg, i32 immarg)
diff --git a/llvm/test/CodeGen/AMDGPU/ran-out-of-registers-errors.ll b/llvm/test/CodeGen/AMDGPU/ran-out-of-registers-errors.ll
index 11b5363aa7ca1..dc2f3fb6bd447 100644
--- a/llvm/test/CodeGen/AMDGPU/ran-out-of-registers-errors.ll
+++ b/llvm/test/CodeGen/AMDGPU/ran-out-of-registers-errors.ll
@@ -1,9 +1,9 @@
-; RUN: not llc -mtriple=amdgpu9.08-amd-amdhsa -stress-regalloc=1 -vgpr-regalloc=greedy -filetype=null %s 2>&1 | FileCheck -check-prefixes=CHECK,GREEDY -implicit-check-not=error %s
-; RUN: not llc -mtriple=amdgpu9.08-amd-amdhsa -stress-regalloc=1 -vgpr-regalloc=basic -filetype=null %s 2>&1 | FileCheck -implicit-check-not=error -check-prefixes=CHECK,BASIC %s
-; RUN: not llc -mtriple=amdgpu9.08-amd-amdhsa -stress-regalloc=1 -vgpr-regalloc=fast -filetype=null %s 2>&1 | FileCheck -implicit-check-not=error -check-prefixes=CHECK,FAST %s
+; RUN: not llc -mtriple=amdgpu9.08-amd-amdhsa -stress-regalloc=1 -vgpr-regalloc-npm=greedy -filetype=null %s 2>&1 | FileCheck -check-prefixes=CHECK,GREEDY -implicit-check-not=error %s
+; RUN: not llc -enable-new-pm=0 -mtriple=amdgpu9.08-amd-amdhsa -stress-regalloc=1 -vgpr-regalloc=basic -filetype=null %s 2>&1 | FileCheck -implicit-check-not=error -check-prefixes=CHECK,BASIC %s
+; RUN: not llc -mtriple=amdgpu9.08-amd-amdhsa -stress-regalloc=1 -vgpr-regalloc-npm=fast -filetype=null %s 2>&1 | FileCheck -implicit-check-not=error -check-prefixes=CHECK,FAST %s
; RUN: opt -passes=debugify -o %t.bc %s
-; RUN: not llc -mtriple=amdgpu9.08-amd-amdhsa -stress-regalloc=1 -vgpr-regalloc=greedy -filetype=null %t.bc 2>&1 | FileCheck -implicit-check-not=error -check-prefixes=DBGINFO-CHECK,DBGINFO-GREEDY %s
-; RUN: not llc -mtriple=amdgpu9.08-amd-amdhsa -stress-regalloc=1 -vgpr-regalloc=basic -filetype=null %t.bc 2>&1 | FileCheck -implicit-check-not=error -check-prefixes=DBGINFO-CHECK,DBGINFO-BASIC %s
+; RUN: not llc -mtriple=amdgpu9.08-amd-amdhsa -stress-regalloc=1 -vgpr-regalloc-npm=greedy -filetype=null %t.bc 2>&1 | FileCheck -implicit-check-not=error -check-prefixes=DBGINFO-CHECK,DBGINFO-GREEDY %s
+; RUN: not llc -enable-new-pm=0 -mtriple=amdgpu9.08-amd-amdhsa -stress-regalloc=1 -vgpr-regalloc=basic -filetype=null %t.bc 2>&1 | FileCheck -implicit-check-not=error -check-prefixes=DBGINFO-CHECK,DBGINFO-BASIC %s
; FIXME: Asserts when using -O2 + -vgpr-regalloc=fast
; RUN: not llc -mtriple=amdgpu9.08-amd-amdhsa -stress-regalloc=1 -O0 -filetype=null %t.bc 2>&1 | FileCheck -implicit-check-not=error -check-prefixes=DBGINFO-CHECK,DBGINFO-FAST %s
diff --git a/llvm/test/CodeGen/AMDGPU/ran-out-of-sgprs-allocation-failure.mir b/llvm/test/CodeGen/AMDGPU/ran-out-of-sgprs-allocation-failure.mir
index 7ecbecec82898..878bc6db21569 100644
--- a/llvm/test/CodeGen/AMDGPU/ran-out-of-sgprs-allocation-failure.mir
+++ b/llvm/test/CodeGen/AMDGPU/ran-out-of-sgprs-allocation-failure.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu9.0a-amd-amdhsa -verify-regalloc -start-before=greedy,0 -stop-after=virtregrewriter,0 -greedy-regclass-priority-trumps-globalness=1 -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.0a-amd-amdhsa -verify-regalloc -start-before=greedy,1 -stop-after=virt-reg-rewriter,1 -greedy-regclass-priority-trumps-globalness=1 -o - %s | FileCheck %s
# The allocation would previously fail due to poor ordering based on
# register class. The super wide tuples should be allocated first so
diff --git a/llvm/test/CodeGen/AMDGPU/recursive-resource-usage-mcexpr.ll b/llvm/test/CodeGen/AMDGPU/recursive-resource-usage-mcexpr.ll
index b26e2784fab17..329b3d4d2ceab 100644
--- a/llvm/test/CodeGen/AMDGPU/recursive-resource-usage-mcexpr.ll
+++ b/llvm/test/CodeGen/AMDGPU/recursive-resource-usage-mcexpr.ll
@@ -2,27 +2,19 @@
; Recursion: foo -> bar -> baz -> qux -> foo
-; CHECK-LABEL: {{^}}qux
-; CHECK: .set .Lqux.num_vgpr, max(71, .Lfoo.num_vgpr)
-; CHECK: .set .Lqux.num_agpr, max(0, .Lfoo.num_agpr)
-; CHECK: .set .Lqux.numbered_sgpr, max(46, .Lfoo.numbered_sgpr)
-; CHECK: .set .Lqux.private_seg_size, 16
-; CHECK: .set .Lqux.uses_vcc, or(1, .Lfoo.uses_vcc)
-; CHECK: .set .Lqux.uses_flat_scratch, or(0, .Lfoo.uses_flat_scratch)
-; CHECK: .set .Lqux.has_dyn_sized_stack, or(0, .Lfoo.has_dyn_sized_stack)
-; CHECK: .set .Lqux.has_recursion, or(1, .Lfoo.has_recursion)
-; CHECK: .set .Lqux.has_indirect_call, or(0, .Lfoo.has_indirect_call)
-
-; CHECK-LABEL: {{^}}baz
-; CHECK: .set .Lbaz.num_vgpr, max(61, .Lqux.num_vgpr)
-; CHECK: .set .Lbaz.num_agpr, max(0, .Lqux.num_agpr)
-; CHECK: .set .Lbaz.numbered_sgpr, max(51, .Lqux.numbered_sgpr)
-; CHECK: .set .Lbaz.private_seg_size, 16+max(.Lqux.private_seg_size)
-; CHECK: .set .Lbaz.uses_vcc, or(1, .Lqux.uses_vcc)
-; CHECK: .set .Lbaz.uses_flat_scratch, or(0, .Lqux.uses_flat_scratch)
-; CHECK: .set .Lbaz.has_dyn_sized_stack, or(0, .Lqux.has_dyn_sized_stack)
-; CHECK: .set .Lbaz.has_recursion, or(1, .Lqux.has_recursion)
-; CHECK: .set .Lbaz.has_indirect_call, or(0, .Lqux.has_indirect_call)
+; The new pass manager emits an SCC in reverse order compared to the legacy
+; pass manager: the entry (foo) is emitted first and refers to its successors
+; symbolically, while the last-emitted member (qux) is the fully-resolved base.
+; CHECK-LABEL: {{^}}foo
+; CHECK: .set .Lfoo.num_vgpr, max(46, .Lbar.num_vgpr)
+; CHECK: .set .Lfoo.num_agpr, max(0, .Lbar.num_agpr)
+; CHECK: .set .Lfoo.numbered_sgpr, max(71, .Lbar.numbered_sgpr)
+; CHECK: .set .Lfoo.private_seg_size, 16+max(.Lbar.private_seg_size)
+; CHECK: .set .Lfoo.uses_vcc, or(1, .Lbar.uses_vcc)
+; CHECK: .set .Lfoo.uses_flat_scratch, or(0, .Lbar.uses_flat_scratch)
+; CHECK: .set .Lfoo.has_dyn_sized_stack, or(0, .Lbar.has_dyn_sized_stack)
+; CHECK: .set .Lfoo.has_recursion, or(1, .Lbar.has_recursion)
+; CHECK: .set .Lfoo.has_indirect_call, or(0, .Lbar.has_indirect_call)
; CHECK-LABEL: {{^}}bar
; CHECK: .set .Lbar.num_vgpr, max(51, .Lbaz.num_vgpr)
@@ -35,16 +27,27 @@
; CHECK: .set .Lbar.has_recursion, or(1, .Lbaz.has_recursion)
; CHECK: .set .Lbar.has_indirect_call, or(0, .Lbaz.has_indirect_call)
-; CHECK-LABEL: {{^}}foo
-; CHECK: .set .Lfoo.num_vgpr, max(46, 71)
-; CHECK: .set .Lfoo.num_agpr, max(0, 0)
-; CHECK: .set .Lfoo.numbered_sgpr, max(71, 61)
-; CHECK: .set .Lfoo.private_seg_size, 16
-; CHECK: .set .Lfoo.uses_vcc, 1
-; CHECK: .set .Lfoo.uses_flat_scratch, 0
-; CHECK: .set .Lfoo.has_dyn_sized_stack, 0
-; CHECK: .set .Lfoo.has_recursion, 1
-; CHECK: .set .Lfoo.has_indirect_call, 0
+; CHECK-LABEL: {{^}}baz
+; CHECK: .set .Lbaz.num_vgpr, max(61, .Lqux.num_vgpr)
+; CHECK: .set .Lbaz.num_agpr, max(0, .Lqux.num_agpr)
+; CHECK: .set .Lbaz.numbered_sgpr, max(51, .Lqux.numbered_sgpr)
+; CHECK: .set .Lbaz.private_seg_size, 16+max(.Lqux.private_seg_size)
+; CHECK: .set .Lbaz.uses_vcc, or(1, .Lqux.uses_vcc)
+; CHECK: .set .Lbaz.uses_flat_scratch, or(0, .Lqux.uses_flat_scratch)
+; CHECK: .set .Lbaz.has_dyn_sized_stack, or(0, .Lqux.has_dyn_sized_stack)
+; CHECK: .set .Lbaz.has_recursion, or(1, .Lqux.has_recursion)
+; CHECK: .set .Lbaz.has_indirect_call, or(0, .Lqux.has_indirect_call)
+
+; CHECK-LABEL: {{^}}qux
+; CHECK: .set .Lqux.num_vgpr, max(71, 61)
+; CHECK: .set .Lqux.num_agpr, max(0, 0)
+; CHECK: .set .Lqux.numbered_sgpr, max(71, 71)
+; CHECK: .set .Lqux.private_seg_size, 16
+; CHECK: .set .Lqux.uses_vcc, 1
+; CHECK: .set .Lqux.uses_flat_scratch, 0
+; CHECK: .set .Lqux.has_dyn_sized_stack, 0
+; CHECK: .set .Lqux.has_recursion, 1
+; CHECK: .set .Lqux.has_indirect_call, 0
define void @foo() {
entry:
@@ -95,32 +98,21 @@ define amdgpu_kernel void @usefoo() {
; Recursion: A -> B -> C -> A && C -> D -> C
-; CHECK-LABEL: {{^}}D
-; CHECK: .set .LD.num_vgpr, max(71, .LC.num_vgpr)
-; CHECK: .set .LD.num_agpr, max(0, .LC.num_agpr)
-; CHECK: .set .LD.numbered_sgpr, max(71, .LC.numbered_sgpr)
-; CHECK: .set .LD.private_seg_size, 16+max(.LC.private_seg_size)
-; CHECK: .set .LD.uses_vcc, or(1, .LC.uses_vcc)
-; CHECK: .set .LD.uses_flat_scratch, or(0, .LC.uses_flat_scratch)
-; CHECK: .set .LD.has_dyn_sized_stack, or(0, .LC.has_dyn_sized_stack)
-; CHECK: .set .LD.has_recursion, or(1, .LC.has_recursion)
-; CHECK: .set .LD.has_indirect_call, or(0, .LC.has_indirect_call)
-
-; CHECK-LABEL: {{^}}C
-; CHECK: .set .LC.num_vgpr, max(42, .LA.num_vgpr, 71)
-; CHECK: .set .LC.num_agpr, max(0, .LA.num_agpr, 0)
-; CHECK: .set .LC.numbered_sgpr, max(71, .LA.numbered_sgpr, 71)
-; CHECK: .set .LC.private_seg_size, 16+max(.LA.private_seg_size)
-; CHECK: .set .LC.uses_vcc, or(1, .LA.uses_vcc)
-; CHECK: .set .LC.uses_flat_scratch, or(0, .LA.uses_flat_scratch)
-; CHECK: .set .LC.has_dyn_sized_stack, or(0, .LA.has_dyn_sized_stack)
-; CHECK: .set .LC.has_recursion, or(1, .LA.has_recursion)
-; CHECK: .set .LC.has_indirect_call, or(0, .LA.has_indirect_call)
+; CHECK-LABEL: {{^}}A
+; CHECK: .set .LA.num_vgpr, max(41, .LB.num_vgpr)
+; CHECK: .set .LA.num_agpr, max(0, .LB.num_agpr)
+; CHECK: .set .LA.numbered_sgpr, max(51, .LB.numbered_sgpr)
+; CHECK: .set .LA.private_seg_size, 16+max(.LB.private_seg_size)
+; CHECK: .set .LA.uses_vcc, or(1, .LB.uses_vcc)
+; CHECK: .set .LA.uses_flat_scratch, or(0, .LB.uses_flat_scratch)
+; CHECK: .set .LA.has_dyn_sized_stack, or(0, .LB.has_dyn_sized_stack)
+; CHECK: .set .LA.has_recursion, or(1, .LB.has_recursion)
+; CHECK: .set .LA.has_indirect_call, or(0, .LB.has_indirect_call)
; CHECK-LABEL: {{^}}B
-; CHECK: .set .LB.num_vgpr, max(42, .LC.num_vgpr)
+; CHECK: .set .LB.num_vgpr, max(41, .LC.num_vgpr)
; CHECK: .set .LB.num_agpr, max(0, .LC.num_agpr)
-; CHECK: .set .LB.numbered_sgpr, max(71, .LC.numbered_sgpr)
+; CHECK: .set .LB.numbered_sgpr, max(34, .LC.numbered_sgpr)
; CHECK: .set .LB.private_seg_size, 16+max(.LC.private_seg_size)
; CHECK: .set .LB.uses_vcc, or(1, .LC.uses_vcc)
; CHECK: .set .LB.uses_flat_scratch, or(0, .LC.uses_flat_scratch)
@@ -128,16 +120,27 @@ define amdgpu_kernel void @usefoo() {
; CHECK: .set .LB.has_recursion, or(1, .LC.has_recursion)
; CHECK: .set .LB.has_indirect_call, or(0, .LC.has_indirect_call)
-; CHECK-LABEL: {{^}}A
-; CHECK: .set .LA.num_vgpr, max(42, 71)
-; CHECK: .set .LA.num_agpr, max(0, 0)
-; CHECK: .set .LA.numbered_sgpr, max(71, 71)
-; CHECK: .set .LA.private_seg_size, 16
-; CHECK: .set .LA.uses_vcc, 1
-; CHECK: .set .LA.uses_flat_scratch, 0
-; CHECK: .set .LA.has_dyn_sized_stack, 0
-; CHECK: .set .LA.has_recursion, 1
-; CHECK: .set .LA.has_indirect_call, 0
+; CHECK-LABEL: {{^}}C
+; CHECK: .set .LC.num_vgpr, max(43, 41, .LD.num_vgpr)
+; CHECK: .set .LC.num_agpr, max(0, 0, .LD.num_agpr)
+; CHECK: .set .LC.numbered_sgpr, max(55, 51, .LD.numbered_sgpr)
+; CHECK: .set .LC.private_seg_size, 16+max(.LD.private_seg_size)
+; CHECK: .set .LC.uses_vcc, or(1, .LD.uses_vcc)
+; CHECK: .set .LC.uses_flat_scratch, or(0, .LD.uses_flat_scratch)
+; CHECK: .set .LC.has_dyn_sized_stack, or(0, .LD.has_dyn_sized_stack)
+; CHECK: .set .LC.has_recursion, or(1, .LD.has_recursion)
+; CHECK: .set .LC.has_indirect_call, or(0, .LD.has_indirect_call)
+
+; CHECK-LABEL: {{^}}D
+; CHECK: .set .LD.num_vgpr, max(71, 43)
+; CHECK: .set .LD.num_agpr, max(0, 0)
+; CHECK: .set .LD.numbered_sgpr, max(71, 55)
+; CHECK: .set .LD.private_seg_size, 16
+; CHECK: .set .LD.uses_vcc, 1
+; CHECK: .set .LD.uses_flat_scratch, 0
+; CHECK: .set .LD.has_dyn_sized_stack, 0
+; CHECK: .set .LD.has_recursion, 1
+; CHECK: .set .LD.has_indirect_call, 0
define void @A() {
call void @B()
diff --git a/llvm/test/CodeGen/AMDGPU/regalloc-failure-overlapping-insert-assert.mir b/llvm/test/CodeGen/AMDGPU/regalloc-failure-overlapping-insert-assert.mir
index 4f702f1356588..7dc4b5336b193 100644
--- a/llvm/test/CodeGen/AMDGPU/regalloc-failure-overlapping-insert-assert.mir
+++ b/llvm/test/CodeGen/AMDGPU/regalloc-failure-overlapping-insert-assert.mir
@@ -1,4 +1,4 @@
-# RUN: not llc -mtriple=amdgpu9.08 -verify-machineinstrs -start-before=greedy,1 -stop-after=virtregrewriter,2 %s -filetype=null 2>&1 | FileCheck -check-prefix=ERR %s
+# RUN: not llc -mtriple=amdgpu9.08 -verify-machineinstrs -start-before=greedy,2 -stop-after=virt-reg-rewriter,3 %s -filetype=null 2>&1 | FileCheck -check-prefix=ERR %s
# Make sure there's no machine verifier error after failure.
diff --git a/llvm/test/CodeGen/AMDGPU/regalloc-fast-dont-drop-subreg-index-issue61134.mir b/llvm/test/CodeGen/AMDGPU/regalloc-fast-dont-drop-subreg-index-issue61134.mir
index 27e0747fdd232..3f7b79275930a 100644
--- a/llvm/test/CodeGen/AMDGPU/regalloc-fast-dont-drop-subreg-index-issue61134.mir
+++ b/llvm/test/CodeGen/AMDGPU/regalloc-fast-dont-drop-subreg-index-issue61134.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -sgpr-regalloc=fast -start-before=regallocfast,0 -stop-before=greedy -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -sgpr-regalloc-npm=fast -start-before=regallocfast,1 -stop-before=greedy -o - %s | FileCheck %s
# RegAllocFast was incorrectly dropping subregister indexes on
# unassigned virtual registers. Make sure they are passed through
diff --git a/llvm/test/CodeGen/AMDGPU/regalloc-introduces-copy-sgpr-to-agpr.mir b/llvm/test/CodeGen/AMDGPU/regalloc-introduces-copy-sgpr-to-agpr.mir
index 578a1f8e261e6..f0349ddff548a 100644
--- a/llvm/test/CodeGen/AMDGPU/regalloc-introduces-copy-sgpr-to-agpr.mir
+++ b/llvm/test/CodeGen/AMDGPU/regalloc-introduces-copy-sgpr-to-agpr.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu9.08-amd-amdhsa -start-before=greedy,0 -stop-after=postrapseudos -o - %s | FileCheck -check-prefix=GFX908 %s
+# RUN: llc -mtriple=amdgpu9.08-amd-amdhsa -start-before=greedy,1 -stop-after=post-ra-pseudos -o - %s | FileCheck -check-prefix=GFX908 %s
# This testcase has a long sequence of sgpr to vgpr copies with a lot of vgpr
# pressure, encouraging the allocator to displace some into AGPRs. This
diff --git a/llvm/test/CodeGen/AMDGPU/regalloc-undef-copy-fold.mir b/llvm/test/CodeGen/AMDGPU/regalloc-undef-copy-fold.mir
index f9ad9ecf51a6f..4f7380565ab89 100644
--- a/llvm/test/CodeGen/AMDGPU/regalloc-undef-copy-fold.mir
+++ b/llvm/test/CodeGen/AMDGPU/regalloc-undef-copy-fold.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
-# RUN: llc -mtriple=amdgpu9.08 -verify-machineinstrs --start-before=greedy,2 --stop-after=greedy,2 %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.08 -verify-machineinstrs --start-before=greedy,3 --stop-after=greedy,3 %s -o - | FileCheck %s
# Make sure there's no machine verifier error
diff --git a/llvm/test/CodeGen/AMDGPU/register-killed-error-after-alloc-failure0.mir b/llvm/test/CodeGen/AMDGPU/register-killed-error-after-alloc-failure0.mir
index fb1866c379e5c..c92df2e1baed6 100644
--- a/llvm/test/CodeGen/AMDGPU/register-killed-error-after-alloc-failure0.mir
+++ b/llvm/test/CodeGen/AMDGPU/register-killed-error-after-alloc-failure0.mir
@@ -1,5 +1,5 @@
-# RUN: not llc -mtriple=amdgpu9.00-amd-amdhsa -vgpr-regalloc=basic -sgpr-regalloc=basic -start-before=regallocbasic,0 -stop-after=virtregrewriter,2 -verify-regalloc -o - %s 2> %t.basic.err | FileCheck -check-prefix=BASIC %s
-# RUN: not llc -mtriple=amdgpu9.00-amd-amdhsa -start-before=greedy,0 -stop-after=virtregrewriter,2 -verify-regalloc -o - %s 2> %t.greedy.err | FileCheck -check-prefix=GREEDY %s
+# RUN: not llc -enable-new-pm=0 -mtriple=amdgpu9.00-amd-amdhsa -vgpr-regalloc=basic -sgpr-regalloc=basic -start-before=regallocbasic,0 -stop-after=virtregrewriter,2 -verify-regalloc -o - %s 2> %t.basic.err | FileCheck -check-prefix=BASIC %s
+# RUN: not llc -mtriple=amdgpu9.00-amd-amdhsa -start-before=greedy,1 -stop-after=virt-reg-rewriter,3 -verify-regalloc -o - %s 2> %t.greedy.err | FileCheck -check-prefix=GREEDY %s
# RUN: FileCheck -check-prefix=ERR -implicit-check-not=error %s < %t.basic.err
# RUN: FileCheck -check-prefix=ERR -implicit-check-not=error %s < %t.greedy.err
diff --git a/llvm/test/CodeGen/AMDGPU/remaining-virtual-register-operands.mir b/llvm/test/CodeGen/AMDGPU/remaining-virtual-register-operands.mir
index 3476342177d5e..4bc178f3c5bdb 100644
--- a/llvm/test/CodeGen/AMDGPU/remaining-virtual-register-operands.mir
+++ b/llvm/test/CodeGen/AMDGPU/remaining-virtual-register-operands.mir
@@ -1,4 +1,4 @@
-# RUN: not llc -mtriple=amdgpu9.08-amd-amdhsa -verify-machineinstrs -start-before=greedy,2 -filetype=null %s 2>&1 | FileCheck %s
+# RUN: not llc -mtriple=amdgpu9.08-amd-amdhsa -verify-machineinstrs -start-before=greedy,3 -filetype=null %s 2>&1 | FileCheck %s
# This testcase fails register allocation at the same time it performs
# virtual register splitting (by introducing VGPR to AGPR copies). We
diff --git a/llvm/test/CodeGen/AMDGPU/remat-vop.mir b/llvm/test/CodeGen/AMDGPU/remat-vop.mir
index 418b6e68a24e9..7720ee85ef6c9 100644
--- a/llvm/test/CodeGen/AMDGPU/remat-vop.mir
+++ b/llvm/test/CodeGen/AMDGPU/remat-vop.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu9.0a-amd-amdhsa -verify-regalloc --stress-regalloc=2 -start-before=greedy,0 -stop-after=virtregrewriter,1 -o - %s | FileCheck -check-prefix=GCN %s
+# RUN: llc -mtriple=amdgpu9.0a-amd-amdhsa -verify-regalloc --stress-regalloc=2 -start-before=greedy,1 -stop-after=virt-reg-rewriter,2 -o - %s | FileCheck -check-prefix=GCN %s
---
name: test_remat_v_mov_b32_e32
diff --git a/llvm/test/CodeGen/AMDGPU/schedule-regpressure-ilp-metric-spills.mir b/llvm/test/CodeGen/AMDGPU/schedule-regpressure-ilp-metric-spills.mir
index fec3c1b3a6e1d..be824db78af78 100644
--- a/llvm/test/CodeGen/AMDGPU/schedule-regpressure-ilp-metric-spills.mir
+++ b/llvm/test/CodeGen/AMDGPU/schedule-regpressure-ilp-metric-spills.mir
@@ -1,5 +1,5 @@
-# RUN: llc -mtriple=amdgpu11.00 -start-before=machine-scheduler -stop-after=greedy,2 -verify-machineinstrs -o - %s | FileCheck -check-prefix=GCN %s
-# RUN: llc -mtriple=amdgpu11.00 -start-before=machine-scheduler -stop-after=greedy,2 -amdgpu-use-amdgpu-trackers=1 -verify-machineinstrs -o - %s | FileCheck -check-prefix=GCN-GCNTRACKER %s
+# RUN: llc -mtriple=amdgpu11.00 -start-before=machine-scheduler -stop-after=greedy,3 -verify-machineinstrs -o - %s | FileCheck -check-prefix=GCN %s
+# RUN: llc -mtriple=amdgpu11.00 -start-before=machine-scheduler -stop-after=greedy,3 -amdgpu-use-amdgpu-trackers=1 -verify-machineinstrs -o - %s | FileCheck -check-prefix=GCN-GCNTRACKER %s
--- |
define amdgpu_kernel void @no_sched_metric_due_to_spills() #0 {
diff --git a/llvm/test/CodeGen/AMDGPU/schedule-regpressure-no-unclustered-regions.mir b/llvm/test/CodeGen/AMDGPU/schedule-regpressure-no-unclustered-regions.mir
index 8b70b5c320eb2..7e7615f052cc1 100644
--- a/llvm/test/CodeGen/AMDGPU/schedule-regpressure-no-unclustered-regions.mir
+++ b/llvm/test/CodeGen/AMDGPU/schedule-regpressure-no-unclustered-regions.mir
@@ -1,5 +1,5 @@
# REQUIRES: asserts
-# RUN: llc -mtriple=amdgpu11.00 -start-before=machine-scheduler -stop-after=greedy,2 -stress-regalloc=4 -debug-only=machine-scheduler %s -o - 2>&1 | FileCheck %s
+# RUN: llc -mtriple=amdgpu11.00 -start-before=machine-scheduler -stop-after=greedy,3 -stress-regalloc=4 -debug-only=machine-scheduler %s -o - 2>&1 | FileCheck %s
--- |
define amdgpu_kernel void @no_sched_metric_due_to_spills() #0 {
diff --git a/llvm/test/CodeGen/AMDGPU/sgpr-regalloc-flags.ll b/llvm/test/CodeGen/AMDGPU/sgpr-regalloc-flags.ll
index a41a31e539ee7..966b4d4ac6b61 100644
--- a/llvm/test/CodeGen/AMDGPU/sgpr-regalloc-flags.ll
+++ b/llvm/test/CodeGen/AMDGPU/sgpr-regalloc-flags.ll
@@ -1,16 +1,16 @@
; REQUIRES: asserts
-; RUN: llc -verify-machineinstrs=0 -mtriple=amdgpu7.00-amd-amdhsa -debug-pass=Structure -filetype=null %s 2>&1 | FileCheck -check-prefix=DEFAULT %s
-; RUN: llc -verify-machineinstrs=0 -sgpr-regalloc=greedy -wwm-regalloc=greedy -vgpr-regalloc=greedy -mtriple=amdgpu7.00-amd-amdhsa -debug-pass=Structure -filetype=null %s 2>&1 | FileCheck -check-prefix=DEFAULT %s
+; RUN: llc -enable-new-pm=0 -verify-machineinstrs=0 -mtriple=amdgpu7.00-amd-amdhsa -debug-pass=Structure -filetype=null %s 2>&1 | FileCheck -check-prefix=DEFAULT %s
+; RUN: llc -enable-new-pm=0 -verify-machineinstrs=0 -sgpr-regalloc=greedy -wwm-regalloc=greedy -vgpr-regalloc=greedy -mtriple=amdgpu7.00-amd-amdhsa -debug-pass=Structure -filetype=null %s 2>&1 | FileCheck -check-prefix=DEFAULT %s
-; RUN: llc -verify-machineinstrs=0 -O0 -mtriple=amdgpu7.00-amd-amdhsa -debug-pass=Structure -filetype=null %s 2>&1 | FileCheck -check-prefix=O0 %s
+; RUN: llc -enable-new-pm=0 -verify-machineinstrs=0 -O0 -mtriple=amdgpu7.00-amd-amdhsa -debug-pass=Structure -filetype=null %s 2>&1 | FileCheck -check-prefix=O0 %s
-; RUN: llc -verify-machineinstrs=0 -wwm-regalloc=basic -vgpr-regalloc=basic -mtriple=amdgpu7.00-amd-amdhsa -debug-pass=Structure -filetype=null %s 2>&1 | FileCheck -check-prefix=DEFAULT-BASIC %s
-; RUN: llc -verify-machineinstrs=0 -sgpr-regalloc=basic -mtriple=amdgpu7.00-amd-amdhsa -debug-pass=Structure -filetype=null %s 2>&1 | FileCheck -check-prefix=BASIC-DEFAULT %s
-; RUN: llc -verify-machineinstrs=0 -sgpr-regalloc=basic -wwm-regalloc=basic -vgpr-regalloc=basic -mtriple=amdgpu7.00-amd-amdhsa -debug-pass=Structure -filetype=null %s 2>&1 | FileCheck -check-prefix=BASIC-BASIC %s
+; RUN: llc -enable-new-pm=0 -verify-machineinstrs=0 -wwm-regalloc=basic -vgpr-regalloc=basic -mtriple=amdgpu7.00-amd-amdhsa -debug-pass=Structure -filetype=null %s 2>&1 | FileCheck -check-prefix=DEFAULT-BASIC %s
+; RUN: llc -enable-new-pm=0 -verify-machineinstrs=0 -sgpr-regalloc=basic -mtriple=amdgpu7.00-amd-amdhsa -debug-pass=Structure -filetype=null %s 2>&1 | FileCheck -check-prefix=BASIC-DEFAULT %s
+; RUN: llc -enable-new-pm=0 -verify-machineinstrs=0 -sgpr-regalloc=basic -wwm-regalloc=basic -vgpr-regalloc=basic -mtriple=amdgpu7.00-amd-amdhsa -debug-pass=Structure -filetype=null %s 2>&1 | FileCheck -check-prefix=BASIC-BASIC %s
-; RUN: not llc -verify-machineinstrs=0 -regalloc=basic -mtriple=amdgpu7.00-amd-amdhsa -debug-pass=Structure -filetype=null %s 2>&1 | FileCheck -check-prefix=REGALLOC %s
-; RUN: not llc -verify-machineinstrs=0 -regalloc=fast -O0 -mtriple=amdgpu7.00-amd-amdhsa -debug-pass=Structure -filetype=null %s 2>&1 | FileCheck -check-prefix=REGALLOC %s
+; RUN: not llc -enable-new-pm=0 -verify-machineinstrs=0 -regalloc=basic -mtriple=amdgpu7.00-amd-amdhsa -debug-pass=Structure -filetype=null %s 2>&1 | FileCheck -check-prefix=REGALLOC %s
+; RUN: not llc -enable-new-pm=0 -verify-machineinstrs=0 -regalloc=fast -O0 -mtriple=amdgpu7.00-amd-amdhsa -debug-pass=Structure -filetype=null %s 2>&1 | FileCheck -check-prefix=REGALLOC %s
; REGALLOC: -regalloc not supported with amdgcn. Use -sgpr-regalloc, -wwm-regalloc, and -vgpr-regalloc
diff --git a/llvm/test/CodeGen/AMDGPU/sgpr-spill-update-only-slot-indexes.ll b/llvm/test/CodeGen/AMDGPU/sgpr-spill-update-only-slot-indexes.ll
index 7d1ba484442ff..e4d14180dfe82 100644
--- a/llvm/test/CodeGen/AMDGPU/sgpr-spill-update-only-slot-indexes.ll
+++ b/llvm/test/CodeGen/AMDGPU/sgpr-spill-update-only-slot-indexes.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=amdgpu9.00 -sgpr-regalloc=fast -vgpr-regalloc=fast -wwm-regalloc=fast < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -mtriple=amdgpu9.00 -sgpr-regalloc-npm=fast -vgpr-regalloc-npm=fast -wwm-regalloc-npm=fast < %s | FileCheck -check-prefix=GCN %s
; Make sure there's no verifier error from improperly updated
; SlotIndexes if regalloc fast is manually used.
diff --git a/llvm/test/CodeGen/AMDGPU/si-lower-sgpr-spills-cycle-header.mir b/llvm/test/CodeGen/AMDGPU/si-lower-sgpr-spills-cycle-header.mir
index 469a10d47fc1e..20dfa7537f4df 100644
--- a/llvm/test/CodeGen/AMDGPU/si-lower-sgpr-spills-cycle-header.mir
+++ b/llvm/test/CodeGen/AMDGPU/si-lower-sgpr-spills-cycle-header.mir
@@ -1,7 +1,7 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
# RUN: llc -mtriple=amdgpu7.00-amd-amdhsa -verify-machineinstrs -run-pass=si-lower-sgpr-spills -o - %s | FileCheck -check-prefix=SGPR-SPILL %s
-# RUN: llc -mtriple=amdgpu7.00-amd-amdhsa -O0 -start-before=si-lower-sgpr-spills -stop-after=regallocfast,1 -verify-machineinstrs %s -o - | FileCheck -check-prefix=WWM-REGALLOC %s
+# RUN: llc -mtriple=amdgpu7.00-amd-amdhsa -O0 -start-before=si-lower-sgpr-spills -stop-after=regallocfast,2 -verify-machineinstrs %s -o - | FileCheck -check-prefix=WWM-REGALLOC %s
# When SGPR spills to a virtual VGPR lane occur in both a loop header and the latch,
diff --git a/llvm/test/CodeGen/AMDGPU/si-lower-sgpr-spills-multi-entry-cycle.mir b/llvm/test/CodeGen/AMDGPU/si-lower-sgpr-spills-multi-entry-cycle.mir
index 63939ff5d69ae..e544210ed5815 100644
--- a/llvm/test/CodeGen/AMDGPU/si-lower-sgpr-spills-multi-entry-cycle.mir
+++ b/llvm/test/CodeGen/AMDGPU/si-lower-sgpr-spills-multi-entry-cycle.mir
@@ -1,6 +1,6 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
# RUN: llc -mtriple=amdgpu7.00-amd-amdhsa -verify-machineinstrs -run-pass=si-lower-sgpr-spills -o - %s | FileCheck -check-prefix=SGPR-SPILL %s
-# RUN: llc -mtriple=amdgpu7.00-amd-amdhsa -O0 -start-before=si-lower-sgpr-spills -stop-after=regallocfast,1 -verify-machineinstrs %s -o - | FileCheck -check-prefix=WWM-REGALLOC %s
+# RUN: llc -mtriple=amdgpu7.00-amd-amdhsa -O0 -start-before=si-lower-sgpr-spills -stop-after=regallocfast,2 -verify-machineinstrs %s -o - | FileCheck -check-prefix=WWM-REGALLOC %s
# Ensure that for a multi-entry cycle si-lower-sgpr-spills inserts
# IMPLICIT_DEF into the NCD of the cycle's entries.
diff --git a/llvm/test/CodeGen/AMDGPU/si-lower-sgpr-spills-vgpr-lanes-usage.mir b/llvm/test/CodeGen/AMDGPU/si-lower-sgpr-spills-vgpr-lanes-usage.mir
index 570469d0955ce..d156bbc8a2919 100644
--- a/llvm/test/CodeGen/AMDGPU/si-lower-sgpr-spills-vgpr-lanes-usage.mir
+++ b/llvm/test/CodeGen/AMDGPU/si-lower-sgpr-spills-vgpr-lanes-usage.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
-# RUN: llc -mtriple=amdgpu9.08-amd-amdhsa -verify-machineinstrs -stress-regalloc=3 -start-before=greedy,0 -stop-after=si-lower-sgpr-spills -o - %s | FileCheck -check-prefix=SGPR_SPILLED %s
+# RUN: llc -mtriple=amdgpu9.08-amd-amdhsa -verify-machineinstrs -stress-regalloc=3 -start-before=greedy,1 -stop-after=si-lower-sgpr-spills -o - %s | FileCheck -check-prefix=SGPR_SPILLED %s
# INFO : The test starts from the sgpr-regalloc pipeline.
# INFO : Now, StackSlotColoring pass comes just after sgpr-regalloc pipeline.
diff --git a/llvm/test/CodeGen/AMDGPU/snippet-copy-bundle-regression.mir b/llvm/test/CodeGen/AMDGPU/snippet-copy-bundle-regression.mir
index 3c99fab308ad0..50789cffb4cfa 100644
--- a/llvm/test/CodeGen/AMDGPU/snippet-copy-bundle-regression.mir
+++ b/llvm/test/CodeGen/AMDGPU/snippet-copy-bundle-regression.mir
@@ -1,6 +1,6 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 2
# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa \
-# RUN: -start-before=greedy,0 -stop-after=virtregrewriter,1 -verify-regalloc -o - %s | FileCheck %s
+# RUN: -start-before=greedy,1 -stop-after=virt-reg-rewriter,2 -verify-regalloc -o - %s | FileCheck %s
# This hit an assert when identifying snippet copy bundles from
# running off the end of the block due to using
diff --git a/llvm/test/CodeGen/AMDGPU/spill-empty-live-interval.mir b/llvm/test/CodeGen/AMDGPU/spill-empty-live-interval.mir
index 75b68900a10a7..2b7f1af29e41e 100644
--- a/llvm/test/CodeGen/AMDGPU/spill-empty-live-interval.mir
+++ b/llvm/test/CodeGen/AMDGPU/spill-empty-live-interval.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu7.00-amd-amdhsa -verify-machineinstrs -amdgpu-enable-rewrite-partial-reg-uses=false -amdgpu-dce-in-ra=0 -stress-regalloc=1 -start-before=register-coalescer -stop-after=greedy,1 -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu7.00-amd-amdhsa -verify-machineinstrs -amdgpu-enable-rewrite-partial-reg-uses=false -amdgpu-dce-in-ra=0 -stress-regalloc=1 -start-before=register-coalescer -stop-after=greedy,2 -o - %s | FileCheck %s
# https://bugs.llvm.org/show_bug.cgi?id=33620
---
diff --git a/llvm/test/CodeGen/AMDGPU/spill-scavenge-offset.ll b/llvm/test/CodeGen/AMDGPU/spill-scavenge-offset.ll
index 9c119f6ad326c..f748474bbe0d7 100644
--- a/llvm/test/CodeGen/AMDGPU/spill-scavenge-offset.ll
+++ b/llvm/test/CodeGen/AMDGPU/spill-scavenge-offset.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=amdgpu6.01 -enable-misched=0 -post-RA-scheduler=0 -amdgpu-spill-sgpr-to-vgpr=0 < %s | FileCheck -check-prefixes=CHECK,GFX6 %s
-; RUN: llc -sgpr-regalloc=basic -vgpr-regalloc=basic -mtriple=amdgpu8.02 -enable-misched=0 -post-RA-scheduler=0 -amdgpu-spill-sgpr-to-vgpr=0 < %s | FileCheck --check-prefix=CHECK %s
+; RUN: llc -enable-new-pm=0 -sgpr-regalloc=basic -vgpr-regalloc=basic -mtriple=amdgpu8.02 -enable-misched=0 -post-RA-scheduler=0 -amdgpu-spill-sgpr-to-vgpr=0 < %s | FileCheck --check-prefix=CHECK %s
; RUN: llc -mtriple=amdgpu9.00 -mattr=-xnack,+enable-flat-scratch -enable-misched=0 -post-RA-scheduler=0 -amdgpu-spill-sgpr-to-vgpr=0 < %s | FileCheck -check-prefixes=CHECK,GFX9-FLATSCR,FLATSCR %s
; RUN: llc -mtriple=amdgpu10.30 -enable-misched=0 -post-RA-scheduler=0 -amdgpu-spill-sgpr-to-vgpr=0 -mattr=+enable-flat-scratch < %s | FileCheck -check-prefixes=CHECK,GFX10-FLATSCR,FLATSCR %s
;
diff --git a/llvm/test/CodeGen/AMDGPU/spill-vector-superclass.ll b/llvm/test/CodeGen/AMDGPU/spill-vector-superclass.ll
index 29c5092609a82..ae3bbd6e3eded 100644
--- a/llvm/test/CodeGen/AMDGPU/spill-vector-superclass.ll
+++ b/llvm/test/CodeGen/AMDGPU/spill-vector-superclass.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -mtriple=amdgpu9.08-amd-amdhsa -stop-after=greedy,1 -o - %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -mtriple=amdgpu9.08-amd-amdhsa -stop-after=greedy,2 -o - %s | FileCheck -check-prefix=GCN %s
; Convert AV spills into VGPR spills by introducing appropriate copies in between.
define amdgpu_kernel void @test_spill_av_class(<4 x i32> %arg) #0 {
diff --git a/llvm/test/CodeGen/AMDGPU/splitkit-do-not-undo-subclass-split-with-remat.mir b/llvm/test/CodeGen/AMDGPU/splitkit-do-not-undo-subclass-split-with-remat.mir
index 70d6975dad8f3..c01688ae02c52 100644
--- a/llvm/test/CodeGen/AMDGPU/splitkit-do-not-undo-subclass-split-with-remat.mir
+++ b/llvm/test/CodeGen/AMDGPU/splitkit-do-not-undo-subclass-split-with-remat.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
-# RUN: llc -mtriple=amdgpu9.0a-amd-amdhsa -verify-regalloc -start-before=greedy,2 -stop-before=virtregrewriter,2 -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.0a-amd-amdhsa -verify-regalloc -start-before=greedy,3 -stop-before=virt-reg-rewriter,3 -o - %s | FileCheck %s
# FIXME: Assert if run to end
# The V_MFMA_F32_32X32X1F32_vgprcd_e64 as written requires 66 VGPRs
diff --git a/llvm/test/CodeGen/AMDGPU/swdev502267-use-after-free-last-chance-recoloring-alloc-succeeds.mir b/llvm/test/CodeGen/AMDGPU/swdev502267-use-after-free-last-chance-recoloring-alloc-succeeds.mir
index 2ef6908d3701d..57b65681557fa 100644
--- a/llvm/test/CodeGen/AMDGPU/swdev502267-use-after-free-last-chance-recoloring-alloc-succeeds.mir
+++ b/llvm/test/CodeGen/AMDGPU/swdev502267-use-after-free-last-chance-recoloring-alloc-succeeds.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
-# RUN: llc -mtriple=amdgpu9.42-amd-amdhsa -stress-regalloc=4 -verify-regalloc -start-before=greedy,2 -stop-after=virtregrewriter,2 -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.42-amd-amdhsa -stress-regalloc=4 -verify-regalloc -start-before=greedy,3 -stop-after=virt-reg-rewriter,3 -o - %s | FileCheck %s
# This testcase hit a situation where greedy would hit a use after
# free during last chance recoloring. This case successfully allocates
diff --git a/llvm/test/CodeGen/AMDGPU/true16-ra-pre-gfx11-regression-test.mir b/llvm/test/CodeGen/AMDGPU/true16-ra-pre-gfx11-regression-test.mir
index 90cbbaef3ab2e..d97415df573e3 100644
--- a/llvm/test/CodeGen/AMDGPU/true16-ra-pre-gfx11-regression-test.mir
+++ b/llvm/test/CodeGen/AMDGPU/true16-ra-pre-gfx11-regression-test.mir
@@ -1,4 +1,4 @@
-# RUN: llc -mtriple=amdgpu10.10 -start-before=greedy,0 -stop-after=virtregrewriter,2 -verify-machineinstrs -o - %s | FileCheck --check-prefixes=GCN %s
+# RUN: llc -mtriple=amdgpu10.10 -start-before=greedy,1 -stop-after=virt-reg-rewriter,3 -verify-machineinstrs -o - %s | FileCheck --check-prefixes=GCN %s
--- |
define amdgpu_ps void @e32() #0 {
diff --git a/llvm/test/CodeGen/AMDGPU/vgpr-agpr-limit-gfx90a.ll b/llvm/test/CodeGen/AMDGPU/vgpr-agpr-limit-gfx90a.ll
index 12ffc853188e7..6138ba6df8771 100644
--- a/llvm/test/CodeGen/AMDGPU/vgpr-agpr-limit-gfx90a.ll
+++ b/llvm/test/CodeGen/AMDGPU/vgpr-agpr-limit-gfx90a.ll
@@ -1,6 +1,6 @@
; -enable-misched=false makes the register usage more predictable
; -regalloc=fast just makes the test run faster
-; RUN: llc -mtriple=amdgpu9.0a -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc=fast -vgpr-regalloc=fast -wwm-regalloc=fast < %s | FileCheck %s --check-prefixes=GCN,GFX90A
+; RUN: llc -mtriple=amdgpu9.0a -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc-npm=fast -vgpr-regalloc-npm=fast -wwm-regalloc-npm=fast < %s | FileCheck %s --check-prefixes=GCN,GFX90A
define internal void @use256vgprs() {
%v0 = call i32 asm sideeffect "; def $0", "=v"()
diff --git a/llvm/test/CodeGen/AMDGPU/vgpr-limit-gfx1250.ll b/llvm/test/CodeGen/AMDGPU/vgpr-limit-gfx1250.ll
index 8c690f6579d89..416e89554ee89 100644
--- a/llvm/test/CodeGen/AMDGPU/vgpr-limit-gfx1250.ll
+++ b/llvm/test/CodeGen/AMDGPU/vgpr-limit-gfx1250.ll
@@ -1,6 +1,6 @@
; -enable-misched=false makes the register usage more predictable
; -regalloc=fast just makes the test run faster
-; RUN: llc -mtriple=amdgpu12.50 -O0 -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc=fast -vgpr-regalloc=fast -wwm-regalloc=fast < %s | FileCheck %s --check-prefixes=GCN,GFX1250
+; RUN: llc -mtriple=amdgpu12.50 -O0 -amdgpu-function-calls=false -enable-misched=false -sgpr-regalloc-npm=fast -vgpr-regalloc-npm=fast -wwm-regalloc-npm=fast < %s | FileCheck %s --check-prefixes=GCN,GFX1250
define internal void @use256vgprs_asm() {
%v0 = call i32 asm sideeffect "; def $0", "=v"()
diff --git a/llvm/test/CodeGen/AMDGPU/virtregrewrite-undef-identity-copy.mir b/llvm/test/CodeGen/AMDGPU/virtregrewrite-undef-identity-copy.mir
index 6d57b9dae6e32..ace8bd0e2a554 100644
--- a/llvm/test/CodeGen/AMDGPU/virtregrewrite-undef-identity-copy.mir
+++ b/llvm/test/CodeGen/AMDGPU/virtregrewrite-undef-identity-copy.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu7.00-amd-amdhsa -start-before=greedy,0 -stop-after=virtregrewriter,1 -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu7.00-amd-amdhsa -start-before=greedy,1 -stop-after=virt-reg-rewriter,2 -verify-machineinstrs -o - %s | FileCheck %s
# The undef copy of %4 is allocated to $vgpr3, and the identity copy
# was deleted, and $vgpr3 was considered undef. The code to replace
diff --git a/llvm/test/CodeGen/AMDGPU/wave32.ll b/llvm/test/CodeGen/AMDGPU/wave32.ll
index ba191cd41be04..0a18c6c6c9ad0 100644
--- a/llvm/test/CodeGen/AMDGPU/wave32.ll
+++ b/llvm/test/CodeGen/AMDGPU/wave32.ll
@@ -2310,11 +2310,12 @@ define amdgpu_kernel void @test_branch_true() #2 {
; GFX1032-NEXT: s_cbranch_execnz .LBB45_2
; GFX1032-NEXT: ; %bb.1: ; %for.body.lr.ph
; GFX1032-NEXT: s_branch .LBB45_3
-; GFX1032-NEXT: .LBB45_2: ; %Flow
+; GFX1032-NEXT: .LBB45_2: ; %Flow1
; GFX1032-NEXT: s_branch .LBB45_5
; GFX1032-NEXT: .LBB45_3: ; %for.body
-; GFX1032-NEXT: s_mov_b32 vcc_lo, 0
-; GFX1032-NEXT: ; %bb.4: ; %for.end.loopexit
+; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_cbranch_scc1 .LBB45_3
+; GFX1032-NEXT: ; %bb.4: ; %Flow
; GFX1032-NEXT: s_branch .LBB45_2
; GFX1032-NEXT: .LBB45_5: ; %for.end
; GFX1032-NEXT: s_endpgm
@@ -2325,11 +2326,12 @@ define amdgpu_kernel void @test_branch_true() #2 {
; GFX1064-NEXT: s_cbranch_execnz .LBB45_2
; GFX1064-NEXT: ; %bb.1: ; %for.body.lr.ph
; GFX1064-NEXT: s_branch .LBB45_3
-; GFX1064-NEXT: .LBB45_2: ; %Flow
+; GFX1064-NEXT: .LBB45_2: ; %Flow1
; GFX1064-NEXT: s_branch .LBB45_5
; GFX1064-NEXT: .LBB45_3: ; %for.body
-; GFX1064-NEXT: s_mov_b64 vcc, 0
-; GFX1064-NEXT: ; %bb.4: ; %for.end.loopexit
+; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_cbranch_scc1 .LBB45_3
+; GFX1064-NEXT: ; %bb.4: ; %Flow
; GFX1064-NEXT: s_branch .LBB45_2
; GFX1064-NEXT: .LBB45_5: ; %for.end
; GFX1064-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/wwm-spill-superclass-pseudo.mir b/llvm/test/CodeGen/AMDGPU/wwm-spill-superclass-pseudo.mir
index ce8aebe7458ed..3c445919d7a26 100644
--- a/llvm/test/CodeGen/AMDGPU/wwm-spill-superclass-pseudo.mir
+++ b/llvm/test/CodeGen/AMDGPU/wwm-spill-superclass-pseudo.mir
@@ -1,5 +1,5 @@
-# RUN: llc -mtriple=amdgpu9.08-amd-amdhsa -start-before=si-lower-sgpr-spills -stop-after=greedy,1 -verify-machineinstrs --stress-regalloc=2 -o - %s | FileCheck -check-prefix GCN-REGALLOC %s
-# RUN: llc -mtriple=amdgpu9.08-amd-amdhsa -start-before=si-lower-sgpr-spills -stop-after=virtregrewriter,1 -verify-machineinstrs --stress-regalloc=2 -o - %s | FileCheck -check-prefix GCN-REWRITER %s
+# RUN: llc -mtriple=amdgpu9.08-amd-amdhsa -start-before=si-lower-sgpr-spills -stop-after=greedy,2 -verify-machineinstrs --stress-regalloc=2 -o - %s | FileCheck -check-prefix GCN-REGALLOC %s
+# RUN: llc -mtriple=amdgpu9.08-amd-amdhsa -start-before=si-lower-sgpr-spills -stop-after=virt-reg-rewriter,2 -verify-machineinstrs --stress-regalloc=2 -o - %s | FileCheck -check-prefix GCN-REWRITER %s
name: test_wwm_reg_superclass_spill
tracksRegLiveness: true
diff --git a/llvm/test/DebugInfo/AMDGPU/debug-loc-copy.ll b/llvm/test/DebugInfo/AMDGPU/debug-loc-copy.ll
index f81a57f12173b..d35b727edb3ef 100644
--- a/llvm/test/DebugInfo/AMDGPU/debug-loc-copy.ll
+++ b/llvm/test/DebugInfo/AMDGPU/debug-loc-copy.ll
@@ -24,7 +24,6 @@ define void @_Z12lane_pc_testj() #0 !dbg !9 {
; GCN-NEXT: .cfi_undefined 39
; GCN-NEXT: .cfi_undefined 40
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: ; %bb.1: ; %lab
; GCN-NEXT: .Ltmp0:
; GCN-NEXT: .loc 0 12 1 prologue_end ; t.cpp:12:1
; GCN-NEXT: s_mov_b64 s[4:5], 0
@@ -34,13 +33,14 @@ define void @_Z12lane_pc_testj() #0 !dbg !9 {
; GCN-NEXT: s_cmp_lg_u32 s8, s6
; GCN-NEXT: s_cselect_b32 s5, s7, s5
; GCN-NEXT: s_cselect_b32 s4, s8, s4
-; GCN-NEXT: v_mov_b32_e32 v2, 0
+; GCN-NEXT: ; %bb.1: ; %lab
; GCN-NEXT: .loc 0 13 1 ; t.cpp:13:1
+; GCN-NEXT: v_mov_b32_e32 v2, 0
; GCN-NEXT: v_mov_b32_e32 v0, s4
; GCN-NEXT: v_mov_b32_e32 v1, s5
; GCN-NEXT: flat_store_dword v[0:1], v2
-; GCN-NEXT: v_mov_b32_e32 v2, 1
; GCN-NEXT: .loc 0 14 1 ; t.cpp:14:1
+; GCN-NEXT: v_mov_b32_e32 v2, 1
; GCN-NEXT: v_mov_b32_e32 v0, s4
; GCN-NEXT: v_mov_b32_e32 v1, s5
; GCN-NEXT: flat_store_dword v[0:1], v2
diff --git a/llvm/tools/llc/NewPMDriver.cpp b/llvm/tools/llc/NewPMDriver.cpp
index 97a7a8a2a6ba8..dec23a5438188 100644
--- a/llvm/tools/llc/NewPMDriver.cpp
+++ b/llvm/tools/llc/NewPMDriver.cpp
@@ -87,18 +87,19 @@ bool LLCDiagnosticHandler::handleDiagnostics(const DiagnosticInfo &DI) {
static llvm::ExitOnError ExitOnErr;
-int llvm::compileModuleWithNewPM(
- StringRef Arg0, std::unique_ptr<Module> M, std::unique_ptr<MIRParser> MIR,
- std::unique_ptr<TargetMachine> Target, std::unique_ptr<ToolOutputFile> Out,
- std::unique_ptr<ToolOutputFile> DwoOut, LLVMContext &Context,
+LLCExitState llvm::compileModuleWithNewPM(
+ StringRef Arg0, std::unique_ptr<Module> &M, std::unique_ptr<MIRParser> &MIR,
+ std::unique_ptr<TargetMachine> &Target,
+ std::unique_ptr<ToolOutputFile> &Out,
+ std::unique_ptr<ToolOutputFile> &DwoOut, LLVMContext &Context,
const TargetLibraryInfoImpl &TLII, VerifierKind VK, StringRef PassPipeline,
- CodeGenFileType FileType) {
+ CodeGenFileType FileType, bool RetryWithLegacyPM) {
if (!PassPipeline.empty() && TargetPassConfig::hasLimitedCodeGenPipeline()) {
WithColor::error(errs(), Arg0)
<< "--passes cannot be used with "
<< TargetPassConfig::getLimitedCodeGenPipelineReason() << ".\n";
- return 1;
+ return LLC_Error;
}
raw_pwrite_stream *OS = &Out->os();
@@ -159,7 +160,7 @@ int llvm::compileModuleWithNewPM(
if (!MIR) {
WithColor::error(errs(), Arg0) << "-passes is for .mir file only.\n";
- return 1;
+ return LLC_Error;
}
// FIXME: verify that there are no IR passes.
@@ -173,9 +174,14 @@ int llvm::compileModuleWithNewPM(
MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
} else {
- ExitOnErr(Target->buildCodeGenPipeline(
+ Error Err = Target->buildCodeGenPipeline(
MPM, MAM, *OS, DwoOut ? &DwoOut->os() : nullptr, FileType, Opt,
- MMI.getContext(), &PIC));
+ MMI.getContext(), &PIC);
+ if (Err && RetryWithLegacyPM) {
+ consumeError(std::move(Err));
+ return LLC_Retry;
+ }
+ ExitOnErr(std::move(Err));
}
// If user only wants to print the pipeline, print it before parsing the MIR.
@@ -188,11 +194,11 @@ int llvm::compileModuleWithNewPM(
});
printFormattedPipelinePasses(outs(), PipelineStr, *PrintPipelinePasses);
outs() << '\n';
- return 0;
+ return LLC_Success;
}
if (MIR && MIR->parseMachineFunctions(*M, MAM))
- return 1;
+ return LLC_Error;
// Before executing passes, print the final values of the LLVM options.
cl::PrintOptionValues();
@@ -200,12 +206,12 @@ int llvm::compileModuleWithNewPM(
MPM.run(*M, MAM);
if (Context.getDiagHandlerPtr()->HasErrors)
- return 1;
+ return LLC_Error;
// Declare success.
Out->keep();
if (DwoOut)
DwoOut->keep();
- return 0;
+ return LLC_Success;
}
diff --git a/llvm/tools/llc/NewPMDriver.h b/llvm/tools/llc/NewPMDriver.h
index 0dbd46797dabc..f8a332f96871f 100644
--- a/llvm/tools/llc/NewPMDriver.h
+++ b/llvm/tools/llc/NewPMDriver.h
@@ -37,14 +37,19 @@ struct LLCDiagnosticHandler : public DiagnosticHandler {
bool handleDiagnostics(const DiagnosticInfo &DI) override;
};
-int compileModuleWithNewPM(StringRef Arg0, std::unique_ptr<Module> M,
- std::unique_ptr<MIRParser> MIR,
- std::unique_ptr<TargetMachine> Target,
- std::unique_ptr<ToolOutputFile> Out,
- std::unique_ptr<ToolOutputFile> DwoOut,
- LLVMContext &Context,
- const TargetLibraryInfoImpl &TLII, VerifierKind VK,
- StringRef PassPipeline, CodeGenFileType FileType);
+enum LLCExitState {
+ LLC_Success = 0,
+ LLC_Error = 1,
+ LLC_Retry = 2,
+};
+
+LLCExitState compileModuleWithNewPM(
+ StringRef Arg0, std::unique_ptr<Module> &M, std::unique_ptr<MIRParser> &MIR,
+ std::unique_ptr<TargetMachine> &Target,
+ std::unique_ptr<ToolOutputFile> &Out,
+ std::unique_ptr<ToolOutputFile> &DwoOut, LLVMContext &Context,
+ const TargetLibraryInfoImpl &TLII, VerifierKind VK, StringRef PassPipeline,
+ CodeGenFileType FileType, bool RetryWithLegacyPM);
} // namespace llvm
#endif
diff --git a/llvm/tools/llc/llc.cpp b/llvm/tools/llc/llc.cpp
index dcd54c18401d0..054223739f747 100644
--- a/llvm/tools/llc/llc.cpp
+++ b/llvm/tools/llc/llc.cpp
@@ -495,6 +495,8 @@ static bool addPass(PassManagerBase &PM, const char *argv0, StringRef PassName,
return false;
}
+bool targetSupportsNPMBackend(const Triple &T) { return T.isAMDGCN(); }
+
static int compileModule(char **argv, SmallVectorImpl<PassPlugin> &PluginList,
LLVMContext &Context, std::string &OutputFilename) {
// Load the module to be compiled...
@@ -749,11 +751,31 @@ static int compileModule(char **argv, SmallVectorImpl<PassPlugin> &PluginList,
else if (VerifyEach)
VK = VerifierKind::EachPass;
- if (EnableNewPassManager || !PassPipeline.empty()) {
- return compileModuleWithNewPM(argv[0], std::move(M), std::move(MIR),
- std::move(Target), std::move(Out),
- std::move(DwoOut), Context, TLII, VK,
- PassPipeline, codegen::getFileType());
+ bool UseNPM = false;
+ bool RetryWithLegacyPM = false;
+ if (EnableNewPassManager.getNumOccurrences()) {
+ UseNPM = EnableNewPassManager;
+ } else if (!PassPipeline.empty()) {
+ UseNPM = true;
+ } else if (RunPass.getNumOccurrences())
+ UseNPM = false;
+ else if (targetSupportsNPMBackend(TheTriple)) {
+ RetryWithLegacyPM = true;
+ UseNPM = true;
+ }
+
+ if (UseNPM) {
+ auto Exit = compileModuleWithNewPM(
+ argv[0], M, MIR, Target, Out, DwoOut, Context, TLII, VK, PassPipeline,
+ codegen::getFileType(), RetryWithLegacyPM);
+ // Fall back to the legacy pass manager only when the new PM explicitly
+ // requested a retry and legacy fallback is permitted for this target.
+ // Otherwise (success or a genuine error) return the new PM's result.
+ if (!RetryWithLegacyPM || Exit != LLC_Retry)
+ return static_cast<int>(Exit);
+
+ WithColor::warning(errs(), argv[0])
+ << "New pass manager failed, falling back to legacy pass manager.\n";
}
// Build up all of the passes that we want to do to the module.
More information about the llvm-branch-commits
mailing list