[llvm] [MISched][RISCV] Add anti-dependency breaking to PostMachineScheduler (PR #221984)
Pengcheng Wang via llvm-commits
llvm-commits at lists.llvm.org
Wed Sep 9 02:35:16 PDT 2026
https://github.com/wangpc-pp updated https://github.com/llvm/llvm-project/pull/221984
>From 6a9b1d5c2bce67e9b06adfe10b600aba0c101a62 Mon Sep 17 00:00:00 2001
From: Pengcheng Wang <wangpengcheng.pp at bytedance.com>
Date: Wed, 9 Sep 2026 16:46:43 +0800
Subject: [PATCH 1/5] [MISched] Apply debug filters before starting a block
Apply the misched-only function and block filters before calling
startBlock. This avoids opening scheduler state for blocks that will not
be visited.
Assisted-by: TRAE CLI (GPT-5.6)
---
llvm/lib/CodeGen/MachineScheduler.cpp | 5 ++---
.../CodeGen/SystemZ/postra-sched-filter.mir | 21 +++++++++++++++++++
2 files changed, 23 insertions(+), 3 deletions(-)
create mode 100644 llvm/test/CodeGen/SystemZ/postra-sched-filter.mir
diff --git a/llvm/lib/CodeGen/MachineScheduler.cpp b/llvm/lib/CodeGen/MachineScheduler.cpp
index 960c75d8050b1..8d0130973b81b 100644
--- a/llvm/lib/CodeGen/MachineScheduler.cpp
+++ b/llvm/lib/CodeGen/MachineScheduler.cpp
@@ -815,9 +815,6 @@ void MachineSchedulerBase::scheduleRegions(ScheduleDAGInstrs &Scheduler,
// loop tree. Then we can optionally compute global RegPressure.
for (MachineFunction::iterator MBB = MF->begin(), MBBEnd = MF->end();
MBB != MBBEnd; ++MBB) {
-
- Scheduler.startBlock(&*MBB);
-
#ifndef NDEBUG
if (SchedOnlyFunc.getNumOccurrences() && SchedOnlyFunc != MF->getName())
continue;
@@ -826,6 +823,8 @@ void MachineSchedulerBase::scheduleRegions(ScheduleDAGInstrs &Scheduler,
continue;
#endif
+ Scheduler.startBlock(&*MBB);
+
// Break the block into scheduling regions [I, RegionEnd). RegionEnd
// points to the scheduling boundary at the bottom of the region. The DAG
// does not include RegionEnd, but the region does (i.e. the next
diff --git a/llvm/test/CodeGen/SystemZ/postra-sched-filter.mir b/llvm/test/CodeGen/SystemZ/postra-sched-filter.mir
new file mode 100644
index 0000000000000..01e0dadde5465
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/postra-sched-filter.mir
@@ -0,0 +1,21 @@
+# REQUIRES: asserts
+# RUN: llc -mtriple=s390x-linux-gnu -mcpu=z13 -run-pass=postmisched \
+# RUN: -misched-only-block=1 -debug-only=machine-scheduler -o /dev/null %s \
+# RUN: 2>&1 | FileCheck %s
+
+# CHECK-NOT: ** Entering %bb.0
+# CHECK: ** Entering %bb.1:
+
+---
+name: test
+tracksRegLiveness: true
+body: |
+ bb.0:
+ successors: %bb.1(0x80000000)
+
+ J %bb.1
+
+ bb.1:
+ $r2l = LHI 0
+ Return implicit killed $r2l
+...
>From d57a50d9decaa5111a83a26a11364c099c5320bf Mon Sep 17 00:00:00 2001
From: Pengcheng Wang <wangpengcheng.pp at bytedance.com>
Date: Wed, 9 Sep 2026 16:46:47 +0800
Subject: [PATCH 2/5] [CodeGen] Skip pseudo instructions in
AggressiveAntiDepBreaker
Pseudo instructions do not have SUnits in the post-RA scheduling DAG.
Skipping only debug instructions can therefore dereference a null SUnit.
Ignore pseudo instructions and add a PSEUDO_PROBE regression.
Assisted-by: TRAE CLI (GPT-5.6)
---
llvm/lib/CodeGen/AggressiveAntiDepBreaker.cpp | 2 +-
llvm/test/CodeGen/Hexagon/anti-dep-partial.mir | 4 ++++
2 files changed, 5 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/CodeGen/AggressiveAntiDepBreaker.cpp b/llvm/lib/CodeGen/AggressiveAntiDepBreaker.cpp
index 6be353bb8da63..a425864921416 100644
--- a/llvm/lib/CodeGen/AggressiveAntiDepBreaker.cpp
+++ b/llvm/lib/CodeGen/AggressiveAntiDepBreaker.cpp
@@ -790,7 +790,7 @@ unsigned AggressiveAntiDepBreaker::BreakAntiDependencies(
I != E; --Count) {
MachineInstr &MI = *--I;
- if (MI.isDebugInstr())
+ if (MI.isDebugOrPseudoInstr())
continue;
LLVM_DEBUG(dbgs() << "Anti: ");
diff --git a/llvm/test/CodeGen/Hexagon/anti-dep-partial.mir b/llvm/test/CodeGen/Hexagon/anti-dep-partial.mir
index b7a3f8051f698..b8f987818441c 100644
--- a/llvm/test/CodeGen/Hexagon/anti-dep-partial.mir
+++ b/llvm/test/CodeGen/Hexagon/anti-dep-partial.mir
@@ -1,4 +1,5 @@
# RUN: llc -mtriple=hexagon -post-RA-scheduler -run-pass post-RA-sched %s -o - | FileCheck %s
+# RUN: llc -mtriple=hexagon -post-RA-scheduler -passes=post-RA-sched %s -o - | FileCheck %s
--- |
declare void @check(i64, i32, i32, i64)
@@ -15,6 +16,8 @@ body: |
successors:
liveins: $r0, $r1, $d1, $d2, $r16, $r17, $r19, $r22, $r23
$r2 = A2_add $r23, killed $r17
+ ; PSEUDO_PROBE has no SUnit and must be ignored by the breaker.
+ PSEUDO_PROBE 1, 1, 0, 0
$r6 = M2_mpyi $r16, $r16
$r22 = M2_accii $r22, killed $r2, 2
$r7 = A2_tfrsi 12345678
@@ -27,6 +30,7 @@ body: |
; The anti-dependency on r23 between the first A2_add and the
; S2_asr_i_r was causing d11 to be renamed, while r22 remained
; unchanged. Check that the renaming of d11 does not happen.
+ ; CHECK: PSEUDO_PROBE 1, 1, 0, 0
; CHECK: d11
$d0 = A2_tfrp killed $d11
J2_call @check, implicit-def $d0, implicit-def $d1, implicit-def $d2, implicit $d0, implicit $d1, implicit $d2
>From b5e1a4d8c2a59f6b961d0b93019d70e85717bff4 Mon Sep 17 00:00:00 2001
From: Pengcheng Wang <wangpengcheng.pp at bytedance.com>
Date: Tue, 8 Sep 2026 20:30:56 +0800
Subject: [PATCH 3/5] [RISCV] Add precommit tests for post-RA anti-dependencies
Add coverage for region-local and loop-carried GPR, FPR, and vector WAR
hazards in the post-RA machine scheduler. Also cover conservative
bailout conditions, bundled instructions, pseudo probes, an in-order
processor without V, and out-of-order scheduling models.
Assisted-by: TRAE CLI (GPT-5.6)
---
.../CodeGen/RISCV/postmisched-antidep.mir | 852 ++++++++++++++++++
1 file changed, 852 insertions(+)
create mode 100644 llvm/test/CodeGen/RISCV/postmisched-antidep.mir
diff --git a/llvm/test/CodeGen/RISCV/postmisched-antidep.mir b/llvm/test/CodeGen/RISCV/postmisched-antidep.mir
new file mode 100644
index 0000000000000..18fdd840fe9e0
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/postmisched-antidep.mir
@@ -0,0 +1,852 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=riscv64 -mcpu=sifive-x280 -run-pass=postmisched \
+# RUN: -enable-post-misched -verify-machineinstrs -o - %s \
+# RUN: | FileCheck %s --check-prefix=INORDER
+# RUN: llc -mtriple=riscv64 -mcpu=sifive-x280 -passes=postmisched \
+# RUN: -enable-post-misched -verify-machineinstrs -o - %s \
+# RUN: | FileCheck %s --check-prefix=INORDER
+# RUN: llc -mtriple=riscv64 -mcpu=sifive-u74 -run-pass=postmisched \
+# RUN: -enable-post-misched -verify-machineinstrs -o - %s \
+# RUN: | FileCheck %s --check-prefix=INORDER
+# RUN: llc -mtriple=riscv64 -mcpu=sifive-p870-d -run-pass=postmisched \
+# RUN: -enable-post-misched -verify-machineinstrs -o - %s \
+# RUN: | FileCheck %s --check-prefix=OOO
+# RUN: llc -mtriple=riscv64 -mcpu=sifive-p870-d -passes=postmisched \
+# RUN: -enable-post-misched -verify-machineinstrs -o - %s \
+# RUN: | FileCheck %s --check-prefix=OOO
+
+# These functions have a WAR hazard within a single non-loop scheduling
+# region.
+
+---
+name: gpr_region_war
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $x10, $x11, $x12, $x13, $x15
+
+ ; INORDER-LABEL: name: gpr_region_war
+ ; INORDER: liveins: $x10, $x11, $x12, $x13, $x15
+ ; INORDER-NEXT: {{ $}}
+ ; INORDER-NEXT: renamable $x14 = ADD killed renamable $x15, killed renamable $x10
+ ; INORDER-NEXT: renamable $x15 = LD killed renamable $x11, 0 :: (load (s64))
+ ; INORDER-NEXT: SD killed renamable $x15, killed renamable $x12, 0 :: (store (s64))
+ ; INORDER-NEXT: SD killed renamable $x14, killed renamable $x13, 0 :: (store (s64))
+ ; INORDER-NEXT: PseudoRET
+ ;
+ ; OOO-LABEL: name: gpr_region_war
+ ; OOO: liveins: $x10, $x11, $x12, $x13, $x15
+ ; OOO-NEXT: {{ $}}
+ ; OOO-NEXT: renamable $x14 = ADD killed renamable $x15, killed renamable $x10
+ ; OOO-NEXT: renamable $x15 = LD killed renamable $x11, 0 :: (load (s64))
+ ; OOO-NEXT: SD killed renamable $x15, killed renamable $x12, 0 :: (store (s64))
+ ; OOO-NEXT: SD killed renamable $x14, killed renamable $x13, 0 :: (store (s64))
+ ; OOO-NEXT: PseudoRET
+ renamable $x14 = ADD renamable $x15, renamable $x10
+ renamable $x15 = LD renamable $x11, 0 :: (load (s64))
+ SD renamable $x15, renamable $x12, 0 :: (store (s64))
+ SD renamable $x14, renamable $x13, 0 :: (store (s64))
+ PseudoRET
+...
+
+---
+name: fpr_region_war
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $x10, $x11, $x12, $f10_d, $f11_d
+
+ ; INORDER-LABEL: name: fpr_region_war
+ ; INORDER: liveins: $x10, $x11, $x12, $f10_d, $f11_d
+ ; INORDER-NEXT: {{ $}}
+ ; INORDER-NEXT: renamable $f12_d = nofpexcept FADD_D killed renamable $f10_d, killed renamable $f11_d, 7, implicit $frm
+ ; INORDER-NEXT: renamable $f10_d = FLD killed renamable $x10, 0 :: (load (s64))
+ ; INORDER-NEXT: FSD killed renamable $f10_d, killed renamable $x11, 0 :: (store (s64))
+ ; INORDER-NEXT: FSD killed renamable $f12_d, killed renamable $x12, 0 :: (store (s64))
+ ; INORDER-NEXT: PseudoRET
+ ;
+ ; OOO-LABEL: name: fpr_region_war
+ ; OOO: liveins: $x10, $x11, $x12, $f10_d, $f11_d
+ ; OOO-NEXT: {{ $}}
+ ; OOO-NEXT: renamable $f12_d = nofpexcept FADD_D killed renamable $f10_d, killed renamable $f11_d, 7, implicit $frm
+ ; OOO-NEXT: renamable $f10_d = FLD killed renamable $x10, 0 :: (load (s64))
+ ; OOO-NEXT: FSD killed renamable $f10_d, killed renamable $x11, 0 :: (store (s64))
+ ; OOO-NEXT: FSD killed renamable $f12_d, killed renamable $x12, 0 :: (store (s64))
+ ; OOO-NEXT: PseudoRET
+ renamable $f12_d = nofpexcept FADD_D renamable $f10_d, renamable $f11_d, 7, implicit $frm
+ renamable $f10_d = FLD renamable $x10, 0 :: (load (s64))
+ FSD renamable $f10_d, renamable $x11, 0 :: (store (s64))
+ FSD renamable $f12_d, renamable $x12, 0 :: (store (s64))
+ PseudoRET
+...
+
+---
+name: rvv_region_war
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $x10, $x11, $v8
+
+ ; INORDER-LABEL: name: rvv_region_war
+ ; INORDER: liveins: $x10, $x11, $v8
+ ; INORDER-NEXT: {{ $}}
+ ; INORDER-NEXT: $x12 = PseudoVMV_X_S killed renamable $v8, 5 /* e32 */, implicit $vtype
+ ; INORDER-NEXT: renamable $v8 = VL1RE8_V killed renamable $x10 :: (load unknown-size, align 1)
+ ; INORDER-NEXT: VS1R_V killed renamable $v8, killed renamable $x11 :: (store unknown-size, align 1)
+ ; INORDER-NEXT: PseudoRET
+ ;
+ ; OOO-LABEL: name: rvv_region_war
+ ; OOO: liveins: $x10, $x11, $v8
+ ; OOO-NEXT: {{ $}}
+ ; OOO-NEXT: $x12 = PseudoVMV_X_S killed renamable $v8, 5 /* e32 */, implicit $vtype
+ ; OOO-NEXT: renamable $v8 = VL1RE8_V killed renamable $x10 :: (load unknown-size, align 1)
+ ; OOO-NEXT: VS1R_V killed renamable $v8, killed renamable $x11 :: (store unknown-size, align 1)
+ ; OOO-NEXT: PseudoRET
+ $x12 = PseudoVMV_X_S renamable $v8, 5 /* e32 */, implicit $vtype
+ renamable $v8 = VL1RE8_V renamable $x10 :: (load unknown-size, align 1)
+ VS1R_V renamable $v8, renamable $x11 :: (store unknown-size, align 1)
+ PseudoRET
+...
+
+# These single-block loops have a WAR hazard across the backedge: the first
+# instruction of the next iteration overwrites a register that is read near the
+# end of the current iteration. The checks record how postmisched handles the
+# hazard for in-order and out-of-order processors.
+
+---
+name: gpr_loop_carried_war
+tracksRegLiveness: true
+body: |
+ ; INORDER-LABEL: name: gpr_loop_carried_war
+ ; INORDER: bb.0:
+ ; INORDER-NEXT: successors: %bb.1(0x80000000)
+ ; INORDER-NEXT: {{ $}}
+ ; INORDER-NEXT: PseudoBR %bb.1
+ ; INORDER-NEXT: {{ $}}
+ ; INORDER-NEXT: bb.1:
+ ; INORDER-NEXT: successors: %bb.1(0x7c000000), %bb.2(0x04000000)
+ ; INORDER-NEXT: liveins: $x10, $x11, $x12, $x13
+ ; INORDER-NEXT: {{ $}}
+ ; INORDER-NEXT: renamable $x15 = LD renamable $x10, 0 :: (load (s64))
+ ; INORDER-NEXT: renamable $x15 = ADD killed renamable $x15, renamable $x11
+ ; INORDER-NEXT: SD killed renamable $x15, renamable $x12, 0 :: (store (s64))
+ ; INORDER-NEXT: BNE renamable $x13, $x0, %bb.1
+ ; INORDER-NEXT: {{ $}}
+ ; INORDER-NEXT: bb.2:
+ ; INORDER-NEXT: PseudoRET
+ ;
+ ; OOO-LABEL: name: gpr_loop_carried_war
+ ; OOO: bb.0:
+ ; OOO-NEXT: successors: %bb.1(0x80000000)
+ ; OOO-NEXT: {{ $}}
+ ; OOO-NEXT: PseudoBR %bb.1
+ ; OOO-NEXT: {{ $}}
+ ; OOO-NEXT: bb.1:
+ ; OOO-NEXT: successors: %bb.1(0x7c000000), %bb.2(0x04000000)
+ ; OOO-NEXT: liveins: $x10, $x11, $x12, $x13
+ ; OOO-NEXT: {{ $}}
+ ; OOO-NEXT: renamable $x15 = LD renamable $x10, 0 :: (load (s64))
+ ; OOO-NEXT: renamable $x15 = ADD killed renamable $x15, renamable $x11
+ ; OOO-NEXT: SD killed renamable $x15, renamable $x12, 0 :: (store (s64))
+ ; OOO-NEXT: BNE renamable $x13, $x0, %bb.1
+ ; OOO-NEXT: {{ $}}
+ ; OOO-NEXT: bb.2:
+ ; OOO-NEXT: PseudoRET
+ bb.0:
+ successors: %bb.1(0x80000000)
+
+ PseudoBR %bb.1
+
+ bb.1:
+ successors: %bb.1(0x7c000000), %bb.2(0x04000000)
+ liveins: $x10, $x11, $x12, $x13
+
+ renamable $x15 = LD renamable $x10, 0 :: (load (s64))
+ renamable $x15 = ADD killed renamable $x15, renamable $x11
+ SD renamable $x15, renamable $x12, 0 :: (store (s64))
+ BNE renamable $x13, $x0, %bb.1
+
+ bb.2:
+ PseudoRET
+...
+---
+name: fpr_loop_carried_war
+tracksRegLiveness: true
+body: |
+ ; INORDER-LABEL: name: fpr_loop_carried_war
+ ; INORDER: bb.0:
+ ; INORDER-NEXT: successors: %bb.1(0x80000000)
+ ; INORDER-NEXT: {{ $}}
+ ; INORDER-NEXT: PseudoBR %bb.1
+ ; INORDER-NEXT: {{ $}}
+ ; INORDER-NEXT: bb.1:
+ ; INORDER-NEXT: successors: %bb.1(0x7c000000), %bb.2(0x04000000)
+ ; INORDER-NEXT: liveins: $x10, $x11, $x12, $f11_d
+ ; INORDER-NEXT: {{ $}}
+ ; INORDER-NEXT: renamable $f10_d = FLD renamable $x10, 0 :: (load (s64))
+ ; INORDER-NEXT: renamable $f10_d = nofpexcept FADD_D killed renamable $f10_d, renamable $f11_d, 7, implicit $frm
+ ; INORDER-NEXT: FSD killed renamable $f10_d, renamable $x11, 0 :: (store (s64))
+ ; INORDER-NEXT: BNE renamable $x12, $x0, %bb.1
+ ; INORDER-NEXT: {{ $}}
+ ; INORDER-NEXT: bb.2:
+ ; INORDER-NEXT: PseudoRET
+ ;
+ ; OOO-LABEL: name: fpr_loop_carried_war
+ ; OOO: bb.0:
+ ; OOO-NEXT: successors: %bb.1(0x80000000)
+ ; OOO-NEXT: {{ $}}
+ ; OOO-NEXT: PseudoBR %bb.1
+ ; OOO-NEXT: {{ $}}
+ ; OOO-NEXT: bb.1:
+ ; OOO-NEXT: successors: %bb.1(0x7c000000), %bb.2(0x04000000)
+ ; OOO-NEXT: liveins: $x10, $x11, $x12, $f11_d
+ ; OOO-NEXT: {{ $}}
+ ; OOO-NEXT: renamable $f10_d = FLD renamable $x10, 0 :: (load (s64))
+ ; OOO-NEXT: renamable $f10_d = nofpexcept FADD_D killed renamable $f10_d, renamable $f11_d, 7, implicit $frm
+ ; OOO-NEXT: FSD killed renamable $f10_d, renamable $x11, 0 :: (store (s64))
+ ; OOO-NEXT: BNE renamable $x12, $x0, %bb.1
+ ; OOO-NEXT: {{ $}}
+ ; OOO-NEXT: bb.2:
+ ; OOO-NEXT: PseudoRET
+ bb.0:
+ successors: %bb.1(0x80000000)
+
+ PseudoBR %bb.1
+
+ bb.1:
+ successors: %bb.1(0x7c000000), %bb.2(0x04000000)
+ liveins: $x10, $x11, $x12, $f11_d
+
+ renamable $f10_d = FLD renamable $x10, 0 :: (load (s64))
+ renamable $f10_d = nofpexcept FADD_D killed renamable $f10_d, renamable $f11_d, 7, implicit $frm
+ FSD renamable $f10_d, renamable $x11, 0 :: (store (s64))
+ BNE renamable $x12, $x0, %bb.1
+
+ bb.2:
+ PseudoRET
+...
+
+---
+name: rvv_loop_carried_war
+tracksRegLiveness: true
+body: |
+ ; INORDER-LABEL: name: rvv_loop_carried_war
+ ; INORDER: bb.0:
+ ; INORDER-NEXT: successors: %bb.1(0x80000000)
+ ; INORDER-NEXT: {{ $}}
+ ; INORDER-NEXT: PseudoBR %bb.1
+ ; INORDER-NEXT: {{ $}}
+ ; INORDER-NEXT: bb.1:
+ ; INORDER-NEXT: successors: %bb.1(0x7c000000), %bb.2(0x04000000)
+ ; INORDER-NEXT: liveins: $x10, $x11, $x12, $v16m8
+ ; INORDER-NEXT: {{ $}}
+ ; INORDER-NEXT: dead $x0 = PseudoVSETIVLI 0, 211 /* e32, m8, ta, ma */, implicit-def $vl, implicit-def $vtype
+ ; INORDER-NEXT: renamable $v8m8 = PseudoVLE32_V_M8 undef renamable $v8m8, renamable $x10, $noreg /* vl */, 5 /* e32 */, 2 /* tu, ma */, implicit $vl, implicit $vtype :: (load unknown-size, align 4)
+ ; INORDER-NEXT: renamable $v8m8 = nofpexcept PseudoVFADD_VV_M8_E32 undef renamable $v8m8, killed renamable $v8m8, renamable $v16m8, 7 /* frm=dyn */, $noreg /* vl */, 5 /* e32 */, 0 /* tu, mu */, implicit $frm, implicit $vl, implicit $vtype
+ ; INORDER-NEXT: PseudoVSE32_V_M8 killed renamable $v8m8, renamable $x11, $noreg /* vl */, 5 /* e32 */, implicit $vl, implicit $vtype :: (store unknown-size, align 4)
+ ; INORDER-NEXT: BNE renamable $x12, $x0, %bb.1
+ ; INORDER-NEXT: {{ $}}
+ ; INORDER-NEXT: bb.2:
+ ; INORDER-NEXT: PseudoRET
+ ;
+ ; OOO-LABEL: name: rvv_loop_carried_war
+ ; OOO: bb.0:
+ ; OOO-NEXT: successors: %bb.1(0x80000000)
+ ; OOO-NEXT: {{ $}}
+ ; OOO-NEXT: PseudoBR %bb.1
+ ; OOO-NEXT: {{ $}}
+ ; OOO-NEXT: bb.1:
+ ; OOO-NEXT: successors: %bb.1(0x7c000000), %bb.2(0x04000000)
+ ; OOO-NEXT: liveins: $x10, $x11, $x12, $v16m8
+ ; OOO-NEXT: {{ $}}
+ ; OOO-NEXT: dead $x0 = PseudoVSETIVLI 0, 211 /* e32, m8, ta, ma */, implicit-def $vl, implicit-def $vtype
+ ; OOO-NEXT: renamable $v8m8 = PseudoVLE32_V_M8 undef renamable $v8m8, renamable $x10, $noreg /* vl */, 5 /* e32 */, 2 /* tu, ma */, implicit $vl, implicit $vtype :: (load unknown-size, align 4)
+ ; OOO-NEXT: renamable $v8m8 = nofpexcept PseudoVFADD_VV_M8_E32 undef renamable $v8m8, killed renamable $v8m8, renamable $v16m8, 7 /* frm=dyn */, $noreg /* vl */, 5 /* e32 */, 0 /* tu, mu */, implicit $frm, implicit $vl, implicit $vtype
+ ; OOO-NEXT: PseudoVSE32_V_M8 killed renamable $v8m8, renamable $x11, $noreg /* vl */, 5 /* e32 */, implicit $vl, implicit $vtype :: (store unknown-size, align 4)
+ ; OOO-NEXT: BNE renamable $x12, $x0, %bb.1
+ ; OOO-NEXT: {{ $}}
+ ; OOO-NEXT: bb.2:
+ ; OOO-NEXT: PseudoRET
+ bb.0:
+ successors: %bb.1(0x80000000)
+
+ PseudoBR %bb.1
+
+ bb.1:
+ successors: %bb.1(0x7c000000), %bb.2(0x04000000)
+ liveins: $x10, $x11, $x12, $v16m8
+
+ dead $x0 = PseudoVSETIVLI 0, 211 /* e32, m8, ta, ma */, implicit-def $vl, implicit-def $vtype
+ renamable $v8m8 = PseudoVLE32_V_M8 undef renamable $v8m8, renamable $x10, $noreg /* vl */, 5 /* e32 */, 2 /* tu, ma */, implicit $vl, implicit $vtype :: (load unknown-size, align 4)
+ renamable $v8m8 = nofpexcept PseudoVFADD_VV_M8_E32 undef renamable $v8m8, killed renamable $v8m8, renamable $v16m8, 7 /* frm=dyn */, $noreg /* vl */, 5 /* e32 */, 0 /* tu, mu */, implicit $frm, implicit $vl, implicit $vtype
+ PseudoVSE32_V_M8 killed renamable $v8m8, renamable $x11, $noreg /* vl */, 5 /* e32 */, implicit $vl, implicit $vtype :: (store unknown-size, align 4)
+ BNE renamable $x12, $x0, %bb.1
+
+ bb.2:
+ PseudoRET
+...
+
+# Negative tests
+
+# Renaming a loop with only one definition would only move the same backedge
+# WAR to another register, so a second complete definition is required.
+---
+name: single_def_loop_carried_war
+tracksRegLiveness: true
+body: |
+ ; INORDER-LABEL: name: single_def_loop_carried_war
+ ; INORDER: bb.0:
+ ; INORDER-NEXT: successors: %bb.1(0x80000000)
+ ; INORDER-NEXT: {{ $}}
+ ; INORDER-NEXT: PseudoBR %bb.1
+ ; INORDER-NEXT: {{ $}}
+ ; INORDER-NEXT: bb.1:
+ ; INORDER-NEXT: successors: %bb.1(0x7c000000), %bb.2(0x04000000)
+ ; INORDER-NEXT: liveins: $x10, $x11, $x12
+ ; INORDER-NEXT: {{ $}}
+ ; INORDER-NEXT: renamable $x15 = LD renamable $x10, 0 :: (load (s64))
+ ; INORDER-NEXT: SD killed renamable $x15, renamable $x11, 0 :: (store (s64))
+ ; INORDER-NEXT: BNE renamable $x12, $x0, %bb.1
+ ; INORDER-NEXT: {{ $}}
+ ; INORDER-NEXT: bb.2:
+ ; INORDER-NEXT: PseudoRET
+ ;
+ ; OOO-LABEL: name: single_def_loop_carried_war
+ ; OOO: bb.0:
+ ; OOO-NEXT: successors: %bb.1(0x80000000)
+ ; OOO-NEXT: {{ $}}
+ ; OOO-NEXT: PseudoBR %bb.1
+ ; OOO-NEXT: {{ $}}
+ ; OOO-NEXT: bb.1:
+ ; OOO-NEXT: successors: %bb.1(0x7c000000), %bb.2(0x04000000)
+ ; OOO-NEXT: liveins: $x10, $x11, $x12
+ ; OOO-NEXT: {{ $}}
+ ; OOO-NEXT: renamable $x15 = LD renamable $x10, 0 :: (load (s64))
+ ; OOO-NEXT: SD killed renamable $x15, renamable $x11, 0 :: (store (s64))
+ ; OOO-NEXT: BNE renamable $x12, $x0, %bb.1
+ ; OOO-NEXT: {{ $}}
+ ; OOO-NEXT: bb.2:
+ ; OOO-NEXT: PseudoRET
+ bb.0:
+ successors: %bb.1(0x80000000)
+
+ PseudoBR %bb.1
+
+ bb.1:
+ successors: %bb.1(0x7c000000), %bb.2(0x04000000)
+ liveins: $x10, $x11, $x12
+
+ renamable $x15 = LD renamable $x10, 0 :: (load (s64))
+ SD renamable $x15, renamable $x11, 0 :: (store (s64))
+ BNE renamable $x12, $x0, %bb.1
+
+ bb.2:
+ PseudoRET
+...
+
+# A use before the first definition means that the old value is live across the
+# backedge, so the def-use segment cannot be split safely.
+---
+name: live_in_use_loop_carried_war
+tracksRegLiveness: true
+body: |
+ ; INORDER-LABEL: name: live_in_use_loop_carried_war
+ ; INORDER: bb.0:
+ ; INORDER-NEXT: successors: %bb.1(0x80000000)
+ ; INORDER-NEXT: {{ $}}
+ ; INORDER-NEXT: PseudoBR %bb.1
+ ; INORDER-NEXT: {{ $}}
+ ; INORDER-NEXT: bb.1:
+ ; INORDER-NEXT: successors: %bb.1(0x7c000000), %bb.2(0x04000000)
+ ; INORDER-NEXT: liveins: $x10, $x11, $x12, $x13, $x15
+ ; INORDER-NEXT: {{ $}}
+ ; INORDER-NEXT: SD killed renamable $x15, renamable $x12, 0 :: (store (s64))
+ ; INORDER-NEXT: renamable $x15 = LD renamable $x10, 0 :: (load (s64))
+ ; INORDER-NEXT: renamable $x15 = ADD killed renamable $x15, renamable $x11
+ ; INORDER-NEXT: SD renamable $x15, renamable $x12, 8 :: (store (s64))
+ ; INORDER-NEXT: BNE renamable $x13, $x0, %bb.1
+ ; INORDER-NEXT: {{ $}}
+ ; INORDER-NEXT: bb.2:
+ ; INORDER-NEXT: PseudoRET
+ ;
+ ; OOO-LABEL: name: live_in_use_loop_carried_war
+ ; OOO: bb.0:
+ ; OOO-NEXT: successors: %bb.1(0x80000000)
+ ; OOO-NEXT: {{ $}}
+ ; OOO-NEXT: PseudoBR %bb.1
+ ; OOO-NEXT: {{ $}}
+ ; OOO-NEXT: bb.1:
+ ; OOO-NEXT: successors: %bb.1(0x7c000000), %bb.2(0x04000000)
+ ; OOO-NEXT: liveins: $x10, $x11, $x12, $x13, $x15
+ ; OOO-NEXT: {{ $}}
+ ; OOO-NEXT: SD killed renamable $x15, renamable $x12, 0 :: (store (s64))
+ ; OOO-NEXT: renamable $x15 = LD renamable $x10, 0 :: (load (s64))
+ ; OOO-NEXT: renamable $x15 = ADD killed renamable $x15, renamable $x11
+ ; OOO-NEXT: SD renamable $x15, renamable $x12, 8 :: (store (s64))
+ ; OOO-NEXT: BNE renamable $x13, $x0, %bb.1
+ ; OOO-NEXT: {{ $}}
+ ; OOO-NEXT: bb.2:
+ ; OOO-NEXT: PseudoRET
+ bb.0:
+ successors: %bb.1(0x80000000)
+
+ PseudoBR %bb.1
+
+ bb.1:
+ successors: %bb.1(0x7c000000), %bb.2(0x04000000)
+ liveins: $x10, $x11, $x12, $x13, $x15
+
+ SD renamable $x15, renamable $x12, 0 :: (store (s64))
+ renamable $x15 = LD renamable $x10, 0 :: (load (s64))
+ renamable $x15 = ADD killed renamable $x15, renamable $x11
+ SD renamable $x15, renamable $x12, 8 :: (store (s64))
+ BNE renamable $x13, $x0, %bb.1
+
+ bb.2:
+ PseudoRET
+...
+
+# Physical operands without the renamable flag must remain unchanged.
+---
+name: nonrenamable_loop_carried_war
+tracksRegLiveness: true
+body: |
+ ; INORDER-LABEL: name: nonrenamable_loop_carried_war
+ ; INORDER: bb.0:
+ ; INORDER-NEXT: successors: %bb.1(0x80000000)
+ ; INORDER-NEXT: {{ $}}
+ ; INORDER-NEXT: PseudoBR %bb.1
+ ; INORDER-NEXT: {{ $}}
+ ; INORDER-NEXT: bb.1:
+ ; INORDER-NEXT: successors: %bb.1(0x7c000000), %bb.2(0x04000000)
+ ; INORDER-NEXT: liveins: $x10, $x11, $x12, $x13
+ ; INORDER-NEXT: {{ $}}
+ ; INORDER-NEXT: $x15 = LD renamable $x10, 0 :: (load (s64))
+ ; INORDER-NEXT: renamable $x15 = ADD killed renamable $x15, renamable $x11
+ ; INORDER-NEXT: SD killed renamable $x15, renamable $x12, 0 :: (store (s64))
+ ; INORDER-NEXT: BNE renamable $x13, $x0, %bb.1
+ ; INORDER-NEXT: {{ $}}
+ ; INORDER-NEXT: bb.2:
+ ; INORDER-NEXT: PseudoRET
+ ;
+ ; OOO-LABEL: name: nonrenamable_loop_carried_war
+ ; OOO: bb.0:
+ ; OOO-NEXT: successors: %bb.1(0x80000000)
+ ; OOO-NEXT: {{ $}}
+ ; OOO-NEXT: PseudoBR %bb.1
+ ; OOO-NEXT: {{ $}}
+ ; OOO-NEXT: bb.1:
+ ; OOO-NEXT: successors: %bb.1(0x7c000000), %bb.2(0x04000000)
+ ; OOO-NEXT: liveins: $x10, $x11, $x12, $x13
+ ; OOO-NEXT: {{ $}}
+ ; OOO-NEXT: $x15 = LD renamable $x10, 0 :: (load (s64))
+ ; OOO-NEXT: renamable $x15 = ADD killed renamable $x15, renamable $x11
+ ; OOO-NEXT: SD killed renamable $x15, renamable $x12, 0 :: (store (s64))
+ ; OOO-NEXT: BNE renamable $x13, $x0, %bb.1
+ ; OOO-NEXT: {{ $}}
+ ; OOO-NEXT: bb.2:
+ ; OOO-NEXT: PseudoRET
+ bb.0:
+ successors: %bb.1(0x80000000)
+
+ PseudoBR %bb.1
+
+ bb.1:
+ successors: %bb.1(0x7c000000), %bb.2(0x04000000)
+ liveins: $x10, $x11, $x12, $x13
+
+ $x15 = LD renamable $x10, 0 :: (load (s64))
+ renamable $x15 = ADD killed renamable $x15, renamable $x11
+ SD renamable $x15, renamable $x12, 0 :: (store (s64))
+ BNE renamable $x13, $x0, %bb.1
+
+ bb.2:
+ PseudoRET
+...
+
+# Only single-block loops are handled; a multi-block loop requires cross-block
+# liveness reasoning.
+---
+name: multiblock_loop_carried_war
+tracksRegLiveness: true
+body: |
+ ; INORDER-LABEL: name: multiblock_loop_carried_war
+ ; INORDER: bb.0:
+ ; INORDER-NEXT: successors: %bb.1(0x80000000)
+ ; INORDER-NEXT: {{ $}}
+ ; INORDER-NEXT: PseudoBR %bb.1
+ ; INORDER-NEXT: {{ $}}
+ ; INORDER-NEXT: bb.1:
+ ; INORDER-NEXT: successors: %bb.2(0x80000000)
+ ; INORDER-NEXT: liveins: $x10, $x11, $x12, $x13
+ ; INORDER-NEXT: {{ $}}
+ ; INORDER-NEXT: renamable $x15 = LD renamable $x10, 0 :: (load (s64))
+ ; INORDER-NEXT: renamable $x15 = ADD killed renamable $x15, renamable $x11
+ ; INORDER-NEXT: SD killed renamable $x15, renamable $x12, 0 :: (store (s64))
+ ; INORDER-NEXT: PseudoBR %bb.2
+ ; INORDER-NEXT: {{ $}}
+ ; INORDER-NEXT: bb.2:
+ ; INORDER-NEXT: successors: %bb.1(0x7c000000), %bb.3(0x04000000)
+ ; INORDER-NEXT: liveins: $x10, $x11, $x12, $x13
+ ; INORDER-NEXT: {{ $}}
+ ; INORDER-NEXT: BNE renamable $x13, $x0, %bb.1
+ ; INORDER-NEXT: {{ $}}
+ ; INORDER-NEXT: bb.3:
+ ; INORDER-NEXT: PseudoRET
+ ;
+ ; OOO-LABEL: name: multiblock_loop_carried_war
+ ; OOO: bb.0:
+ ; OOO-NEXT: successors: %bb.1(0x80000000)
+ ; OOO-NEXT: {{ $}}
+ ; OOO-NEXT: PseudoBR %bb.1
+ ; OOO-NEXT: {{ $}}
+ ; OOO-NEXT: bb.1:
+ ; OOO-NEXT: successors: %bb.2(0x80000000)
+ ; OOO-NEXT: liveins: $x10, $x11, $x12, $x13
+ ; OOO-NEXT: {{ $}}
+ ; OOO-NEXT: renamable $x15 = LD renamable $x10, 0 :: (load (s64))
+ ; OOO-NEXT: renamable $x15 = ADD killed renamable $x15, renamable $x11
+ ; OOO-NEXT: SD killed renamable $x15, renamable $x12, 0 :: (store (s64))
+ ; OOO-NEXT: PseudoBR %bb.2
+ ; OOO-NEXT: {{ $}}
+ ; OOO-NEXT: bb.2:
+ ; OOO-NEXT: successors: %bb.1(0x7c000000), %bb.3(0x04000000)
+ ; OOO-NEXT: liveins: $x10, $x11, $x12, $x13
+ ; OOO-NEXT: {{ $}}
+ ; OOO-NEXT: BNE renamable $x13, $x0, %bb.1
+ ; OOO-NEXT: {{ $}}
+ ; OOO-NEXT: bb.3:
+ ; OOO-NEXT: PseudoRET
+ bb.0:
+ successors: %bb.1(0x80000000)
+
+ PseudoBR %bb.1
+
+ bb.1:
+ successors: %bb.2(0x80000000)
+ liveins: $x10, $x11, $x12, $x13
+
+ renamable $x15 = LD renamable $x10, 0 :: (load (s64))
+ renamable $x15 = ADD killed renamable $x15, renamable $x11
+ SD renamable $x15, renamable $x12, 0 :: (store (s64))
+ PseudoBR %bb.2
+
+ bb.2:
+ successors: %bb.1(0x7c000000), %bb.3(0x04000000)
+ liveins: $x10, $x11, $x12, $x13
+
+ BNE renamable $x13, $x0, %bb.1
+
+ bb.3:
+ PseudoRET
+...
+
+# Without a use before the second definition, there is no def-use segment to
+# split.
+---
+name: no_use_before_second_def
+tracksRegLiveness: true
+body: |
+ ; INORDER-LABEL: name: no_use_before_second_def
+ ; INORDER: bb.0:
+ ; INORDER-NEXT: successors: %bb.1(0x80000000)
+ ; INORDER-NEXT: {{ $}}
+ ; INORDER-NEXT: PseudoBR %bb.1
+ ; INORDER-NEXT: {{ $}}
+ ; INORDER-NEXT: bb.1:
+ ; INORDER-NEXT: successors: %bb.1(0x7c000000), %bb.2(0x04000000)
+ ; INORDER-NEXT: liveins: $x10, $x11, $x12, $x13
+ ; INORDER-NEXT: {{ $}}
+ ; INORDER-NEXT: dead renamable $x15 = LD renamable $x10, 0 :: (load (s64))
+ ; INORDER-NEXT: renamable $x15 = ADD renamable $x11, renamable $x12
+ ; INORDER-NEXT: SD killed renamable $x15, renamable $x12, 0 :: (store (s64))
+ ; INORDER-NEXT: BNE renamable $x13, $x0, %bb.1
+ ; INORDER-NEXT: {{ $}}
+ ; INORDER-NEXT: bb.2:
+ ; INORDER-NEXT: PseudoRET
+ ;
+ ; OOO-LABEL: name: no_use_before_second_def
+ ; OOO: bb.0:
+ ; OOO-NEXT: successors: %bb.1(0x80000000)
+ ; OOO-NEXT: {{ $}}
+ ; OOO-NEXT: PseudoBR %bb.1
+ ; OOO-NEXT: {{ $}}
+ ; OOO-NEXT: bb.1:
+ ; OOO-NEXT: successors: %bb.1(0x7c000000), %bb.2(0x04000000)
+ ; OOO-NEXT: liveins: $x10, $x11, $x12, $x13
+ ; OOO-NEXT: {{ $}}
+ ; OOO-NEXT: dead renamable $x15 = LD renamable $x10, 0 :: (load (s64))
+ ; OOO-NEXT: renamable $x15 = ADD renamable $x11, renamable $x12
+ ; OOO-NEXT: SD killed renamable $x15, renamable $x12, 0 :: (store (s64))
+ ; OOO-NEXT: BNE renamable $x13, $x0, %bb.1
+ ; OOO-NEXT: {{ $}}
+ ; OOO-NEXT: bb.2:
+ ; OOO-NEXT: PseudoRET
+ bb.0:
+ successors: %bb.1(0x80000000)
+
+ PseudoBR %bb.1
+
+ bb.1:
+ successors: %bb.1(0x7c000000), %bb.2(0x04000000)
+ liveins: $x10, $x11, $x12, $x13
+
+ dead renamable $x15 = LD renamable $x10, 0 :: (load (s64))
+ renamable $x15 = ADD renamable $x11, renamable $x12
+ SD renamable $x15, renamable $x12, 0 :: (store (s64))
+ BNE renamable $x13, $x0, %bb.1
+
+ bb.2:
+ PseudoRET
+...
+
+# Without a use after the second definition, there is no loop-carried WAR to
+# break.
+---
+name: no_use_after_second_def
+tracksRegLiveness: true
+body: |
+ ; INORDER-LABEL: name: no_use_after_second_def
+ ; INORDER: bb.0:
+ ; INORDER-NEXT: successors: %bb.1(0x80000000)
+ ; INORDER-NEXT: {{ $}}
+ ; INORDER-NEXT: PseudoBR %bb.1
+ ; INORDER-NEXT: {{ $}}
+ ; INORDER-NEXT: bb.1:
+ ; INORDER-NEXT: successors: %bb.1(0x7c000000), %bb.2(0x04000000)
+ ; INORDER-NEXT: liveins: $x10, $x11, $x12, $x13
+ ; INORDER-NEXT: {{ $}}
+ ; INORDER-NEXT: renamable $x15 = LD renamable $x10, 0 :: (load (s64))
+ ; INORDER-NEXT: SD killed renamable $x15, renamable $x12, 0 :: (store (s64))
+ ; INORDER-NEXT: dead renamable $x15 = ADD renamable $x11, renamable $x12
+ ; INORDER-NEXT: BNE renamable $x13, $x0, %bb.1
+ ; INORDER-NEXT: {{ $}}
+ ; INORDER-NEXT: bb.2:
+ ; INORDER-NEXT: PseudoRET
+ ;
+ ; OOO-LABEL: name: no_use_after_second_def
+ ; OOO: bb.0:
+ ; OOO-NEXT: successors: %bb.1(0x80000000)
+ ; OOO-NEXT: {{ $}}
+ ; OOO-NEXT: PseudoBR %bb.1
+ ; OOO-NEXT: {{ $}}
+ ; OOO-NEXT: bb.1:
+ ; OOO-NEXT: successors: %bb.1(0x7c000000), %bb.2(0x04000000)
+ ; OOO-NEXT: liveins: $x10, $x11, $x12, $x13
+ ; OOO-NEXT: {{ $}}
+ ; OOO-NEXT: renamable $x15 = LD renamable $x10, 0 :: (load (s64))
+ ; OOO-NEXT: SD killed renamable $x15, renamable $x12, 0 :: (store (s64))
+ ; OOO-NEXT: dead renamable $x15 = ADD renamable $x11, renamable $x12
+ ; OOO-NEXT: BNE renamable $x13, $x0, %bb.1
+ ; OOO-NEXT: {{ $}}
+ ; OOO-NEXT: bb.2:
+ ; OOO-NEXT: PseudoRET
+ bb.0:
+ successors: %bb.1(0x80000000)
+
+ PseudoBR %bb.1
+
+ bb.1:
+ successors: %bb.1(0x7c000000), %bb.2(0x04000000)
+ liveins: $x10, $x11, $x12, $x13
+
+ renamable $x15 = LD renamable $x10, 0 :: (load (s64))
+ SD killed renamable $x15, renamable $x12, 0 :: (store (s64))
+ dead renamable $x15 = ADD renamable $x11, renamable $x12
+ BNE renamable $x13, $x0, %bb.1
+
+ bb.2:
+ PseudoRET
+...
+
+# A tied use at the second definition cannot be renamed independently of its
+# tied def.
+---
+name: tied_use_at_second_def
+tracksRegLiveness: true
+body: |
+ ; INORDER-LABEL: name: tied_use_at_second_def
+ ; INORDER: bb.0:
+ ; INORDER-NEXT: successors: %bb.1(0x80000000)
+ ; INORDER-NEXT: {{ $}}
+ ; INORDER-NEXT: PseudoBR %bb.1
+ ; INORDER-NEXT: {{ $}}
+ ; INORDER-NEXT: bb.1:
+ ; INORDER-NEXT: successors: %bb.1(0x7c000000), %bb.2(0x04000000)
+ ; INORDER-NEXT: liveins: $x10, $x11, $x12, $v16m8
+ ; INORDER-NEXT: {{ $}}
+ ; INORDER-NEXT: dead $x0 = PseudoVSETIVLI 0, 211 /* e32, m8, ta, ma */, implicit-def $vl, implicit-def $vtype
+ ; INORDER-NEXT: renamable $v8m8 = PseudoVLE32_V_M8 undef renamable $v8m8, renamable $x10, $noreg /* vl */, 5 /* e32 */, 2 /* tu, ma */, implicit $vl, implicit $vtype :: (load unknown-size, align 4)
+ ; INORDER-NEXT: renamable $v8m8 = nofpexcept PseudoVFADD_VV_M8_E32 killed renamable $v8m8, renamable $v8m8, renamable $v16m8, 7 /* frm=dyn */, $noreg /* vl */, 5 /* e32 */, 0 /* tu, mu */, implicit $frm, implicit $vl, implicit $vtype
+ ; INORDER-NEXT: PseudoVSE32_V_M8 killed renamable $v8m8, renamable $x11, $noreg /* vl */, 5 /* e32 */, implicit $vl, implicit $vtype :: (store unknown-size, align 4)
+ ; INORDER-NEXT: BNE renamable $x12, $x0, %bb.1
+ ; INORDER-NEXT: {{ $}}
+ ; INORDER-NEXT: bb.2:
+ ; INORDER-NEXT: PseudoRET
+ ;
+ ; OOO-LABEL: name: tied_use_at_second_def
+ ; OOO: bb.0:
+ ; OOO-NEXT: successors: %bb.1(0x80000000)
+ ; OOO-NEXT: {{ $}}
+ ; OOO-NEXT: PseudoBR %bb.1
+ ; OOO-NEXT: {{ $}}
+ ; OOO-NEXT: bb.1:
+ ; OOO-NEXT: successors: %bb.1(0x7c000000), %bb.2(0x04000000)
+ ; OOO-NEXT: liveins: $x10, $x11, $x12, $v16m8
+ ; OOO-NEXT: {{ $}}
+ ; OOO-NEXT: dead $x0 = PseudoVSETIVLI 0, 211 /* e32, m8, ta, ma */, implicit-def $vl, implicit-def $vtype
+ ; OOO-NEXT: renamable $v8m8 = PseudoVLE32_V_M8 undef renamable $v8m8, renamable $x10, $noreg /* vl */, 5 /* e32 */, 2 /* tu, ma */, implicit $vl, implicit $vtype :: (load unknown-size, align 4)
+ ; OOO-NEXT: renamable $v8m8 = nofpexcept PseudoVFADD_VV_M8_E32 killed renamable $v8m8, renamable $v8m8, renamable $v16m8, 7 /* frm=dyn */, $noreg /* vl */, 5 /* e32 */, 0 /* tu, mu */, implicit $frm, implicit $vl, implicit $vtype
+ ; OOO-NEXT: PseudoVSE32_V_M8 killed renamable $v8m8, renamable $x11, $noreg /* vl */, 5 /* e32 */, implicit $vl, implicit $vtype :: (store unknown-size, align 4)
+ ; OOO-NEXT: BNE renamable $x12, $x0, %bb.1
+ ; OOO-NEXT: {{ $}}
+ ; OOO-NEXT: bb.2:
+ ; OOO-NEXT: PseudoRET
+ bb.0:
+ successors: %bb.1(0x80000000)
+
+ PseudoBR %bb.1
+
+ bb.1:
+ successors: %bb.1(0x7c000000), %bb.2(0x04000000)
+ liveins: $x10, $x11, $x12, $v16m8
+
+ dead $x0 = PseudoVSETIVLI 0, 211 /* e32, m8, ta, ma */, implicit-def $vl, implicit-def $vtype
+ renamable $v8m8 = PseudoVLE32_V_M8 undef renamable $v8m8, renamable $x10, $noreg /* vl */, 5 /* e32 */, 2 /* tu, ma */, implicit $vl, implicit $vtype :: (load unknown-size, align 4)
+ renamable $v8m8 = nofpexcept PseudoVFADD_VV_M8_E32 renamable $v8m8, killed renamable $v8m8, renamable $v16m8, 7 /* frm=dyn */, $noreg /* vl */, 5 /* e32 */, 0 /* tu, mu */, implicit $frm, implicit $vl, implicit $vtype
+ PseudoVSE32_V_M8 killed renamable $v8m8, renamable $x11, $noreg /* vl */, 5 /* e32 */, implicit $vl, implicit $vtype :: (store unknown-size, align 4)
+ BNE renamable $x12, $x0, %bb.1
+
+ bb.2:
+ PseudoRET
+...
+
+# The only otherwise available VRM8 group has a live-in alias, so there is no
+# safe register to use for the first def-use segment.
+---
+name: no_free_rvv_register
+tracksRegLiveness: true
+body: |
+ ; INORDER-LABEL: name: no_free_rvv_register
+ ; INORDER: bb.0:
+ ; INORDER-NEXT: successors: %bb.1(0x80000000)
+ ; INORDER-NEXT: {{ $}}
+ ; INORDER-NEXT: PseudoBR %bb.1
+ ; INORDER-NEXT: {{ $}}
+ ; INORDER-NEXT: bb.1:
+ ; INORDER-NEXT: successors: %bb.1(0x7c000000), %bb.2(0x04000000)
+ ; INORDER-NEXT: liveins: $x10, $x11, $x12, $v0m8, $v16m8, $v24
+ ; INORDER-NEXT: {{ $}}
+ ; INORDER-NEXT: dead $x0 = PseudoVSETIVLI 0, 211 /* e32, m8, ta, ma */, implicit-def $vl, implicit-def $vtype
+ ; INORDER-NEXT: renamable $v8m8 = PseudoVLE32_V_M8 undef renamable $v8m8, renamable $x10, $noreg /* vl */, 5 /* e32 */, 2 /* tu, ma */, implicit $vl, implicit $vtype :: (load unknown-size, align 4)
+ ; INORDER-NEXT: renamable $v8m8 = nofpexcept PseudoVFADD_VV_M8_E32 undef renamable $v8m8, killed renamable $v8m8, renamable $v16m8, 7 /* frm=dyn */, $noreg /* vl */, 5 /* e32 */, 0 /* tu, mu */, implicit $frm, implicit $vl, implicit $vtype
+ ; INORDER-NEXT: PseudoVSE32_V_M8 killed renamable $v8m8, renamable $x11, $noreg /* vl */, 5 /* e32 */, implicit $vl, implicit $vtype :: (store unknown-size, align 4)
+ ; INORDER-NEXT: BNE renamable $x12, $x0, %bb.1
+ ; INORDER-NEXT: {{ $}}
+ ; INORDER-NEXT: bb.2:
+ ; INORDER-NEXT: PseudoRET
+ ;
+ ; OOO-LABEL: name: no_free_rvv_register
+ ; OOO: bb.0:
+ ; OOO-NEXT: successors: %bb.1(0x80000000)
+ ; OOO-NEXT: {{ $}}
+ ; OOO-NEXT: PseudoBR %bb.1
+ ; OOO-NEXT: {{ $}}
+ ; OOO-NEXT: bb.1:
+ ; OOO-NEXT: successors: %bb.1(0x7c000000), %bb.2(0x04000000)
+ ; OOO-NEXT: liveins: $x10, $x11, $x12, $v0m8, $v16m8, $v24
+ ; OOO-NEXT: {{ $}}
+ ; OOO-NEXT: dead $x0 = PseudoVSETIVLI 0, 211 /* e32, m8, ta, ma */, implicit-def $vl, implicit-def $vtype
+ ; OOO-NEXT: renamable $v8m8 = PseudoVLE32_V_M8 undef renamable $v8m8, renamable $x10, $noreg /* vl */, 5 /* e32 */, 2 /* tu, ma */, implicit $vl, implicit $vtype :: (load unknown-size, align 4)
+ ; OOO-NEXT: renamable $v8m8 = nofpexcept PseudoVFADD_VV_M8_E32 undef renamable $v8m8, killed renamable $v8m8, renamable $v16m8, 7 /* frm=dyn */, $noreg /* vl */, 5 /* e32 */, 0 /* tu, mu */, implicit $frm, implicit $vl, implicit $vtype
+ ; OOO-NEXT: PseudoVSE32_V_M8 killed renamable $v8m8, renamable $x11, $noreg /* vl */, 5 /* e32 */, implicit $vl, implicit $vtype :: (store unknown-size, align 4)
+ ; OOO-NEXT: BNE renamable $x12, $x0, %bb.1
+ ; OOO-NEXT: {{ $}}
+ ; OOO-NEXT: bb.2:
+ ; OOO-NEXT: PseudoRET
+ bb.0:
+ successors: %bb.1(0x80000000)
+
+ PseudoBR %bb.1
+
+ bb.1:
+ successors: %bb.1(0x7c000000), %bb.2(0x04000000)
+ liveins: $x10, $x11, $x12, $v0m8, $v16m8, $v24
+
+ dead $x0 = PseudoVSETIVLI 0, 211 /* e32, m8, ta, ma */, implicit-def $vl, implicit-def $vtype
+ renamable $v8m8 = PseudoVLE32_V_M8 undef renamable $v8m8, renamable $x10, $noreg /* vl */, 5 /* e32 */, 2 /* tu, ma */, implicit $vl, implicit $vtype :: (load unknown-size, align 4)
+ renamable $v8m8 = nofpexcept PseudoVFADD_VV_M8_E32 undef renamable $v8m8, killed renamable $v8m8, renamable $v16m8, 7 /* frm=dyn */, $noreg /* vl */, 5 /* e32 */, 0 /* tu, mu */, implicit $frm, implicit $vl, implicit $vtype
+ PseudoVSE32_V_M8 killed renamable $v8m8, renamable $x11, $noreg /* vl */, 5 /* e32 */, implicit $vl, implicit $vtype :: (store unknown-size, align 4)
+ BNE renamable $x12, $x0, %bb.1
+
+ bb.2:
+ PseudoRET
+...
+
+# Bundles are skipped because renaming a member independently could make the
+# bundle header's register operands and liveness stale.
+---
+name: bundled_loop_carried_war
+tracksRegLiveness: true
+body: |
+ ; INORDER-LABEL: name: bundled_loop_carried_war
+ ; INORDER: bb.0:
+ ; INORDER-NEXT: successors: %bb.1(0x80000000)
+ ; INORDER-NEXT: {{ $}}
+ ; INORDER-NEXT: PseudoBR %bb.1
+ ; INORDER-NEXT: {{ $}}
+ ; INORDER-NEXT: bb.1:
+ ; INORDER-NEXT: successors: %bb.1(0x7c000000), %bb.2(0x04000000)
+ ; INORDER-NEXT: liveins: $x10, $x11, $x12, $x13
+ ; INORDER-NEXT: {{ $}}
+ ; INORDER-NEXT: renamable $x15 = LD renamable $x10, 0 :: (load (s64))
+ ; INORDER-NEXT: BUNDLE implicit-def $x14, implicit $x11 {
+ ; INORDER-NEXT: renamable $x14 = ADD renamable $x11, renamable $x11
+ ; INORDER-NEXT: }
+ ; INORDER-NEXT: renamable $x15 = ADD killed renamable $x15, renamable $x11
+ ; INORDER-NEXT: SD killed renamable $x15, renamable $x12, 0 :: (store (s64))
+ ; INORDER-NEXT: BNE renamable $x13, $x0, %bb.1
+ ; INORDER-NEXT: {{ $}}
+ ; INORDER-NEXT: bb.2:
+ ; INORDER-NEXT: PseudoRET
+ ;
+ ; OOO-LABEL: name: bundled_loop_carried_war
+ ; OOO: bb.0:
+ ; OOO-NEXT: successors: %bb.1(0x80000000)
+ ; OOO-NEXT: {{ $}}
+ ; OOO-NEXT: PseudoBR %bb.1
+ ; OOO-NEXT: {{ $}}
+ ; OOO-NEXT: bb.1:
+ ; OOO-NEXT: successors: %bb.1(0x7c000000), %bb.2(0x04000000)
+ ; OOO-NEXT: liveins: $x10, $x11, $x12, $x13
+ ; OOO-NEXT: {{ $}}
+ ; OOO-NEXT: renamable $x15 = LD renamable $x10, 0 :: (load (s64))
+ ; OOO-NEXT: BUNDLE implicit-def $x14, implicit $x11 {
+ ; OOO-NEXT: renamable $x14 = ADD renamable $x11, renamable $x11
+ ; OOO-NEXT: }
+ ; OOO-NEXT: renamable $x15 = ADD killed renamable $x15, renamable $x11
+ ; OOO-NEXT: SD killed renamable $x15, renamable $x12, 0 :: (store (s64))
+ ; OOO-NEXT: BNE renamable $x13, $x0, %bb.1
+ ; OOO-NEXT: {{ $}}
+ ; OOO-NEXT: bb.2:
+ ; OOO-NEXT: PseudoRET
+ bb.0:
+ successors: %bb.1(0x80000000)
+
+ PseudoBR %bb.1
+
+ bb.1:
+ successors: %bb.1(0x7c000000), %bb.2(0x04000000)
+ liveins: $x10, $x11, $x12, $x13
+
+ renamable $x15 = LD renamable $x10, 0 :: (load (s64))
+ BUNDLE implicit-def $x14, implicit $x11 {
+ renamable $x14 = ADD renamable $x11, renamable $x11
+ }
+ renamable $x15 = ADD killed renamable $x15, renamable $x11
+ SD renamable $x15, renamable $x12, 0 :: (store (s64))
+ BNE renamable $x13, $x0, %bb.1
+
+ bb.2:
+ PseudoRET
+...
>From dec3ab59cf0a826b8f88310b8eee14e2773abfb5 Mon Sep 17 00:00:00 2001
From: Pengcheng Wang <wangpengcheng.pp at bytedance.com>
Date: Tue, 8 Sep 2026 20:31:41 +0800
Subject: [PATCH 4/5] [MISched][RISCV] Add anti-dependency breaking to
PostMachineScheduler
Port the post-RA anti-dependency breaker lifecycle to ScheduleDAGMI and
add conservative single-block loop-carried WAR renaming for aggressive
mode.
Enable aggressive anti-dependency breaking for RISC-V processors with an
in-order scheduling model. This applies uniformly to allocatable GPR,
FPR, and vector register classes while leaving out-of-order and unknown
models unchanged.
Fixes #221163
Assisted-by: TRAE CLI (GPT-5.6)
---
llvm/include/llvm/CodeGen/MachineScheduler.h | 18 +-
llvm/lib/CodeGen/AggressiveAntiDepBreaker.cpp | 227 ++++++++++++++++++
llvm/lib/CodeGen/AggressiveAntiDepBreaker.h | 4 +
llvm/lib/CodeGen/MachineScheduler.cpp | 102 +++++++-
llvm/lib/Target/RISCV/RISCVSubtarget.h | 6 +
llvm/lib/Target/RISCV/RISCVTargetMachine.cpp | 4 +-
.../CodeGen/RISCV/postmisched-antidep.mir | 24 +-
7 files changed, 365 insertions(+), 20 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/MachineScheduler.h b/llvm/include/llvm/CodeGen/MachineScheduler.h
index 5280a2376c34a..b80c8afff397f 100644
--- a/llvm/include/llvm/CodeGen/MachineScheduler.h
+++ b/llvm/include/llvm/CodeGen/MachineScheduler.h
@@ -316,6 +316,9 @@ class LLVM_ABI ScheduleDAGMI : public ScheduleDAGInstrs {
MachineBlockFrequencyInfo *MBFI;
std::unique_ptr<MachineSchedStrategy> SchedImpl;
+ struct AntiDepState;
+ std::unique_ptr<AntiDepState> AntiDep;
+
/// Ordered list of DAG postprocessing steps.
std::vector<std::unique_ptr<ScheduleDAGMutation>> Mutations;
@@ -332,10 +335,11 @@ class LLVM_ABI ScheduleDAGMI : public ScheduleDAGInstrs {
#endif
public:
+ /// Construct a post-RA scheduler with anti-dependency breaking enabled.
ScheduleDAGMI(MachineSchedContext *C, std::unique_ptr<MachineSchedStrategy> S,
- bool RemoveKillFlags)
- : ScheduleDAGInstrs(*C->MF, C->MLI, RemoveKillFlags), AA(C->AA),
- LIS(C->LIS), MBFI(C->MBFI), SchedImpl(std::move(S)) {}
+ bool RemoveKillFlags,
+ TargetSubtargetInfo::AntiDepBreakMode AntiDepMode =
+ TargetSubtargetInfo::ANTIDEP_NONE);
// Provide a vtable anchor
~ScheduleDAGMI() override;
@@ -373,6 +377,7 @@ class LLVM_ABI ScheduleDAGMI : public ScheduleDAGInstrs {
MachineBasicBlock::iterator begin,
MachineBasicBlock::iterator end,
unsigned regioninstrs) override;
+ void exitRegion() override;
/// Implement ScheduleDAGInstrs interface for scheduling a sequence of
/// reorderable instructions.
@@ -1443,9 +1448,12 @@ ScheduleDAGMILive *createSchedLive(MachineSchedContext *C) {
/// Create a generic scheduler with no vreg liveness or DAG mutation passes.
template <typename Strategy = PostGenericScheduler>
-ScheduleDAGMI *createSchedPostRA(MachineSchedContext *C) {
+ScheduleDAGMI *
+createSchedPostRA(MachineSchedContext *C,
+ TargetSubtargetInfo::AntiDepBreakMode AntiDepMode =
+ TargetSubtargetInfo::ANTIDEP_NONE) {
return new ScheduleDAGMI(C, std::make_unique<Strategy>(C),
- /*RemoveKillFlags=*/true);
+ /*RemoveKillFlags=*/true, AntiDepMode);
}
class MachineSchedulerPass
diff --git a/llvm/lib/CodeGen/AggressiveAntiDepBreaker.cpp b/llvm/lib/CodeGen/AggressiveAntiDepBreaker.cpp
index a425864921416..bd8f7ecd07ba6 100644
--- a/llvm/lib/CodeGen/AggressiveAntiDepBreaker.cpp
+++ b/llvm/lib/CodeGen/AggressiveAntiDepBreaker.cpp
@@ -21,6 +21,7 @@
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineInstr.h"
+#include "llvm/CodeGen/MachineLoopInfo.h"
#include "llvm/CodeGen/MachineOperand.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/RegisterClassInfo.h"
@@ -34,6 +35,7 @@
#include "llvm/Support/Debug.h"
#include "llvm/Support/raw_ostream.h"
#include <cassert>
+#include <optional>
using namespace llvm;
@@ -184,6 +186,231 @@ void AggressiveAntiDepBreaker::FinishBlock() {
State = nullptr;
}
+namespace {
+
+struct LoopCarriedRename {
+ SmallVector<MachineOperand *, 8> Refs;
+ BitVector Candidates;
+};
+
+static bool overlapsReg(const MachineOperand &MO, MCRegister Reg,
+ const TargetRegisterInfo &TRI) {
+ return MO.isReg() && MO.getReg().isPhysical() &&
+ TRI.regsOverlap(MO.getReg(), Reg);
+}
+
+static std::optional<LoopCarriedRename>
+findLoopCarriedRename(MachineBasicBlock &MBB, MCRegister Reg,
+ const TargetInstrInfo &TII,
+ const TargetRegisterInfo &TRI) {
+ LoopCarriedRename Rename{{}, BitVector(TRI.getNumRegs(), true)};
+ bool FoundFirstDef = false;
+ bool FoundSecondDef = false;
+ bool FoundUse = false;
+ bool FoundUseAfterSecondDef = false;
+
+ for (MachineInstr &MI : MBB) {
+ if (MI.isBundle())
+ return std::nullopt;
+ if (MI.isDebugInstr()) {
+ if (FoundFirstDef && !FoundSecondDef) {
+ for (MachineOperand &MO : MI.operands()) {
+ if (!overlapsReg(MO, Reg, TRI))
+ continue;
+ if (MI.isDebugPHI())
+ continue;
+ if (MO.getReg() != Reg)
+ return std::nullopt;
+ Rename.Refs.push_back(&MO);
+ }
+ }
+ continue;
+ }
+ if (MI.isCall() || MI.isInlineAsm() || TII.isPredicated(MI))
+ return std::nullopt;
+ for (const MachineOperand &MO : MI.operands())
+ if (MO.isRegMask())
+ return std::nullopt;
+
+ bool HasFullDef = false;
+ for (MachineOperand &MO : MI.operands()) {
+ if (!overlapsReg(MO, Reg, TRI))
+ continue;
+ if (MO.getReg() != Reg || MO.getSubReg())
+ return std::nullopt;
+ if (MO.isDef() && !MO.isImplicit())
+ HasFullDef = true;
+ }
+
+ if (!FoundFirstDef) {
+ for (MachineOperand &MO : MI.operands())
+ if (overlapsReg(MO, Reg, TRI) && MO.readsReg())
+ return std::nullopt;
+ if (!HasFullDef)
+ continue;
+
+ FoundFirstDef = true;
+ for (MachineOperand &MO : MI.operands()) {
+ if (!overlapsReg(MO, Reg, TRI))
+ continue;
+ if (MO.readsReg() || MO.isImplicit() || !MO.isRenamable())
+ return std::nullopt;
+ Rename.Refs.push_back(&MO);
+ }
+ continue;
+ }
+
+ if (!FoundSecondDef) {
+ if (HasFullDef) {
+ FoundSecondDef = true;
+ for (MachineOperand &MO : MI.operands()) {
+ if (!overlapsReg(MO, Reg, TRI))
+ continue;
+ if (MO.getReg() != Reg || MO.getSubReg() || MO.isImplicit())
+ return std::nullopt;
+ if (MO.isDef()) {
+ if (MO.isEarlyClobber())
+ return std::nullopt;
+ continue;
+ }
+ if (MO.readsReg()) {
+ if (MO.isTied() || !MO.isRenamable())
+ return std::nullopt;
+ Rename.Refs.push_back(&MO);
+ FoundUse = true;
+ }
+ }
+ continue;
+ }
+
+ for (MachineOperand &MO : MI.operands()) {
+ if (!overlapsReg(MO, Reg, TRI))
+ continue;
+ if (MO.getReg() != Reg || MO.getSubReg() || MO.isDef() ||
+ MO.isImplicit() || !MO.readsReg() || !MO.isRenamable())
+ return std::nullopt;
+ FoundUse = true;
+ Rename.Refs.push_back(&MO);
+ }
+ continue;
+ }
+
+ for (MachineOperand &MO : MI.operands())
+ if (overlapsReg(MO, Reg, TRI) && MO.readsReg())
+ FoundUseAfterSecondDef = true;
+ }
+
+ if (!FoundSecondDef || !FoundUse || !FoundUseAfterSecondDef)
+ return std::nullopt;
+
+ bool HasConstraint = false;
+ for (MachineOperand *MO : Rename.Refs) {
+ if (MO->isDebug())
+ continue;
+ const TargetRegisterClass *RC =
+ MO->getParent()->getRegClassConstraint(MO->getOperandNo(), &TII, &TRI);
+ if (!RC)
+ return std::nullopt;
+ BitVector Allocatable = TRI.getAllocatableSet(*MBB.getParent(), RC);
+ if (!HasConstraint) {
+ Rename.Candidates = std::move(Allocatable);
+ HasConstraint = true;
+ } else {
+ Rename.Candidates &= Allocatable;
+ }
+ }
+ if (!HasConstraint)
+ return std::nullopt;
+
+ return Rename;
+}
+
+static bool isUnusedLoopRegister(MCRegister Reg, const MachineBasicBlock &MBB,
+ const MachineRegisterInfo &MRI,
+ const TargetRegisterInfo &TRI) {
+ if (!MRI.isAllocatable(Reg))
+ return false;
+
+ for (const auto &LI : MBB.liveins())
+ if (TRI.regsOverlap(Reg, LI.PhysReg))
+ return false;
+ for (const MachineBasicBlock *Succ : MBB.successors())
+ for (const auto &LI : Succ->liveins())
+ if (TRI.regsOverlap(Reg, LI.PhysReg))
+ return false;
+
+ const BitVector Pristine =
+ MBB.getParent()->getFrameInfo().getPristineRegs(*MBB.getParent());
+ for (MCRegAliasIterator AI(Reg, &TRI, true); AI.isValid(); ++AI)
+ if (Pristine.test((*AI).id()))
+ return false;
+
+ for (const MachineInstr &MI : MBB) {
+ if (MI.isBundle())
+ return false;
+ for (const MachineOperand &MO : MI.operands()) {
+ if (overlapsReg(MO, Reg, TRI))
+ return false;
+ if (!MO.isRegMask())
+ continue;
+ for (MCRegAliasIterator AI(Reg, &TRI, true); AI.isValid(); ++AI)
+ if (MO.clobbersPhysReg(*AI))
+ return false;
+ }
+ }
+ return true;
+}
+
+} // namespace
+
+void AggressiveAntiDepBreaker::BreakLoopCarriedAntiDependencies(
+ MachineBasicBlock &MBB, const MachineLoopInfo &MLI) {
+ const MachineLoop *Loop = MLI.getLoopFor(&MBB);
+ if (!Loop || Loop->getHeader() != &MBB || Loop->getNumBlocks() != 1 ||
+ !MBB.isSuccessor(&MBB))
+ return;
+
+ SmallVector<MCRegister, 8> DefRegs;
+ BitVector Seen(TRI->getNumRegs());
+ for (MachineInstr &MI : MBB) {
+ if (MI.isBundle())
+ return;
+ if (MI.isDebugOrPseudoInstr())
+ continue;
+ for (MachineOperand &MO : MI.operands()) {
+ if (!MO.isReg() || !MO.isDef() || MO.isImplicit() ||
+ !MO.getReg().isPhysical() || !MO.getReg() || MO.getSubReg())
+ continue;
+ MCRegister Reg = MO.getReg().asMCReg();
+ if (!Seen.test(Reg.id())) {
+ Seen.set(Reg.id());
+ DefRegs.push_back(Reg);
+ }
+ }
+ }
+
+ for (MCRegister Reg : DefRegs) {
+ std::optional<LoopCarriedRename> Rename =
+ findLoopCarriedRename(MBB, Reg, *TII, *TRI);
+ if (!Rename)
+ continue;
+
+ const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg);
+ for (MCRegister NewReg : RegClassInfo.getOrder(RC)) {
+ if (NewReg == Reg || !Rename->Candidates.test(NewReg.id()) ||
+ !isUnusedLoopRegister(NewReg, MBB, MRI, *TRI))
+ continue;
+
+ LLVM_DEBUG(dbgs() << "Breaking loop-carried anti-dependence on "
+ << printReg(Reg, TRI) << " using "
+ << printReg(NewReg, TRI) << '\n');
+ for (MachineOperand *MO : Rename->Refs)
+ MO->setReg(NewReg);
+ break;
+ }
+ }
+}
+
void AggressiveAntiDepBreaker::Observe(MachineInstr &MI, unsigned Count,
unsigned InsertPosIndex) {
assert(Count < InsertPosIndex && "Instruction index out of expected range!");
diff --git a/llvm/lib/CodeGen/AggressiveAntiDepBreaker.h b/llvm/lib/CodeGen/AggressiveAntiDepBreaker.h
index 4f474ebb6b89e..687a3fe999183 100644
--- a/llvm/lib/CodeGen/AggressiveAntiDepBreaker.h
+++ b/llvm/lib/CodeGen/AggressiveAntiDepBreaker.h
@@ -29,6 +29,7 @@ namespace llvm {
class MachineBasicBlock;
class MachineFunction;
class MachineInstr;
+class MachineLoopInfo;
class MachineOperand;
class MachineRegisterInfo;
class RegisterClassInfo;
@@ -150,6 +151,9 @@ class LLVM_LIBRARY_VISIBILITY AggressiveAntiDepState {
unsigned InsertPosIndex,
DbgValueVector &DbgValues) override;
+ void BreakLoopCarriedAntiDependencies(MachineBasicBlock &MBB,
+ const MachineLoopInfo &MLI);
+
/// Update liveness information to account for the current
/// instruction, which will not be scheduled.
void Observe(MachineInstr &MI, unsigned Count,
diff --git a/llvm/lib/CodeGen/MachineScheduler.cpp b/llvm/lib/CodeGen/MachineScheduler.cpp
index 8d0130973b81b..e99734f6fcd86 100644
--- a/llvm/lib/CodeGen/MachineScheduler.cpp
+++ b/llvm/lib/CodeGen/MachineScheduler.cpp
@@ -12,6 +12,7 @@
//===----------------------------------------------------------------------===//
#include "llvm/CodeGen/MachineScheduler.h"
+#include "AggressiveAntiDepBreaker.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/BitVector.h"
#include "llvm/ADT/DenseMap.h"
@@ -352,6 +353,7 @@ class PostMachineSchedulerImpl : public MachineSchedulerBase {
struct RequiredAnalyses {
MachineLoopInfo &MLI;
AAResults &AA;
+ RegisterClassInfo &RegClassInfo;
};
PostMachineSchedulerImpl() = default;
// Migration only
@@ -448,6 +450,7 @@ void PostMachineSchedulerLegacy::getAnalysisUsage(AnalysisUsage &AU) const {
AU.addRequired<MachineLoopInfoWrapperPass>();
AU.addRequired<AAResultsWrapperPass>();
AU.addRequired<TargetPassConfig>();
+ AU.addRequired<MachineRegisterClassInfoWrapperPass>();
MachineFunctionPass::getAnalysisUsage(AU);
}
@@ -592,6 +595,7 @@ bool PostMachineSchedulerImpl::run(MachineFunction &Func,
MLI = &Analyses.MLI;
this->TM = &TM;
AA = &Analyses.AA;
+ RegClassInfo = &Analyses.RegClassInfo;
if (VerifyScheduling) {
const char *PostMSchedBanner = "Before post machine scheduling.";
@@ -714,8 +718,10 @@ bool PostMachineSchedulerLegacy::runOnMachineFunction(MachineFunction &MF) {
auto &MLI = getAnalysis<MachineLoopInfoWrapperPass>().getLI();
auto &TM = getAnalysis<TargetPassConfig>().getTM<TargetMachine>();
auto &AA = getAnalysis<AAResultsWrapperPass>().getAAResults();
+ auto &RegClassInfo =
+ getAnalysis<MachineRegisterClassInfoWrapperPass>().getRCI();
Impl.setLegacyPass(this);
- return Impl.run(MF, TM, {MLI, AA});
+ return Impl.run(MF, TM, {MLI, AA, RegClassInfo});
}
PreservedAnalyses
@@ -733,9 +739,10 @@ PostMachineSchedulerPass::run(MachineFunction &MF,
auto &FAM = MFAM.getResult<FunctionAnalysisManagerMachineFunctionProxy>(MF)
.getManager();
auto &AA = FAM.getResult<AAManager>(MF.getFunction());
+ auto &RegClassInfo = MFAM.getResult<MachineRegisterClassAnalysis>(MF);
Impl->setMFAM(&MFAM);
- bool Changed = Impl->run(MF, *TM, {MLI, AA});
+ bool Changed = Impl->run(MF, *TM, {MLI, AA, RegClassInfo});
if (!Changed)
return PreservedAnalyses::all();
@@ -912,6 +919,43 @@ LLVM_DUMP_METHOD void ReadyQueue::dump() const {
// ===----------------------------------------------------------------------===/
// Provide a vtable anchor.
+struct ScheduleDAGMI::AntiDepState {
+ std::unique_ptr<AntiDepBreaker> Breaker;
+ AggressiveAntiDepBreaker *Aggressive = nullptr;
+ unsigned EndIndex = 0;
+ MachineBasicBlock::iterator ScanPos;
+ bool RegionProcessed = false;
+};
+
+ScheduleDAGMI::ScheduleDAGMI(MachineSchedContext *C,
+ std::unique_ptr<MachineSchedStrategy> S,
+ bool RemoveKillFlags,
+ TargetSubtargetInfo::AntiDepBreakMode AntiDepMode)
+ : ScheduleDAGInstrs(*C->MF, C->MLI, RemoveKillFlags), AA(C->AA),
+ LIS(C->LIS), MBFI(C->MBFI), SchedImpl(std::move(S)) {
+ if (SchedImpl->doMBBSchedRegionsTopDown())
+ AntiDepMode = TargetSubtargetInfo::ANTIDEP_NONE;
+ assert((AntiDepMode == TargetSubtargetInfo::ANTIDEP_NONE ||
+ C->MF->getRegInfo().tracksLiveness()) &&
+ "Live-ins must be accurate for anti-dependency breaking");
+ assert(
+ (AntiDepMode == TargetSubtargetInfo::ANTIDEP_NONE || C->RegClassInfo) &&
+ "Register class info is required for anti-dependency breaking");
+ if (AntiDepMode == TargetSubtargetInfo::ANTIDEP_ALL) {
+ SmallVector<const TargetRegisterClass *, 4> CriticalPathRCs;
+ C->MF->getSubtarget().getCriticalPathRCs(CriticalPathRCs);
+ AntiDep = std::make_unique<AntiDepState>();
+ auto Breaker = std::make_unique<AggressiveAntiDepBreaker>(
+ *C->MF, *C->RegClassInfo, CriticalPathRCs);
+ AntiDep->Aggressive = Breaker.get();
+ AntiDep->Breaker = std::move(Breaker);
+ } else if (AntiDepMode == TargetSubtargetInfo::ANTIDEP_CRITICAL) {
+ AntiDep = std::make_unique<AntiDepState>();
+ AntiDep->Breaker.reset(
+ createCriticalAntiDepBreaker(*C->MF, *C->RegClassInfo));
+ }
+}
+
ScheduleDAGMI::~ScheduleDAGMI() = default;
/// ReleaseSucc - Decrement the NumPredsLeft count of a successor. When
@@ -987,10 +1031,19 @@ void ScheduleDAGMI::releasePredecessors(SUnit *SU) {
void ScheduleDAGMI::startBlock(MachineBasicBlock *bb) {
ScheduleDAGInstrs::startBlock(bb);
SchedImpl->enterMBB(bb);
+ if (AntiDep) {
+ AntiDep->Breaker->StartBlock(bb);
+ AntiDep->ScanPos = bb->end();
+ AntiDep->EndIndex = bb->size();
+ }
}
void ScheduleDAGMI::finishBlock() {
SchedImpl->leaveMBB();
+ if (AntiDep && AntiDep->Aggressive)
+ AntiDep->Aggressive->BreakLoopCarriedAntiDependencies(*BB, *MLI);
+ if (AntiDep)
+ AntiDep->Breaker->FinishBlock();
ScheduleDAGInstrs::finishBlock();
}
@@ -1004,6 +1057,21 @@ void ScheduleDAGMI::enterRegion(MachineBasicBlock *bb,
unsigned regioninstrs)
{
ScheduleDAGInstrs::enterRegion(bb, begin, end, regioninstrs);
+ if (AntiDep) {
+ unsigned Count =
+ std::distance(bb->instr_begin(), AntiDep->ScanPos.getInstrIterator());
+ for (auto I = AntiDep->ScanPos; I != end;) {
+ MachineInstr &MI = *--I;
+ --Count;
+ if (!MI.isDebugOrPseudoInstr())
+ AntiDep->Breaker->Observe(MI, Count, AntiDep->EndIndex);
+ if (MI.isBundle())
+ Count -= MI.getBundleSize();
+ }
+ AntiDep->EndIndex =
+ std::distance(bb->instr_begin(), end.getInstrIterator());
+ AntiDep->RegionProcessed = false;
+ }
SchedImpl->initPolicy(begin, end, regioninstrs);
@@ -1018,6 +1086,24 @@ void ScheduleDAGMI::enterRegion(MachineBasicBlock *bb,
setDumpDirection(D);
}
+void ScheduleDAGMI::exitRegion() {
+ if (AntiDep) {
+ if (!AntiDep->RegionProcessed) {
+ unsigned Count = AntiDep->EndIndex;
+ for (auto I = RegionEnd; I != RegionBegin;) {
+ MachineInstr &MI = *--I;
+ --Count;
+ if (!MI.isDebugOrPseudoInstr())
+ AntiDep->Breaker->Observe(MI, Count, AntiDep->EndIndex);
+ if (MI.isBundle())
+ Count -= MI.getBundleSize();
+ }
+ }
+ AntiDep->ScanPos = RegionBegin;
+ }
+ ScheduleDAGInstrs::exitRegion();
+}
+
/// This is normally called from the main scheduler loop but may also be invoked
/// by the scheduling strategy to perform additional code motion.
void ScheduleDAGMI::moveInstruction(
@@ -1060,6 +1146,15 @@ void ScheduleDAGMI::schedule() {
// Build the DAG.
buildSchedGraph(AA);
+ if (AntiDep) {
+ unsigned Broken = AntiDep->Breaker->BreakAntiDependencies(
+ SUnits, RegionBegin, RegionEnd, AntiDep->EndIndex, DbgValues);
+ if (Broken != 0) {
+ ScheduleDAG::clearDAG();
+ buildSchedGraph(AA);
+ }
+ }
+
postProcessDAG();
SmallVector<SUnit*, 8> TopRoots, BotRoots;
@@ -1125,6 +1220,9 @@ void ScheduleDAGMI::schedule() {
dumpSchedule();
dbgs() << '\n';
});
+
+ if (AntiDep)
+ AntiDep->RegionProcessed = true;
}
/// Apply each ScheduleDAGMutation step in order.
diff --git a/llvm/lib/Target/RISCV/RISCVSubtarget.h b/llvm/lib/Target/RISCV/RISCVSubtarget.h
index 14c33ee8691ec..20734e2bd553b 100644
--- a/llvm/lib/Target/RISCV/RISCVSubtarget.h
+++ b/llvm/lib/Target/RISCV/RISCVSubtarget.h
@@ -158,6 +158,12 @@ class RISCVSubtarget : public RISCVGenSubtargetInfo {
bool enablePostRAScheduler() const override { return UsePostRAScheduler; }
+ AntiDepBreakMode getPostMachineSchedulerAntiDepBreakMode() const {
+ const MCSchedModel &Model = getSchedModel();
+ return Model.hasInstrSchedModel() && !Model.isOutOfOrder() ? ANTIDEP_ALL
+ : ANTIDEP_NONE;
+ }
+
Align getPrefFunctionAlignment() const {
return Align(TuneInfo->PrefFunctionAlignment);
}
diff --git a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
index 74dad048be7cb..3f91579fe6f7f 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
@@ -305,7 +305,9 @@ RISCVTargetMachine::createMachineScheduler(MachineSchedContext *C) const {
ScheduleDAGInstrs *
RISCVTargetMachine::createPostMachineScheduler(MachineSchedContext *C) const {
const RISCVSubtarget &ST = C->MF->getSubtarget<RISCVSubtarget>();
- ScheduleDAGMI *DAG = createSchedPostRA(C);
+ TargetSubtargetInfo::AntiDepBreakMode AntiDepMode =
+ ST.getPostMachineSchedulerAntiDepBreakMode();
+ ScheduleDAGMI *DAG = createSchedPostRA(C, AntiDepMode);
// Add MacroFusion mutation first with a higher priority than later clustering
const auto &MacroFusions = ST.getMacroFusions();
diff --git a/llvm/test/CodeGen/RISCV/postmisched-antidep.mir b/llvm/test/CodeGen/RISCV/postmisched-antidep.mir
index 18fdd840fe9e0..5b7457162c59f 100644
--- a/llvm/test/CodeGen/RISCV/postmisched-antidep.mir
+++ b/llvm/test/CodeGen/RISCV/postmisched-antidep.mir
@@ -29,8 +29,8 @@ body: |
; INORDER: liveins: $x10, $x11, $x12, $x13, $x15
; INORDER-NEXT: {{ $}}
; INORDER-NEXT: renamable $x14 = ADD killed renamable $x15, killed renamable $x10
- ; INORDER-NEXT: renamable $x15 = LD killed renamable $x11, 0 :: (load (s64))
- ; INORDER-NEXT: SD killed renamable $x15, killed renamable $x12, 0 :: (store (s64))
+ ; INORDER-NEXT: $x11 = LD killed renamable $x11, 0 :: (load (s64))
+ ; INORDER-NEXT: SD killed $x11, killed renamable $x12, 0 :: (store (s64))
; INORDER-NEXT: SD killed renamable $x14, killed renamable $x13, 0 :: (store (s64))
; INORDER-NEXT: PseudoRET
;
@@ -60,8 +60,8 @@ body: |
; INORDER: liveins: $x10, $x11, $x12, $f10_d, $f11_d
; INORDER-NEXT: {{ $}}
; INORDER-NEXT: renamable $f12_d = nofpexcept FADD_D killed renamable $f10_d, killed renamable $f11_d, 7, implicit $frm
- ; INORDER-NEXT: renamable $f10_d = FLD killed renamable $x10, 0 :: (load (s64))
- ; INORDER-NEXT: FSD killed renamable $f10_d, killed renamable $x11, 0 :: (store (s64))
+ ; INORDER-NEXT: $f11_d = FLD killed renamable $x10, 0 :: (load (s64))
+ ; INORDER-NEXT: FSD killed $f11_d, killed renamable $x11, 0 :: (store (s64))
; INORDER-NEXT: FSD killed renamable $f12_d, killed renamable $x12, 0 :: (store (s64))
; INORDER-NEXT: PseudoRET
;
@@ -90,9 +90,9 @@ body: |
; INORDER-LABEL: name: rvv_region_war
; INORDER: liveins: $x10, $x11, $v8
; INORDER-NEXT: {{ $}}
+ ; INORDER-NEXT: $v31 = VL1RE8_V killed renamable $x10 :: (load unknown-size, align 1)
; INORDER-NEXT: $x12 = PseudoVMV_X_S killed renamable $v8, 5 /* e32 */, implicit $vtype
- ; INORDER-NEXT: renamable $v8 = VL1RE8_V killed renamable $x10 :: (load unknown-size, align 1)
- ; INORDER-NEXT: VS1R_V killed renamable $v8, killed renamable $x11 :: (store unknown-size, align 1)
+ ; INORDER-NEXT: VS1R_V killed $v31, killed renamable $x11 :: (store unknown-size, align 1)
; INORDER-NEXT: PseudoRET
;
; OOO-LABEL: name: rvv_region_war
@@ -127,8 +127,8 @@ body: |
; INORDER-NEXT: successors: %bb.1(0x7c000000), %bb.2(0x04000000)
; INORDER-NEXT: liveins: $x10, $x11, $x12, $x13
; INORDER-NEXT: {{ $}}
- ; INORDER-NEXT: renamable $x15 = LD renamable $x10, 0 :: (load (s64))
- ; INORDER-NEXT: renamable $x15 = ADD killed renamable $x15, renamable $x11
+ ; INORDER-NEXT: $x14 = LD renamable $x10, 0 :: (load (s64))
+ ; INORDER-NEXT: renamable $x15 = ADD killed $x14, renamable $x11
; INORDER-NEXT: SD killed renamable $x15, renamable $x12, 0 :: (store (s64))
; INORDER-NEXT: BNE renamable $x13, $x0, %bb.1
; INORDER-NEXT: {{ $}}
@@ -183,8 +183,8 @@ body: |
; INORDER-NEXT: successors: %bb.1(0x7c000000), %bb.2(0x04000000)
; INORDER-NEXT: liveins: $x10, $x11, $x12, $f11_d
; INORDER-NEXT: {{ $}}
- ; INORDER-NEXT: renamable $f10_d = FLD renamable $x10, 0 :: (load (s64))
- ; INORDER-NEXT: renamable $f10_d = nofpexcept FADD_D killed renamable $f10_d, renamable $f11_d, 7, implicit $frm
+ ; INORDER-NEXT: $f15_d = FLD renamable $x10, 0 :: (load (s64))
+ ; INORDER-NEXT: renamable $f10_d = nofpexcept FADD_D killed $f15_d, renamable $f11_d, 7, implicit $frm
; INORDER-NEXT: FSD killed renamable $f10_d, renamable $x11, 0 :: (store (s64))
; INORDER-NEXT: BNE renamable $x12, $x0, %bb.1
; INORDER-NEXT: {{ $}}
@@ -241,8 +241,8 @@ body: |
; INORDER-NEXT: liveins: $x10, $x11, $x12, $v16m8
; INORDER-NEXT: {{ $}}
; INORDER-NEXT: dead $x0 = PseudoVSETIVLI 0, 211 /* e32, m8, ta, ma */, implicit-def $vl, implicit-def $vtype
- ; INORDER-NEXT: renamable $v8m8 = PseudoVLE32_V_M8 undef renamable $v8m8, renamable $x10, $noreg /* vl */, 5 /* e32 */, 2 /* tu, ma */, implicit $vl, implicit $vtype :: (load unknown-size, align 4)
- ; INORDER-NEXT: renamable $v8m8 = nofpexcept PseudoVFADD_VV_M8_E32 undef renamable $v8m8, killed renamable $v8m8, renamable $v16m8, 7 /* frm=dyn */, $noreg /* vl */, 5 /* e32 */, 0 /* tu, mu */, implicit $frm, implicit $vl, implicit $vtype
+ ; INORDER-NEXT: $v24m8 = PseudoVLE32_V_M8 undef $v24m8, renamable $x10, $noreg /* vl */, 5 /* e32 */, 2 /* tu, ma */, implicit $vl, implicit $vtype :: (load unknown-size, align 4)
+ ; INORDER-NEXT: renamable $v8m8 = nofpexcept PseudoVFADD_VV_M8_E32 undef renamable $v8m8, killed $v24m8, renamable $v16m8, 7 /* frm=dyn */, $noreg /* vl */, 5 /* e32 */, 0 /* tu, mu */, implicit $frm, implicit $vl, implicit $vtype
; INORDER-NEXT: PseudoVSE32_V_M8 killed renamable $v8m8, renamable $x11, $noreg /* vl */, 5 /* e32 */, implicit $vl, implicit $vtype :: (store unknown-size, align 4)
; INORDER-NEXT: BNE renamable $x12, $x0, %bb.1
; INORDER-NEXT: {{ $}}
>From 5ce35383d5612b2a3d9800d5bbcd1c3852df5e4b Mon Sep 17 00:00:00 2001
From: Pengcheng Wang <wangpengcheng.pp at bytedance.com>
Date: Wed, 9 Sep 2026 16:22:05 +0800
Subject: [PATCH 5/5] [MISched] Use LiveRegUnits for loop rename candidates
Use LiveRegUnits to collect physical register units referenced or
clobbered in a loop block. This replaces repeated alias and regmask
scans while preserving conservative boundary, debug, and undef operand
handling.
Assisted-by: TRAE CLI (GPT-5.6)
---
llvm/lib/CodeGen/AggressiveAntiDepBreaker.cpp | 59 +++++++------------
1 file changed, 22 insertions(+), 37 deletions(-)
diff --git a/llvm/lib/CodeGen/AggressiveAntiDepBreaker.cpp b/llvm/lib/CodeGen/AggressiveAntiDepBreaker.cpp
index bd8f7ecd07ba6..1a4b4063054fe 100644
--- a/llvm/lib/CodeGen/AggressiveAntiDepBreaker.cpp
+++ b/llvm/lib/CodeGen/AggressiveAntiDepBreaker.cpp
@@ -325,42 +325,6 @@ findLoopCarriedRename(MachineBasicBlock &MBB, MCRegister Reg,
return Rename;
}
-static bool isUnusedLoopRegister(MCRegister Reg, const MachineBasicBlock &MBB,
- const MachineRegisterInfo &MRI,
- const TargetRegisterInfo &TRI) {
- if (!MRI.isAllocatable(Reg))
- return false;
-
- for (const auto &LI : MBB.liveins())
- if (TRI.regsOverlap(Reg, LI.PhysReg))
- return false;
- for (const MachineBasicBlock *Succ : MBB.successors())
- for (const auto &LI : Succ->liveins())
- if (TRI.regsOverlap(Reg, LI.PhysReg))
- return false;
-
- const BitVector Pristine =
- MBB.getParent()->getFrameInfo().getPristineRegs(*MBB.getParent());
- for (MCRegAliasIterator AI(Reg, &TRI, true); AI.isValid(); ++AI)
- if (Pristine.test((*AI).id()))
- return false;
-
- for (const MachineInstr &MI : MBB) {
- if (MI.isBundle())
- return false;
- for (const MachineOperand &MO : MI.operands()) {
- if (overlapsReg(MO, Reg, TRI))
- return false;
- if (!MO.isRegMask())
- continue;
- for (MCRegAliasIterator AI(Reg, &TRI, true); AI.isValid(); ++AI)
- if (MO.clobbersPhysReg(*AI))
- return false;
- }
- }
- return true;
-}
-
} // namespace
void AggressiveAntiDepBreaker::BreakLoopCarriedAntiDependencies(
@@ -370,11 +334,31 @@ void AggressiveAntiDepBreaker::BreakLoopCarriedAntiDependencies(
!MBB.isSuccessor(&MBB))
return;
+ // Track every register unit referenced or clobbered anywhere in the block.
+ LiveRegUnits UnavailableUnits(*TRI);
+ // Preserve the conservative whole-register treatment of boundary liveness.
+ for (const auto &LI : MBB.liveins())
+ UnavailableUnits.addReg(LI.PhysReg);
+ for (const MachineBasicBlock *Succ : MBB.successors())
+ for (const auto &LI : Succ->liveins())
+ UnavailableUnits.addReg(LI.PhysReg);
+ const BitVector Pristine =
+ MBB.getParent()->getFrameInfo().getPristineRegs(*MBB.getParent());
+ for (unsigned Reg : Pristine.set_bits())
+ UnavailableUnits.addReg(MCRegister(Reg));
+
SmallVector<MCRegister, 8> DefRegs;
BitVector Seen(TRI->getNumRegs());
for (MachineInstr &MI : MBB) {
if (MI.isBundle())
return;
+ UnavailableUnits.accumulate(MI);
+ // Keep the whole-block-unused policy for physical register operands that
+ // accumulate() ignores, such as undef uses.
+ for (const MachineOperand &MO : MI.operands())
+ if (MO.isReg() && MO.getReg() && MO.getReg().isPhysical() &&
+ !MO.isDef() && !MO.readsReg())
+ UnavailableUnits.addReg(MO.getReg().asMCReg());
if (MI.isDebugOrPseudoInstr())
continue;
for (MachineOperand &MO : MI.operands()) {
@@ -398,7 +382,7 @@ void AggressiveAntiDepBreaker::BreakLoopCarriedAntiDependencies(
const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg);
for (MCRegister NewReg : RegClassInfo.getOrder(RC)) {
if (NewReg == Reg || !Rename->Candidates.test(NewReg.id()) ||
- !isUnusedLoopRegister(NewReg, MBB, MRI, *TRI))
+ !MRI.isAllocatable(NewReg) || !UnavailableUnits.available(NewReg))
continue;
LLVM_DEBUG(dbgs() << "Breaking loop-carried anti-dependence on "
@@ -406,6 +390,7 @@ void AggressiveAntiDepBreaker::BreakLoopCarriedAntiDependencies(
<< printReg(NewReg, TRI) << '\n');
for (MachineOperand *MO : Rename->Refs)
MO->setReg(NewReg);
+ UnavailableUnits.addReg(NewReg);
break;
}
}
More information about the llvm-commits
mailing list