[llvm] [EarlyIfConversion] Add cascading conditional branches as a conversion pattern (PR #218415)
Jonathan Cohen via llvm-commits
llvm-commits at lists.llvm.org
Mon Aug 24 23:47:56 PDT 2026
https://github.com/jcohen-apple updated https://github.com/llvm/llvm-project/pull/218415
>From d7176de84ee032c7468680d4d2800dee00c1611f Mon Sep 17 00:00:00 2001
From: Jonathan Cohen <joncoh at apple.com>
Date: Thu, 16 Apr 2026 15:29:47 +0300
Subject: [PATCH 1/2] [EarlyIfConversion] Unit tests documenting current state
before cascading conversion logic
---
.../CodeGen/AArch64/early-ifcvt-cascade.mir | 843 ++++++++++++++++++
.../CodeGen/AArch64/early-ifcvt-remarks.mir | 159 ++++
2 files changed, 1002 insertions(+)
create mode 100644 llvm/test/CodeGen/AArch64/early-ifcvt-cascade.mir
diff --git a/llvm/test/CodeGen/AArch64/early-ifcvt-cascade.mir b/llvm/test/CodeGen/AArch64/early-ifcvt-cascade.mir
new file mode 100644
index 0000000000000..99c0ea79267a1
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/early-ifcvt-cascade.mir
@@ -0,0 +1,843 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=aarch64 -mcpu=apple-m1 -run-pass=early-ifcvt -enable-early-ifcvt-data-dependent %s -o - | FileCheck %s
+
+--- |
+ define void @cascade_selector_loop() { ret void }
+ define void @cascade_branches_from_args_not_dd() { ret void }
+ define void @cascade_exceeds_crit_limit() { ret void }
+ define void @cascade_backedge_to_earlier_block() { ret void }
+ define void @cascade_forward_skip_to_later_block() { ret void }
+...
+---
+name: cascade_selector_loop
+tracksRegLiveness: true
+liveins:
+ - { reg: '$x0', virtual-reg: '%0' }
+ - { reg: '$x1', virtual-reg: '%1' }
+ - { reg: '$w2', virtual-reg: '%2' }
+ - { reg: '$w3', virtual-reg: '%3' }
+ - { reg: '$w4', virtual-reg: '%4' }
+body: |
+ ; CHECK-LABEL: name: cascade_selector_loop
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.10(0x30000000), %bb.1(0x50000000)
+ ; CHECK-NEXT: liveins: $x0, $x1, $w2, $w3, $w4
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr32 = COPY $w2
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gpr32 = COPY $w3
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:gpr32 = COPY $w4
+ ; CHECK-NEXT: [[COPY5:%[0-9]+]]:gpr64all = COPY $xzr
+ ; CHECK-NEXT: CBZW [[COPY2]], %bb.10
+ ; CHECK-NEXT: B %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.2(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 0 :: (load (s32))
+ ; CHECK-NEXT: [[LDRWui1:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 1 :: (load (s32))
+ ; CHECK-NEXT: [[LDRWui2:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 2 :: (load (s32))
+ ; CHECK-NEXT: [[LDRWui3:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 3 :: (load (s32))
+ ; CHECK-NEXT: [[LDRWui4:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 4 :: (load (s32))
+ ; CHECK-NEXT: [[LDRWui5:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 5 :: (load (s32))
+ ; CHECK-NEXT: [[LDRWui6:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 6 :: (load (s32))
+ ; CHECK-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: successors: %bb.3(0x40000000), %bb.9(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[PHI:%[0-9]+]]:gpr32 = PHI [[COPY2]], %bb.1, %15, %bb.9
+ ; CHECK-NEXT: [[LDRWui7:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load (s32))
+ ; CHECK-NEXT: [[MADDWrrr:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[LDRWui7]], $wzr
+ ; CHECK-NEXT: [[MADDWrrr1:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY4]], [[LDRWui7]], [[MADDWrrr]]
+ ; CHECK-NEXT: [[MADDWrrr2:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[LDRWui7]], [[MADDWrrr1]]
+ ; CHECK-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 7
+ ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; CHECK-NEXT: B %bb.3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.3:
+ ; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.9(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui1]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 6
+ ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; CHECK-NEXT: B %bb.4
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.4:
+ ; CHECK-NEXT: successors: %bb.5(0x40000000), %bb.9(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[SUBSWrr2:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui2]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 5
+ ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; CHECK-NEXT: B %bb.5
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.5:
+ ; CHECK-NEXT: successors: %bb.6(0x40000000), %bb.9(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[SUBSWrr3:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui3]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 4
+ ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; CHECK-NEXT: B %bb.6
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.6:
+ ; CHECK-NEXT: successors: %bb.7(0x40000000), %bb.9(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[SUBSWrr4:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui4]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm4:%[0-9]+]]:gpr64 = MOVi64imm 3
+ ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; CHECK-NEXT: B %bb.7
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.7:
+ ; CHECK-NEXT: successors: %bb.8(0x40000000), %bb.9(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[SUBSWrr5:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui5]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm5:%[0-9]+]]:gpr64 = MOVi64imm 2
+ ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; CHECK-NEXT: B %bb.8
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.8:
+ ; CHECK-NEXT: successors: %bb.9(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[MOVi64imm6:%[0-9]+]]:gpr64 = MOVi64imm 1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.9:
+ ; CHECK-NEXT: successors: %bb.10(0x04000000), %bb.2(0x7c000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[PHI1:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.2, [[MOVi64imm1]], %bb.3, [[MOVi64imm2]], %bb.4, [[MOVi64imm3]], %bb.5, [[MOVi64imm4]], %bb.6, [[MOVi64imm5]], %bb.7, [[MOVi64imm6]], %bb.8
+ ; CHECK-NEXT: [[SUBSWrr6:%[0-9]+]]:gpr32 = SUBSWrr [[PHI]], [[MOVi32imm]], implicit-def $nzcv
+ ; CHECK-NEXT: Bcc 1, %bb.2, implicit $nzcv
+ ; CHECK-NEXT: B %bb.10
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.10:
+ ; CHECK-NEXT: [[PHI2:%[0-9]+]]:gpr64 = PHI [[COPY5]], %bb.0, [[PHI1]], %bb.9
+ ; CHECK-NEXT: $x0 = COPY [[PHI2]]
+ ; CHECK-NEXT: RET_ReallyLR implicit $x0
+ bb.0:
+ successors: %bb.10(0x30000000), %bb.1(0x50000000)
+ liveins: $x0, $x1, $w2, $w3, $w4
+
+ %0:gpr64common = COPY $x0
+ %1:gpr64common = COPY $x1
+ %2:gpr32 = COPY $w2
+ %3:gpr32 = COPY $w3
+ %4:gpr32 = COPY $w4
+ %99:gpr64all = COPY $xzr
+ CBZW %2, %bb.10
+ B %bb.1
+
+ bb.1:
+ successors: %bb.2(0x80000000)
+
+ %10:gpr32 = LDRWui %1, 0 :: (load (s32))
+ %11:gpr32 = LDRWui %1, 1 :: (load (s32))
+ %12:gpr32 = LDRWui %1, 2 :: (load (s32))
+ %13:gpr32 = LDRWui %1, 3 :: (load (s32))
+ %14:gpr32 = LDRWui %1, 4 :: (load (s32))
+ %15:gpr32 = LDRWui %1, 5 :: (load (s32))
+ %16:gpr32 = LDRWui %1, 6 :: (load (s32))
+ %17:gpr32 = MOVi32imm 1
+
+ bb.2:
+ successors: %bb.3(0x40000000), %bb.9(0x40000000)
+
+ %50:gpr32 = PHI %2, %bb.1, %51, %bb.9
+ %20:gpr32 = LDRWui %0, 0 :: (load (s32))
+ %31:gpr32 = nsw MADDWrrr %3, %20, $wzr
+ %32:gpr32 = nsw MADDWrrr %4, %20, %31
+ %30:gpr32 = nsw MADDWrrr %3, %20, %32
+ %21:gpr32 = SUBSWrr %30, %10, implicit-def $nzcv
+ %40:gpr64 = MOVi64imm 7
+ Bcc 10, %bb.9, implicit $nzcv
+ B %bb.3
+
+ bb.3:
+ successors: %bb.4(0x40000000), %bb.9(0x40000000)
+
+ %22:gpr32 = SUBSWrr %30, %11, implicit-def $nzcv
+ %41:gpr64 = MOVi64imm 6
+ Bcc 10, %bb.9, implicit $nzcv
+ B %bb.4
+
+ bb.4:
+ successors: %bb.5(0x40000000), %bb.9(0x40000000)
+
+ %23:gpr32 = SUBSWrr %30, %12, implicit-def $nzcv
+ %42:gpr64 = MOVi64imm 5
+ Bcc 10, %bb.9, implicit $nzcv
+ B %bb.5
+
+ bb.5:
+ successors: %bb.6(0x40000000), %bb.9(0x40000000)
+
+ %24:gpr32 = SUBSWrr %30, %13, implicit-def $nzcv
+ %43:gpr64 = MOVi64imm 4
+ Bcc 10, %bb.9, implicit $nzcv
+ B %bb.6
+
+ bb.6:
+ successors: %bb.7(0x40000000), %bb.9(0x40000000)
+
+ %25:gpr32 = SUBSWrr %30, %14, implicit-def $nzcv
+ %44:gpr64 = MOVi64imm 3
+ Bcc 10, %bb.9, implicit $nzcv
+ B %bb.7
+
+ bb.7:
+ successors: %bb.8(0x40000000), %bb.9(0x40000000)
+
+ %26:gpr32 = SUBSWrr %30, %15, implicit-def $nzcv
+ %45:gpr64 = MOVi64imm 2
+ Bcc 10, %bb.9, implicit $nzcv
+ B %bb.8
+
+ bb.8:
+ successors: %bb.9(0x80000000)
+
+ %47:gpr64 = MOVi64imm 1
+
+ bb.9:
+ successors: %bb.10(0x04000000), %bb.2(0x7c000000)
+
+ %90:gpr64 = PHI %40, %bb.2, %41, %bb.3, %42, %bb.4, %43, %bb.5, %44, %bb.6, %45, %bb.7, %47, %bb.8
+ %51:gpr32 = SUBSWrr %50, %17, implicit-def $nzcv
+ Bcc 1, %bb.2, implicit $nzcv
+ B %bb.10
+
+ bb.10:
+ %95:gpr64 = PHI %99, %bb.0, %90, %bb.9
+ $x0 = COPY %95
+ RET_ReallyLR implicit $x0
+...
+---
+name: cascade_branches_from_args_not_dd
+tracksRegLiveness: true
+liveins:
+ - { reg: '$x1', virtual-reg: '%1' }
+ - { reg: '$w2', virtual-reg: '%2' }
+ - { reg: '$w3', virtual-reg: '%3' }
+ - { reg: '$w4', virtual-reg: '%4' }
+ - { reg: '$w5', virtual-reg: '%5' }
+body: |
+ ; CHECK-LABEL: name: cascade_branches_from_args_not_dd
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.10(0x30000000), %bb.1(0x50000000)
+ ; CHECK-NEXT: liveins: $x1, $w2, $w3, $w4, $w5
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x1
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr32 = COPY $w2
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr32 = COPY $w3
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gpr32 = COPY $w4
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:gpr32 = COPY $w5
+ ; CHECK-NEXT: [[COPY5:%[0-9]+]]:gpr64all = COPY $xzr
+ ; CHECK-NEXT: CBZW [[COPY1]], %bb.10
+ ; CHECK-NEXT: B %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.2(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY6:%[0-9]+]]:gpr32 = COPY [[COPY2]]
+ ; CHECK-NEXT: [[COPY7:%[0-9]+]]:gpr32 = COPY [[COPY3]]
+ ; CHECK-NEXT: [[COPY8:%[0-9]+]]:gpr32 = COPY [[COPY4]]
+ ; CHECK-NEXT: [[COPY9:%[0-9]+]]:gpr32 = COPY [[COPY2]]
+ ; CHECK-NEXT: [[COPY10:%[0-9]+]]:gpr32 = COPY [[COPY3]]
+ ; CHECK-NEXT: [[COPY11:%[0-9]+]]:gpr32 = COPY [[COPY4]]
+ ; CHECK-NEXT: [[COPY12:%[0-9]+]]:gpr32 = COPY [[COPY2]]
+ ; CHECK-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: successors: %bb.3(0x40000000), %bb.9(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[PHI:%[0-9]+]]:gpr32 = PHI [[COPY1]], %bb.1, %15, %bb.9
+ ; CHECK-NEXT: [[MADDWrrr:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY2]], [[COPY4]], $wzr
+ ; CHECK-NEXT: [[MADDWrrr1:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[COPY4]], [[MADDWrrr]]
+ ; CHECK-NEXT: [[MADDWrrr2:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY2]], [[COPY3]], [[MADDWrrr1]]
+ ; CHECK-NEXT: [[MADDWrrr3:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[COPY2]], [[MADDWrrr2]]
+ ; CHECK-NEXT: [[MADDWrrr4:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY4]], [[COPY3]], [[MADDWrrr3]]
+ ; CHECK-NEXT: [[MADDWrrr5:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY2]], [[COPY4]], [[MADDWrrr4]]
+ ; CHECK-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr5]], [[COPY6]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 7
+ ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; CHECK-NEXT: B %bb.3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.3:
+ ; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.9(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr5]], [[COPY7]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 6
+ ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; CHECK-NEXT: B %bb.4
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.4:
+ ; CHECK-NEXT: successors: %bb.5(0x40000000), %bb.9(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[SUBSWrr2:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr5]], [[COPY8]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 5
+ ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; CHECK-NEXT: B %bb.5
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.5:
+ ; CHECK-NEXT: successors: %bb.6(0x40000000), %bb.9(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[SUBSWrr3:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr5]], [[COPY9]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 4
+ ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; CHECK-NEXT: B %bb.6
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.6:
+ ; CHECK-NEXT: successors: %bb.7(0x40000000), %bb.9(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[SUBSWrr4:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr5]], [[COPY10]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm4:%[0-9]+]]:gpr64 = MOVi64imm 3
+ ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; CHECK-NEXT: B %bb.7
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.7:
+ ; CHECK-NEXT: successors: %bb.8(0x40000000), %bb.9(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[SUBSWrr5:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr5]], [[COPY11]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm5:%[0-9]+]]:gpr64 = MOVi64imm 2
+ ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; CHECK-NEXT: B %bb.8
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.8:
+ ; CHECK-NEXT: successors: %bb.9(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[MOVi64imm6:%[0-9]+]]:gpr64 = MOVi64imm 1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.9:
+ ; CHECK-NEXT: successors: %bb.10(0x04000000), %bb.2(0x7c000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[PHI1:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.2, [[MOVi64imm1]], %bb.3, [[MOVi64imm2]], %bb.4, [[MOVi64imm3]], %bb.5, [[MOVi64imm4]], %bb.6, [[MOVi64imm5]], %bb.7, [[MOVi64imm6]], %bb.8
+ ; CHECK-NEXT: [[SUBSWrr6:%[0-9]+]]:gpr32 = SUBSWrr [[PHI]], [[MOVi32imm]], implicit-def $nzcv
+ ; CHECK-NEXT: Bcc 1, %bb.2, implicit $nzcv
+ ; CHECK-NEXT: B %bb.10
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.10:
+ ; CHECK-NEXT: [[PHI2:%[0-9]+]]:gpr64 = PHI [[COPY5]], %bb.0, [[PHI1]], %bb.9
+ ; CHECK-NEXT: $x0 = COPY [[PHI2]]
+ ; CHECK-NEXT: RET_ReallyLR implicit $x0
+ bb.0:
+ successors: %bb.10(0x30000000), %bb.1(0x50000000)
+ liveins: $x1, $w2, $w3, $w4, $w5
+
+ %1:gpr64common = COPY $x1
+ %2:gpr32 = COPY $w2
+ %3:gpr32 = COPY $w3
+ %4:gpr32 = COPY $w4
+ %5:gpr32 = COPY $w5
+ %99:gpr64all = COPY $xzr
+ CBZW %2, %bb.10
+ B %bb.1
+
+ bb.1:
+ successors: %bb.2(0x80000000)
+
+ ; Thresholds from registers, not memory loads
+ %10:gpr32 = COPY %3
+ %11:gpr32 = COPY %4
+ %12:gpr32 = COPY %5
+ %13:gpr32 = COPY %3
+ %14:gpr32 = COPY %4
+ %15:gpr32 = COPY %5
+ %16:gpr32 = COPY %3
+ %17:gpr32 = MOVi32imm 1
+
+ ; Cascade head: NOT data-dependent (computed from register args, no loads)
+ bb.2:
+ successors: %bb.3(0x40000000), %bb.9(0x40000000)
+
+ %50:gpr32 = PHI %2, %bb.1, %51, %bb.9
+ %31:gpr32 = nsw MADDWrrr %3, %5, $wzr
+ %32:gpr32 = nsw MADDWrrr %4, %5, %31
+ %33:gpr32 = nsw MADDWrrr %3, %4, %32
+ %34:gpr32 = nsw MADDWrrr %4, %3, %33
+ %35:gpr32 = nsw MADDWrrr %5, %4, %34
+ %30:gpr32 = nsw MADDWrrr %3, %5, %35
+ %21:gpr32 = SUBSWrr %30, %10, implicit-def $nzcv
+ %40:gpr64 = MOVi64imm 7
+ Bcc 10, %bb.9, implicit $nzcv
+ B %bb.3
+
+ bb.3:
+ successors: %bb.4(0x40000000), %bb.9(0x40000000)
+
+ %22:gpr32 = SUBSWrr %30, %11, implicit-def $nzcv
+ %41:gpr64 = MOVi64imm 6
+ Bcc 10, %bb.9, implicit $nzcv
+ B %bb.4
+
+ bb.4:
+ successors: %bb.5(0x40000000), %bb.9(0x40000000)
+
+ %23:gpr32 = SUBSWrr %30, %12, implicit-def $nzcv
+ %42:gpr64 = MOVi64imm 5
+ Bcc 10, %bb.9, implicit $nzcv
+ B %bb.5
+
+ bb.5:
+ successors: %bb.6(0x40000000), %bb.9(0x40000000)
+
+ %24:gpr32 = SUBSWrr %30, %13, implicit-def $nzcv
+ %43:gpr64 = MOVi64imm 4
+ Bcc 10, %bb.9, implicit $nzcv
+ B %bb.6
+
+ bb.6:
+ successors: %bb.7(0x40000000), %bb.9(0x40000000)
+
+ %25:gpr32 = SUBSWrr %30, %14, implicit-def $nzcv
+ %44:gpr64 = MOVi64imm 3
+ Bcc 10, %bb.9, implicit $nzcv
+ B %bb.7
+
+ bb.7:
+ successors: %bb.8(0x40000000), %bb.9(0x40000000)
+
+ %26:gpr32 = SUBSWrr %30, %15, implicit-def $nzcv
+ %45:gpr64 = MOVi64imm 2
+ Bcc 10, %bb.9, implicit $nzcv
+ B %bb.8
+
+ bb.8:
+ successors: %bb.9(0x80000000)
+
+ %47:gpr64 = MOVi64imm 1
+
+ bb.9:
+ successors: %bb.10(0x04000000), %bb.2(0x7c000000)
+
+ %90:gpr64 = PHI %40, %bb.2, %41, %bb.3, %42, %bb.4, %43, %bb.5, %44, %bb.6, %45, %bb.7, %47, %bb.8
+ %51:gpr32 = SUBSWrr %50, %17, implicit-def $nzcv
+ Bcc 1, %bb.2, implicit $nzcv
+ B %bb.10
+
+ bb.10:
+ %95:gpr64 = PHI %99, %bb.0, %90, %bb.9
+ $x0 = COPY %95
+ RET_ReallyLR implicit $x0
+...
+---
+name: cascade_exceeds_crit_limit
+tracksRegLiveness: true
+liveins:
+ - { reg: '$x0', virtual-reg: '%0' }
+ - { reg: '$x1', virtual-reg: '%1' }
+ - { reg: '$w2', virtual-reg: '%2' }
+ - { reg: '$w3', virtual-reg: '%3' }
+ - { reg: '$w4', virtual-reg: '%4' }
+body: |
+ ; CHECK-LABEL: name: cascade_exceeds_crit_limit
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.11(0x30000000), %bb.1(0x50000000)
+ ; CHECK-NEXT: liveins: $x0, $x1, $w2, $w3, $w4
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr32 = COPY $w2
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gpr32 = COPY $w3
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:gpr32 = COPY $w4
+ ; CHECK-NEXT: [[COPY5:%[0-9]+]]:gpr64all = COPY $xzr
+ ; CHECK-NEXT: CBZW [[COPY2]], %bb.11
+ ; CHECK-NEXT: B %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.2(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 0 :: (load (s32))
+ ; CHECK-NEXT: [[LDRWui1:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 1 :: (load (s32))
+ ; CHECK-NEXT: [[LDRWui2:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 2 :: (load (s32))
+ ; CHECK-NEXT: [[LDRWui3:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 3 :: (load (s32))
+ ; CHECK-NEXT: [[LDRWui4:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 4 :: (load (s32))
+ ; CHECK-NEXT: [[LDRWui5:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 5 :: (load (s32))
+ ; CHECK-NEXT: [[LDRWui6:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 6 :: (load (s32))
+ ; CHECK-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: successors: %bb.3(0x40000000), %bb.10(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[PHI:%[0-9]+]]:gpr32 = PHI [[COPY2]], %bb.1, %15, %bb.10
+ ; CHECK-NEXT: [[LDRWui7:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load (s32))
+ ; CHECK-NEXT: [[MADDWrrr:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[LDRWui7]], $wzr
+ ; CHECK-NEXT: [[MADDWrrr1:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY4]], [[LDRWui7]], [[MADDWrrr]]
+ ; CHECK-NEXT: [[MADDWrrr2:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[LDRWui7]], [[MADDWrrr1]]
+ ; CHECK-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 7
+ ; CHECK-NEXT: Bcc 10, %bb.10, implicit $nzcv
+ ; CHECK-NEXT: B %bb.3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.3:
+ ; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.10(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui1]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 6
+ ; CHECK-NEXT: Bcc 10, %bb.10, implicit $nzcv
+ ; CHECK-NEXT: B %bb.4
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.4:
+ ; CHECK-NEXT: successors: %bb.6(0x40000000), %bb.5(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[SUBSWrr2:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui2]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 5
+ ; CHECK-NEXT: Bcc 10, %bb.5, implicit $nzcv
+ ; CHECK-NEXT: B %bb.6
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.5:
+ ; CHECK-NEXT: successors: %bb.10(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: B %bb.10
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.6:
+ ; CHECK-NEXT: successors: %bb.7(0x40000000), %bb.10(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[SUBSWrr3:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui3]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 4
+ ; CHECK-NEXT: Bcc 10, %bb.10, implicit $nzcv
+ ; CHECK-NEXT: B %bb.7
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.7:
+ ; CHECK-NEXT: successors: %bb.8(0x40000000), %bb.10(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[SUBSWrr4:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui4]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm4:%[0-9]+]]:gpr64 = MOVi64imm 3
+ ; CHECK-NEXT: Bcc 10, %bb.10, implicit $nzcv
+ ; CHECK-NEXT: B %bb.8
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.8:
+ ; CHECK-NEXT: successors: %bb.9(0x40000000), %bb.10(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[SUBSWrr5:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui5]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm5:%[0-9]+]]:gpr64 = MOVi64imm 2
+ ; CHECK-NEXT: Bcc 10, %bb.10, implicit $nzcv
+ ; CHECK-NEXT: B %bb.9
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.9:
+ ; CHECK-NEXT: successors: %bb.10(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[MOVi64imm6:%[0-9]+]]:gpr64 = MOVi64imm 1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.10:
+ ; CHECK-NEXT: successors: %bb.11(0x04000000), %bb.2(0x7c000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[PHI1:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.2, [[MOVi64imm1]], %bb.3, [[MOVi64imm2]], %bb.5, [[MOVi64imm3]], %bb.6, [[MOVi64imm4]], %bb.7, [[MOVi64imm5]], %bb.8, [[MOVi64imm6]], %bb.9
+ ; CHECK-NEXT: [[SUBSWrr6:%[0-9]+]]:gpr32 = SUBSWrr [[PHI]], [[MOVi32imm]], implicit-def $nzcv
+ ; CHECK-NEXT: Bcc 1, %bb.2, implicit $nzcv
+ ; CHECK-NEXT: B %bb.11
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.11:
+ ; CHECK-NEXT: [[PHI2:%[0-9]+]]:gpr64 = PHI [[COPY5]], %bb.0, [[PHI1]], %bb.10
+ ; CHECK-NEXT: $x0 = COPY [[PHI2]]
+ ; CHECK-NEXT: RET_ReallyLR implicit $x0
+ bb.0:
+ successors: %bb.11(0x30000000), %bb.1(0x50000000)
+ liveins: $x0, $x1, $w2, $w3, $w4
+
+ %0:gpr64common = COPY $x0
+ %1:gpr64common = COPY $x1
+ %2:gpr32 = COPY $w2
+ %3:gpr32 = COPY $w3
+ %4:gpr32 = COPY $w4
+ %99:gpr64all = COPY $xzr
+ CBZW %2, %bb.11
+ B %bb.1
+
+ bb.1:
+ successors: %bb.2(0x80000000)
+
+ %10:gpr32 = LDRWui %1, 0 :: (load (s32))
+ %11:gpr32 = LDRWui %1, 1 :: (load (s32))
+ %12:gpr32 = LDRWui %1, 2 :: (load (s32))
+ %13:gpr32 = LDRWui %1, 3 :: (load (s32))
+ %14:gpr32 = LDRWui %1, 4 :: (load (s32))
+ %15:gpr32 = LDRWui %1, 5 :: (load (s32))
+ %16:gpr32 = LDRWui %1, 6 :: (load (s32))
+ %17:gpr32 = MOVi32imm 1
+
+ ; Cascade head: data-dependent
+ bb.2:
+ successors: %bb.3(0x40000000), %bb.10(0x40000000)
+
+ %50:gpr32 = PHI %2, %bb.1, %51, %bb.10
+ %20:gpr32 = LDRWui %0, 0 :: (load (s32))
+ %31:gpr32 = nsw MADDWrrr %3, %20, $wzr
+ %32:gpr32 = nsw MADDWrrr %4, %20, %31
+ %30:gpr32 = nsw MADDWrrr %3, %20, %32
+ %21:gpr32 = SUBSWrr %30, %10, implicit-def $nzcv
+ %40:gpr64 = MOVi64imm 7
+ Bcc 10, %bb.10, implicit $nzcv
+ B %bb.3
+
+ bb.3:
+ successors: %bb.4(0x40000000), %bb.10(0x40000000)
+
+ %22:gpr32 = SUBSWrr %30, %11, implicit-def $nzcv
+ %41:gpr64 = MOVi64imm 6
+ Bcc 10, %bb.10, implicit $nzcv
+ B %bb.4
+
+ ; bb.4 branches to bb.11 (not directly to tail) - breaks cascade
+ bb.4:
+ successors: %bb.6(0x40000000), %bb.5(0x40000000)
+
+ %23:gpr32 = SUBSWrr %30, %12, implicit-def $nzcv
+ %42:gpr64 = MOVi64imm 5
+ Bcc 10, %bb.5, implicit $nzcv
+ B %bb.6
+
+ ; Extra block that breaks the cascade chain
+ bb.5:
+ successors: %bb.10(0x80000000)
+
+ B %bb.10
+
+ bb.6:
+ successors: %bb.7(0x40000000), %bb.10(0x40000000)
+
+ %24:gpr32 = SUBSWrr %30, %13, implicit-def $nzcv
+ %43:gpr64 = MOVi64imm 4
+ Bcc 10, %bb.10, implicit $nzcv
+ B %bb.7
+
+ bb.7:
+ successors: %bb.8(0x40000000), %bb.10(0x40000000)
+
+ %25:gpr32 = SUBSWrr %30, %14, implicit-def $nzcv
+ %44:gpr64 = MOVi64imm 3
+ Bcc 10, %bb.10, implicit $nzcv
+ B %bb.8
+
+ bb.8:
+ successors: %bb.9(0x40000000), %bb.10(0x40000000)
+
+ %26:gpr32 = SUBSWrr %30, %15, implicit-def $nzcv
+ %45:gpr64 = MOVi64imm 2
+ Bcc 10, %bb.10, implicit $nzcv
+ B %bb.9
+
+ bb.9:
+ successors: %bb.10(0x80000000)
+
+ %47:gpr64 = MOVi64imm 1
+
+ bb.10:
+ successors: %bb.11(0x04000000), %bb.2(0x7c000000)
+
+ %90:gpr64 = PHI %40, %bb.2, %41, %bb.3, %42, %bb.5, %43, %bb.6, %44, %bb.7, %45, %bb.8, %47, %bb.9
+ %51:gpr32 = SUBSWrr %50, %17, implicit-def $nzcv
+ Bcc 1, %bb.2, implicit $nzcv
+ B %bb.11
+
+ bb.11:
+ %95:gpr64 = PHI %99, %bb.0, %90, %bb.10
+ $x0 = COPY %95
+ RET_ReallyLR implicit $x0
+...
+---
+name: cascade_backedge_to_earlier_block
+alignment: 4
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: cascade_backedge_to_earlier_block
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: liveins: $x0, $x1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.2(0x40000000), %bb.5(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load (s32))
+ ; CHECK-NEXT: [[LDRWui1:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 0 :: (load (s32))
+ ; CHECK-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui1]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 100
+ ; CHECK-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 200
+ ; CHECK-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 400
+ ; CHECK-NEXT: Bcc 10, %bb.5, implicit $nzcv
+ ; CHECK-NEXT: B %bb.2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: successors: %bb.3(0x40000000), %bb.5(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[LDRWui2:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 1 :: (load (s32))
+ ; CHECK-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui2]], implicit-def $nzcv
+ ; CHECK-NEXT: Bcc 10, %bb.5, implicit $nzcv
+ ; CHECK-NEXT: B %bb.3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.3:
+ ; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[LDRWui3:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 2 :: (load (s32))
+ ; CHECK-NEXT: [[SUBSWrr2:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui3]], implicit-def $nzcv
+ ; CHECK-NEXT: Bcc 10, %bb.2, implicit $nzcv
+ ; CHECK-NEXT: B %bb.4
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.4:
+ ; CHECK-NEXT: successors: %bb.5(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.5:
+ ; CHECK-NEXT: [[PHI:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.1, [[MOVi64imm1]], %bb.2, [[MOVi64imm2]], %bb.4
+ ; CHECK-NEXT: $x0 = COPY [[PHI]]
+ ; CHECK-NEXT: RET_ReallyLR implicit $x0
+ bb.0:
+ successors: %bb.1(0x80000000)
+ liveins: $x0, $x1
+
+ %0:gpr64common = COPY $x0
+ %1:gpr64common = COPY $x1
+ bb.1:
+ successors: %bb.2(0x40000000), %bb.5(0x40000000)
+
+ %10:gpr32 = LDRWui %0, 0 :: (load (s32))
+ %11:gpr32 = LDRWui %1, 0 :: (load (s32))
+ %12:gpr32 = SUBSWrr %10, %11, implicit-def $nzcv
+ %40:gpr64 = MOVi64imm 100
+ %41:gpr64 = MOVi64imm 200
+ %43:gpr64 = MOVi64imm 400
+ Bcc 10, %bb.5, implicit $nzcv
+ B %bb.2
+ bb.2:
+ successors: %bb.3(0x40000000), %bb.5(0x40000000)
+
+ %20:gpr32 = LDRWui %1, 1 :: (load (s32))
+ %21:gpr32 = SUBSWrr %10, %20, implicit-def $nzcv
+ Bcc 10, %bb.5, implicit $nzcv
+ B %bb.3
+
+ bb.3:
+ successors: %bb.4(0x40000000), %bb.2(0x40000000)
+
+ %30:gpr32 = LDRWui %1, 2 :: (load (s32))
+ %31:gpr32 = SUBSWrr %10, %30, implicit-def $nzcv
+ ; Branches to bb.2 instead of bb.5 (Tail) - creates back-edge
+ Bcc 10, %bb.2, implicit $nzcv
+ B %bb.4
+
+ bb.4:
+ successors: %bb.5(0x80000000)
+
+ bb.5:
+ %90:gpr64 = PHI %40, %bb.1, %41, %bb.2, %43, %bb.4
+ $x0 = COPY %90
+ RET_ReallyLR implicit $x0
+...
+---
+name: cascade_forward_skip_to_later_block
+alignment: 4
+tracksRegLiveness: true
+liveins:
+ - { reg: '$x0', virtual-reg: '%0' }
+ - { reg: '$x1', virtual-reg: '%1' }
+body: |
+ ; CHECK-LABEL: name: cascade_forward_skip_to_later_block
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: liveins: $x0, $x1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.2(0x40000000), %bb.6(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load (s32))
+ ; CHECK-NEXT: [[LDRWui1:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 0 :: (load (s32))
+ ; CHECK-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui1]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 100
+ ; CHECK-NEXT: Bcc 10, %bb.6, implicit $nzcv
+ ; CHECK-NEXT: B %bb.2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: successors: %bb.3(0x40000000), %bb.6(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[LDRWui2:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 1 :: (load (s32))
+ ; CHECK-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui2]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 200
+ ; CHECK-NEXT: Bcc 10, %bb.6, implicit $nzcv
+ ; CHECK-NEXT: B %bb.3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.3:
+ ; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.5(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[LDRWui3:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 2 :: (load (s32))
+ ; CHECK-NEXT: [[SUBSWrr2:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui3]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 300
+ ; CHECK-NEXT: Bcc 10, %bb.5, implicit $nzcv
+ ; CHECK-NEXT: B %bb.4
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.4:
+ ; CHECK-NEXT: successors: %bb.5(0x40000000), %bb.6(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[LDRWui4:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 3 :: (load (s32))
+ ; CHECK-NEXT: [[SUBSWrr3:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui4]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 400
+ ; CHECK-NEXT: Bcc 10, %bb.6, implicit $nzcv
+ ; CHECK-NEXT: B %bb.5
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.5:
+ ; CHECK-NEXT: successors: %bb.6(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[MOVi64imm4:%[0-9]+]]:gpr64 = MOVi64imm 500
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.6:
+ ; CHECK-NEXT: [[PHI:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.1, [[MOVi64imm1]], %bb.2, [[MOVi64imm3]], %bb.4, [[MOVi64imm4]], %bb.5
+ ; CHECK-NEXT: $x0 = COPY [[PHI]]
+ ; CHECK-NEXT: RET_ReallyLR implicit $x0
+ bb.0:
+ successors: %bb.1(0x80000000)
+ liveins: $x0, $x1
+
+ %0:gpr64common = COPY $x0
+ %1:gpr64common = COPY $x1
+
+ bb.1:
+ successors: %bb.2(0x40000000), %bb.6(0x40000000)
+
+ %10:gpr32 = LDRWui %0, 0 :: (load (s32))
+ %11:gpr32 = LDRWui %1, 0 :: (load (s32))
+ %12:gpr32 = SUBSWrr %10, %11, implicit-def $nzcv
+ %40:gpr64 = MOVi64imm 100
+ Bcc 10, %bb.6, implicit $nzcv
+ B %bb.2
+
+ bb.2:
+ successors: %bb.3(0x40000000), %bb.6(0x40000000)
+
+ %20:gpr32 = LDRWui %1, 1 :: (load (s32))
+ %21:gpr32 = SUBSWrr %10, %20, implicit-def $nzcv
+ %41:gpr64 = MOVi64imm 200
+ Bcc 10, %bb.6, implicit $nzcv
+ B %bb.3
+
+ bb.3:
+ successors: %bb.4(0x40000000), %bb.5(0x40000000)
+
+ %30:gpr32 = LDRWui %1, 2 :: (load (s32))
+ %31:gpr32 = SUBSWrr %10, %30, implicit-def $nzcv
+ %42:gpr64 = MOVi64imm 300
+ ; Branches to bb.5 instead of bb.6 (Tail) - forward skip
+ Bcc 10, %bb.5, implicit $nzcv
+ B %bb.4
+
+ bb.4:
+ successors: %bb.5(0x40000000), %bb.6(0x40000000)
+
+ %32:gpr32 = LDRWui %1, 3 :: (load (s32))
+ %33:gpr32 = SUBSWrr %10, %32, implicit-def $nzcv
+ %43:gpr64 = MOVi64imm 400
+ Bcc 10, %bb.6, implicit $nzcv
+ B %bb.5
+
+ bb.5:
+ successors: %bb.6(0x80000000)
+
+ %44:gpr64 = MOVi64imm 500
+
+ bb.6:
+ %90:gpr64 = PHI %40, %bb.1, %41, %bb.2, %43, %bb.4, %44, %bb.5
+ $x0 = COPY %90
+ RET_ReallyLR implicit $x0
+...
+
diff --git a/llvm/test/CodeGen/AArch64/early-ifcvt-remarks.mir b/llvm/test/CodeGen/AArch64/early-ifcvt-remarks.mir
index 93460d5822ab4..6f01d48601fa2 100644
--- a/llvm/test/CodeGen/AArch64/early-ifcvt-remarks.mir
+++ b/llvm/test/CodeGen/AArch64/early-ifcvt-remarks.mir
@@ -11,6 +11,12 @@
define i32 @data_dependent_branch(ptr %p, i32 %x, i32 %y) {
ret i32 0
}
+ define i64 @cascade_success(ptr %p, ptr %q, i32 %n) {
+ ret i64 0
+ }
+ define i64 @cascade_increasing_cond_depth(ptr %p) {
+ ret i64 0
+ }
...
---
name: data_dependent_branch
@@ -43,3 +49,156 @@ body: |
$w0 = COPY %100
RET_ReallyLR implicit $w0
...
+---
+name: cascade_success
+alignment: 4
+tracksRegLiveness: true
+liveins:
+ - { reg: '$x0', virtual-reg: '%0' }
+ - { reg: '$x1', virtual-reg: '%1' }
+ - { reg: '$w2', virtual-reg: '%2' }
+body: |
+ ; Entry - check loop count
+ bb.0:
+ successors: %bb.6(0x30000000), %bb.1(0x50000000)
+ liveins: $x0, $x1, $w2
+
+ %0:gpr64common = COPY $x0
+ %1:gpr64common = COPY $x1
+ %2:gpr32 = COPY $w2
+ %50:gpr64all = COPY $xzr
+ %51:gpr64all = COPY %50
+ CBZW %2, %bb.6
+ B %bb.1
+
+ ; Loop header / cascade head - ALL loads here, branches to bb.2 or bb.5
+ bb.1:
+ successors: %bb.2(0x40000000), %bb.5(0x40000000)
+
+ %60:gpr64 = PHI %51, %bb.0, %81, %bb.5
+ %61:gpr32 = PHI %2, %bb.0, %80, %bb.5
+ ; Load ALL data in head - makes branch data-dependent
+ %62:gpr32 = LDRWui %0, 0 :: (load (s32))
+ %63:gpr32 = LDRWui %1, 0 :: (load (s32))
+ %66:gpr32 = LDRWui %1, 1 :: (load (s32))
+ %69:gpr32 = LDRWui %1, 2 :: (load (s32))
+ %90:gpr32 = LDRWui %1, 3 :: (load (s32))
+ %64:gpr64 = MOVi64imm 4
+ %65:gpr32 = SUBSWrr %62, %63, implicit-def $nzcv
+ Bcc 10, %bb.5, implicit $nzcv
+ B %bb.2
+
+ ; First cascade block (branches to bb.3 or bb.5)
+ bb.2:
+ successors: %bb.3(0x40000000), %bb.5(0x40000000)
+
+ %67:gpr32 = SUBSWrr %62, %66, implicit-def $nzcv
+ %68:gpr64 = MOVi64imm 3
+ Bcc 10, %bb.5, implicit $nzcv
+ B %bb.3
+
+ ; Second cascade block (branches to bb.4 or bb.5)
+ bb.3:
+ successors: %bb.4(0x40000000), %bb.5(0x40000000)
+
+ %70:gpr32 = SUBSWrr %62, %69, implicit-def $nzcv
+ %91:gpr64 = MOVi64imm 2
+ Bcc 10, %bb.5, implicit $nzcv
+ B %bb.4
+
+ ; Third cascade block (only branches to bb.5)
+ bb.4:
+ successors: %bb.5(0x80000000)
+
+ %92:gpr32 = SUBSWrr %62, %90, implicit-def $nzcv
+ %71:gpr32 = CSINCWr $wzr, $wzr, 11, implicit $nzcv
+ %72:gpr64 = SUBREG_TO_REG %71, %subreg.sub_32
+
+ ; Loop latch / cascade tail - PHI merge and back edge
+ bb.5:
+ successors: %bb.6(0x04000000), %bb.1(0x7c000000)
+
+ %73:gpr64 = PHI %64, %bb.1, %68, %bb.2, %91, %bb.3, %72, %bb.4
+ %74:gpr64 = ADDXrr %60, %73
+ %75:gpr32 = MOVi32imm 1
+ %80:gpr32 = SUBSWrr %61, %75, implicit-def $nzcv
+ %81:gpr64all = COPY %74
+ Bcc 1, %bb.1, implicit $nzcv
+ B %bb.6
+
+ ; Exit
+ bb.6:
+ %82:gpr64 = PHI %51, %bb.0, %81, %bb.5
+ $x0 = COPY %82
+ RET_ReallyLR implicit $x0
+...
+---
+name: cascade_increasing_cond_depth
+alignment: 4
+tracksRegLiveness: true
+liveins:
+ - { reg: '$x0', virtual-reg: '%0' }
+body: |
+ bb.0:
+ successors: %bb.1(0x80000000)
+ liveins: $x0
+
+ %0:gpr64common = COPY $x0
+
+ bb.1:
+ successors: %bb.2(0x40000000), %bb.7(0x40000000)
+
+ %10:gpr32 = LDRWui %0, 0 :: (load (s32))
+ %11:gpr32 = MADDWrrr %10, %10, $wzr
+ %12:gpr32 = MADDWrrr %11, %11, $wzr
+ %20:gpr32 = ADDWrr %12, %12
+ %21:gpr32 = ADDWrr %20, %20
+ %22:gpr32 = ADDWrr %21, %21
+ %23:gpr32 = ADDWrr %22, %22
+ %40:gpr64 = MOVi64imm 100
+ %30:gpr32 = SUBSWrr %12, %12, implicit-def $nzcv
+ Bcc 10, %bb.7, implicit $nzcv
+ B %bb.2
+
+ bb.2:
+ successors: %bb.3(0x40000000), %bb.7(0x40000000)
+
+ %41:gpr64 = MOVi64imm 200
+ %31:gpr32 = SUBSWrr %20, %20, implicit-def $nzcv
+ Bcc 10, %bb.7, implicit $nzcv
+ B %bb.3
+
+ bb.3:
+ successors: %bb.4(0x40000000), %bb.7(0x40000000)
+
+ %42:gpr64 = MOVi64imm 300
+ %32:gpr32 = SUBSWrr %21, %21, implicit-def $nzcv
+ Bcc 10, %bb.7, implicit $nzcv
+ B %bb.4
+
+ bb.4:
+ successors: %bb.5(0x40000000), %bb.7(0x40000000)
+
+ %43:gpr64 = MOVi64imm 400
+ %33:gpr32 = SUBSWrr %22, %22, implicit-def $nzcv
+ Bcc 10, %bb.7, implicit $nzcv
+ B %bb.5
+
+ bb.5:
+ successors: %bb.6(0x40000000), %bb.7(0x40000000)
+
+ %44:gpr64 = MOVi64imm 500
+ %34:gpr32 = SUBSWrr %23, %23, implicit-def $nzcv
+ Bcc 10, %bb.7, implicit $nzcv
+ B %bb.6
+
+ bb.6:
+ successors: %bb.7(0x80000000)
+
+ %45:gpr64 = MOVi64imm 600
+
+ bb.7:
+ %90:gpr64 = PHI %40, %bb.1, %41, %bb.2, %42, %bb.3, %43, %bb.4, %44, %bb.5, %45, %bb.6
+ $x0 = COPY %90
+ RET_ReallyLR implicit $x0
+...
>From 05249d5fdfbbde1e313207eee2f4ec5734e7b047 Mon Sep 17 00:00:00 2001
From: Jonathan Cohen <joncoh at apple.com>
Date: Thu, 16 Apr 2026 16:21:31 +0300
Subject: [PATCH 2/2] [EarlyIfConvert] Add heuristic to convert cascades of
conditional branches
Recognize and convert cascade patterns: chains of conditional branches where each block branches to either the next block or a shared tail. These patterns arise from if-else-if chains and switch-like constructs, and may suffer from serial misprediction when the branch conditions do not follow any predictable pattern.
When enabled via -enable-early-ifcvt-cascade, the pass walks up from single-successor blocks to identify cascades, validates each block for speculative execution, and converts the entire chain if profitable.
The profitability model requires all cascade branches to be data-dependent (conditions which have recently been loaded from memory and therefore may not be correlated to other branches, see PR #174457), checks that sufficient ILP exists to hide the speculated instructions, and bounds the CSEL chain depth against a scaled misprediction penalty.
---
llvm/lib/CodeGen/EarlyIfConversion.cpp | 658 +++++++-
.../CodeGen/AArch64/early-ifcvt-cascade.mir | 1320 ++++++++++++-----
.../early-ifcvt-load-to-cond-br-limit.mir | 577 +++++++
.../CodeGen/AArch64/early-ifcvt-remarks.mir | 97 ++
4 files changed, 2184 insertions(+), 468 deletions(-)
diff --git a/llvm/lib/CodeGen/EarlyIfConversion.cpp b/llvm/lib/CodeGen/EarlyIfConversion.cpp
index bf2664fd4225c..af2323a7cc0ee 100644
--- a/llvm/lib/CodeGen/EarlyIfConversion.cpp
+++ b/llvm/lib/CodeGen/EarlyIfConversion.cpp
@@ -16,11 +16,16 @@
//===----------------------------------------------------------------------===//
#include "llvm/CodeGen/EarlyIfConversion.h"
+#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/BitVector.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/DepthFirstIterator.h"
#include "llvm/ADT/PostOrderIterator.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/SparseSet.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
@@ -65,6 +70,16 @@ static cl::opt<bool> EnableDataDependentBranchAnalysis(
"enable-early-ifcvt-data-dependent", cl::Hidden, cl::init(false),
cl::desc("Enable hard-to-predict branch analysis for if-conversion"));
+// Enable recognition of cascade if-else patterns (chains of triangles).
+static cl::opt<bool> EnableCascadeIfConv(
+ "enable-early-ifcvt-cascade", cl::Hidden, cl::init(false),
+ cl::desc("Enable recognition of cascade if-else patterns"));
+
+// Maximum depth for cascade patterns.
+static cl::opt<unsigned>
+ MaxCascadeDepth("early-ifcvt-max-cascade-depth", cl::Hidden, cl::init(8),
+ cl::desc("Maximum cascade depth for early if-conversion"));
+
// Limit the number steps we take when searching conditions that depend on
// values recently loaded from memory.
static cl::opt<unsigned>
@@ -86,6 +101,8 @@ STATISTIC(NumTrianglesConv, "Number of triangles converted");
STATISTIC(NumDataDependant,
"Number of data dependent conditional branches encountered");
STATISTIC(NumLikelyBiased, "Number of branches with a hot path encountered");
+STATISTIC(NumCascadesSeen, "Number of cascade patterns detected");
+STATISTIC(NumCascadesConv, "Number of cascade patterns converted");
//===----------------------------------------------------------------------===//
// SSAIfConv
@@ -109,6 +126,20 @@ STATISTIC(NumLikelyBiased, "Number of branches with a hot path encountered");
// Head block, and phis in the Tail block are converted to select instructions.
//
namespace {
+
+/// What one cascade collapse step would do to one Tail phi.
+struct CascadeSelectInfo {
+ int CondCycles = 0;
+ bool NeedsSelect = false;
+};
+
+struct CascadeResult {
+ MachineBasicBlock *Head = nullptr;
+ SmallVector<MachineBasicBlock *> Blocks;
+ SmallVector<SmallVector<CascadeSelectInfo, 4>> Selects;
+ explicit operator bool() const { return Head != nullptr; }
+};
+
class SSAIfConv {
const TargetInstrInfo *TII;
const TargetRegisterInfo *TRI;
@@ -213,12 +244,36 @@ class SSAIfConv {
/// initialize the internal state, and return true.
/// If predicate is set try to predicate the block otherwise try to
/// speculatively execute it.
- bool canConvertIf(MachineBasicBlock *MBB, bool Predicate = false);
+ bool canConvertIf(MachineBasicBlock *MBB, bool Predicate = false,
+ bool AllowMultiSuccTBB = false,
+ MachineBasicBlock *ExplicitTail = nullptr);
/// convertIf - If-convert the last block passed to canConvertIf(), assuming
/// it is possible. Add any blocks that are to be erased to RemoveBlocks.
void convertIf(SmallVectorImpl<MachineBasicBlock *> &RemoveBlocks,
bool Predicate = false);
+
+ /// matchCascade - match a cascade of conditional branches by walking up
+ /// from MBB as a potential cascade end. A cascade is a chain of triangles
+ /// where each block has 2 successors (next cascade block + Tail) and the
+ /// last block has 1 successor (Tail). Returns the discovered Head and list
+ /// of cascade blocks [BB1, ..., BBn] if valid, empty result otherwise.
+ ///
+ /// Head --------+
+ /// | |
+ /// v |
+ /// BB1 ----------+
+ /// | |
+ /// v |
+ /// BB2 ----------+
+ /// | |
+ /// ...
+ /// ...
+ /// | |
+ /// v v
+ /// BBn (MBB) -> Tail
+ ///
+ CascadeResult matchCascade(MachineBasicBlock *MBB);
};
} // end anonymous namespace
@@ -469,7 +524,9 @@ bool SSAIfConv::findInsertionPoint() {
/// canConvertIf - analyze the sub-cfg rooted in MBB, and return true if it is
/// a potential candidate for if-conversion. Fill out the internal state.
///
-bool SSAIfConv::canConvertIf(MachineBasicBlock *MBB, bool Predicate) {
+bool SSAIfConv::canConvertIf(MachineBasicBlock *MBB, bool Predicate,
+ bool AllowMultiSuccTBB,
+ MachineBasicBlock *ExplicitTail) {
Head = MBB;
TBB = FBB = Tail = nullptr;
@@ -482,31 +539,40 @@ bool SSAIfConv::canConvertIf(MachineBasicBlock *MBB, bool Predicate) {
if (Succ0->pred_size() != 1)
std::swap(Succ0, Succ1);
- if (Succ0->pred_size() != 1 || Succ0->succ_size() != 1)
+ if (Succ0->pred_size() != 1 ||
+ (!AllowMultiSuccTBB && Succ0->succ_size() != 1))
return false;
- Tail = Succ0->succ_begin()[0];
-
- // This is not a triangle.
- if (Tail != Succ1) {
- // Check for a diamond. We won't deal with any critical edges.
- if (Succ1->pred_size() != 1 || Succ1->succ_size() != 1 ||
- Succ1->succ_begin()[0] != Tail)
- return false;
- LLVM_DEBUG(dbgs() << "\nDiamond: " << printMBBReference(*Head) << " -> "
- << printMBBReference(*Succ0) << "/"
- << printMBBReference(*Succ1) << " -> "
+ // Use explicit tail if provided (for cascade validation), otherwise compute.
+ if (ExplicitTail) {
+ Tail = ExplicitTail;
+ LLVM_DEBUG(dbgs() << "\nCascade triangle: " << printMBBReference(*Head)
+ << " -> " << printMBBReference(*Succ0) << " -> "
<< printMBBReference(*Tail) << '\n');
-
- // Live-in physregs are tricky to get right when speculating code.
- if (!Tail->livein_empty()) {
- LLVM_DEBUG(dbgs() << "Tail has live-ins.\n");
- return false;
- }
} else {
- LLVM_DEBUG(dbgs() << "\nTriangle: " << printMBBReference(*Head) << " -> "
- << printMBBReference(*Succ0) << " -> "
- << printMBBReference(*Tail) << '\n');
+ Tail = Succ0->succ_begin()[0];
+
+ // This is not a triangle.
+ if (Tail != Succ1) {
+ // Check for a diamond. We won't deal with any critical edges.
+ if (Succ1->pred_size() != 1 || Succ1->succ_size() != 1 ||
+ Succ1->succ_begin()[0] != Tail)
+ return false;
+ LLVM_DEBUG(dbgs() << "\nDiamond: " << printMBBReference(*Head) << " -> "
+ << printMBBReference(*Succ0) << "/"
+ << printMBBReference(*Succ1) << " -> "
+ << printMBBReference(*Tail) << '\n');
+
+ // Live-in physregs are tricky to get right when speculating code.
+ if (!Tail->livein_empty()) {
+ LLVM_DEBUG(dbgs() << "Tail has live-ins.\n");
+ return false;
+ }
+ } else {
+ LLVM_DEBUG(dbgs() << "\nTriangle: " << printMBBReference(*Head) << " -> "
+ << printMBBReference(*Succ0) << " -> "
+ << printMBBReference(*Tail) << '\n');
+ }
}
// This is a triangle or a diamond.
@@ -588,10 +654,15 @@ bool SSAIfConv::canConvertIf(MachineBasicBlock *MBB, bool Predicate) {
if (!findInsertionPoint())
return false;
- if (isTriangle())
- ++NumTrianglesSeen;
- else
- ++NumDiamondsSeen;
+ // ExplicitTail means this is a validation call. Don't count towards the
+ // triangle / diamonds seen stat - that should happen only when we consider
+ // them for conversion seperately.
+ if (!ExplicitTail) {
+ if (isTriangle())
+ ++NumTrianglesSeen;
+ else
+ ++NumDiamondsSeen;
+ }
return true;
}
@@ -641,6 +712,122 @@ static bool hasSameValue(const MachineRegisterInfo &MRI,
return TIdx == FIdx;
}
+/// matchCascade - match a cascade pattern by walking up from MBB.
+/// MBB is a potential cascade end (last block before Tail).
+/// A cascade is a chain of triangles where:
+/// - Head has 2 successors: one to first cascade block, one to Tail
+/// - Each cascade block has 2 successors: next cascade + Tail (or 1 for last)
+/// - Last cascade block (MBB) has 1 successor: Tail
+/// Returns the discovered Head and cascade blocks if valid, empty otherwise.
+///
+/// Head --------+
+/// | |
+/// v |
+/// BB1 ----------+
+/// | |
+/// v |
+/// BB2 ----------+
+/// | |
+/// ...
+/// ...
+/// | |
+/// v v
+/// BBn (MBB) -> Tail
+///
+CascadeResult SSAIfConv::matchCascade(MachineBasicBlock *MBB) {
+ // Cascade end must have exactly 1 successor (to Tail) and 1 predecessor
+ if (MBB->succ_size() != 1 || MBB->pred_size() != 1)
+ return {};
+
+ MachineBasicBlock *Tail = MBB->succ_begin()[0];
+
+ // Collect cascade blocks from bottom to top: [BBn, BB(n-1), ..., BB1, Head]
+ SmallVector<MachineBasicBlock *> Blocks;
+ Blocks.push_back(MBB);
+ MachineBasicBlock *Current = MBB;
+
+ // Walk up the cascade chain
+ while (true) {
+ MachineBasicBlock *Pred = Current->pred_begin()[0];
+
+ // Predecessor must have 2 successors: Current and Tail
+ if (Pred->succ_size() != 2)
+ break;
+
+ MachineBasicBlock *S0 = Pred->succ_begin()[0];
+ MachineBasicBlock *S1 = Pred->succ_begin()[1];
+
+ // One successor must be Current, the other must be Tail
+ if (!((S0 == Current && S1 == Tail) || (S1 == Current && S0 == Tail)))
+ break;
+
+ Blocks.push_back(Pred);
+
+ // If Pred has multiple predecessors, it's the Head - stop here
+ if (Pred->pred_size() != 1)
+ break;
+
+ // Pred has 1 predecessor, so the cascade continues. If we've reached
+ // max depth, bail out entirely - we don't want to convert a partial
+ // cascade.
+ if (Blocks.size() > MaxCascadeDepth) {
+ LLVM_DEBUG(dbgs() << "Cascade extends beyond max depth "
+ << MaxCascadeDepth << ", not converting.\n");
+ return {};
+ }
+
+ Current = Pred;
+ }
+
+ // Need at least 3 blocks: [BBn, BB1, Head] for a minimal 2-block cascade
+ if (Blocks.size() < 3)
+ return {};
+
+ // Reverse to get [HEAD, BB1, BB2, ..., BBn] order
+ std::reverse(Blocks.begin(), Blocks.end());
+
+ // Check that we can actually convert all the blocks in the cascade.
+ // We skip the last cascade block (BBn) since it has a single successor
+ // (to Tail), and is only used as a TBB (not a HEAD) - its speculatability
+ // is validated when we check its predecessor.
+ SmallVector<SmallVector<CascadeSelectInfo, 4>> CollectedSelects;
+ size_t NumBlocksToValidate = Blocks.size() - 1;
+ for (size_t I = 0; I < NumBlocksToValidate; ++I) {
+ if (!canConvertIf(Blocks[I], /*Predicate=*/false,
+ /*AllowMultiSuccTBB=*/true,
+ /*ExplicitTail=*/Tail)) {
+ LLVM_DEBUG(dbgs() << "Cannot convert cascade, block "
+ << printMBBReference(*Blocks[I])
+ << " is not if-convertible.\n");
+ return {};
+ }
+ assert((CollectedSelects.empty() ||
+ CollectedSelects.back().size() == PHIs.size()) &&
+ "Tail phi list changed between cascade blocks");
+
+ // Capture PHI info from canConvertIf's canInsertSelect call.
+ auto &StepSelects = CollectedSelects.emplace_back();
+ for (const PHIInfo &PI : PHIs)
+ StepSelects.push_back(
+ {PI.CondCycles, !hasSameValue(*MRI, TII, PI.TReg, PI.FReg)});
+ }
+
+ // The first block in Blocks is the cascade Head
+ MachineBasicBlock *Head = Blocks.front();
+
+ // Remove Head from the cascade blocks to convert
+ Blocks.erase(Blocks.begin());
+
+ LLVM_DEBUG({
+ dbgs() << "\nCascade found: " << printMBBReference(*Head) << " -> [";
+ for (auto *BB : Blocks)
+ dbgs() << printMBBReference(*BB) << ", ";
+ dbgs() << "] -> " << printMBBReference(*Tail) << "\n";
+ });
+
+ return {Head, std::move(Blocks), std::move(CollectedSelects)};
+}
+
/// replacePHIInstrs - Completely replace PHI instructions with selects.
/// This is possible when the only Tail predecessors are the if-converted
/// blocks.
@@ -843,6 +1030,10 @@ class EarlyIfConverter {
/// each block to the number of instructions scanned in it.
DenseMap<const MachineBasicBlock *, unsigned> NoCallBlocksCache;
+ /// Set of blocks that must be converted (part of a cascade).
+ /// These blocks bypass normal profitability checks in shouldConvertIf().
+ SmallPtrSet<MachineBasicBlock *, 16> MustConvertBlocks;
+
public:
EarlyIfConverter(MachineDominatorTree &DT, MachineLoopInfo &LI,
MachineTraceMetrics &MTM, MachineBranchProbabilityInfo *MBPI)
@@ -853,11 +1044,20 @@ class EarlyIfConverter {
private:
bool tryConvertIf(MachineBasicBlock *);
+ void detectCascades(MachineBasicBlock *);
+ void convertIf();
void invalidateTraces();
bool shouldConvertIf();
- bool isConditionDataDependent();
- bool doOperandsComeFromMemory(const MachineInstr *ConditionDef);
+ bool isConditionDataDependent(MachineBasicBlock *BB, bool RecordStats = true);
+ bool isCascadeDataDependent(MachineBasicBlock *Head,
+ ArrayRef<MachineBasicBlock *> CascadeBlocks);
+ bool doOperandsComeFromMemory(const MachineInstr *ConditionDef,
+ MachineBasicBlock *BB);
bool hasCallOrLoopInRange(const MachineInstr *From, const MachineInstr *To);
+ bool shouldConvertCascade(CascadeResult &Cascade, MachineBasicBlock *Tail);
+ bool hasEnoughILP(MachineBasicBlock *TraceBlock,
+ SmallVectorImpl<MachineBasicBlock *> &ExtraBlocks,
+ unsigned CritLimit);
};
class EarlyIfConverterLegacy : public MachineFunctionPass {
@@ -1047,9 +1247,10 @@ bool EarlyIfConverter::hasCallOrLoopInRange(const MachineInstr *From,
/// depend on values loaded from memory (unless they are loop invariant,
/// or come from a constant pool). The walk starts from the definition of
/// ConditionDef's first operand, which is not ConditionDef itself for
-/// instructions such as FCMPSrr, where that operand is a use.
+/// instructions such as FCMPSrr, where that operand is a use. BB is the block
+/// whose terminator consumes the condition.
bool EarlyIfConverter::doOperandsComeFromMemory(
- const MachineInstr *ConditionDef) {
+ const MachineInstr *ConditionDef, MachineBasicBlock *BB) {
Register Reg = ConditionDef->getOperand(0).getReg();
if (!Reg.isVirtual())
return false;
@@ -1057,10 +1258,10 @@ bool EarlyIfConverter::doOperandsComeFromMemory(
LLVM_DEBUG(dbgs() << " doOperandsComeFromMemory starting from reg "
<< printReg(Reg) << "\n");
- // The condition is consumed by the branch terminating Head, so this is the
+ // The condition is consumed by the branch terminating BB, so this is the
// end of the interval a load has to survive without a call in between.
- const MachineInstr *Br = &*IfConv.Head->getFirstTerminator();
- MachineLoop *IfConvLoop = Loops->getLoopFor(IfConv.Head);
+ const MachineInstr *Br = &*BB->getFirstTerminator();
+ MachineLoop *IfConvLoop = Loops->getLoopFor(BB);
// Walk the def-use chain.
SmallPtrSet<const MachineInstr *, 8> VisitedInstrs;
@@ -1127,9 +1328,12 @@ bool EarlyIfConverter::doOperandsComeFromMemory(
}
/// Check if the branch condition is data-dependent (comes from memory loads).
-bool EarlyIfConverter::isConditionDataDependent() {
+/// RecordStats should be false when the same branch may be examined again
+/// later, so that it is only counted once.
+bool EarlyIfConverter::isConditionDataDependent(MachineBasicBlock *BB,
+ bool RecordStats) {
TargetInstrInfo::MachineBranchPredicate MBP;
- if (TII->analyzeBranchPredicate(*IfConv.Head, MBP, /*AllowModify=*/false))
+ if (TII->analyzeBranchPredicate(*BB, MBP, /*AllowModify=*/false))
return false;
if (!MBP.ConditionDef)
@@ -1138,29 +1342,52 @@ bool EarlyIfConverter::isConditionDataDependent() {
// If the branch is biased (not 50/50), don't consider it data dependent.
// This is to prevent converting unprofitable checks such as
// `x[i] != 0;`
- auto TBBProb = MBPI->getEdgeProbability(IfConv.Head, IfConv.TBB);
- auto FBBProb = MBPI->getEdgeProbability(IfConv.Head, IfConv.FBB);
- if (TBBProb != FBBProb) {
- ++NumLikelyBiased;
- return false;
+ if (MBP.TrueDest && MBP.FalseDest && MBPI) {
+ auto TBBProb = MBPI->getEdgeProbability(BB, MBP.TrueDest);
+ auto FBBProb = MBPI->getEdgeProbability(BB, MBP.FalseDest);
+ if (TBBProb != FBBProb) {
+ if (RecordStats)
+ ++NumLikelyBiased;
+ return false;
+ }
}
// Check if operands used to compute the branch condition were loaded recently
// from memory, starting by the ConditionDef itself and walking up the use-def
// chain.
- if (doOperandsComeFromMemory(MBP.ConditionDef)) {
- ++NumDataDependant;
+ if (doOperandsComeFromMemory(MBP.ConditionDef, BB)) {
+ if (RecordStats)
+ ++NumDataDependant;
return true;
}
return false;
}
-// Adjust cycles with downward saturation.
-static unsigned adjCycles(unsigned Cyc, int Delta) {
- if (Delta < 0 && Cyc + Delta > Cyc)
- return 0;
- return Cyc + Delta;
+/// Check if ALL branches in a cascade are data-dependent (come from loads).
+/// Cascade conversion always requires data-dependent branches, so this check
+/// is enabled whenever cascade conversion is enabled.
+bool EarlyIfConverter::isCascadeDataDependent(
+ MachineBasicBlock *Head, ArrayRef<MachineBasicBlock *> CascadeBlocks) {
+
+ // Check Head block
+ if (!isConditionDataDependent(Head, /*RecordStats=*/false)) {
+ LLVM_DEBUG(dbgs() << "Cascade: Head not data-dependent\n");
+ return false;
+ }
+
+ // Check all cascade blocks except the last one (which has only 1 successor).
+ // Don't record stats because we are not converting anything yet.
+ for (auto *CascadeBlock : CascadeBlocks.drop_back()) {
+ if (!isConditionDataDependent(CascadeBlock, /*RecordStats=*/false)) {
+ LLVM_DEBUG(dbgs() << "Cascade: " << printMBBReference(*CascadeBlock)
+ << " not data-dependent\n");
+ return false;
+ }
+ }
+
+ LLVM_DEBUG(dbgs() << "Cascade: all branches are data-dependent\n");
+ return true;
}
namespace {
@@ -1168,20 +1395,250 @@ namespace {
struct Cycles {
const char *Key;
unsigned Value;
+ Cycles(const char *K, unsigned V) : Key(K), Value(V) {}
};
template <typename Remark> Remark &operator<<(Remark &R, Cycles C) {
return R << ore::NV(C.Key, C.Value) << (C.Value == 1 ? " cycle" : " cycles");
}
} // anonymous namespace
+// Adjust cycles with downward saturation.
+static unsigned adjCycles(unsigned Cyc, int Delta) {
+ if (Delta < 0 && Cyc + Delta > Cyc)
+ return 0;
+ return Cyc + Delta;
+}
+
+/// Count the instructions of MBB that a predecessor would have to speculate to
+/// absorb it. This matches what canSpeculateInstrs() checks against
+/// BlockInstrLimit: the non-debug instructions ahead of the terminators.
+/// If we don't perform this check we could bail out of a cascade conversion
+/// midway by exceeding `BlockInstrLimit`.
+static unsigned countSpeculatedInstrs(const MachineBasicBlock &MBB) {
+ return count_if(make_range(MBB.begin(), MBB.getFirstTerminator()),
+ [](const MachineInstr &MI) { return !MI.isDebugInstr(); });
+}
+
+/// Apply profitability check for cascade conversion.
+bool EarlyIfConverter::shouldConvertCascade(CascadeResult &Cascade,
+ MachineBasicBlock *Tail) {
+ MachineBasicBlock *Head = Cascade.Head;
+ auto &CascadeBlocks = Cascade.Blocks;
+
+ if (!isCascadeDataDependent(Head, CascadeBlocks)) {
+ LLVM_DEBUG(dbgs() << "Cascade: not all branches are data-dependent\n");
+ return false;
+ }
+
+ // Calculate CritLimit using cascade formula.
+ unsigned CascadeSize = CascadeBlocks.size();
+ unsigned MispredictPenalty = STI->getMispredictionPenalty();
+ unsigned CritLimit = std::min(MispredictPenalty * 25 * CascadeSize / 100,
+ 2 * MispredictPenalty);
+
+ LLVM_DEBUG(dbgs() << "Cascade: size=" << CascadeSize
+ << ", CritLimit=" << CritLimit
+ << " (MispredictPenalty=" << MispredictPenalty << ")\n");
+
+ if (CritLimit == 0) {
+ LLVM_DEBUG(dbgs() << "Cascade: CritLimit is 0, skipping\n");
+ return false;
+ }
+
+ MachineOptimizationRemarkEmitter MORE(*Head->getParent(), nullptr);
+ SmallVector<MachineBasicBlock *, 8> CondBlocks;
+ CondBlocks.push_back(Head);
+ append_range(CondBlocks, ArrayRef(CascadeBlocks).drop_back());
+ assert(CondBlocks.size() == Cascade.Selects.size() &&
+ "Mismatched cascade condition info");
+
+ unsigned NumPHIs = Cascade.Selects.front().size();
+ assert(NumPHIs == range_size(Tail->phis()) &&
+ "Tail phi list changed since matchCascade()");
+
+ // A step only needs a select if the phi's two incoming values differ. But
+ // once some deeper step has produced a select, every shallower step takes
+ // that select's result as an incoming value and needs one too.
+ SmallVector<int, 4> DeepestSelectStep(NumPHIs, -1);
+ for (auto [I, Step] : enumerate(Cascade.Selects))
+ for (auto [J, Sel] : enumerate(Step))
+ if (Sel.NeedsSelect)
+ DeepestSelectStep[J] = I;
+
+ if (!Stress) {
+ unsigned NumSelects = 0;
+ for (int Deepest : DeepestSelectStep)
+ NumSelects += Deepest + 1;
+ unsigned TotalInstrs = NumSelects;
+ for (MachineBasicBlock *BB : CascadeBlocks)
+ TotalInstrs += countSpeculatedInstrs(*BB);
+
+ LLVM_DEBUG(dbgs() << "Cascade: collapsed cascade holds " << TotalInstrs
+ << " instructions (" << NumSelects
+ << " of them selects), limit is " << BlockInstrLimit
+ << '\n');
+
+ if (TotalInstrs > BlockInstrLimit) {
+ LLVM_DEBUG(dbgs() << "Cascade: collapsed cascade exceeds instruction "
+ "limit, skipping\n");
+ MORE.emit([&]() {
+ return MachineOptimizationRemarkMissed(DEBUG_TYPE,
+ "CascadeIfConversion",
+ Head->back().getDebugLoc(), Head)
+ << "did not if-convert cascade with "
+ << ore::NV("CascadeSize", CascadeSize)
+ << " blocks: the speculated blocks would hold "
+ << ore::NV("TotalInstrs", TotalInstrs)
+ << " instructions, exceeding the limit of "
+ << ore::NV("BlockInstrLimit", BlockInstrLimit) << ".";
+ });
+ return false;
+ }
+ }
+
+ // Check if there's enough ILP to hide the speculated cascade blocks.
+ if (!hasEnoughILP(Tail, CascadeBlocks, CritLimit)) {
+ LLVM_DEBUG(dbgs() << "Cascade: not enough ILP, skipping\n");
+ MORE.emit([&]() {
+ return MachineOptimizationRemarkMissed(DEBUG_TYPE, "CascadeIfConversion",
+ Head->back().getDebugLoc(), Head)
+ << "did not if-convert cascade with "
+ << ore::NV("CascadeSize", CascadeSize)
+ << " blocks: not enough ILP to hide speculated instructions.";
+ });
+ return false;
+ }
+
+ // Compute CSEL chain critical path depth.
+ // After conversion, we get a chain of CSELs where each CSEL depends on:
+ // 1. The condition from that block's branch
+ // 2. The previous CSEL's output (chain dependency)
+ // The chain is rooted at the deepest cascade block and ends at Head, which is
+ // the order convertIf() creates the selects in.
+ if (!MinInstr)
+ MinInstr = Traces->getEnsemble(MachineTraceStrategy::TS_MinInstrCount);
+
+ MachineTraceMetrics::Trace TailTrace = MinInstr->getTrace(Tail);
+
+ SmallVector<unsigned, 8> BranchDepths;
+ for (MachineBasicBlock *CondBlock : CondBlocks) {
+ MachineTraceMetrics::Trace BlockTrace = MinInstr->getTrace(CondBlock);
+ BranchDepths.push_back(
+ BlockTrace.getInstrCycles(*CondBlock->getFirstTerminator()).Depth);
+ }
+
+ // Each phi gets its own budget and its own chain: select latency depends on
+ // the phi's register class, and a phi with a lot of slack must not raise the
+ // bar for a phi with none. All of them have to be profitable.
+ unsigned MaxExtension = 0;
+ bool ExceedsLimit = false;
+ for (auto [J, PHI] : enumerate(Tail->phis())) {
+ unsigned Slack = TailTrace.getInstrSlack(PHI);
+ unsigned MaxDepth = Slack + TailTrace.getInstrCycles(PHI).Depth;
+
+ // convertIf() collapses the cascade bottom-up, so the deepest block's
+ // condition roots the CSEL chain and Head's condition feeds the last
+ // select.
+ LLVM_DEBUG(dbgs() << "CSEL chain depth computation for " << PHI);
+ unsigned CSELChainDepth = 0;
+ for (int I = DeepestSelectStep[J]; I >= 0; --I) {
+ int CondCycles = Cascade.Selects[I][J].CondCycles;
+ CSELChainDepth =
+ adjCycles(std::max(BranchDepths[I], CSELChainDepth), CondCycles);
+
+ LLVM_DEBUG(dbgs() << " " << printMBBReference(*CondBlocks[I])
+ << ": branch depth=" << BranchDepths[I]
+ << ", CondCycles=" << CondCycles
+ << ", new CSEL depth=" << CSELChainDepth << "\n");
+ }
+
+ unsigned Extension =
+ CSELChainDepth > MaxDepth ? CSELChainDepth - MaxDepth : 0;
+ MaxExtension = std::max(MaxExtension, Extension);
+
+ LLVM_DEBUG(dbgs() << "Final CSEL chain depth: " << CSELChainDepth
+ << ", MaxDepth: " << MaxDepth << ", Extension: "
+ << Extension << ", CritLimit: " << CritLimit << '\n');
+
+ if (Extension > CritLimit) {
+ LLVM_DEBUG(dbgs() << "Cascade: critical path extension exceeds "
+ "limit, skipping\n");
+ ExceedsLimit = true;
+ }
+ }
+
+ if (ExceedsLimit) {
+ MORE.emit([&]() {
+ MachineOptimizationRemarkMissed R(DEBUG_TYPE, "CascadeIfConversion",
+ Head->back().getDebugLoc(), Head);
+ R << "did not if-convert cascade with "
+ << ore::NV("CascadeSize", CascadeSize)
+ << " blocks: critical path extension of "
+ << Cycles("CritPathExtension", MaxExtension)
+ << " exceeds the threshold of " << Cycles("CritLimit", CritLimit)
+ << ".";
+ return R;
+ });
+ return false;
+ }
+
+ MORE.emit([&]() {
+ MachineOptimizationRemark R(DEBUG_TYPE, "CascadeIfConversion",
+ Head->back().getDebugLoc(), Head);
+ R << "performing cascade if-conversion on "
+ << ore::NV("CascadeSize", CascadeSize)
+ << " blocks: critical path extension is "
+ << Cycles("CritPathExtension", MaxExtension)
+ << ", staying under the threshold of " << Cycles("CritLimit", CritLimit)
+ << ".";
+ return R;
+ });
+
+ return true;
+}
+
+/// Check if there is enough ILP to hide the latency of speculatively executing
+/// the extra blocks
+bool EarlyIfConverter::hasEnoughILP(
+ MachineBasicBlock *TraceBlock,
+ SmallVectorImpl<MachineBasicBlock *> &ExtraBlocks, unsigned CritLimit) {
+ if (!MinInstr)
+ MinInstr = Traces->getEnsemble(MachineTraceStrategy::TS_MinInstrCount);
+
+ MachineTraceMetrics::Trace Trace = MinInstr->getTrace(TraceBlock);
+ unsigned CritPath = Trace.getCriticalPath();
+ SmallVector<const MachineBasicBlock *, 8> ConstExtraBlocks(
+ ExtraBlocks.begin(), ExtraBlocks.end());
+ unsigned ResLength = Trace.getResourceLength(ConstExtraBlocks);
+
+ LLVM_DEBUG(dbgs() << "ILP check: CriticalPath=" << CritPath
+ << ", ResourceLength=" << ResLength
+ << ", CritLimit=" << CritLimit << "\n");
+
+ if (ResLength > CritPath + CritLimit) {
+ LLVM_DEBUG(dbgs() << "Not enough ILP: resource length " << ResLength
+ << " exceeds critical path " << CritPath << " + limit "
+ << CritLimit << "\n");
+ return false;
+ }
+ return true;
+}
+
/// Apply cost model and heuristics to the if-conversion in IfConv.
/// Return true if the conversion is a good idea.
///
bool EarlyIfConverter::shouldConvertIf() {
+ bool InCascade = MustConvertBlocks.erase(IfConv.Head);
+
// Stress testing mode disables all cost considerations.
if (Stress)
return true;
+ if (InCascade) {
+ LLVM_DEBUG(dbgs() << "Block is part of cascade, skipping profitability\n");
+ return true;
+ }
+
// Do not try to if-convert if the condition has a high chance of being
// predictable.
MachineLoop *CurrentLoop = Loops->getLoopFor(IfConv.Head);
@@ -1228,7 +1685,7 @@ bool EarlyIfConverter::shouldConvertIf() {
// hard-to-predict branches, half for others. Otherwise use half for all.
bool DataDependent = false;
if (EnableDataDependentBranchAnalysis)
- DataDependent = isConditionDataDependent();
+ DataDependent = isConditionDataDependent(IfConv.Head);
unsigned CritLimit = DataDependent ? STI->getMispredictionPenalty()
: STI->getMispredictionPenalty() / 2;
@@ -1263,10 +1720,10 @@ bool EarlyIfConverter::shouldConvertIf() {
MachineOptimizationRemarkMissed R(DEBUG_TYPE, "IfConversion",
MBB.findDebugLoc(MBB.back()), &MBB);
R << "did not if-convert branch: the resulting critical path ("
- << Cycles{"ResLength", ResLength}
+ << Cycles("ResLength", ResLength)
<< ") would extend the shorter leg's critical path ("
- << Cycles{"MinCrit", MinCrit} << ") by more than the threshold of "
- << Cycles{"CritLimit", CritLimit}
+ << Cycles("MinCrit", MinCrit) << ") by more than the threshold of "
+ << Cycles("CritLimit", CritLimit)
<< ", which cannot be hidden by available ILP.";
return R;
});
@@ -1348,15 +1805,15 @@ bool EarlyIfConverter::shouldConvertIf() {
MachineOptimizationRemark R(DEBUG_TYPE, "IfConversion",
MBB.back().getDebugLoc(), &MBB);
R << "performing if-conversion on branch: the condition adds "
- << Cycles{"CondCycles", Cond.Extra} << " to the critical path";
+ << Cycles("CondCycles", Cond.Extra) << " to the critical path";
if (Short.Extra > 0)
R << ", and the short leg adds another "
- << Cycles{"ShortCycles", Short.Extra};
+ << Cycles("ShortCycles", Short.Extra);
if (Long.Extra > 0)
R << ", and the long leg adds another "
- << Cycles{"LongCycles", Long.Extra};
+ << Cycles("LongCycles", Long.Extra);
R << ", each staying under the threshold of "
- << Cycles{"CritLimit", CritLimit} << ".";
+ << Cycles("CritLimit", CritLimit) << ".";
return R;
});
} else {
@@ -1364,20 +1821,20 @@ bool EarlyIfConverter::shouldConvertIf() {
MachineOptimizationRemarkMissed R(DEBUG_TYPE, "IfConversion",
MBB.back().getDebugLoc(), &MBB);
R << "did not if-convert branch: the condition would add "
- << Cycles{"CondCycles", Cond.Extra} << " to the critical path";
+ << Cycles("CondCycles", Cond.Extra) << " to the critical path";
if (Cond.Extra > CritLimit)
- R << " exceeding the limit of " << Cycles{"CritLimit", CritLimit};
+ R << " exceeding the limit of " << Cycles("CritLimit", CritLimit);
if (Short.Extra > 0) {
R << ", and the short leg would add another "
- << Cycles{"ShortCycles", Short.Extra};
+ << Cycles("ShortCycles", Short.Extra);
if (Short.Extra > CritLimit)
- R << " exceeding the limit of " << Cycles{"CritLimit", CritLimit};
+ R << " exceeding the limit of " << Cycles("CritLimit", CritLimit);
}
if (Long.Extra > 0) {
R << ", and the long leg would add another "
- << Cycles{"LongCycles", Long.Extra};
+ << Cycles("LongCycles", Long.Extra);
if (Long.Extra > CritLimit)
- R << " exceeding the limit of " << Cycles{"CritLimit", CritLimit};
+ R << " exceeding the limit of " << Cycles("CritLimit", CritLimit);
}
R << ".";
return R;
@@ -1387,26 +1844,73 @@ bool EarlyIfConverter::shouldConvertIf() {
return ShouldConvert;
}
+/// Perform the actual if-conversion and update analyses.
+void EarlyIfConverter::convertIf() {
+ SmallVector<MachineBasicBlock *, 4> RemoveBlocks;
+ invalidateTraces();
+ IfConv.convertIf(RemoveBlocks);
+ updateDomTree(DomTree, IfConv, RemoveBlocks);
+ updateLoops(Loops, RemoveBlocks);
+ // Head absorbs the instructions of the removed blocks, including any calls,
+ // so a Head cached as call-free may no longer be.
+ NoCallBlocksCache.erase(IfConv.Head);
+ for (MachineBasicBlock *MBB : RemoveBlocks) {
+ NoCallBlocksCache.erase(MBB);
+ MustConvertBlocks.erase(MBB);
+ MBB->eraseFromParent();
+ }
+}
+
+/// Try to detect a cascade ending at MBB and mark blocks for conversion.
+/// Since we visit blocks in postorder, the cascade end is visited before
+/// its head, allowing us to mark cascade blocks before they are visited.
+void EarlyIfConverter::detectCascades(MachineBasicBlock *MBB) {
+ if (!EnableCascadeIfConv || !EnableDataDependentBranchAnalysis ||
+ MustConvertBlocks.count(MBB))
+ return;
+
+ auto Cascade = IfConv.matchCascade(MBB);
+ if (!Cascade)
+ return;
+
+ ++NumCascadesSeen;
+ MachineBasicBlock *Tail = MBB->succ_begin()[0];
+ LLVM_DEBUG(dbgs() << "Found cascade with " << Cascade.Blocks.size()
+ << " blocks, Head=" << printMBBReference(*Cascade.Head)
+ << ", Tail=" << printMBBReference(*Tail) << "\n");
+
+ if (!shouldConvertCascade(Cascade, Tail))
+ return;
+
+ ++NumCascadesConv;
+ NumDataDependant += Cascade.Blocks.size();
+
+ // Mark Head and all cascade blocks except the last one.
+ // The last cascade block (BBn) is never a head - it's always the TBB.
+ MustConvertBlocks.insert(Cascade.Head);
+ for (size_t I = 0; I + 1 < Cascade.Blocks.size(); ++I)
+ MustConvertBlocks.insert(Cascade.Blocks[I]);
+ LLVM_DEBUG(dbgs() << "Marked " << (1 + Cascade.Blocks.size() - 1)
+ << " blocks for cascade conversion\n");
+}
+
/// Attempt repeated if-conversion on MBB, return true if successful.
///
bool EarlyIfConverter::tryConvertIf(MachineBasicBlock *MBB) {
+ detectCascades(MBB);
+
bool Changed = false;
while (IfConv.canConvertIf(MBB) && shouldConvertIf()) {
- // If-convert MBB and update analyses.
- invalidateTraces();
- SmallVector<MachineBasicBlock *, 4> RemoveBlocks;
- IfConv.convertIf(RemoveBlocks);
+ convertIf();
Changed = true;
- updateDomTree(DomTree, IfConv, RemoveBlocks);
- updateLoops(Loops, RemoveBlocks);
- // Head absorbs the instructions of the removed blocks, including any calls,
- // so a Head cached as call-free may no longer be.
- NoCallBlocksCache.erase(IfConv.Head);
- for (MachineBasicBlock *MBB : RemoveBlocks) {
- NoCallBlocksCache.erase(MBB);
- MBB->eraseFromParent();
- }
}
+
+ // A mark that survives means a cascade block was not converted. We only
+ // convert cascades if the entire cascade conversion is profitable, so
+ // something went wrong.
+ assert(!MustConvertBlocks.contains(MBB) &&
+ "cascade block failed to if-convert");
+
return Changed;
}
@@ -1427,6 +1931,8 @@ bool EarlyIfConverter::run(MachineFunction &MF) {
bool Changed = false;
IfConv.init(MF);
+ MustConvertBlocks.clear();
+
// Visit blocks in dominator tree post-order. The post-order enables nested
// if-conversion in a single pass. The tryConvertIf() function may erase
// blocks, but only blocks dominated by the head block. This makes it safe to
@@ -1435,6 +1941,8 @@ bool EarlyIfConverter::run(MachineFunction &MF) {
if (tryConvertIf(DomNode->getBlock()))
Changed = true;
+ assert(MustConvertBlocks.empty() && "cascade block failed to if-convert");
+
return Changed;
}
diff --git a/llvm/test/CodeGen/AArch64/early-ifcvt-cascade.mir b/llvm/test/CodeGen/AArch64/early-ifcvt-cascade.mir
index 99c0ea79267a1..8939c2cca2235 100644
--- a/llvm/test/CodeGen/AArch64/early-ifcvt-cascade.mir
+++ b/llvm/test/CodeGen/AArch64/early-ifcvt-cascade.mir
@@ -1,5 +1,7 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
-# RUN: llc -mtriple=aarch64 -mcpu=apple-m1 -run-pass=early-ifcvt -enable-early-ifcvt-data-dependent %s -o - | FileCheck %s
+# RUN: llc -mtriple=aarch64 -mcpu=apple-m1 -run-pass=early-ifcvt -enable-early-ifcvt-cascade -enable-early-ifcvt-data-dependent %s -o - | FileCheck %s --check-prefixes=COMMON,CASCADE-WITH-DD
+# RUN: llc -mtriple=aarch64 -mcpu=apple-m1 -run-pass=early-ifcvt -enable-early-ifcvt-cascade -enable-early-ifcvt-data-dependent -early-ifcvt-max-cascade-depth=3 %s -o - | FileCheck %s --check-prefixes=COMMON,DEPTH3
+# RUN: llc -mtriple=aarch64 -mcpu=apple-m1 -run-pass=early-ifcvt -enable-early-ifcvt-cascade -enable-early-ifcvt-data-dependent -early-ifcvt-limit=3 %s -o - | FileCheck %s --check-prefixes=COMMON,LIMIT3
--- |
define void @cascade_selector_loop() { ret void }
@@ -7,6 +9,8 @@
define void @cascade_exceeds_crit_limit() { ret void }
define void @cascade_backedge_to_earlier_block() { ret void }
define void @cascade_forward_skip_to_later_block() { ret void }
+ define void @cascade_per_phi_slack() { ret void }
+ define void @cascade_phi_subset() { ret void }
...
---
name: cascade_selector_loop
@@ -18,102 +22,261 @@ liveins:
- { reg: '$w3', virtual-reg: '%3' }
- { reg: '$w4', virtual-reg: '%4' }
body: |
- ; CHECK-LABEL: name: cascade_selector_loop
- ; CHECK: bb.0:
- ; CHECK-NEXT: successors: %bb.10(0x30000000), %bb.1(0x50000000)
- ; CHECK-NEXT: liveins: $x0, $x1, $w2, $w3, $w4
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr32 = COPY $w2
- ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gpr32 = COPY $w3
- ; CHECK-NEXT: [[COPY4:%[0-9]+]]:gpr32 = COPY $w4
- ; CHECK-NEXT: [[COPY5:%[0-9]+]]:gpr64all = COPY $xzr
- ; CHECK-NEXT: CBZW [[COPY2]], %bb.10
- ; CHECK-NEXT: B %bb.1
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.1:
- ; CHECK-NEXT: successors: %bb.2(0x80000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 0 :: (load (s32))
- ; CHECK-NEXT: [[LDRWui1:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 1 :: (load (s32))
- ; CHECK-NEXT: [[LDRWui2:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 2 :: (load (s32))
- ; CHECK-NEXT: [[LDRWui3:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 3 :: (load (s32))
- ; CHECK-NEXT: [[LDRWui4:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 4 :: (load (s32))
- ; CHECK-NEXT: [[LDRWui5:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 5 :: (load (s32))
- ; CHECK-NEXT: [[LDRWui6:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 6 :: (load (s32))
- ; CHECK-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 1
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.2:
- ; CHECK-NEXT: successors: %bb.3(0x40000000), %bb.9(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[PHI:%[0-9]+]]:gpr32 = PHI [[COPY2]], %bb.1, %15, %bb.9
- ; CHECK-NEXT: [[LDRWui7:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load (s32))
- ; CHECK-NEXT: [[MADDWrrr:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[LDRWui7]], $wzr
- ; CHECK-NEXT: [[MADDWrrr1:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY4]], [[LDRWui7]], [[MADDWrrr]]
- ; CHECK-NEXT: [[MADDWrrr2:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[LDRWui7]], [[MADDWrrr1]]
- ; CHECK-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 7
- ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
- ; CHECK-NEXT: B %bb.3
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.3:
- ; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.9(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui1]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 6
- ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
- ; CHECK-NEXT: B %bb.4
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.4:
- ; CHECK-NEXT: successors: %bb.5(0x40000000), %bb.9(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[SUBSWrr2:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui2]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 5
- ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
- ; CHECK-NEXT: B %bb.5
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.5:
- ; CHECK-NEXT: successors: %bb.6(0x40000000), %bb.9(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[SUBSWrr3:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui3]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 4
- ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
- ; CHECK-NEXT: B %bb.6
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.6:
- ; CHECK-NEXT: successors: %bb.7(0x40000000), %bb.9(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[SUBSWrr4:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui4]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm4:%[0-9]+]]:gpr64 = MOVi64imm 3
- ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
- ; CHECK-NEXT: B %bb.7
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.7:
- ; CHECK-NEXT: successors: %bb.8(0x40000000), %bb.9(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[SUBSWrr5:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui5]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm5:%[0-9]+]]:gpr64 = MOVi64imm 2
- ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
- ; CHECK-NEXT: B %bb.8
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.8:
- ; CHECK-NEXT: successors: %bb.9(0x80000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[MOVi64imm6:%[0-9]+]]:gpr64 = MOVi64imm 1
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.9:
- ; CHECK-NEXT: successors: %bb.10(0x04000000), %bb.2(0x7c000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[PHI1:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.2, [[MOVi64imm1]], %bb.3, [[MOVi64imm2]], %bb.4, [[MOVi64imm3]], %bb.5, [[MOVi64imm4]], %bb.6, [[MOVi64imm5]], %bb.7, [[MOVi64imm6]], %bb.8
- ; CHECK-NEXT: [[SUBSWrr6:%[0-9]+]]:gpr32 = SUBSWrr [[PHI]], [[MOVi32imm]], implicit-def $nzcv
- ; CHECK-NEXT: Bcc 1, %bb.2, implicit $nzcv
- ; CHECK-NEXT: B %bb.10
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.10:
- ; CHECK-NEXT: [[PHI2:%[0-9]+]]:gpr64 = PHI [[COPY5]], %bb.0, [[PHI1]], %bb.9
- ; CHECK-NEXT: $x0 = COPY [[PHI2]]
- ; CHECK-NEXT: RET_ReallyLR implicit $x0
+ ; CASCADE-WITH-DD-LABEL: name: cascade_selector_loop
+ ; CASCADE-WITH-DD: bb.0:
+ ; CASCADE-WITH-DD-NEXT: successors: %bb.10(0x30000000), %bb.1(0x50000000)
+ ; CASCADE-WITH-DD-NEXT: liveins: $x0, $x1, $w2, $w3, $w4
+ ; CASCADE-WITH-DD-NEXT: {{ $}}
+ ; CASCADE-WITH-DD-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; CASCADE-WITH-DD-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1
+ ; CASCADE-WITH-DD-NEXT: [[COPY2:%[0-9]+]]:gpr32 = COPY $w2
+ ; CASCADE-WITH-DD-NEXT: [[COPY3:%[0-9]+]]:gpr32 = COPY $w3
+ ; CASCADE-WITH-DD-NEXT: [[COPY4:%[0-9]+]]:gpr32 = COPY $w4
+ ; CASCADE-WITH-DD-NEXT: [[COPY5:%[0-9]+]]:gpr64all = COPY $xzr
+ ; CASCADE-WITH-DD-NEXT: CBZW [[COPY2]], %bb.10
+ ; CASCADE-WITH-DD-NEXT: B %bb.1
+ ; CASCADE-WITH-DD-NEXT: {{ $}}
+ ; CASCADE-WITH-DD-NEXT: bb.1:
+ ; CASCADE-WITH-DD-NEXT: successors: %bb.2(0x80000000)
+ ; CASCADE-WITH-DD-NEXT: {{ $}}
+ ; CASCADE-WITH-DD-NEXT: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 0 :: (load (s32))
+ ; CASCADE-WITH-DD-NEXT: [[LDRWui1:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 1 :: (load (s32))
+ ; CASCADE-WITH-DD-NEXT: [[LDRWui2:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 2 :: (load (s32))
+ ; CASCADE-WITH-DD-NEXT: [[LDRWui3:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 3 :: (load (s32))
+ ; CASCADE-WITH-DD-NEXT: [[LDRWui4:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 4 :: (load (s32))
+ ; CASCADE-WITH-DD-NEXT: [[LDRWui5:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 5 :: (load (s32))
+ ; CASCADE-WITH-DD-NEXT: [[LDRWui6:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 6 :: (load (s32))
+ ; CASCADE-WITH-DD-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 1
+ ; CASCADE-WITH-DD-NEXT: {{ $}}
+ ; CASCADE-WITH-DD-NEXT: bb.2:
+ ; CASCADE-WITH-DD-NEXT: successors: %bb.10(0x04000000), %bb.2(0x7c000000)
+ ; CASCADE-WITH-DD-NEXT: {{ $}}
+ ; CASCADE-WITH-DD-NEXT: [[PHI:%[0-9]+]]:gpr32 = PHI [[COPY2]], %bb.1, %15, %bb.2
+ ; CASCADE-WITH-DD-NEXT: [[LDRWui7:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load (s32))
+ ; CASCADE-WITH-DD-NEXT: [[MADDWrrr:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[LDRWui7]], $wzr
+ ; CASCADE-WITH-DD-NEXT: [[MADDWrrr1:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY4]], [[LDRWui7]], [[MADDWrrr]]
+ ; CASCADE-WITH-DD-NEXT: [[MADDWrrr2:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[LDRWui7]], [[MADDWrrr1]]
+ ; CASCADE-WITH-DD-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui5]], implicit-def $nzcv
+ ; CASCADE-WITH-DD-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 2
+ ; CASCADE-WITH-DD-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 1
+ ; CASCADE-WITH-DD-NEXT: [[CSINCXr:%[0-9]+]]:gpr64 = CSINCXr [[MOVi64imm]], $xzr, 10, implicit $nzcv
+ ; CASCADE-WITH-DD-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui4]], implicit-def $nzcv
+ ; CASCADE-WITH-DD-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 3
+ ; CASCADE-WITH-DD-NEXT: [[CSELXr:%[0-9]+]]:gpr64 = CSELXr [[MOVi64imm2]], [[CSINCXr]], 10, implicit $nzcv
+ ; CASCADE-WITH-DD-NEXT: [[SUBSWrr2:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui3]], implicit-def $nzcv
+ ; CASCADE-WITH-DD-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 4
+ ; CASCADE-WITH-DD-NEXT: [[CSELXr1:%[0-9]+]]:gpr64 = CSELXr [[MOVi64imm3]], [[CSELXr]], 10, implicit $nzcv
+ ; CASCADE-WITH-DD-NEXT: [[SUBSWrr3:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui2]], implicit-def $nzcv
+ ; CASCADE-WITH-DD-NEXT: [[MOVi64imm4:%[0-9]+]]:gpr64 = MOVi64imm 5
+ ; CASCADE-WITH-DD-NEXT: [[CSELXr2:%[0-9]+]]:gpr64 = CSELXr [[MOVi64imm4]], [[CSELXr1]], 10, implicit $nzcv
+ ; CASCADE-WITH-DD-NEXT: [[SUBSWrr4:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui1]], implicit-def $nzcv
+ ; CASCADE-WITH-DD-NEXT: [[MOVi64imm5:%[0-9]+]]:gpr64 = MOVi64imm 6
+ ; CASCADE-WITH-DD-NEXT: [[CSELXr3:%[0-9]+]]:gpr64 = CSELXr [[MOVi64imm5]], [[CSELXr2]], 10, implicit $nzcv
+ ; CASCADE-WITH-DD-NEXT: [[SUBSWrr5:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui]], implicit-def $nzcv
+ ; CASCADE-WITH-DD-NEXT: [[MOVi64imm6:%[0-9]+]]:gpr64 = MOVi64imm 7
+ ; CASCADE-WITH-DD-NEXT: [[CSELXr4:%[0-9]+]]:gpr64 = CSELXr [[MOVi64imm6]], [[CSELXr3]], 10, implicit $nzcv
+ ; CASCADE-WITH-DD-NEXT: [[SUBSWrr6:%[0-9]+]]:gpr32 = SUBSWrr [[PHI]], [[MOVi32imm]], implicit-def $nzcv
+ ; CASCADE-WITH-DD-NEXT: Bcc 1, %bb.2, implicit $nzcv
+ ; CASCADE-WITH-DD-NEXT: B %bb.10
+ ; CASCADE-WITH-DD-NEXT: {{ $}}
+ ; CASCADE-WITH-DD-NEXT: bb.10:
+ ; CASCADE-WITH-DD-NEXT: [[PHI1:%[0-9]+]]:gpr64 = PHI [[COPY5]], %bb.0, [[CSELXr4]], %bb.2
+ ; CASCADE-WITH-DD-NEXT: $x0 = COPY [[PHI1]]
+ ; CASCADE-WITH-DD-NEXT: RET_ReallyLR implicit $x0
+ ;
+ ; DEPTH3-LABEL: name: cascade_selector_loop
+ ; DEPTH3: bb.0:
+ ; DEPTH3-NEXT: successors: %bb.10(0x30000000), %bb.1(0x50000000)
+ ; DEPTH3-NEXT: liveins: $x0, $x1, $w2, $w3, $w4
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; DEPTH3-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1
+ ; DEPTH3-NEXT: [[COPY2:%[0-9]+]]:gpr32 = COPY $w2
+ ; DEPTH3-NEXT: [[COPY3:%[0-9]+]]:gpr32 = COPY $w3
+ ; DEPTH3-NEXT: [[COPY4:%[0-9]+]]:gpr32 = COPY $w4
+ ; DEPTH3-NEXT: [[COPY5:%[0-9]+]]:gpr64all = COPY $xzr
+ ; DEPTH3-NEXT: CBZW [[COPY2]], %bb.10
+ ; DEPTH3-NEXT: B %bb.1
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: bb.1:
+ ; DEPTH3-NEXT: successors: %bb.2(0x80000000)
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 0 :: (load (s32))
+ ; DEPTH3-NEXT: [[LDRWui1:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 1 :: (load (s32))
+ ; DEPTH3-NEXT: [[LDRWui2:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 2 :: (load (s32))
+ ; DEPTH3-NEXT: [[LDRWui3:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 3 :: (load (s32))
+ ; DEPTH3-NEXT: [[LDRWui4:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 4 :: (load (s32))
+ ; DEPTH3-NEXT: [[LDRWui5:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 5 :: (load (s32))
+ ; DEPTH3-NEXT: [[LDRWui6:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 6 :: (load (s32))
+ ; DEPTH3-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 1
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: bb.2:
+ ; DEPTH3-NEXT: successors: %bb.3(0x40000000), %bb.9(0x40000000)
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: [[PHI:%[0-9]+]]:gpr32 = PHI [[COPY2]], %bb.1, %15, %bb.9
+ ; DEPTH3-NEXT: [[LDRWui7:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load (s32))
+ ; DEPTH3-NEXT: [[MADDWrrr:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[LDRWui7]], $wzr
+ ; DEPTH3-NEXT: [[MADDWrrr1:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY4]], [[LDRWui7]], [[MADDWrrr]]
+ ; DEPTH3-NEXT: [[MADDWrrr2:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[LDRWui7]], [[MADDWrrr1]]
+ ; DEPTH3-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui]], implicit-def $nzcv
+ ; DEPTH3-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 7
+ ; DEPTH3-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; DEPTH3-NEXT: B %bb.3
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: bb.3:
+ ; DEPTH3-NEXT: successors: %bb.4(0x40000000), %bb.9(0x40000000)
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui1]], implicit-def $nzcv
+ ; DEPTH3-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 6
+ ; DEPTH3-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; DEPTH3-NEXT: B %bb.4
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: bb.4:
+ ; DEPTH3-NEXT: successors: %bb.5(0x40000000), %bb.9(0x40000000)
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: [[SUBSWrr2:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui2]], implicit-def $nzcv
+ ; DEPTH3-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 5
+ ; DEPTH3-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; DEPTH3-NEXT: B %bb.5
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: bb.5:
+ ; DEPTH3-NEXT: successors: %bb.6(0x40000000), %bb.9(0x40000000)
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: [[SUBSWrr3:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui3]], implicit-def $nzcv
+ ; DEPTH3-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 4
+ ; DEPTH3-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; DEPTH3-NEXT: B %bb.6
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: bb.6:
+ ; DEPTH3-NEXT: successors: %bb.7(0x40000000), %bb.9(0x40000000)
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: [[SUBSWrr4:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui4]], implicit-def $nzcv
+ ; DEPTH3-NEXT: [[MOVi64imm4:%[0-9]+]]:gpr64 = MOVi64imm 3
+ ; DEPTH3-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; DEPTH3-NEXT: B %bb.7
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: bb.7:
+ ; DEPTH3-NEXT: successors: %bb.8(0x40000000), %bb.9(0x40000000)
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: [[SUBSWrr5:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui5]], implicit-def $nzcv
+ ; DEPTH3-NEXT: [[MOVi64imm5:%[0-9]+]]:gpr64 = MOVi64imm 2
+ ; DEPTH3-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; DEPTH3-NEXT: B %bb.8
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: bb.8:
+ ; DEPTH3-NEXT: successors: %bb.9(0x80000000)
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: [[MOVi64imm6:%[0-9]+]]:gpr64 = MOVi64imm 1
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: bb.9:
+ ; DEPTH3-NEXT: successors: %bb.10(0x04000000), %bb.2(0x7c000000)
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: [[PHI1:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.2, [[MOVi64imm1]], %bb.3, [[MOVi64imm2]], %bb.4, [[MOVi64imm3]], %bb.5, [[MOVi64imm4]], %bb.6, [[MOVi64imm5]], %bb.7, [[MOVi64imm6]], %bb.8
+ ; DEPTH3-NEXT: [[SUBSWrr6:%[0-9]+]]:gpr32 = SUBSWrr [[PHI]], [[MOVi32imm]], implicit-def $nzcv
+ ; DEPTH3-NEXT: Bcc 1, %bb.2, implicit $nzcv
+ ; DEPTH3-NEXT: B %bb.10
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: bb.10:
+ ; DEPTH3-NEXT: [[PHI2:%[0-9]+]]:gpr64 = PHI [[COPY5]], %bb.0, [[PHI1]], %bb.9
+ ; DEPTH3-NEXT: $x0 = COPY [[PHI2]]
+ ; DEPTH3-NEXT: RET_ReallyLR implicit $x0
+ ;
+ ; LIMIT3-LABEL: name: cascade_selector_loop
+ ; LIMIT3: bb.0:
+ ; LIMIT3-NEXT: successors: %bb.10(0x30000000), %bb.1(0x50000000)
+ ; LIMIT3-NEXT: liveins: $x0, $x1, $w2, $w3, $w4
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; LIMIT3-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1
+ ; LIMIT3-NEXT: [[COPY2:%[0-9]+]]:gpr32 = COPY $w2
+ ; LIMIT3-NEXT: [[COPY3:%[0-9]+]]:gpr32 = COPY $w3
+ ; LIMIT3-NEXT: [[COPY4:%[0-9]+]]:gpr32 = COPY $w4
+ ; LIMIT3-NEXT: [[COPY5:%[0-9]+]]:gpr64all = COPY $xzr
+ ; LIMIT3-NEXT: CBZW [[COPY2]], %bb.10
+ ; LIMIT3-NEXT: B %bb.1
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: bb.1:
+ ; LIMIT3-NEXT: successors: %bb.2(0x80000000)
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 0 :: (load (s32))
+ ; LIMIT3-NEXT: [[LDRWui1:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 1 :: (load (s32))
+ ; LIMIT3-NEXT: [[LDRWui2:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 2 :: (load (s32))
+ ; LIMIT3-NEXT: [[LDRWui3:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 3 :: (load (s32))
+ ; LIMIT3-NEXT: [[LDRWui4:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 4 :: (load (s32))
+ ; LIMIT3-NEXT: [[LDRWui5:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 5 :: (load (s32))
+ ; LIMIT3-NEXT: [[LDRWui6:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 6 :: (load (s32))
+ ; LIMIT3-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 1
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: bb.2:
+ ; LIMIT3-NEXT: successors: %bb.3(0x40000000), %bb.9(0x40000000)
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: [[PHI:%[0-9]+]]:gpr32 = PHI [[COPY2]], %bb.1, %15, %bb.9
+ ; LIMIT3-NEXT: [[LDRWui7:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load (s32))
+ ; LIMIT3-NEXT: [[MADDWrrr:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[LDRWui7]], $wzr
+ ; LIMIT3-NEXT: [[MADDWrrr1:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY4]], [[LDRWui7]], [[MADDWrrr]]
+ ; LIMIT3-NEXT: [[MADDWrrr2:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[LDRWui7]], [[MADDWrrr1]]
+ ; LIMIT3-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui]], implicit-def $nzcv
+ ; LIMIT3-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 7
+ ; LIMIT3-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; LIMIT3-NEXT: B %bb.3
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: bb.3:
+ ; LIMIT3-NEXT: successors: %bb.4(0x40000000), %bb.9(0x40000000)
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui1]], implicit-def $nzcv
+ ; LIMIT3-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 6
+ ; LIMIT3-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; LIMIT3-NEXT: B %bb.4
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: bb.4:
+ ; LIMIT3-NEXT: successors: %bb.5(0x40000000), %bb.9(0x40000000)
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: [[SUBSWrr2:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui2]], implicit-def $nzcv
+ ; LIMIT3-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 5
+ ; LIMIT3-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; LIMIT3-NEXT: B %bb.5
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: bb.5:
+ ; LIMIT3-NEXT: successors: %bb.6(0x40000000), %bb.9(0x40000000)
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: [[SUBSWrr3:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui3]], implicit-def $nzcv
+ ; LIMIT3-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 4
+ ; LIMIT3-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; LIMIT3-NEXT: B %bb.6
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: bb.6:
+ ; LIMIT3-NEXT: successors: %bb.7(0x40000000), %bb.9(0x40000000)
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: [[SUBSWrr4:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui4]], implicit-def $nzcv
+ ; LIMIT3-NEXT: [[MOVi64imm4:%[0-9]+]]:gpr64 = MOVi64imm 3
+ ; LIMIT3-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; LIMIT3-NEXT: B %bb.7
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: bb.7:
+ ; LIMIT3-NEXT: successors: %bb.8(0x40000000), %bb.9(0x40000000)
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: [[SUBSWrr5:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui5]], implicit-def $nzcv
+ ; LIMIT3-NEXT: [[MOVi64imm5:%[0-9]+]]:gpr64 = MOVi64imm 2
+ ; LIMIT3-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; LIMIT3-NEXT: B %bb.8
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: bb.8:
+ ; LIMIT3-NEXT: successors: %bb.9(0x80000000)
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: [[MOVi64imm6:%[0-9]+]]:gpr64 = MOVi64imm 1
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: bb.9:
+ ; LIMIT3-NEXT: successors: %bb.10(0x04000000), %bb.2(0x7c000000)
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: [[PHI1:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.2, [[MOVi64imm1]], %bb.3, [[MOVi64imm2]], %bb.4, [[MOVi64imm3]], %bb.5, [[MOVi64imm4]], %bb.6, [[MOVi64imm5]], %bb.7, [[MOVi64imm6]], %bb.8
+ ; LIMIT3-NEXT: [[SUBSWrr6:%[0-9]+]]:gpr32 = SUBSWrr [[PHI]], [[MOVi32imm]], implicit-def $nzcv
+ ; LIMIT3-NEXT: Bcc 1, %bb.2, implicit $nzcv
+ ; LIMIT3-NEXT: B %bb.10
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: bb.10:
+ ; LIMIT3-NEXT: [[PHI2:%[0-9]+]]:gpr64 = PHI [[COPY5]], %bb.0, [[PHI1]], %bb.9
+ ; LIMIT3-NEXT: $x0 = COPY [[PHI2]]
+ ; LIMIT3-NEXT: RET_ReallyLR implicit $x0
bb.0:
successors: %bb.10(0x30000000), %bb.1(0x50000000)
liveins: $x0, $x1, $w2, $w3, $w4
@@ -139,6 +302,7 @@ body: |
%16:gpr32 = LDRWui %1, 6 :: (load (s32))
%17:gpr32 = MOVi32imm 1
+ ; Cascade head: data-dependent condition (derived from load + compute chain)
bb.2:
successors: %bb.3(0x40000000), %bb.9(0x40000000)
@@ -220,104 +384,104 @@ liveins:
- { reg: '$w4', virtual-reg: '%4' }
- { reg: '$w5', virtual-reg: '%5' }
body: |
- ; CHECK-LABEL: name: cascade_branches_from_args_not_dd
- ; CHECK: bb.0:
- ; CHECK-NEXT: successors: %bb.10(0x30000000), %bb.1(0x50000000)
- ; CHECK-NEXT: liveins: $x1, $w2, $w3, $w4, $w5
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x1
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr32 = COPY $w2
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr32 = COPY $w3
- ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gpr32 = COPY $w4
- ; CHECK-NEXT: [[COPY4:%[0-9]+]]:gpr32 = COPY $w5
- ; CHECK-NEXT: [[COPY5:%[0-9]+]]:gpr64all = COPY $xzr
- ; CHECK-NEXT: CBZW [[COPY1]], %bb.10
- ; CHECK-NEXT: B %bb.1
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.1:
- ; CHECK-NEXT: successors: %bb.2(0x80000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[COPY6:%[0-9]+]]:gpr32 = COPY [[COPY2]]
- ; CHECK-NEXT: [[COPY7:%[0-9]+]]:gpr32 = COPY [[COPY3]]
- ; CHECK-NEXT: [[COPY8:%[0-9]+]]:gpr32 = COPY [[COPY4]]
- ; CHECK-NEXT: [[COPY9:%[0-9]+]]:gpr32 = COPY [[COPY2]]
- ; CHECK-NEXT: [[COPY10:%[0-9]+]]:gpr32 = COPY [[COPY3]]
- ; CHECK-NEXT: [[COPY11:%[0-9]+]]:gpr32 = COPY [[COPY4]]
- ; CHECK-NEXT: [[COPY12:%[0-9]+]]:gpr32 = COPY [[COPY2]]
- ; CHECK-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 1
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.2:
- ; CHECK-NEXT: successors: %bb.3(0x40000000), %bb.9(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[PHI:%[0-9]+]]:gpr32 = PHI [[COPY1]], %bb.1, %15, %bb.9
- ; CHECK-NEXT: [[MADDWrrr:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY2]], [[COPY4]], $wzr
- ; CHECK-NEXT: [[MADDWrrr1:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[COPY4]], [[MADDWrrr]]
- ; CHECK-NEXT: [[MADDWrrr2:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY2]], [[COPY3]], [[MADDWrrr1]]
- ; CHECK-NEXT: [[MADDWrrr3:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[COPY2]], [[MADDWrrr2]]
- ; CHECK-NEXT: [[MADDWrrr4:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY4]], [[COPY3]], [[MADDWrrr3]]
- ; CHECK-NEXT: [[MADDWrrr5:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY2]], [[COPY4]], [[MADDWrrr4]]
- ; CHECK-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr5]], [[COPY6]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 7
- ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
- ; CHECK-NEXT: B %bb.3
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.3:
- ; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.9(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr5]], [[COPY7]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 6
- ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
- ; CHECK-NEXT: B %bb.4
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.4:
- ; CHECK-NEXT: successors: %bb.5(0x40000000), %bb.9(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[SUBSWrr2:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr5]], [[COPY8]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 5
- ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
- ; CHECK-NEXT: B %bb.5
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.5:
- ; CHECK-NEXT: successors: %bb.6(0x40000000), %bb.9(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[SUBSWrr3:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr5]], [[COPY9]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 4
- ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
- ; CHECK-NEXT: B %bb.6
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.6:
- ; CHECK-NEXT: successors: %bb.7(0x40000000), %bb.9(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[SUBSWrr4:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr5]], [[COPY10]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm4:%[0-9]+]]:gpr64 = MOVi64imm 3
- ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
- ; CHECK-NEXT: B %bb.7
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.7:
- ; CHECK-NEXT: successors: %bb.8(0x40000000), %bb.9(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[SUBSWrr5:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr5]], [[COPY11]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm5:%[0-9]+]]:gpr64 = MOVi64imm 2
- ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
- ; CHECK-NEXT: B %bb.8
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.8:
- ; CHECK-NEXT: successors: %bb.9(0x80000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[MOVi64imm6:%[0-9]+]]:gpr64 = MOVi64imm 1
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.9:
- ; CHECK-NEXT: successors: %bb.10(0x04000000), %bb.2(0x7c000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[PHI1:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.2, [[MOVi64imm1]], %bb.3, [[MOVi64imm2]], %bb.4, [[MOVi64imm3]], %bb.5, [[MOVi64imm4]], %bb.6, [[MOVi64imm5]], %bb.7, [[MOVi64imm6]], %bb.8
- ; CHECK-NEXT: [[SUBSWrr6:%[0-9]+]]:gpr32 = SUBSWrr [[PHI]], [[MOVi32imm]], implicit-def $nzcv
- ; CHECK-NEXT: Bcc 1, %bb.2, implicit $nzcv
- ; CHECK-NEXT: B %bb.10
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.10:
- ; CHECK-NEXT: [[PHI2:%[0-9]+]]:gpr64 = PHI [[COPY5]], %bb.0, [[PHI1]], %bb.9
- ; CHECK-NEXT: $x0 = COPY [[PHI2]]
- ; CHECK-NEXT: RET_ReallyLR implicit $x0
+ ; COMMON-LABEL: name: cascade_branches_from_args_not_dd
+ ; COMMON: bb.0:
+ ; COMMON-NEXT: successors: %bb.10(0x30000000), %bb.1(0x50000000)
+ ; COMMON-NEXT: liveins: $x1, $w2, $w3, $w4, $w5
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x1
+ ; COMMON-NEXT: [[COPY1:%[0-9]+]]:gpr32 = COPY $w2
+ ; COMMON-NEXT: [[COPY2:%[0-9]+]]:gpr32 = COPY $w3
+ ; COMMON-NEXT: [[COPY3:%[0-9]+]]:gpr32 = COPY $w4
+ ; COMMON-NEXT: [[COPY4:%[0-9]+]]:gpr32 = COPY $w5
+ ; COMMON-NEXT: [[COPY5:%[0-9]+]]:gpr64all = COPY $xzr
+ ; COMMON-NEXT: CBZW [[COPY1]], %bb.10
+ ; COMMON-NEXT: B %bb.1
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.1:
+ ; COMMON-NEXT: successors: %bb.2(0x80000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[COPY6:%[0-9]+]]:gpr32 = COPY [[COPY2]]
+ ; COMMON-NEXT: [[COPY7:%[0-9]+]]:gpr32 = COPY [[COPY3]]
+ ; COMMON-NEXT: [[COPY8:%[0-9]+]]:gpr32 = COPY [[COPY4]]
+ ; COMMON-NEXT: [[COPY9:%[0-9]+]]:gpr32 = COPY [[COPY2]]
+ ; COMMON-NEXT: [[COPY10:%[0-9]+]]:gpr32 = COPY [[COPY3]]
+ ; COMMON-NEXT: [[COPY11:%[0-9]+]]:gpr32 = COPY [[COPY4]]
+ ; COMMON-NEXT: [[COPY12:%[0-9]+]]:gpr32 = COPY [[COPY2]]
+ ; COMMON-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 1
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.2:
+ ; COMMON-NEXT: successors: %bb.3(0x40000000), %bb.9(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[PHI:%[0-9]+]]:gpr32 = PHI [[COPY1]], %bb.1, %15, %bb.9
+ ; COMMON-NEXT: [[MADDWrrr:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY2]], [[COPY4]], $wzr
+ ; COMMON-NEXT: [[MADDWrrr1:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[COPY4]], [[MADDWrrr]]
+ ; COMMON-NEXT: [[MADDWrrr2:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY2]], [[COPY3]], [[MADDWrrr1]]
+ ; COMMON-NEXT: [[MADDWrrr3:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[COPY2]], [[MADDWrrr2]]
+ ; COMMON-NEXT: [[MADDWrrr4:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY4]], [[COPY3]], [[MADDWrrr3]]
+ ; COMMON-NEXT: [[MADDWrrr5:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY2]], [[COPY4]], [[MADDWrrr4]]
+ ; COMMON-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr5]], [[COPY6]], implicit-def $nzcv
+ ; COMMON-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 7
+ ; COMMON-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; COMMON-NEXT: B %bb.3
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.3:
+ ; COMMON-NEXT: successors: %bb.4(0x40000000), %bb.9(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr5]], [[COPY7]], implicit-def $nzcv
+ ; COMMON-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 6
+ ; COMMON-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; COMMON-NEXT: B %bb.4
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.4:
+ ; COMMON-NEXT: successors: %bb.5(0x40000000), %bb.9(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[SUBSWrr2:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr5]], [[COPY8]], implicit-def $nzcv
+ ; COMMON-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 5
+ ; COMMON-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; COMMON-NEXT: B %bb.5
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.5:
+ ; COMMON-NEXT: successors: %bb.6(0x40000000), %bb.9(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[SUBSWrr3:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr5]], [[COPY9]], implicit-def $nzcv
+ ; COMMON-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 4
+ ; COMMON-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; COMMON-NEXT: B %bb.6
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.6:
+ ; COMMON-NEXT: successors: %bb.7(0x40000000), %bb.9(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[SUBSWrr4:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr5]], [[COPY10]], implicit-def $nzcv
+ ; COMMON-NEXT: [[MOVi64imm4:%[0-9]+]]:gpr64 = MOVi64imm 3
+ ; COMMON-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; COMMON-NEXT: B %bb.7
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.7:
+ ; COMMON-NEXT: successors: %bb.8(0x40000000), %bb.9(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[SUBSWrr5:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr5]], [[COPY11]], implicit-def $nzcv
+ ; COMMON-NEXT: [[MOVi64imm5:%[0-9]+]]:gpr64 = MOVi64imm 2
+ ; COMMON-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; COMMON-NEXT: B %bb.8
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.8:
+ ; COMMON-NEXT: successors: %bb.9(0x80000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[MOVi64imm6:%[0-9]+]]:gpr64 = MOVi64imm 1
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.9:
+ ; COMMON-NEXT: successors: %bb.10(0x04000000), %bb.2(0x7c000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[PHI1:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.2, [[MOVi64imm1]], %bb.3, [[MOVi64imm2]], %bb.4, [[MOVi64imm3]], %bb.5, [[MOVi64imm4]], %bb.6, [[MOVi64imm5]], %bb.7, [[MOVi64imm6]], %bb.8
+ ; COMMON-NEXT: [[SUBSWrr6:%[0-9]+]]:gpr32 = SUBSWrr [[PHI]], [[MOVi32imm]], implicit-def $nzcv
+ ; COMMON-NEXT: Bcc 1, %bb.2, implicit $nzcv
+ ; COMMON-NEXT: B %bb.10
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.10:
+ ; COMMON-NEXT: [[PHI2:%[0-9]+]]:gpr64 = PHI [[COPY5]], %bb.0, [[PHI1]], %bb.9
+ ; COMMON-NEXT: $x0 = COPY [[PHI2]]
+ ; COMMON-NEXT: RET_ReallyLR implicit $x0
bb.0:
successors: %bb.10(0x30000000), %bb.1(0x50000000)
liveins: $x1, $w2, $w3, $w4, $w5
@@ -428,107 +592,107 @@ liveins:
- { reg: '$w3', virtual-reg: '%3' }
- { reg: '$w4', virtual-reg: '%4' }
body: |
- ; CHECK-LABEL: name: cascade_exceeds_crit_limit
- ; CHECK: bb.0:
- ; CHECK-NEXT: successors: %bb.11(0x30000000), %bb.1(0x50000000)
- ; CHECK-NEXT: liveins: $x0, $x1, $w2, $w3, $w4
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr32 = COPY $w2
- ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gpr32 = COPY $w3
- ; CHECK-NEXT: [[COPY4:%[0-9]+]]:gpr32 = COPY $w4
- ; CHECK-NEXT: [[COPY5:%[0-9]+]]:gpr64all = COPY $xzr
- ; CHECK-NEXT: CBZW [[COPY2]], %bb.11
- ; CHECK-NEXT: B %bb.1
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.1:
- ; CHECK-NEXT: successors: %bb.2(0x80000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 0 :: (load (s32))
- ; CHECK-NEXT: [[LDRWui1:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 1 :: (load (s32))
- ; CHECK-NEXT: [[LDRWui2:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 2 :: (load (s32))
- ; CHECK-NEXT: [[LDRWui3:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 3 :: (load (s32))
- ; CHECK-NEXT: [[LDRWui4:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 4 :: (load (s32))
- ; CHECK-NEXT: [[LDRWui5:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 5 :: (load (s32))
- ; CHECK-NEXT: [[LDRWui6:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 6 :: (load (s32))
- ; CHECK-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 1
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.2:
- ; CHECK-NEXT: successors: %bb.3(0x40000000), %bb.10(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[PHI:%[0-9]+]]:gpr32 = PHI [[COPY2]], %bb.1, %15, %bb.10
- ; CHECK-NEXT: [[LDRWui7:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load (s32))
- ; CHECK-NEXT: [[MADDWrrr:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[LDRWui7]], $wzr
- ; CHECK-NEXT: [[MADDWrrr1:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY4]], [[LDRWui7]], [[MADDWrrr]]
- ; CHECK-NEXT: [[MADDWrrr2:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[LDRWui7]], [[MADDWrrr1]]
- ; CHECK-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 7
- ; CHECK-NEXT: Bcc 10, %bb.10, implicit $nzcv
- ; CHECK-NEXT: B %bb.3
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.3:
- ; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.10(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui1]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 6
- ; CHECK-NEXT: Bcc 10, %bb.10, implicit $nzcv
- ; CHECK-NEXT: B %bb.4
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.4:
- ; CHECK-NEXT: successors: %bb.6(0x40000000), %bb.5(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[SUBSWrr2:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui2]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 5
- ; CHECK-NEXT: Bcc 10, %bb.5, implicit $nzcv
- ; CHECK-NEXT: B %bb.6
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.5:
- ; CHECK-NEXT: successors: %bb.10(0x80000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: B %bb.10
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.6:
- ; CHECK-NEXT: successors: %bb.7(0x40000000), %bb.10(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[SUBSWrr3:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui3]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 4
- ; CHECK-NEXT: Bcc 10, %bb.10, implicit $nzcv
- ; CHECK-NEXT: B %bb.7
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.7:
- ; CHECK-NEXT: successors: %bb.8(0x40000000), %bb.10(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[SUBSWrr4:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui4]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm4:%[0-9]+]]:gpr64 = MOVi64imm 3
- ; CHECK-NEXT: Bcc 10, %bb.10, implicit $nzcv
- ; CHECK-NEXT: B %bb.8
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.8:
- ; CHECK-NEXT: successors: %bb.9(0x40000000), %bb.10(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[SUBSWrr5:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui5]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm5:%[0-9]+]]:gpr64 = MOVi64imm 2
- ; CHECK-NEXT: Bcc 10, %bb.10, implicit $nzcv
- ; CHECK-NEXT: B %bb.9
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.9:
- ; CHECK-NEXT: successors: %bb.10(0x80000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[MOVi64imm6:%[0-9]+]]:gpr64 = MOVi64imm 1
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.10:
- ; CHECK-NEXT: successors: %bb.11(0x04000000), %bb.2(0x7c000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[PHI1:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.2, [[MOVi64imm1]], %bb.3, [[MOVi64imm2]], %bb.5, [[MOVi64imm3]], %bb.6, [[MOVi64imm4]], %bb.7, [[MOVi64imm5]], %bb.8, [[MOVi64imm6]], %bb.9
- ; CHECK-NEXT: [[SUBSWrr6:%[0-9]+]]:gpr32 = SUBSWrr [[PHI]], [[MOVi32imm]], implicit-def $nzcv
- ; CHECK-NEXT: Bcc 1, %bb.2, implicit $nzcv
- ; CHECK-NEXT: B %bb.11
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.11:
- ; CHECK-NEXT: [[PHI2:%[0-9]+]]:gpr64 = PHI [[COPY5]], %bb.0, [[PHI1]], %bb.10
- ; CHECK-NEXT: $x0 = COPY [[PHI2]]
- ; CHECK-NEXT: RET_ReallyLR implicit $x0
+ ; COMMON-LABEL: name: cascade_exceeds_crit_limit
+ ; COMMON: bb.0:
+ ; COMMON-NEXT: successors: %bb.11(0x30000000), %bb.1(0x50000000)
+ ; COMMON-NEXT: liveins: $x0, $x1, $w2, $w3, $w4
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; COMMON-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1
+ ; COMMON-NEXT: [[COPY2:%[0-9]+]]:gpr32 = COPY $w2
+ ; COMMON-NEXT: [[COPY3:%[0-9]+]]:gpr32 = COPY $w3
+ ; COMMON-NEXT: [[COPY4:%[0-9]+]]:gpr32 = COPY $w4
+ ; COMMON-NEXT: [[COPY5:%[0-9]+]]:gpr64all = COPY $xzr
+ ; COMMON-NEXT: CBZW [[COPY2]], %bb.11
+ ; COMMON-NEXT: B %bb.1
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.1:
+ ; COMMON-NEXT: successors: %bb.2(0x80000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 0 :: (load (s32))
+ ; COMMON-NEXT: [[LDRWui1:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 1 :: (load (s32))
+ ; COMMON-NEXT: [[LDRWui2:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 2 :: (load (s32))
+ ; COMMON-NEXT: [[LDRWui3:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 3 :: (load (s32))
+ ; COMMON-NEXT: [[LDRWui4:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 4 :: (load (s32))
+ ; COMMON-NEXT: [[LDRWui5:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 5 :: (load (s32))
+ ; COMMON-NEXT: [[LDRWui6:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 6 :: (load (s32))
+ ; COMMON-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 1
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.2:
+ ; COMMON-NEXT: successors: %bb.3(0x40000000), %bb.10(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[PHI:%[0-9]+]]:gpr32 = PHI [[COPY2]], %bb.1, %15, %bb.10
+ ; COMMON-NEXT: [[LDRWui7:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load (s32))
+ ; COMMON-NEXT: [[MADDWrrr:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[LDRWui7]], $wzr
+ ; COMMON-NEXT: [[MADDWrrr1:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY4]], [[LDRWui7]], [[MADDWrrr]]
+ ; COMMON-NEXT: [[MADDWrrr2:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[LDRWui7]], [[MADDWrrr1]]
+ ; COMMON-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui]], implicit-def $nzcv
+ ; COMMON-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 7
+ ; COMMON-NEXT: Bcc 10, %bb.10, implicit $nzcv
+ ; COMMON-NEXT: B %bb.3
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.3:
+ ; COMMON-NEXT: successors: %bb.4(0x40000000), %bb.10(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui1]], implicit-def $nzcv
+ ; COMMON-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 6
+ ; COMMON-NEXT: Bcc 10, %bb.10, implicit $nzcv
+ ; COMMON-NEXT: B %bb.4
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.4:
+ ; COMMON-NEXT: successors: %bb.6(0x40000000), %bb.5(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[SUBSWrr2:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui2]], implicit-def $nzcv
+ ; COMMON-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 5
+ ; COMMON-NEXT: Bcc 10, %bb.5, implicit $nzcv
+ ; COMMON-NEXT: B %bb.6
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.5:
+ ; COMMON-NEXT: successors: %bb.10(0x80000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: B %bb.10
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.6:
+ ; COMMON-NEXT: successors: %bb.7(0x40000000), %bb.10(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[SUBSWrr3:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui3]], implicit-def $nzcv
+ ; COMMON-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 4
+ ; COMMON-NEXT: Bcc 10, %bb.10, implicit $nzcv
+ ; COMMON-NEXT: B %bb.7
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.7:
+ ; COMMON-NEXT: successors: %bb.8(0x40000000), %bb.10(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[SUBSWrr4:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui4]], implicit-def $nzcv
+ ; COMMON-NEXT: [[MOVi64imm4:%[0-9]+]]:gpr64 = MOVi64imm 3
+ ; COMMON-NEXT: Bcc 10, %bb.10, implicit $nzcv
+ ; COMMON-NEXT: B %bb.8
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.8:
+ ; COMMON-NEXT: successors: %bb.9(0x40000000), %bb.10(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[SUBSWrr5:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui5]], implicit-def $nzcv
+ ; COMMON-NEXT: [[MOVi64imm5:%[0-9]+]]:gpr64 = MOVi64imm 2
+ ; COMMON-NEXT: Bcc 10, %bb.10, implicit $nzcv
+ ; COMMON-NEXT: B %bb.9
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.9:
+ ; COMMON-NEXT: successors: %bb.10(0x80000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[MOVi64imm6:%[0-9]+]]:gpr64 = MOVi64imm 1
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.10:
+ ; COMMON-NEXT: successors: %bb.11(0x04000000), %bb.2(0x7c000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[PHI1:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.2, [[MOVi64imm1]], %bb.3, [[MOVi64imm2]], %bb.5, [[MOVi64imm3]], %bb.6, [[MOVi64imm4]], %bb.7, [[MOVi64imm5]], %bb.8, [[MOVi64imm6]], %bb.9
+ ; COMMON-NEXT: [[SUBSWrr6:%[0-9]+]]:gpr32 = SUBSWrr [[PHI]], [[MOVi32imm]], implicit-def $nzcv
+ ; COMMON-NEXT: Bcc 1, %bb.2, implicit $nzcv
+ ; COMMON-NEXT: B %bb.11
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.11:
+ ; COMMON-NEXT: [[PHI2:%[0-9]+]]:gpr64 = PHI [[COPY5]], %bb.0, [[PHI1]], %bb.10
+ ; COMMON-NEXT: $x0 = COPY [[PHI2]]
+ ; COMMON-NEXT: RET_ReallyLR implicit $x0
bb.0:
successors: %bb.11(0x30000000), %bb.1(0x50000000)
liveins: $x0, $x1, $w2, $w3, $w4
@@ -638,49 +802,49 @@ name: cascade_backedge_to_earlier_block
alignment: 4
tracksRegLiveness: true
body: |
- ; CHECK-LABEL: name: cascade_backedge_to_earlier_block
- ; CHECK: bb.0:
- ; CHECK-NEXT: successors: %bb.1(0x80000000)
- ; CHECK-NEXT: liveins: $x0, $x1
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.1:
- ; CHECK-NEXT: successors: %bb.2(0x40000000), %bb.5(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load (s32))
- ; CHECK-NEXT: [[LDRWui1:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 0 :: (load (s32))
- ; CHECK-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui1]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 100
- ; CHECK-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 200
- ; CHECK-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 400
- ; CHECK-NEXT: Bcc 10, %bb.5, implicit $nzcv
- ; CHECK-NEXT: B %bb.2
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.2:
- ; CHECK-NEXT: successors: %bb.3(0x40000000), %bb.5(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[LDRWui2:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 1 :: (load (s32))
- ; CHECK-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui2]], implicit-def $nzcv
- ; CHECK-NEXT: Bcc 10, %bb.5, implicit $nzcv
- ; CHECK-NEXT: B %bb.3
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.3:
- ; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[LDRWui3:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 2 :: (load (s32))
- ; CHECK-NEXT: [[SUBSWrr2:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui3]], implicit-def $nzcv
- ; CHECK-NEXT: Bcc 10, %bb.2, implicit $nzcv
- ; CHECK-NEXT: B %bb.4
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.4:
- ; CHECK-NEXT: successors: %bb.5(0x80000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.5:
- ; CHECK-NEXT: [[PHI:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.1, [[MOVi64imm1]], %bb.2, [[MOVi64imm2]], %bb.4
- ; CHECK-NEXT: $x0 = COPY [[PHI]]
- ; CHECK-NEXT: RET_ReallyLR implicit $x0
+ ; COMMON-LABEL: name: cascade_backedge_to_earlier_block
+ ; COMMON: bb.0:
+ ; COMMON-NEXT: successors: %bb.1(0x80000000)
+ ; COMMON-NEXT: liveins: $x0, $x1
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; COMMON-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.1:
+ ; COMMON-NEXT: successors: %bb.2(0x40000000), %bb.5(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load (s32))
+ ; COMMON-NEXT: [[LDRWui1:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 0 :: (load (s32))
+ ; COMMON-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui1]], implicit-def $nzcv
+ ; COMMON-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 100
+ ; COMMON-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 200
+ ; COMMON-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 400
+ ; COMMON-NEXT: Bcc 10, %bb.5, implicit $nzcv
+ ; COMMON-NEXT: B %bb.2
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.2:
+ ; COMMON-NEXT: successors: %bb.3(0x40000000), %bb.5(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[LDRWui2:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 1 :: (load (s32))
+ ; COMMON-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui2]], implicit-def $nzcv
+ ; COMMON-NEXT: Bcc 10, %bb.5, implicit $nzcv
+ ; COMMON-NEXT: B %bb.3
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.3:
+ ; COMMON-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[LDRWui3:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 2 :: (load (s32))
+ ; COMMON-NEXT: [[SUBSWrr2:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui3]], implicit-def $nzcv
+ ; COMMON-NEXT: Bcc 10, %bb.2, implicit $nzcv
+ ; COMMON-NEXT: B %bb.4
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.4:
+ ; COMMON-NEXT: successors: %bb.5(0x80000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.5:
+ ; COMMON-NEXT: [[PHI:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.1, [[MOVi64imm1]], %bb.2, [[MOVi64imm2]], %bb.4
+ ; COMMON-NEXT: $x0 = COPY [[PHI]]
+ ; COMMON-NEXT: RET_ReallyLR implicit $x0
bb.0:
successors: %bb.1(0x80000000)
liveins: $x0, $x1
@@ -731,60 +895,60 @@ liveins:
- { reg: '$x0', virtual-reg: '%0' }
- { reg: '$x1', virtual-reg: '%1' }
body: |
- ; CHECK-LABEL: name: cascade_forward_skip_to_later_block
- ; CHECK: bb.0:
- ; CHECK-NEXT: successors: %bb.1(0x80000000)
- ; CHECK-NEXT: liveins: $x0, $x1
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.1:
- ; CHECK-NEXT: successors: %bb.2(0x40000000), %bb.6(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load (s32))
- ; CHECK-NEXT: [[LDRWui1:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 0 :: (load (s32))
- ; CHECK-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui1]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 100
- ; CHECK-NEXT: Bcc 10, %bb.6, implicit $nzcv
- ; CHECK-NEXT: B %bb.2
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.2:
- ; CHECK-NEXT: successors: %bb.3(0x40000000), %bb.6(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[LDRWui2:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 1 :: (load (s32))
- ; CHECK-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui2]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 200
- ; CHECK-NEXT: Bcc 10, %bb.6, implicit $nzcv
- ; CHECK-NEXT: B %bb.3
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.3:
- ; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.5(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[LDRWui3:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 2 :: (load (s32))
- ; CHECK-NEXT: [[SUBSWrr2:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui3]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 300
- ; CHECK-NEXT: Bcc 10, %bb.5, implicit $nzcv
- ; CHECK-NEXT: B %bb.4
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.4:
- ; CHECK-NEXT: successors: %bb.5(0x40000000), %bb.6(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[LDRWui4:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 3 :: (load (s32))
- ; CHECK-NEXT: [[SUBSWrr3:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui4]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 400
- ; CHECK-NEXT: Bcc 10, %bb.6, implicit $nzcv
- ; CHECK-NEXT: B %bb.5
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.5:
- ; CHECK-NEXT: successors: %bb.6(0x80000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[MOVi64imm4:%[0-9]+]]:gpr64 = MOVi64imm 500
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.6:
- ; CHECK-NEXT: [[PHI:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.1, [[MOVi64imm1]], %bb.2, [[MOVi64imm3]], %bb.4, [[MOVi64imm4]], %bb.5
- ; CHECK-NEXT: $x0 = COPY [[PHI]]
- ; CHECK-NEXT: RET_ReallyLR implicit $x0
+ ; COMMON-LABEL: name: cascade_forward_skip_to_later_block
+ ; COMMON: bb.0:
+ ; COMMON-NEXT: successors: %bb.1(0x80000000)
+ ; COMMON-NEXT: liveins: $x0, $x1
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; COMMON-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.1:
+ ; COMMON-NEXT: successors: %bb.2(0x40000000), %bb.6(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load (s32))
+ ; COMMON-NEXT: [[LDRWui1:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 0 :: (load (s32))
+ ; COMMON-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui1]], implicit-def $nzcv
+ ; COMMON-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 100
+ ; COMMON-NEXT: Bcc 10, %bb.6, implicit $nzcv
+ ; COMMON-NEXT: B %bb.2
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.2:
+ ; COMMON-NEXT: successors: %bb.3(0x40000000), %bb.6(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[LDRWui2:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 1 :: (load (s32))
+ ; COMMON-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui2]], implicit-def $nzcv
+ ; COMMON-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 200
+ ; COMMON-NEXT: Bcc 10, %bb.6, implicit $nzcv
+ ; COMMON-NEXT: B %bb.3
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.3:
+ ; COMMON-NEXT: successors: %bb.4(0x40000000), %bb.5(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[LDRWui3:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 2 :: (load (s32))
+ ; COMMON-NEXT: [[SUBSWrr2:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui3]], implicit-def $nzcv
+ ; COMMON-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 300
+ ; COMMON-NEXT: Bcc 10, %bb.5, implicit $nzcv
+ ; COMMON-NEXT: B %bb.4
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.4:
+ ; COMMON-NEXT: successors: %bb.5(0x40000000), %bb.6(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[LDRWui4:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 3 :: (load (s32))
+ ; COMMON-NEXT: [[SUBSWrr3:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui4]], implicit-def $nzcv
+ ; COMMON-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 400
+ ; COMMON-NEXT: Bcc 10, %bb.6, implicit $nzcv
+ ; COMMON-NEXT: B %bb.5
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.5:
+ ; COMMON-NEXT: successors: %bb.6(0x80000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[MOVi64imm4:%[0-9]+]]:gpr64 = MOVi64imm 500
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.6:
+ ; COMMON-NEXT: [[PHI:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.1, [[MOVi64imm1]], %bb.2, [[MOVi64imm3]], %bb.4, [[MOVi64imm4]], %bb.5
+ ; COMMON-NEXT: $x0 = COPY [[PHI]]
+ ; COMMON-NEXT: RET_ReallyLR implicit $x0
bb.0:
successors: %bb.1(0x80000000)
liveins: $x0, $x1
@@ -840,4 +1004,374 @@ body: |
$x0 = COPY %90
RET_ReallyLR implicit $x0
...
+---
+name: cascade_per_phi_slack
+alignment: 4
+tracksRegLiveness: true
+body: |
+ ; COMMON-LABEL: name: cascade_per_phi_slack
+ ; COMMON: bb.0:
+ ; COMMON-NEXT: successors: %bb.1(0x80000000)
+ ; COMMON-NEXT: liveins: $x0
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.1:
+ ; COMMON-NEXT: successors: %bb.2(0x40000000), %bb.7(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load (s32))
+ ; COMMON-NEXT: [[MADDWrrr:%[0-9]+]]:gpr32 = MADDWrrr [[LDRWui]], [[LDRWui]], $wzr
+ ; COMMON-NEXT: [[MADDWrrr1:%[0-9]+]]:gpr32 = MADDWrrr [[MADDWrrr]], [[MADDWrrr]], $wzr
+ ; COMMON-NEXT: [[ADDWrr:%[0-9]+]]:gpr32 = ADDWrr [[MADDWrrr1]], [[MADDWrrr1]]
+ ; COMMON-NEXT: [[ADDWrr1:%[0-9]+]]:gpr32 = ADDWrr [[ADDWrr]], [[ADDWrr]]
+ ; COMMON-NEXT: [[ADDWrr2:%[0-9]+]]:gpr32 = ADDWrr [[ADDWrr1]], [[ADDWrr1]]
+ ; COMMON-NEXT: [[ADDWrr3:%[0-9]+]]:gpr32 = ADDWrr [[ADDWrr2]], [[ADDWrr2]]
+ ; COMMON-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 100
+ ; COMMON-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 1100
+ ; COMMON-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr1]], [[MADDWrrr1]], implicit-def $nzcv
+ ; COMMON-NEXT: Bcc 10, %bb.7, implicit $nzcv
+ ; COMMON-NEXT: B %bb.2
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.2:
+ ; COMMON-NEXT: successors: %bb.3(0x40000000), %bb.7(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 200
+ ; COMMON-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 1200
+ ; COMMON-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[ADDWrr]], [[ADDWrr]], implicit-def $nzcv
+ ; COMMON-NEXT: Bcc 10, %bb.7, implicit $nzcv
+ ; COMMON-NEXT: B %bb.3
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.3:
+ ; COMMON-NEXT: successors: %bb.4(0x40000000), %bb.7(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[MOVi64imm4:%[0-9]+]]:gpr64 = MOVi64imm 300
+ ; COMMON-NEXT: [[MOVi64imm5:%[0-9]+]]:gpr64 = MOVi64imm 1300
+ ; COMMON-NEXT: [[SUBSWrr2:%[0-9]+]]:gpr32 = SUBSWrr [[ADDWrr1]], [[ADDWrr1]], implicit-def $nzcv
+ ; COMMON-NEXT: Bcc 10, %bb.7, implicit $nzcv
+ ; COMMON-NEXT: B %bb.4
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.4:
+ ; COMMON-NEXT: successors: %bb.5(0x40000000), %bb.7(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[MOVi64imm6:%[0-9]+]]:gpr64 = MOVi64imm 400
+ ; COMMON-NEXT: [[MOVi64imm7:%[0-9]+]]:gpr64 = MOVi64imm 1400
+ ; COMMON-NEXT: [[SUBSWrr3:%[0-9]+]]:gpr32 = SUBSWrr [[ADDWrr2]], [[ADDWrr2]], implicit-def $nzcv
+ ; COMMON-NEXT: Bcc 10, %bb.7, implicit $nzcv
+ ; COMMON-NEXT: B %bb.5
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.5:
+ ; COMMON-NEXT: successors: %bb.6(0x40000000), %bb.7(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[MOVi64imm8:%[0-9]+]]:gpr64 = MOVi64imm 500
+ ; COMMON-NEXT: [[MOVi64imm9:%[0-9]+]]:gpr64 = MOVi64imm 1500
+ ; COMMON-NEXT: [[SUBSWrr4:%[0-9]+]]:gpr32 = SUBSWrr [[ADDWrr3]], [[ADDWrr3]], implicit-def $nzcv
+ ; COMMON-NEXT: Bcc 10, %bb.7, implicit $nzcv
+ ; COMMON-NEXT: B %bb.6
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.6:
+ ; COMMON-NEXT: successors: %bb.7(0x80000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[MOVi64imm10:%[0-9]+]]:gpr64 = MOVi64imm 600
+ ; COMMON-NEXT: [[MOVi64imm11:%[0-9]+]]:gpr64 = MOVi64imm 1600
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.7:
+ ; COMMON-NEXT: [[PHI:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.1, [[MOVi64imm2]], %bb.2, [[MOVi64imm4]], %bb.3, [[MOVi64imm6]], %bb.4, [[MOVi64imm8]], %bb.5, [[MOVi64imm10]], %bb.6
+ ; COMMON-NEXT: [[PHI1:%[0-9]+]]:gpr64 = PHI [[MOVi64imm1]], %bb.1, [[MOVi64imm3]], %bb.2, [[MOVi64imm5]], %bb.3, [[MOVi64imm7]], %bb.4, [[MOVi64imm9]], %bb.5, [[MOVi64imm11]], %bb.6
+ ; COMMON-NEXT: [[MADDXrrr:%[0-9]+]]:gpr64 = MADDXrrr [[PHI]], [[PHI]], $xzr
+ ; COMMON-NEXT: [[MADDXrrr1:%[0-9]+]]:gpr64 = MADDXrrr [[MADDXrrr]], [[MADDXrrr]], $xzr
+ ; COMMON-NEXT: [[MADDXrrr2:%[0-9]+]]:gpr64 = MADDXrrr [[MADDXrrr1]], [[MADDXrrr1]], $xzr
+ ; COMMON-NEXT: [[MADDXrrr3:%[0-9]+]]:gpr64 = MADDXrrr [[MADDXrrr2]], [[MADDXrrr2]], $xzr
+ ; COMMON-NEXT: [[ADDXrr:%[0-9]+]]:gpr64 = ADDXrr [[MADDXrrr3]], [[PHI1]]
+ ; COMMON-NEXT: $x0 = COPY [[ADDXrr]]
+ ; COMMON-NEXT: RET_ReallyLR implicit $x0
+ bb.0:
+ successors: %bb.1(0x80000000)
+ liveins: $x0
+
+ %0:gpr64common = COPY $x0
+
+ bb.1:
+ successors: %bb.2(0x40000000), %bb.7(0x40000000)
+
+ %10:gpr32 = LDRWui %0, 0 :: (load (s32))
+ %11:gpr32 = MADDWrrr %10, %10, $wzr
+ %12:gpr32 = MADDWrrr %11, %11, $wzr
+ %20:gpr32 = ADDWrr %12, %12
+ %21:gpr32 = ADDWrr %20, %20
+ %22:gpr32 = ADDWrr %21, %21
+ %23:gpr32 = ADDWrr %22, %22
+ %40:gpr64 = MOVi64imm 100
+ %50:gpr64 = MOVi64imm 1100
+ %30:gpr32 = SUBSWrr %12, %12, implicit-def $nzcv
+ Bcc 10, %bb.7, implicit $nzcv
+ B %bb.2
+
+ bb.2:
+ successors: %bb.3(0x40000000), %bb.7(0x40000000)
+
+ %41:gpr64 = MOVi64imm 200
+ %51:gpr64 = MOVi64imm 1200
+ %31:gpr32 = SUBSWrr %20, %20, implicit-def $nzcv
+ Bcc 10, %bb.7, implicit $nzcv
+ B %bb.3
+
+ bb.3:
+ successors: %bb.4(0x40000000), %bb.7(0x40000000)
+
+ %42:gpr64 = MOVi64imm 300
+ %52:gpr64 = MOVi64imm 1300
+ %32:gpr32 = SUBSWrr %21, %21, implicit-def $nzcv
+ Bcc 10, %bb.7, implicit $nzcv
+ B %bb.4
+
+ bb.4:
+ successors: %bb.5(0x40000000), %bb.7(0x40000000)
+
+ %43:gpr64 = MOVi64imm 400
+ %53:gpr64 = MOVi64imm 1400
+ %33:gpr32 = SUBSWrr %22, %22, implicit-def $nzcv
+ Bcc 10, %bb.7, implicit $nzcv
+ B %bb.5
+
+ bb.5:
+ successors: %bb.6(0x40000000), %bb.7(0x40000000)
+
+ %44:gpr64 = MOVi64imm 500
+ %54:gpr64 = MOVi64imm 1500
+ %34:gpr32 = SUBSWrr %23, %23, implicit-def $nzcv
+ Bcc 10, %bb.7, implicit $nzcv
+ B %bb.6
+
+ bb.6:
+ successors: %bb.7(0x80000000)
+
+ %45:gpr64 = MOVi64imm 600
+ %55:gpr64 = MOVi64imm 1600
+
+ bb.7:
+ %90:gpr64 = PHI %40, %bb.1, %41, %bb.2, %42, %bb.3, %43, %bb.4, %44, %bb.5, %45, %bb.6
+ %91:gpr64 = PHI %50, %bb.1, %51, %bb.2, %52, %bb.3, %53, %bb.4, %54, %bb.5, %55, %bb.6
+ %60:gpr64 = MADDXrrr %90, %90, $xzr
+ %61:gpr64 = MADDXrrr %60, %60, $xzr
+ %62:gpr64 = MADDXrrr %61, %61, $xzr
+ %63:gpr64 = MADDXrrr %62, %62, $xzr
+ %64:gpr64 = ADDXrr %63, %91
+ $x0 = COPY %64
+ RET_ReallyLR implicit $x0
+...
+---
+name: cascade_phi_subset
+alignment: 4
+tracksRegLiveness: true
+body: |
+ ; CASCADE-WITH-DD-LABEL: name: cascade_phi_subset
+ ; CASCADE-WITH-DD: bb.0:
+ ; CASCADE-WITH-DD-NEXT: successors: %bb.1(0x80000000)
+ ; CASCADE-WITH-DD-NEXT: liveins: $x0
+ ; CASCADE-WITH-DD-NEXT: {{ $}}
+ ; CASCADE-WITH-DD-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; CASCADE-WITH-DD-NEXT: {{ $}}
+ ; CASCADE-WITH-DD-NEXT: bb.1:
+ ; CASCADE-WITH-DD-NEXT: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load (s32))
+ ; CASCADE-WITH-DD-NEXT: [[MADDWrrr:%[0-9]+]]:gpr32 = MADDWrrr [[LDRWui]], [[LDRWui]], $wzr
+ ; CASCADE-WITH-DD-NEXT: [[MADDWrrr1:%[0-9]+]]:gpr32 = MADDWrrr [[MADDWrrr]], [[MADDWrrr]], $wzr
+ ; CASCADE-WITH-DD-NEXT: [[ADDWrr:%[0-9]+]]:gpr32 = ADDWrr [[MADDWrrr1]], [[MADDWrrr1]]
+ ; CASCADE-WITH-DD-NEXT: [[ADDWrr1:%[0-9]+]]:gpr32 = ADDWrr [[ADDWrr]], [[ADDWrr]]
+ ; CASCADE-WITH-DD-NEXT: [[ADDWrr2:%[0-9]+]]:gpr32 = ADDWrr [[ADDWrr1]], [[ADDWrr1]]
+ ; CASCADE-WITH-DD-NEXT: [[ADDWrr3:%[0-9]+]]:gpr32 = ADDWrr [[ADDWrr2]], [[ADDWrr2]]
+ ; CASCADE-WITH-DD-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 100
+ ; CASCADE-WITH-DD-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 200
+ ; CASCADE-WITH-DD-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 300
+ ; CASCADE-WITH-DD-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 400
+ ; CASCADE-WITH-DD-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[ADDWrr3]], [[ADDWrr3]], implicit-def $nzcv
+ ; CASCADE-WITH-DD-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[ADDWrr2]], [[ADDWrr2]], implicit-def $nzcv
+ ; CASCADE-WITH-DD-NEXT: [[SUBSWrr2:%[0-9]+]]:gpr32 = SUBSWrr [[ADDWrr1]], [[ADDWrr1]], implicit-def $nzcv
+ ; CASCADE-WITH-DD-NEXT: [[CSELXr:%[0-9]+]]:gpr64 = CSELXr [[MOVi64imm2]], [[MOVi64imm3]], 10, implicit $nzcv
+ ; CASCADE-WITH-DD-NEXT: [[SUBSWrr3:%[0-9]+]]:gpr32 = SUBSWrr [[ADDWrr]], [[ADDWrr]], implicit-def $nzcv
+ ; CASCADE-WITH-DD-NEXT: [[CSELXr1:%[0-9]+]]:gpr64 = CSELXr [[MOVi64imm1]], [[CSELXr]], 10, implicit $nzcv
+ ; CASCADE-WITH-DD-NEXT: [[SUBSWrr4:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr1]], [[MADDWrrr1]], implicit-def $nzcv
+ ; CASCADE-WITH-DD-NEXT: [[CSELXr2:%[0-9]+]]:gpr64 = CSELXr [[MOVi64imm]], [[CSELXr1]], 10, implicit $nzcv
+ ; CASCADE-WITH-DD-NEXT: $x0 = COPY [[CSELXr2]]
+ ; CASCADE-WITH-DD-NEXT: RET_ReallyLR implicit $x0
+ ;
+ ; DEPTH3-LABEL: name: cascade_phi_subset
+ ; DEPTH3: bb.0:
+ ; DEPTH3-NEXT: successors: %bb.1(0x80000000)
+ ; DEPTH3-NEXT: liveins: $x0
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: bb.1:
+ ; DEPTH3-NEXT: successors: %bb.2(0x40000000), %bb.7(0x40000000)
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load (s32))
+ ; DEPTH3-NEXT: [[MADDWrrr:%[0-9]+]]:gpr32 = MADDWrrr [[LDRWui]], [[LDRWui]], $wzr
+ ; DEPTH3-NEXT: [[MADDWrrr1:%[0-9]+]]:gpr32 = MADDWrrr [[MADDWrrr]], [[MADDWrrr]], $wzr
+ ; DEPTH3-NEXT: [[ADDWrr:%[0-9]+]]:gpr32 = ADDWrr [[MADDWrrr1]], [[MADDWrrr1]]
+ ; DEPTH3-NEXT: [[ADDWrr1:%[0-9]+]]:gpr32 = ADDWrr [[ADDWrr]], [[ADDWrr]]
+ ; DEPTH3-NEXT: [[ADDWrr2:%[0-9]+]]:gpr32 = ADDWrr [[ADDWrr1]], [[ADDWrr1]]
+ ; DEPTH3-NEXT: [[ADDWrr3:%[0-9]+]]:gpr32 = ADDWrr [[ADDWrr2]], [[ADDWrr2]]
+ ; DEPTH3-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 100
+ ; DEPTH3-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr1]], [[MADDWrrr1]], implicit-def $nzcv
+ ; DEPTH3-NEXT: Bcc 10, %bb.7, implicit $nzcv
+ ; DEPTH3-NEXT: B %bb.2
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: bb.2:
+ ; DEPTH3-NEXT: successors: %bb.3(0x40000000), %bb.7(0x40000000)
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 200
+ ; DEPTH3-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[ADDWrr]], [[ADDWrr]], implicit-def $nzcv
+ ; DEPTH3-NEXT: Bcc 10, %bb.7, implicit $nzcv
+ ; DEPTH3-NEXT: B %bb.3
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: bb.3:
+ ; DEPTH3-NEXT: successors: %bb.4(0x40000000), %bb.7(0x40000000)
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 300
+ ; DEPTH3-NEXT: [[SUBSWrr2:%[0-9]+]]:gpr32 = SUBSWrr [[ADDWrr1]], [[ADDWrr1]], implicit-def $nzcv
+ ; DEPTH3-NEXT: Bcc 10, %bb.7, implicit $nzcv
+ ; DEPTH3-NEXT: B %bb.4
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: bb.4:
+ ; DEPTH3-NEXT: successors: %bb.5(0x40000000), %bb.7(0x40000000)
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 400
+ ; DEPTH3-NEXT: [[SUBSWrr3:%[0-9]+]]:gpr32 = SUBSWrr [[ADDWrr2]], [[ADDWrr2]], implicit-def $nzcv
+ ; DEPTH3-NEXT: Bcc 10, %bb.7, implicit $nzcv
+ ; DEPTH3-NEXT: B %bb.5
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: bb.5:
+ ; DEPTH3-NEXT: successors: %bb.6(0x40000000), %bb.7(0x40000000)
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: [[SUBSWrr4:%[0-9]+]]:gpr32 = SUBSWrr [[ADDWrr3]], [[ADDWrr3]], implicit-def $nzcv
+ ; DEPTH3-NEXT: Bcc 10, %bb.7, implicit $nzcv
+ ; DEPTH3-NEXT: B %bb.6
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: bb.6:
+ ; DEPTH3-NEXT: successors: %bb.7(0x80000000)
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: bb.7:
+ ; DEPTH3-NEXT: [[PHI:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.1, [[MOVi64imm1]], %bb.2, [[MOVi64imm2]], %bb.3, [[MOVi64imm3]], %bb.4, [[MOVi64imm3]], %bb.5, [[MOVi64imm3]], %bb.6
+ ; DEPTH3-NEXT: $x0 = COPY [[PHI]]
+ ; DEPTH3-NEXT: RET_ReallyLR implicit $x0
+ ;
+ ; LIMIT3-LABEL: name: cascade_phi_subset
+ ; LIMIT3: bb.0:
+ ; LIMIT3-NEXT: successors: %bb.1(0x80000000)
+ ; LIMIT3-NEXT: liveins: $x0
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: bb.1:
+ ; LIMIT3-NEXT: successors: %bb.2(0x40000000), %bb.7(0x40000000)
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load (s32))
+ ; LIMIT3-NEXT: [[MADDWrrr:%[0-9]+]]:gpr32 = MADDWrrr [[LDRWui]], [[LDRWui]], $wzr
+ ; LIMIT3-NEXT: [[MADDWrrr1:%[0-9]+]]:gpr32 = MADDWrrr [[MADDWrrr]], [[MADDWrrr]], $wzr
+ ; LIMIT3-NEXT: [[ADDWrr:%[0-9]+]]:gpr32 = ADDWrr [[MADDWrrr1]], [[MADDWrrr1]]
+ ; LIMIT3-NEXT: [[ADDWrr1:%[0-9]+]]:gpr32 = ADDWrr [[ADDWrr]], [[ADDWrr]]
+ ; LIMIT3-NEXT: [[ADDWrr2:%[0-9]+]]:gpr32 = ADDWrr [[ADDWrr1]], [[ADDWrr1]]
+ ; LIMIT3-NEXT: [[ADDWrr3:%[0-9]+]]:gpr32 = ADDWrr [[ADDWrr2]], [[ADDWrr2]]
+ ; LIMIT3-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 100
+ ; LIMIT3-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr1]], [[MADDWrrr1]], implicit-def $nzcv
+ ; LIMIT3-NEXT: Bcc 10, %bb.7, implicit $nzcv
+ ; LIMIT3-NEXT: B %bb.2
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: bb.2:
+ ; LIMIT3-NEXT: successors: %bb.3(0x40000000), %bb.7(0x40000000)
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 200
+ ; LIMIT3-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[ADDWrr]], [[ADDWrr]], implicit-def $nzcv
+ ; LIMIT3-NEXT: Bcc 10, %bb.7, implicit $nzcv
+ ; LIMIT3-NEXT: B %bb.3
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: bb.3:
+ ; LIMIT3-NEXT: successors: %bb.4(0x40000000), %bb.7(0x40000000)
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 300
+ ; LIMIT3-NEXT: [[SUBSWrr2:%[0-9]+]]:gpr32 = SUBSWrr [[ADDWrr1]], [[ADDWrr1]], implicit-def $nzcv
+ ; LIMIT3-NEXT: Bcc 10, %bb.7, implicit $nzcv
+ ; LIMIT3-NEXT: B %bb.4
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: bb.4:
+ ; LIMIT3-NEXT: successors: %bb.5(0x40000000), %bb.7(0x40000000)
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 400
+ ; LIMIT3-NEXT: [[SUBSWrr3:%[0-9]+]]:gpr32 = SUBSWrr [[ADDWrr2]], [[ADDWrr2]], implicit-def $nzcv
+ ; LIMIT3-NEXT: Bcc 10, %bb.7, implicit $nzcv
+ ; LIMIT3-NEXT: B %bb.5
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: bb.5:
+ ; LIMIT3-NEXT: successors: %bb.6(0x40000000), %bb.7(0x40000000)
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: [[SUBSWrr4:%[0-9]+]]:gpr32 = SUBSWrr [[ADDWrr3]], [[ADDWrr3]], implicit-def $nzcv
+ ; LIMIT3-NEXT: Bcc 10, %bb.7, implicit $nzcv
+ ; LIMIT3-NEXT: B %bb.6
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: bb.6:
+ ; LIMIT3-NEXT: successors: %bb.7(0x80000000)
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: bb.7:
+ ; LIMIT3-NEXT: [[PHI:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.1, [[MOVi64imm1]], %bb.2, [[MOVi64imm2]], %bb.3, [[MOVi64imm3]], %bb.4, [[MOVi64imm3]], %bb.5, [[MOVi64imm3]], %bb.6
+ ; LIMIT3-NEXT: $x0 = COPY [[PHI]]
+ ; LIMIT3-NEXT: RET_ReallyLR implicit $x0
+ bb.0:
+ successors: %bb.1(0x80000000)
+ liveins: $x0
+
+ %0:gpr64common = COPY $x0
+
+ bb.1:
+ successors: %bb.2(0x40000000), %bb.7(0x40000000)
+
+ %10:gpr32 = LDRWui %0, 0 :: (load (s32))
+ %11:gpr32 = MADDWrrr %10, %10, $wzr
+ %12:gpr32 = MADDWrrr %11, %11, $wzr
+ %20:gpr32 = ADDWrr %12, %12
+ %21:gpr32 = ADDWrr %20, %20
+ %22:gpr32 = ADDWrr %21, %21
+ %23:gpr32 = ADDWrr %22, %22
+ %40:gpr64 = MOVi64imm 100
+ %30:gpr32 = SUBSWrr %12, %12, implicit-def $nzcv
+ Bcc 10, %bb.7, implicit $nzcv
+ B %bb.2
+
+ bb.2:
+ successors: %bb.3(0x40000000), %bb.7(0x40000000)
+
+ %41:gpr64 = MOVi64imm 200
+ %31:gpr32 = SUBSWrr %20, %20, implicit-def $nzcv
+ Bcc 10, %bb.7, implicit $nzcv
+ B %bb.3
+ bb.3:
+ successors: %bb.4(0x40000000), %bb.7(0x40000000)
+
+ %42:gpr64 = MOVi64imm 300
+ %32:gpr32 = SUBSWrr %21, %21, implicit-def $nzcv
+ Bcc 10, %bb.7, implicit $nzcv
+ B %bb.4
+
+ bb.4:
+ successors: %bb.5(0x40000000), %bb.7(0x40000000)
+
+ %43:gpr64 = MOVi64imm 400
+ %33:gpr32 = SUBSWrr %22, %22, implicit-def $nzcv
+ Bcc 10, %bb.7, implicit $nzcv
+ B %bb.5
+
+ bb.5:
+ successors: %bb.6(0x40000000), %bb.7(0x40000000)
+
+ %34:gpr32 = SUBSWrr %23, %23, implicit-def $nzcv
+ Bcc 10, %bb.7, implicit $nzcv
+ B %bb.6
+
+ bb.6:
+ successors: %bb.7(0x80000000)
+
+ bb.7:
+ %90:gpr64 = PHI %40, %bb.1, %41, %bb.2, %42, %bb.3, %43, %bb.4, %43, %bb.5, %43, %bb.6
+ $x0 = COPY %90
+ RET_ReallyLR implicit $x0
+...
diff --git a/llvm/test/CodeGen/AArch64/early-ifcvt-load-to-cond-br-limit.mir b/llvm/test/CodeGen/AArch64/early-ifcvt-load-to-cond-br-limit.mir
index 12170733e2332..c8ab9bb5da684 100644
--- a/llvm/test/CodeGen/AArch64/early-ifcvt-load-to-cond-br-limit.mir
+++ b/llvm/test/CodeGen/AArch64/early-ifcvt-load-to-cond-br-limit.mir
@@ -2,6 +2,7 @@
# RUN: llc -mtriple=aarch64 -mcpu=apple-m1 -run-pass=early-ifcvt -verify-machine-dom-info -o - %s | FileCheck %s --check-prefix=CHECK-DEFAULT
# RUN: llc -mtriple=aarch64 -mcpu=apple-m1 -run-pass=early-ifcvt -enable-early-ifcvt-data-dependent -early-ifcvt-max-region-instrs=3 -verify-machine-dom-info -o - %s | FileCheck %s --check-prefix=CHECK-SMALL-BUDGET
# RUN: llc -mtriple=aarch64 -mcpu=apple-m1 -run-pass=early-ifcvt -enable-early-ifcvt-data-dependent -verify-machine-dom-info -o - %s | FileCheck %s --check-prefix=CHECK-LARGE-BUDGET
+# RUN: llc -mtriple=aarch64 -mcpu=apple-m1 -run-pass=early-ifcvt -enable-early-ifcvt-cascade -enable-early-ifcvt-data-dependent -early-ifcvt-limit=50 -early-ifcvt-max-region-instrs=65 -verify-machine-dom-info -o - %s | FileCheck %s --check-prefix=CHECK-CASCADE-BUDGET
---
name: load_in_pred_region_over_instruction_budget
@@ -138,6 +139,43 @@ body: |
; CHECK-LARGE-BUDGET-NEXT: [[CSELWr:%[0-9]+]]:gpr32 = CSELWr [[COPY1]], [[MADDWrrr1]], 12, implicit $nzcv
; CHECK-LARGE-BUDGET-NEXT: $w0 = COPY [[CSELWr]]
; CHECK-LARGE-BUDGET-NEXT: RET_ReallyLR implicit $w0
+ ;
+ ; CHECK-CASCADE-BUDGET-LABEL: name: load_in_pred_region_over_instruction_budget
+ ; CHECK-CASCADE-BUDGET: bb.0:
+ ; CHECK-CASCADE-BUDGET-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-CASCADE-BUDGET-NEXT: liveins: $x0, $w1, $w2
+ ; CHECK-CASCADE-BUDGET-NEXT: {{ $}}
+ ; CHECK-CASCADE-BUDGET-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[COPY1:%[0-9]+]]:gpr32 = COPY $w1
+ ; CHECK-CASCADE-BUDGET-NEXT: [[COPY2:%[0-9]+]]:gpr32 = COPY $w2
+ ; CHECK-CASCADE-BUDGET-NEXT: {{ $}}
+ ; CHECK-CASCADE-BUDGET-NEXT: bb.1:
+ ; CHECK-CASCADE-BUDGET-NEXT: successors: %bb.2(0x40000000), %bb.3(0x40000000)
+ ; CHECK-CASCADE-BUDGET-NEXT: {{ $}}
+ ; CHECK-CASCADE-BUDGET-NEXT: [[LDRWui:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 0 :: (load (s32))
+ ; CHECK-CASCADE-BUDGET-NEXT: [[SUBSWri:%[0-9]+]]:gpr32 = SUBSWri [[LDRWui]], 0, 0, implicit-def $nzcv
+ ; CHECK-CASCADE-BUDGET-NEXT: Bcc 0, %bb.2, implicit $nzcv
+ ; CHECK-CASCADE-BUDGET-NEXT: B %bb.3
+ ; CHECK-CASCADE-BUDGET-NEXT: {{ $}}
+ ; CHECK-CASCADE-BUDGET-NEXT: bb.2:
+ ; CHECK-CASCADE-BUDGET-NEXT: successors: %bb.3(0x80000000)
+ ; CHECK-CASCADE-BUDGET-NEXT: {{ $}}
+ ; CHECK-CASCADE-BUDGET-NEXT: STRWui $wzr, [[COPY]], 0 :: (store (s32))
+ ; CHECK-CASCADE-BUDGET-NEXT: STRWui $wzr, [[COPY]], 1 :: (store (s32))
+ ; CHECK-CASCADE-BUDGET-NEXT: STRWui $wzr, [[COPY]], 2 :: (store (s32))
+ ; CHECK-CASCADE-BUDGET-NEXT: STRWui $wzr, [[COPY]], 3 :: (store (s32))
+ ; CHECK-CASCADE-BUDGET-NEXT: STRWui $wzr, [[COPY]], 4 :: (store (s32))
+ ; CHECK-CASCADE-BUDGET-NEXT: STRWui $wzr, [[COPY]], 5 :: (store (s32))
+ ; CHECK-CASCADE-BUDGET-NEXT: B %bb.3
+ ; CHECK-CASCADE-BUDGET-NEXT: {{ $}}
+ ; CHECK-CASCADE-BUDGET-NEXT: bb.3:
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri:%[0-9]+]]:gpr32common = ADDWri [[LDRWui]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[SUBSWri1:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri]], 100, 0, implicit-def $nzcv
+ ; CHECK-CASCADE-BUDGET-NEXT: [[MADDWrrr:%[0-9]+]]:gpr32 = MADDWrrr [[COPY1]], [[COPY2]], $wzr
+ ; CHECK-CASCADE-BUDGET-NEXT: [[MADDWrrr1:%[0-9]+]]:gpr32 = MADDWrrr [[MADDWrrr]], [[COPY1]], $wzr
+ ; CHECK-CASCADE-BUDGET-NEXT: [[CSELWr:%[0-9]+]]:gpr32 = CSELWr [[COPY1]], [[MADDWrrr1]], 12, implicit $nzcv
+ ; CHECK-CASCADE-BUDGET-NEXT: $w0 = COPY [[CSELWr]]
+ ; CHECK-CASCADE-BUDGET-NEXT: RET_ReallyLR implicit $w0
bb.0:
successors: %bb.1(0x80000000)
liveins: $x0, $w1, $w2
@@ -184,3 +222,542 @@ body: |
$w0 = COPY %8
RET_ReallyLR implicit $w0
...
+---
+name: cascade_load_to_cond_br_scan_budget
+alignment: 4
+tracksRegLiveness: true
+body: |
+ ; CHECK-DEFAULT-LABEL: name: cascade_load_to_cond_br_scan_budget
+ ; CHECK-DEFAULT: bb.0:
+ ; CHECK-DEFAULT-NEXT: successors: %bb.8(0x40000000), %bb.1(0x40000000)
+ ; CHECK-DEFAULT-NEXT: liveins: $x0
+ ; CHECK-DEFAULT-NEXT: {{ $}}
+ ; CHECK-DEFAULT-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; CHECK-DEFAULT-NEXT: [[LDRWui:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 0 :: (load (s32))
+ ; CHECK-DEFAULT-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 1
+ ; CHECK-DEFAULT-NEXT: [[MOVi32imm1:%[0-9]+]]:gpr32 = MOVi32imm 2
+ ; CHECK-DEFAULT-NEXT: [[MOVi32imm2:%[0-9]+]]:gpr32 = MOVi32imm 3
+ ; CHECK-DEFAULT-NEXT: [[MOVi32imm3:%[0-9]+]]:gpr32 = MOVi32imm 4
+ ; CHECK-DEFAULT-NEXT: [[LDRWui1:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 1 :: (load (s32))
+ ; CHECK-DEFAULT-NEXT: [[LDRWui2:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 2 :: (load (s32))
+ ; CHECK-DEFAULT-NEXT: [[LDRWui3:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 3 :: (load (s32))
+ ; CHECK-DEFAULT-NEXT: [[LDRWui4:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 4 :: (load (s32))
+ ; CHECK-DEFAULT-NEXT: [[LDRWui5:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 5 :: (load (s32))
+ ; CHECK-DEFAULT-NEXT: [[LDRWui6:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 6 :: (load (s32))
+ ; CHECK-DEFAULT-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 100
+ ; CHECK-DEFAULT-NEXT: [[ADDWri:%[0-9]+]]:gpr32common = ADDWri [[LDRWui]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri1:%[0-9]+]]:gpr32common = ADDWri [[ADDWri]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri2:%[0-9]+]]:gpr32common = ADDWri [[ADDWri1]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri3:%[0-9]+]]:gpr32common = ADDWri [[ADDWri2]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri4:%[0-9]+]]:gpr32common = ADDWri [[ADDWri3]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[SUBSWri:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri4]], 1, 0, implicit-def $nzcv
+ ; CHECK-DEFAULT-NEXT: Bcc 10, %bb.8, implicit $nzcv
+ ; CHECK-DEFAULT-NEXT: B %bb.1
+ ; CHECK-DEFAULT-NEXT: {{ $}}
+ ; CHECK-DEFAULT-NEXT: bb.1:
+ ; CHECK-DEFAULT-NEXT: successors: %bb.8(0x40000000), %bb.2(0x40000000)
+ ; CHECK-DEFAULT-NEXT: {{ $}}
+ ; CHECK-DEFAULT-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 101
+ ; CHECK-DEFAULT-NEXT: [[ADDWri5:%[0-9]+]]:gpr32common = ADDWri [[LDRWui1]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri6:%[0-9]+]]:gpr32common = ADDWri [[ADDWri5]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri7:%[0-9]+]]:gpr32common = ADDWri [[ADDWri6]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri8:%[0-9]+]]:gpr32common = ADDWri [[ADDWri7]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri9:%[0-9]+]]:gpr32common = ADDWri [[ADDWri8]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[SUBSWri1:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri9]], 2, 0, implicit-def $nzcv
+ ; CHECK-DEFAULT-NEXT: Bcc 11, %bb.8, implicit $nzcv
+ ; CHECK-DEFAULT-NEXT: B %bb.2
+ ; CHECK-DEFAULT-NEXT: {{ $}}
+ ; CHECK-DEFAULT-NEXT: bb.2:
+ ; CHECK-DEFAULT-NEXT: successors: %bb.8(0x40000000), %bb.3(0x40000000)
+ ; CHECK-DEFAULT-NEXT: {{ $}}
+ ; CHECK-DEFAULT-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 102
+ ; CHECK-DEFAULT-NEXT: [[ADDWri10:%[0-9]+]]:gpr32common = ADDWri [[LDRWui2]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri11:%[0-9]+]]:gpr32common = ADDWri [[ADDWri10]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri12:%[0-9]+]]:gpr32common = ADDWri [[ADDWri11]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri13:%[0-9]+]]:gpr32common = ADDWri [[ADDWri12]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri14:%[0-9]+]]:gpr32common = ADDWri [[ADDWri13]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[SUBSWri2:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri14]], 3, 0, implicit-def $nzcv
+ ; CHECK-DEFAULT-NEXT: Bcc 12, %bb.8, implicit $nzcv
+ ; CHECK-DEFAULT-NEXT: B %bb.3
+ ; CHECK-DEFAULT-NEXT: {{ $}}
+ ; CHECK-DEFAULT-NEXT: bb.3:
+ ; CHECK-DEFAULT-NEXT: successors: %bb.8(0x40000000), %bb.4(0x40000000)
+ ; CHECK-DEFAULT-NEXT: {{ $}}
+ ; CHECK-DEFAULT-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 103
+ ; CHECK-DEFAULT-NEXT: [[ADDWri15:%[0-9]+]]:gpr32common = ADDWri [[LDRWui3]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri16:%[0-9]+]]:gpr32common = ADDWri [[ADDWri15]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri17:%[0-9]+]]:gpr32common = ADDWri [[ADDWri16]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri18:%[0-9]+]]:gpr32common = ADDWri [[ADDWri17]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri19:%[0-9]+]]:gpr32common = ADDWri [[ADDWri18]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[SUBSWri3:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri19]], 4, 0, implicit-def $nzcv
+ ; CHECK-DEFAULT-NEXT: Bcc 13, %bb.8, implicit $nzcv
+ ; CHECK-DEFAULT-NEXT: B %bb.4
+ ; CHECK-DEFAULT-NEXT: {{ $}}
+ ; CHECK-DEFAULT-NEXT: bb.4:
+ ; CHECK-DEFAULT-NEXT: successors: %bb.8(0x40000000), %bb.5(0x40000000)
+ ; CHECK-DEFAULT-NEXT: {{ $}}
+ ; CHECK-DEFAULT-NEXT: [[MOVi64imm4:%[0-9]+]]:gpr64 = MOVi64imm 104
+ ; CHECK-DEFAULT-NEXT: [[ADDWri20:%[0-9]+]]:gpr32common = ADDWri [[LDRWui4]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri21:%[0-9]+]]:gpr32common = ADDWri [[ADDWri20]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri22:%[0-9]+]]:gpr32common = ADDWri [[ADDWri21]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri23:%[0-9]+]]:gpr32common = ADDWri [[ADDWri22]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri24:%[0-9]+]]:gpr32common = ADDWri [[ADDWri23]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[SUBSWri4:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri24]], 5, 0, implicit-def $nzcv
+ ; CHECK-DEFAULT-NEXT: Bcc 14, %bb.8, implicit $nzcv
+ ; CHECK-DEFAULT-NEXT: B %bb.5
+ ; CHECK-DEFAULT-NEXT: {{ $}}
+ ; CHECK-DEFAULT-NEXT: bb.5:
+ ; CHECK-DEFAULT-NEXT: successors: %bb.8(0x40000000), %bb.6(0x40000000)
+ ; CHECK-DEFAULT-NEXT: {{ $}}
+ ; CHECK-DEFAULT-NEXT: [[MOVi64imm5:%[0-9]+]]:gpr64 = MOVi64imm 105
+ ; CHECK-DEFAULT-NEXT: [[ADDWri25:%[0-9]+]]:gpr32common = ADDWri [[LDRWui5]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri26:%[0-9]+]]:gpr32common = ADDWri [[ADDWri25]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri27:%[0-9]+]]:gpr32common = ADDWri [[ADDWri26]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri28:%[0-9]+]]:gpr32common = ADDWri [[ADDWri27]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri29:%[0-9]+]]:gpr32common = ADDWri [[ADDWri28]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[SUBSWri5:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri29]], 6, 0, implicit-def $nzcv
+ ; CHECK-DEFAULT-NEXT: Bcc 15, %bb.8, implicit $nzcv
+ ; CHECK-DEFAULT-NEXT: B %bb.6
+ ; CHECK-DEFAULT-NEXT: {{ $}}
+ ; CHECK-DEFAULT-NEXT: bb.6:
+ ; CHECK-DEFAULT-NEXT: successors: %bb.8(0x40000000), %bb.7(0x40000000)
+ ; CHECK-DEFAULT-NEXT: {{ $}}
+ ; CHECK-DEFAULT-NEXT: [[MOVi64imm6:%[0-9]+]]:gpr64 = MOVi64imm 106
+ ; CHECK-DEFAULT-NEXT: [[ADDWri30:%[0-9]+]]:gpr32common = ADDWri [[LDRWui6]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri31:%[0-9]+]]:gpr32common = ADDWri [[ADDWri30]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri32:%[0-9]+]]:gpr32common = ADDWri [[ADDWri31]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri33:%[0-9]+]]:gpr32common = ADDWri [[ADDWri32]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri34:%[0-9]+]]:gpr32common = ADDWri [[ADDWri33]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[SUBSWri6:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri34]], 7, 0, implicit-def $nzcv
+ ; CHECK-DEFAULT-NEXT: Bcc 16, %bb.8, implicit $nzcv
+ ; CHECK-DEFAULT-NEXT: B %bb.7
+ ; CHECK-DEFAULT-NEXT: {{ $}}
+ ; CHECK-DEFAULT-NEXT: bb.7:
+ ; CHECK-DEFAULT-NEXT: successors: %bb.8(0x80000000)
+ ; CHECK-DEFAULT-NEXT: {{ $}}
+ ; CHECK-DEFAULT-NEXT: [[MOVi64imm7:%[0-9]+]]:gpr64 = MOVi64imm 107
+ ; CHECK-DEFAULT-NEXT: {{ $}}
+ ; CHECK-DEFAULT-NEXT: bb.8:
+ ; CHECK-DEFAULT-NEXT: [[PHI:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.0, [[MOVi64imm1]], %bb.1, [[MOVi64imm2]], %bb.2, [[MOVi64imm3]], %bb.3, [[MOVi64imm4]], %bb.4, [[MOVi64imm5]], %bb.5, [[MOVi64imm6]], %bb.6, [[MOVi64imm7]], %bb.7
+ ; CHECK-DEFAULT-NEXT: $x0 = COPY [[PHI]]
+ ; CHECK-DEFAULT-NEXT: RET_ReallyLR implicit $x0
+ ;
+ ; CHECK-SMALL-BUDGET-LABEL: name: cascade_load_to_cond_br_scan_budget
+ ; CHECK-SMALL-BUDGET: bb.0:
+ ; CHECK-SMALL-BUDGET-NEXT: successors: %bb.8(0x40000000), %bb.1(0x40000000)
+ ; CHECK-SMALL-BUDGET-NEXT: liveins: $x0
+ ; CHECK-SMALL-BUDGET-NEXT: {{ $}}
+ ; CHECK-SMALL-BUDGET-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; CHECK-SMALL-BUDGET-NEXT: [[LDRWui:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 0 :: (load (s32))
+ ; CHECK-SMALL-BUDGET-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 1
+ ; CHECK-SMALL-BUDGET-NEXT: [[MOVi32imm1:%[0-9]+]]:gpr32 = MOVi32imm 2
+ ; CHECK-SMALL-BUDGET-NEXT: [[MOVi32imm2:%[0-9]+]]:gpr32 = MOVi32imm 3
+ ; CHECK-SMALL-BUDGET-NEXT: [[MOVi32imm3:%[0-9]+]]:gpr32 = MOVi32imm 4
+ ; CHECK-SMALL-BUDGET-NEXT: [[LDRWui1:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 1 :: (load (s32))
+ ; CHECK-SMALL-BUDGET-NEXT: [[LDRWui2:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 2 :: (load (s32))
+ ; CHECK-SMALL-BUDGET-NEXT: [[LDRWui3:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 3 :: (load (s32))
+ ; CHECK-SMALL-BUDGET-NEXT: [[LDRWui4:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 4 :: (load (s32))
+ ; CHECK-SMALL-BUDGET-NEXT: [[LDRWui5:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 5 :: (load (s32))
+ ; CHECK-SMALL-BUDGET-NEXT: [[LDRWui6:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 6 :: (load (s32))
+ ; CHECK-SMALL-BUDGET-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 100
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri:%[0-9]+]]:gpr32common = ADDWri [[LDRWui]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri1:%[0-9]+]]:gpr32common = ADDWri [[ADDWri]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri2:%[0-9]+]]:gpr32common = ADDWri [[ADDWri1]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri3:%[0-9]+]]:gpr32common = ADDWri [[ADDWri2]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri4:%[0-9]+]]:gpr32common = ADDWri [[ADDWri3]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[SUBSWri:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri4]], 1, 0, implicit-def $nzcv
+ ; CHECK-SMALL-BUDGET-NEXT: Bcc 10, %bb.8, implicit $nzcv
+ ; CHECK-SMALL-BUDGET-NEXT: B %bb.1
+ ; CHECK-SMALL-BUDGET-NEXT: {{ $}}
+ ; CHECK-SMALL-BUDGET-NEXT: bb.1:
+ ; CHECK-SMALL-BUDGET-NEXT: successors: %bb.8(0x40000000), %bb.2(0x40000000)
+ ; CHECK-SMALL-BUDGET-NEXT: {{ $}}
+ ; CHECK-SMALL-BUDGET-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 101
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri5:%[0-9]+]]:gpr32common = ADDWri [[LDRWui1]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri6:%[0-9]+]]:gpr32common = ADDWri [[ADDWri5]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri7:%[0-9]+]]:gpr32common = ADDWri [[ADDWri6]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri8:%[0-9]+]]:gpr32common = ADDWri [[ADDWri7]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri9:%[0-9]+]]:gpr32common = ADDWri [[ADDWri8]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[SUBSWri1:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri9]], 2, 0, implicit-def $nzcv
+ ; CHECK-SMALL-BUDGET-NEXT: Bcc 11, %bb.8, implicit $nzcv
+ ; CHECK-SMALL-BUDGET-NEXT: B %bb.2
+ ; CHECK-SMALL-BUDGET-NEXT: {{ $}}
+ ; CHECK-SMALL-BUDGET-NEXT: bb.2:
+ ; CHECK-SMALL-BUDGET-NEXT: successors: %bb.8(0x40000000), %bb.3(0x40000000)
+ ; CHECK-SMALL-BUDGET-NEXT: {{ $}}
+ ; CHECK-SMALL-BUDGET-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 102
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri10:%[0-9]+]]:gpr32common = ADDWri [[LDRWui2]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri11:%[0-9]+]]:gpr32common = ADDWri [[ADDWri10]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri12:%[0-9]+]]:gpr32common = ADDWri [[ADDWri11]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri13:%[0-9]+]]:gpr32common = ADDWri [[ADDWri12]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri14:%[0-9]+]]:gpr32common = ADDWri [[ADDWri13]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[SUBSWri2:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri14]], 3, 0, implicit-def $nzcv
+ ; CHECK-SMALL-BUDGET-NEXT: Bcc 12, %bb.8, implicit $nzcv
+ ; CHECK-SMALL-BUDGET-NEXT: B %bb.3
+ ; CHECK-SMALL-BUDGET-NEXT: {{ $}}
+ ; CHECK-SMALL-BUDGET-NEXT: bb.3:
+ ; CHECK-SMALL-BUDGET-NEXT: successors: %bb.8(0x40000000), %bb.4(0x40000000)
+ ; CHECK-SMALL-BUDGET-NEXT: {{ $}}
+ ; CHECK-SMALL-BUDGET-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 103
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri15:%[0-9]+]]:gpr32common = ADDWri [[LDRWui3]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri16:%[0-9]+]]:gpr32common = ADDWri [[ADDWri15]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri17:%[0-9]+]]:gpr32common = ADDWri [[ADDWri16]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri18:%[0-9]+]]:gpr32common = ADDWri [[ADDWri17]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri19:%[0-9]+]]:gpr32common = ADDWri [[ADDWri18]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[SUBSWri3:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri19]], 4, 0, implicit-def $nzcv
+ ; CHECK-SMALL-BUDGET-NEXT: Bcc 13, %bb.8, implicit $nzcv
+ ; CHECK-SMALL-BUDGET-NEXT: B %bb.4
+ ; CHECK-SMALL-BUDGET-NEXT: {{ $}}
+ ; CHECK-SMALL-BUDGET-NEXT: bb.4:
+ ; CHECK-SMALL-BUDGET-NEXT: successors: %bb.8(0x40000000), %bb.5(0x40000000)
+ ; CHECK-SMALL-BUDGET-NEXT: {{ $}}
+ ; CHECK-SMALL-BUDGET-NEXT: [[MOVi64imm4:%[0-9]+]]:gpr64 = MOVi64imm 104
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri20:%[0-9]+]]:gpr32common = ADDWri [[LDRWui4]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri21:%[0-9]+]]:gpr32common = ADDWri [[ADDWri20]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri22:%[0-9]+]]:gpr32common = ADDWri [[ADDWri21]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri23:%[0-9]+]]:gpr32common = ADDWri [[ADDWri22]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri24:%[0-9]+]]:gpr32common = ADDWri [[ADDWri23]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[SUBSWri4:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri24]], 5, 0, implicit-def $nzcv
+ ; CHECK-SMALL-BUDGET-NEXT: Bcc 14, %bb.8, implicit $nzcv
+ ; CHECK-SMALL-BUDGET-NEXT: B %bb.5
+ ; CHECK-SMALL-BUDGET-NEXT: {{ $}}
+ ; CHECK-SMALL-BUDGET-NEXT: bb.5:
+ ; CHECK-SMALL-BUDGET-NEXT: successors: %bb.8(0x40000000), %bb.6(0x40000000)
+ ; CHECK-SMALL-BUDGET-NEXT: {{ $}}
+ ; CHECK-SMALL-BUDGET-NEXT: [[MOVi64imm5:%[0-9]+]]:gpr64 = MOVi64imm 105
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri25:%[0-9]+]]:gpr32common = ADDWri [[LDRWui5]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri26:%[0-9]+]]:gpr32common = ADDWri [[ADDWri25]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri27:%[0-9]+]]:gpr32common = ADDWri [[ADDWri26]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri28:%[0-9]+]]:gpr32common = ADDWri [[ADDWri27]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri29:%[0-9]+]]:gpr32common = ADDWri [[ADDWri28]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[SUBSWri5:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri29]], 6, 0, implicit-def $nzcv
+ ; CHECK-SMALL-BUDGET-NEXT: Bcc 15, %bb.8, implicit $nzcv
+ ; CHECK-SMALL-BUDGET-NEXT: B %bb.6
+ ; CHECK-SMALL-BUDGET-NEXT: {{ $}}
+ ; CHECK-SMALL-BUDGET-NEXT: bb.6:
+ ; CHECK-SMALL-BUDGET-NEXT: successors: %bb.8(0x40000000), %bb.7(0x40000000)
+ ; CHECK-SMALL-BUDGET-NEXT: {{ $}}
+ ; CHECK-SMALL-BUDGET-NEXT: [[MOVi64imm6:%[0-9]+]]:gpr64 = MOVi64imm 106
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri30:%[0-9]+]]:gpr32common = ADDWri [[LDRWui6]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri31:%[0-9]+]]:gpr32common = ADDWri [[ADDWri30]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri32:%[0-9]+]]:gpr32common = ADDWri [[ADDWri31]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri33:%[0-9]+]]:gpr32common = ADDWri [[ADDWri32]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri34:%[0-9]+]]:gpr32common = ADDWri [[ADDWri33]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[SUBSWri6:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri34]], 7, 0, implicit-def $nzcv
+ ; CHECK-SMALL-BUDGET-NEXT: Bcc 16, %bb.8, implicit $nzcv
+ ; CHECK-SMALL-BUDGET-NEXT: B %bb.7
+ ; CHECK-SMALL-BUDGET-NEXT: {{ $}}
+ ; CHECK-SMALL-BUDGET-NEXT: bb.7:
+ ; CHECK-SMALL-BUDGET-NEXT: successors: %bb.8(0x80000000)
+ ; CHECK-SMALL-BUDGET-NEXT: {{ $}}
+ ; CHECK-SMALL-BUDGET-NEXT: [[MOVi64imm7:%[0-9]+]]:gpr64 = MOVi64imm 107
+ ; CHECK-SMALL-BUDGET-NEXT: {{ $}}
+ ; CHECK-SMALL-BUDGET-NEXT: bb.8:
+ ; CHECK-SMALL-BUDGET-NEXT: [[PHI:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.0, [[MOVi64imm1]], %bb.1, [[MOVi64imm2]], %bb.2, [[MOVi64imm3]], %bb.3, [[MOVi64imm4]], %bb.4, [[MOVi64imm5]], %bb.5, [[MOVi64imm6]], %bb.6, [[MOVi64imm7]], %bb.7
+ ; CHECK-SMALL-BUDGET-NEXT: $x0 = COPY [[PHI]]
+ ; CHECK-SMALL-BUDGET-NEXT: RET_ReallyLR implicit $x0
+ ;
+ ; CHECK-LARGE-BUDGET-LABEL: name: cascade_load_to_cond_br_scan_budget
+ ; CHECK-LARGE-BUDGET: bb.0:
+ ; CHECK-LARGE-BUDGET-NEXT: successors: %bb.8(0x40000000), %bb.1(0x40000000)
+ ; CHECK-LARGE-BUDGET-NEXT: liveins: $x0
+ ; CHECK-LARGE-BUDGET-NEXT: {{ $}}
+ ; CHECK-LARGE-BUDGET-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; CHECK-LARGE-BUDGET-NEXT: [[LDRWui:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 0 :: (load (s32))
+ ; CHECK-LARGE-BUDGET-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 1
+ ; CHECK-LARGE-BUDGET-NEXT: [[MOVi32imm1:%[0-9]+]]:gpr32 = MOVi32imm 2
+ ; CHECK-LARGE-BUDGET-NEXT: [[MOVi32imm2:%[0-9]+]]:gpr32 = MOVi32imm 3
+ ; CHECK-LARGE-BUDGET-NEXT: [[MOVi32imm3:%[0-9]+]]:gpr32 = MOVi32imm 4
+ ; CHECK-LARGE-BUDGET-NEXT: [[LDRWui1:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 1 :: (load (s32))
+ ; CHECK-LARGE-BUDGET-NEXT: [[LDRWui2:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 2 :: (load (s32))
+ ; CHECK-LARGE-BUDGET-NEXT: [[LDRWui3:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 3 :: (load (s32))
+ ; CHECK-LARGE-BUDGET-NEXT: [[LDRWui4:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 4 :: (load (s32))
+ ; CHECK-LARGE-BUDGET-NEXT: [[LDRWui5:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 5 :: (load (s32))
+ ; CHECK-LARGE-BUDGET-NEXT: [[LDRWui6:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 6 :: (load (s32))
+ ; CHECK-LARGE-BUDGET-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 100
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri:%[0-9]+]]:gpr32common = ADDWri [[LDRWui]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri1:%[0-9]+]]:gpr32common = ADDWri [[ADDWri]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri2:%[0-9]+]]:gpr32common = ADDWri [[ADDWri1]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri3:%[0-9]+]]:gpr32common = ADDWri [[ADDWri2]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri4:%[0-9]+]]:gpr32common = ADDWri [[ADDWri3]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[SUBSWri:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri4]], 1, 0, implicit-def $nzcv
+ ; CHECK-LARGE-BUDGET-NEXT: Bcc 10, %bb.8, implicit $nzcv
+ ; CHECK-LARGE-BUDGET-NEXT: B %bb.1
+ ; CHECK-LARGE-BUDGET-NEXT: {{ $}}
+ ; CHECK-LARGE-BUDGET-NEXT: bb.1:
+ ; CHECK-LARGE-BUDGET-NEXT: successors: %bb.8(0x40000000), %bb.2(0x40000000)
+ ; CHECK-LARGE-BUDGET-NEXT: {{ $}}
+ ; CHECK-LARGE-BUDGET-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 101
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri5:%[0-9]+]]:gpr32common = ADDWri [[LDRWui1]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri6:%[0-9]+]]:gpr32common = ADDWri [[ADDWri5]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri7:%[0-9]+]]:gpr32common = ADDWri [[ADDWri6]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri8:%[0-9]+]]:gpr32common = ADDWri [[ADDWri7]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri9:%[0-9]+]]:gpr32common = ADDWri [[ADDWri8]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[SUBSWri1:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri9]], 2, 0, implicit-def $nzcv
+ ; CHECK-LARGE-BUDGET-NEXT: Bcc 11, %bb.8, implicit $nzcv
+ ; CHECK-LARGE-BUDGET-NEXT: B %bb.2
+ ; CHECK-LARGE-BUDGET-NEXT: {{ $}}
+ ; CHECK-LARGE-BUDGET-NEXT: bb.2:
+ ; CHECK-LARGE-BUDGET-NEXT: successors: %bb.8(0x40000000), %bb.3(0x40000000)
+ ; CHECK-LARGE-BUDGET-NEXT: {{ $}}
+ ; CHECK-LARGE-BUDGET-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 102
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri10:%[0-9]+]]:gpr32common = ADDWri [[LDRWui2]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri11:%[0-9]+]]:gpr32common = ADDWri [[ADDWri10]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri12:%[0-9]+]]:gpr32common = ADDWri [[ADDWri11]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri13:%[0-9]+]]:gpr32common = ADDWri [[ADDWri12]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri14:%[0-9]+]]:gpr32common = ADDWri [[ADDWri13]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[SUBSWri2:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri14]], 3, 0, implicit-def $nzcv
+ ; CHECK-LARGE-BUDGET-NEXT: Bcc 12, %bb.8, implicit $nzcv
+ ; CHECK-LARGE-BUDGET-NEXT: B %bb.3
+ ; CHECK-LARGE-BUDGET-NEXT: {{ $}}
+ ; CHECK-LARGE-BUDGET-NEXT: bb.3:
+ ; CHECK-LARGE-BUDGET-NEXT: successors: %bb.8(0x40000000), %bb.4(0x40000000)
+ ; CHECK-LARGE-BUDGET-NEXT: {{ $}}
+ ; CHECK-LARGE-BUDGET-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 103
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri15:%[0-9]+]]:gpr32common = ADDWri [[LDRWui3]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri16:%[0-9]+]]:gpr32common = ADDWri [[ADDWri15]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri17:%[0-9]+]]:gpr32common = ADDWri [[ADDWri16]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri18:%[0-9]+]]:gpr32common = ADDWri [[ADDWri17]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri19:%[0-9]+]]:gpr32common = ADDWri [[ADDWri18]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[SUBSWri3:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri19]], 4, 0, implicit-def $nzcv
+ ; CHECK-LARGE-BUDGET-NEXT: Bcc 13, %bb.8, implicit $nzcv
+ ; CHECK-LARGE-BUDGET-NEXT: B %bb.4
+ ; CHECK-LARGE-BUDGET-NEXT: {{ $}}
+ ; CHECK-LARGE-BUDGET-NEXT: bb.4:
+ ; CHECK-LARGE-BUDGET-NEXT: successors: %bb.8(0x40000000), %bb.5(0x40000000)
+ ; CHECK-LARGE-BUDGET-NEXT: {{ $}}
+ ; CHECK-LARGE-BUDGET-NEXT: [[MOVi64imm4:%[0-9]+]]:gpr64 = MOVi64imm 104
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri20:%[0-9]+]]:gpr32common = ADDWri [[LDRWui4]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri21:%[0-9]+]]:gpr32common = ADDWri [[ADDWri20]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri22:%[0-9]+]]:gpr32common = ADDWri [[ADDWri21]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri23:%[0-9]+]]:gpr32common = ADDWri [[ADDWri22]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri24:%[0-9]+]]:gpr32common = ADDWri [[ADDWri23]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[SUBSWri4:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri24]], 5, 0, implicit-def $nzcv
+ ; CHECK-LARGE-BUDGET-NEXT: Bcc 14, %bb.8, implicit $nzcv
+ ; CHECK-LARGE-BUDGET-NEXT: B %bb.5
+ ; CHECK-LARGE-BUDGET-NEXT: {{ $}}
+ ; CHECK-LARGE-BUDGET-NEXT: bb.5:
+ ; CHECK-LARGE-BUDGET-NEXT: successors: %bb.8(0x40000000), %bb.6(0x40000000)
+ ; CHECK-LARGE-BUDGET-NEXT: {{ $}}
+ ; CHECK-LARGE-BUDGET-NEXT: [[MOVi64imm5:%[0-9]+]]:gpr64 = MOVi64imm 105
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri25:%[0-9]+]]:gpr32common = ADDWri [[LDRWui5]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri26:%[0-9]+]]:gpr32common = ADDWri [[ADDWri25]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri27:%[0-9]+]]:gpr32common = ADDWri [[ADDWri26]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri28:%[0-9]+]]:gpr32common = ADDWri [[ADDWri27]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri29:%[0-9]+]]:gpr32common = ADDWri [[ADDWri28]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[SUBSWri5:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri29]], 6, 0, implicit-def $nzcv
+ ; CHECK-LARGE-BUDGET-NEXT: Bcc 15, %bb.8, implicit $nzcv
+ ; CHECK-LARGE-BUDGET-NEXT: B %bb.6
+ ; CHECK-LARGE-BUDGET-NEXT: {{ $}}
+ ; CHECK-LARGE-BUDGET-NEXT: bb.6:
+ ; CHECK-LARGE-BUDGET-NEXT: successors: %bb.8(0x40000000), %bb.7(0x40000000)
+ ; CHECK-LARGE-BUDGET-NEXT: {{ $}}
+ ; CHECK-LARGE-BUDGET-NEXT: [[MOVi64imm6:%[0-9]+]]:gpr64 = MOVi64imm 106
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri30:%[0-9]+]]:gpr32common = ADDWri [[LDRWui6]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri31:%[0-9]+]]:gpr32common = ADDWri [[ADDWri30]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri32:%[0-9]+]]:gpr32common = ADDWri [[ADDWri31]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri33:%[0-9]+]]:gpr32common = ADDWri [[ADDWri32]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri34:%[0-9]+]]:gpr32common = ADDWri [[ADDWri33]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[SUBSWri6:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri34]], 7, 0, implicit-def $nzcv
+ ; CHECK-LARGE-BUDGET-NEXT: Bcc 16, %bb.8, implicit $nzcv
+ ; CHECK-LARGE-BUDGET-NEXT: B %bb.7
+ ; CHECK-LARGE-BUDGET-NEXT: {{ $}}
+ ; CHECK-LARGE-BUDGET-NEXT: bb.7:
+ ; CHECK-LARGE-BUDGET-NEXT: successors: %bb.8(0x80000000)
+ ; CHECK-LARGE-BUDGET-NEXT: {{ $}}
+ ; CHECK-LARGE-BUDGET-NEXT: [[MOVi64imm7:%[0-9]+]]:gpr64 = MOVi64imm 107
+ ; CHECK-LARGE-BUDGET-NEXT: {{ $}}
+ ; CHECK-LARGE-BUDGET-NEXT: bb.8:
+ ; CHECK-LARGE-BUDGET-NEXT: [[PHI:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.0, [[MOVi64imm1]], %bb.1, [[MOVi64imm2]], %bb.2, [[MOVi64imm3]], %bb.3, [[MOVi64imm4]], %bb.4, [[MOVi64imm5]], %bb.5, [[MOVi64imm6]], %bb.6, [[MOVi64imm7]], %bb.7
+ ; CHECK-LARGE-BUDGET-NEXT: $x0 = COPY [[PHI]]
+ ; CHECK-LARGE-BUDGET-NEXT: RET_ReallyLR implicit $x0
+ ;
+ ; CHECK-CASCADE-BUDGET-LABEL: name: cascade_load_to_cond_br_scan_budget
+ ; CHECK-CASCADE-BUDGET: bb.0:
+ ; CHECK-CASCADE-BUDGET-NEXT: liveins: $x0
+ ; CHECK-CASCADE-BUDGET-NEXT: {{ $}}
+ ; CHECK-CASCADE-BUDGET-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[LDRWui:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 0 :: (load (s32))
+ ; CHECK-CASCADE-BUDGET-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 1
+ ; CHECK-CASCADE-BUDGET-NEXT: [[MOVi32imm1:%[0-9]+]]:gpr32 = MOVi32imm 2
+ ; CHECK-CASCADE-BUDGET-NEXT: [[MOVi32imm2:%[0-9]+]]:gpr32 = MOVi32imm 3
+ ; CHECK-CASCADE-BUDGET-NEXT: [[MOVi32imm3:%[0-9]+]]:gpr32 = MOVi32imm 4
+ ; CHECK-CASCADE-BUDGET-NEXT: [[LDRWui1:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 1 :: (load (s32))
+ ; CHECK-CASCADE-BUDGET-NEXT: [[LDRWui2:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 2 :: (load (s32))
+ ; CHECK-CASCADE-BUDGET-NEXT: [[LDRWui3:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 3 :: (load (s32))
+ ; CHECK-CASCADE-BUDGET-NEXT: [[LDRWui4:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 4 :: (load (s32))
+ ; CHECK-CASCADE-BUDGET-NEXT: [[LDRWui5:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 5 :: (load (s32))
+ ; CHECK-CASCADE-BUDGET-NEXT: [[LDRWui6:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 6 :: (load (s32))
+ ; CHECK-CASCADE-BUDGET-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 100
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri:%[0-9]+]]:gpr32common = ADDWri [[LDRWui]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri1:%[0-9]+]]:gpr32common = ADDWri [[ADDWri]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri2:%[0-9]+]]:gpr32common = ADDWri [[ADDWri1]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri3:%[0-9]+]]:gpr32common = ADDWri [[ADDWri2]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri4:%[0-9]+]]:gpr32common = ADDWri [[ADDWri3]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 101
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri5:%[0-9]+]]:gpr32common = ADDWri [[LDRWui1]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri6:%[0-9]+]]:gpr32common = ADDWri [[ADDWri5]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri7:%[0-9]+]]:gpr32common = ADDWri [[ADDWri6]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri8:%[0-9]+]]:gpr32common = ADDWri [[ADDWri7]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri9:%[0-9]+]]:gpr32common = ADDWri [[ADDWri8]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 102
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri10:%[0-9]+]]:gpr32common = ADDWri [[LDRWui2]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri11:%[0-9]+]]:gpr32common = ADDWri [[ADDWri10]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri12:%[0-9]+]]:gpr32common = ADDWri [[ADDWri11]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri13:%[0-9]+]]:gpr32common = ADDWri [[ADDWri12]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri14:%[0-9]+]]:gpr32common = ADDWri [[ADDWri13]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 103
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri15:%[0-9]+]]:gpr32common = ADDWri [[LDRWui3]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri16:%[0-9]+]]:gpr32common = ADDWri [[ADDWri15]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri17:%[0-9]+]]:gpr32common = ADDWri [[ADDWri16]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri18:%[0-9]+]]:gpr32common = ADDWri [[ADDWri17]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri19:%[0-9]+]]:gpr32common = ADDWri [[ADDWri18]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[MOVi64imm4:%[0-9]+]]:gpr64 = MOVi64imm 104
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri20:%[0-9]+]]:gpr32common = ADDWri [[LDRWui4]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri21:%[0-9]+]]:gpr32common = ADDWri [[ADDWri20]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri22:%[0-9]+]]:gpr32common = ADDWri [[ADDWri21]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri23:%[0-9]+]]:gpr32common = ADDWri [[ADDWri22]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri24:%[0-9]+]]:gpr32common = ADDWri [[ADDWri23]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[MOVi64imm5:%[0-9]+]]:gpr64 = MOVi64imm 105
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri25:%[0-9]+]]:gpr32common = ADDWri [[LDRWui5]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri26:%[0-9]+]]:gpr32common = ADDWri [[ADDWri25]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri27:%[0-9]+]]:gpr32common = ADDWri [[ADDWri26]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri28:%[0-9]+]]:gpr32common = ADDWri [[ADDWri27]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri29:%[0-9]+]]:gpr32common = ADDWri [[ADDWri28]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[MOVi64imm6:%[0-9]+]]:gpr64 = MOVi64imm 106
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri30:%[0-9]+]]:gpr32common = ADDWri [[LDRWui6]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri31:%[0-9]+]]:gpr32common = ADDWri [[ADDWri30]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri32:%[0-9]+]]:gpr32common = ADDWri [[ADDWri31]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri33:%[0-9]+]]:gpr32common = ADDWri [[ADDWri32]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri34:%[0-9]+]]:gpr32common = ADDWri [[ADDWri33]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[SUBSWri:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri34]], 7, 0, implicit-def $nzcv
+ ; CHECK-CASCADE-BUDGET-NEXT: [[MOVi64imm7:%[0-9]+]]:gpr64 = MOVi64imm 107
+ ; CHECK-CASCADE-BUDGET-NEXT: [[CSELXr:%[0-9]+]]:gpr64 = CSELXr [[MOVi64imm6]], [[MOVi64imm7]], 16, implicit $nzcv
+ ; CHECK-CASCADE-BUDGET-NEXT: [[SUBSWri1:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri29]], 6, 0, implicit-def $nzcv
+ ; CHECK-CASCADE-BUDGET-NEXT: [[CSELXr1:%[0-9]+]]:gpr64 = CSELXr [[MOVi64imm5]], [[CSELXr]], 15, implicit $nzcv
+ ; CHECK-CASCADE-BUDGET-NEXT: [[SUBSWri2:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri24]], 5, 0, implicit-def $nzcv
+ ; CHECK-CASCADE-BUDGET-NEXT: [[CSELXr2:%[0-9]+]]:gpr64 = CSELXr [[MOVi64imm4]], [[CSELXr1]], 14, implicit $nzcv
+ ; CHECK-CASCADE-BUDGET-NEXT: [[SUBSWri3:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri19]], 4, 0, implicit-def $nzcv
+ ; CHECK-CASCADE-BUDGET-NEXT: [[CSELXr3:%[0-9]+]]:gpr64 = CSELXr [[MOVi64imm3]], [[CSELXr2]], 13, implicit $nzcv
+ ; CHECK-CASCADE-BUDGET-NEXT: [[SUBSWri4:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri14]], 3, 0, implicit-def $nzcv
+ ; CHECK-CASCADE-BUDGET-NEXT: [[CSELXr4:%[0-9]+]]:gpr64 = CSELXr [[MOVi64imm2]], [[CSELXr3]], 12, implicit $nzcv
+ ; CHECK-CASCADE-BUDGET-NEXT: [[SUBSWri5:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri9]], 2, 0, implicit-def $nzcv
+ ; CHECK-CASCADE-BUDGET-NEXT: [[CSELXr5:%[0-9]+]]:gpr64 = CSELXr [[MOVi64imm1]], [[CSELXr4]], 11, implicit $nzcv
+ ; CHECK-CASCADE-BUDGET-NEXT: [[SUBSWri6:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri4]], 1, 0, implicit-def $nzcv
+ ; CHECK-CASCADE-BUDGET-NEXT: [[CSELXr6:%[0-9]+]]:gpr64 = CSELXr [[MOVi64imm]], [[CSELXr5]], 10, implicit $nzcv
+ ; CHECK-CASCADE-BUDGET-NEXT: $x0 = COPY [[CSELXr6]]
+ ; CHECK-CASCADE-BUDGET-NEXT: RET_ReallyLR implicit $x0
+ bb.0:
+ successors: %bb.8(0x40000000), %bb.1(0x40000000)
+ liveins: $x0
+
+ %0:gpr64common = COPY $x0
+ %1:gpr32common = LDRWui %0, 0 :: (load (s32))
+ %200:gpr32 = MOVi32imm 1
+ %201:gpr32 = MOVi32imm 2
+ %202:gpr32 = MOVi32imm 3
+ %203:gpr32 = MOVi32imm 4
+ %2:gpr32common = LDRWui %0, 1 :: (load (s32))
+ %3:gpr32common = LDRWui %0, 2 :: (load (s32))
+ %4:gpr32common = LDRWui %0, 3 :: (load (s32))
+ %5:gpr32common = LDRWui %0, 4 :: (load (s32))
+ %6:gpr32common = LDRWui %0, 5 :: (load (s32))
+ %7:gpr32common = LDRWui %0, 6 :: (load (s32))
+ %50:gpr64 = MOVi64imm 100
+ %204:gpr32common = ADDWri %1, 1, 0
+ %205:gpr32common = ADDWri %204, 1, 0
+ %206:gpr32common = ADDWri %205, 1, 0
+ %207:gpr32common = ADDWri %206, 1, 0
+ %208:gpr32common = ADDWri %207, 1, 0
+ %209:gpr32 = SUBSWri %208, 1, 0, implicit-def $nzcv
+ Bcc 10, %bb.8, implicit $nzcv
+ B %bb.1
+
+ bb.1:
+ successors: %bb.8(0x40000000), %bb.2(0x40000000)
+
+ %51:gpr64 = MOVi64imm 101
+ %210:gpr32common = ADDWri %2, 1, 0
+ %211:gpr32common = ADDWri %210, 1, 0
+ %212:gpr32common = ADDWri %211, 1, 0
+ %213:gpr32common = ADDWri %212, 1, 0
+ %214:gpr32common = ADDWri %213, 1, 0
+ %215:gpr32 = SUBSWri %214, 2, 0, implicit-def $nzcv
+ Bcc 11, %bb.8, implicit $nzcv
+ B %bb.2
+
+ bb.2:
+ successors: %bb.8(0x40000000), %bb.3(0x40000000)
+
+ %52:gpr64 = MOVi64imm 102
+ %216:gpr32common = ADDWri %3, 1, 0
+ %217:gpr32common = ADDWri %216, 1, 0
+ %218:gpr32common = ADDWri %217, 1, 0
+ %219:gpr32common = ADDWri %218, 1, 0
+ %220:gpr32common = ADDWri %219, 1, 0
+ %221:gpr32 = SUBSWri %220, 3, 0, implicit-def $nzcv
+ Bcc 12, %bb.8, implicit $nzcv
+ B %bb.3
+
+ bb.3:
+ successors: %bb.8(0x40000000), %bb.4(0x40000000)
+
+ %53:gpr64 = MOVi64imm 103
+ %222:gpr32common = ADDWri %4, 1, 0
+ %223:gpr32common = ADDWri %222, 1, 0
+ %224:gpr32common = ADDWri %223, 1, 0
+ %225:gpr32common = ADDWri %224, 1, 0
+ %226:gpr32common = ADDWri %225, 1, 0
+ %227:gpr32 = SUBSWri %226, 4, 0, implicit-def $nzcv
+ Bcc 13, %bb.8, implicit $nzcv
+ B %bb.4
+
+ bb.4:
+ successors: %bb.8(0x40000000), %bb.5(0x40000000)
+
+ %54:gpr64 = MOVi64imm 104
+ %228:gpr32common = ADDWri %5, 1, 0
+ %229:gpr32common = ADDWri %228, 1, 0
+ %230:gpr32common = ADDWri %229, 1, 0
+ %231:gpr32common = ADDWri %230, 1, 0
+ %232:gpr32common = ADDWri %231, 1, 0
+ %233:gpr32 = SUBSWri %232, 5, 0, implicit-def $nzcv
+ Bcc 14, %bb.8, implicit $nzcv
+ B %bb.5
+
+ bb.5:
+ successors: %bb.8(0x40000000), %bb.6(0x40000000)
+
+ %55:gpr64 = MOVi64imm 105
+ %234:gpr32common = ADDWri %6, 1, 0
+ %235:gpr32common = ADDWri %234, 1, 0
+ %236:gpr32common = ADDWri %235, 1, 0
+ %237:gpr32common = ADDWri %236, 1, 0
+ %238:gpr32common = ADDWri %237, 1, 0
+ %239:gpr32 = SUBSWri %238, 6, 0, implicit-def $nzcv
+ Bcc 15, %bb.8, implicit $nzcv
+ B %bb.6
+
+ bb.6:
+ successors: %bb.8(0x40000000), %bb.7(0x40000000)
+
+ %56:gpr64 = MOVi64imm 106
+ %240:gpr32common = ADDWri %7, 1, 0
+ %241:gpr32common = ADDWri %240, 1, 0
+ %242:gpr32common = ADDWri %241, 1, 0
+ %243:gpr32common = ADDWri %242, 1, 0
+ %244:gpr32common = ADDWri %243, 1, 0
+ %245:gpr32 = SUBSWri %244, 7, 0, implicit-def $nzcv
+ Bcc 16, %bb.8, implicit $nzcv
+ B %bb.7
+
+ bb.7:
+ successors: %bb.8(0x80000000)
+
+ %57:gpr64 = MOVi64imm 107
+
+ bb.8:
+ %90:gpr64 = PHI %50, %bb.0, %51, %bb.1, %52, %bb.2, %53, %bb.3, %54, %bb.4, %55, %bb.5, %56, %bb.6, %57, %bb.7
+ $x0 = COPY %90
+ RET_ReallyLR implicit $x0
+...
diff --git a/llvm/test/CodeGen/AArch64/early-ifcvt-remarks.mir b/llvm/test/CodeGen/AArch64/early-ifcvt-remarks.mir
index 6f01d48601fa2..7d4dd2e9f0af6 100644
--- a/llvm/test/CodeGen/AArch64/early-ifcvt-remarks.mir
+++ b/llvm/test/CodeGen/AArch64/early-ifcvt-remarks.mir
@@ -2,11 +2,44 @@
# RUN: -enable-early-ifcvt-data-dependent \
# RUN: -pass-remarks-analysis='early-ifcvt' -o /dev/null %s 2>&1 | FileCheck %s
+# RUN: llc -mtriple=aarch64 -mcpu=apple-m1 -run-pass=early-ifcvt \
+# RUN: -enable-early-ifcvt-cascade -enable-early-ifcvt-data-dependent \
+# RUN: -pass-remarks='early-ifcvt' -o /dev/null %s 2>&1 | FileCheck %s --check-prefix=CASCADE-SUCCESS \
+# RUN: --implicit-check-not="performing cascade if-conversion"
+
+# RUN: llc -mtriple=aarch64 -mcpu=apple-m1 -run-pass=early-ifcvt \
+# RUN: -enable-early-ifcvt-cascade -enable-early-ifcvt-data-dependent \
+# RUN: -pass-remarks-missed='early-ifcvt' -o /dev/null %s 2>&1 | FileCheck %s --check-prefix=CASCADE-MISSED
+
+# RUN: llc -mtriple=aarch64 -mcpu=apple-m1 -run-pass=early-ifcvt \
+# RUN: -enable-early-ifcvt-cascade \
+# RUN: -pass-remarks='early-ifcvt' -o /dev/null %s 2>&1 \
+# RUN: | FileCheck %s --check-prefix=CASCADE-NO-DD --implicit-check-not="cascade if-conversion"
+
+# RUN: llc -mtriple=aarch64 -mcpu=apple-m1 -run-pass=early-ifcvt \
+# RUN: -enable-early-ifcvt-cascade -enable-early-ifcvt-data-dependent \
+# RUN: -early-ifcvt-limit=3 -pass-remarks='early-ifcvt' \
+# RUN: -pass-remarks-missed='early-ifcvt' -o /dev/null %s 2>&1 \
+# RUN: | FileCheck %s --check-prefix=CASCADE-OVER-LIMIT \
+# RUN: --implicit-check-not="performing cascade if-conversion"
+
# Test that we emit an analysis remark when the branch condition is data-dependent
# (derived from a memory load).
# CHECK: remark: <unknown>:0:0: branch condition is data-dependent (from memory load), using higher CritLimit of {{[0-9]+}} cycles
+# Test cascade conversion success remark.
+# CASCADE-SUCCESS: remark: <unknown>:0:0: performing cascade if-conversion on 3 blocks: critical path extension is {{[0-9]+}} cycle
+
+# CASCADE-MISSED: remark: <unknown>:0:0: did not if-convert cascade with 5 blocks: critical path extension of 21 cycles exceeds the threshold of 20 cycles.
+# CASCADE-MISSED: remark: <unknown>:0:0: did not if-convert cascade with 3 blocks: critical path extension of 30 cycles exceeds the threshold of 12 cycles.
+
+# CASCADE-NO-DD: remark: <unknown>:0:0: performing if-conversion on branch: {{.*}} staying under the threshold of 8 cycles.
+
+# CASCADE-OVER-LIMIT: remark: <unknown>:0:0: did not if-convert cascade with 3 blocks: the speculated blocks would hold 10 instructions, exceeding the limit of 3.
+# CASCADE-OVER-LIMIT: remark: <unknown>:0:0: did not if-convert cascade with 5 blocks: the speculated blocks would hold 14 instructions, exceeding the limit of 3.
+# CASCADE-OVER-LIMIT: remark: <unknown>:0:0: did not if-convert cascade with 3 blocks: the speculated blocks would hold 11 instructions, exceeding the limit of 3.
+
--- |
define i32 @data_dependent_branch(ptr %p, i32 %x, i32 %y) {
ret i32 0
@@ -17,6 +50,9 @@
define i64 @cascade_increasing_cond_depth(ptr %p) {
ret i64 0
}
+ define i64 @cascade_mixed_reg_classes(ptr %p) {
+ ret i64 0
+ }
...
---
name: data_dependent_branch
@@ -202,3 +238,64 @@ body: |
$x0 = COPY %90
RET_ReallyLR implicit $x0
...
+---
+name: cascade_mixed_reg_classes
+alignment: 4
+tracksRegLiveness: true
+body: |
+ bb.0:
+ successors: %bb.1(0x80000000)
+ liveins: $x0, $d0, $d1, $d2, $d3
+
+ %0:gpr64common = COPY $x0
+ %5:fpr64 = COPY $d0
+ %6:fpr64 = COPY $d1
+ %7:fpr64 = COPY $d2
+ %8:fpr64 = COPY $d3
+
+ bb.1:
+ successors: %bb.2(0x40000000), %bb.5(0x40000000)
+
+ %10:gpr32 = LDRWui %0, 0 :: (load (s32))
+ %11:gpr32 = MADDWrrr %10, %10, $wzr
+ %12:gpr32 = MADDWrrr %11, %11, $wzr
+ %20:gpr32 = ADDWrr %12, %12
+ %21:gpr32 = ADDWrr %20, %20
+ %40:gpr64 = MOVi64imm 100
+ %30:gpr32 = SUBSWrr %12, %12, implicit-def $nzcv
+ Bcc 10, %bb.5, implicit $nzcv
+ B %bb.2
+
+ bb.2:
+ successors: %bb.3(0x40000000), %bb.5(0x40000000)
+
+ %41:gpr64 = MOVi64imm 200
+ %31:gpr32 = SUBSWrr %20, %20, implicit-def $nzcv
+ Bcc 10, %bb.5, implicit $nzcv
+ B %bb.3
+
+ bb.3:
+ successors: %bb.4(0x40000000), %bb.5(0x40000000)
+
+ %42:gpr64 = MOVi64imm 300
+ %32:gpr32 = SUBSWrr %21, %21, implicit-def $nzcv
+ Bcc 10, %bb.5, implicit $nzcv
+ B %bb.4
+
+ bb.4:
+ successors: %bb.5(0x80000000)
+
+ %43:gpr64 = MOVi64imm 400
+
+ bb.5:
+ %90:gpr64 = PHI %40, %bb.1, %41, %bb.2, %42, %bb.3, %43, %bb.4
+ %91:fpr64 = PHI %5, %bb.1, %6, %bb.2, %7, %bb.3, %8, %bb.4
+ %60:fpr64 = FMULDrr %91, %91, implicit $fpcr
+ %61:fpr64 = FMULDrr %60, %60, implicit $fpcr
+ %62:fpr64 = FMULDrr %61, %61, implicit $fpcr
+ %63:fpr64 = FMULDrr %62, %62, implicit $fpcr
+ %64:fpr64 = FMULDrr %63, %63, implicit $fpcr
+ $x0 = COPY %90
+ $d0 = COPY %64
+ RET_ReallyLR implicit $x0, implicit $d0
+...
More information about the llvm-commits
mailing list