[llvm] [EarlyIfConversion] Add cascading conditional branches as a conversion pattern (PR #218415)
Jonathan Cohen via llvm-commits
llvm-commits at lists.llvm.org
Mon Aug 24 06:52:08 PDT 2026
https://github.com/jcohen-apple created https://github.com/llvm/llvm-project/pull/218415
EarlyIfConversion today only calculates the profitability of converting triangle and diamond CFGs into conditional selects. This works well for a single conditional branch, but in the case of cascading conditional branches which are unpredictable, the current heuristic underestimates the cost of misprediction. In the case of cascading conditional branches we may mispredict multiple times along the cascade, and the cost of these re-steers of the frontend can compound, as we have seen in internal workloads.
This addition to EarlyIfConversion utilizes the previously added data dependent analysis for conditional branches to check if we have a cascade of likely-to-be unpredictable branches, and if so, raise the acceptable critical path extension as a function of the cascade depth. We also add an additional hidden parameter to configure the average mispredict likelihood, which we set experimentally at 25%, but may need additional tuning.
>From d7176de84ee032c7468680d4d2800dee00c1611f Mon Sep 17 00:00:00 2001
From: Jonathan Cohen <joncoh at apple.com>
Date: Thu, 16 Apr 2026 15:29:47 +0300
Subject: [PATCH 1/2] [EarlyIfConversion] Unit tests documenting current state
before cascading conversion logic
---
.../CodeGen/AArch64/early-ifcvt-cascade.mir | 843 ++++++++++++++++++
.../CodeGen/AArch64/early-ifcvt-remarks.mir | 159 ++++
2 files changed, 1002 insertions(+)
create mode 100644 llvm/test/CodeGen/AArch64/early-ifcvt-cascade.mir
diff --git a/llvm/test/CodeGen/AArch64/early-ifcvt-cascade.mir b/llvm/test/CodeGen/AArch64/early-ifcvt-cascade.mir
new file mode 100644
index 0000000000000..99c0ea79267a1
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/early-ifcvt-cascade.mir
@@ -0,0 +1,843 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=aarch64 -mcpu=apple-m1 -run-pass=early-ifcvt -enable-early-ifcvt-data-dependent %s -o - | FileCheck %s
+
+--- |
+ define void @cascade_selector_loop() { ret void }
+ define void @cascade_branches_from_args_not_dd() { ret void }
+ define void @cascade_exceeds_crit_limit() { ret void }
+ define void @cascade_backedge_to_earlier_block() { ret void }
+ define void @cascade_forward_skip_to_later_block() { ret void }
+...
+---
+name: cascade_selector_loop
+tracksRegLiveness: true
+liveins:
+ - { reg: '$x0', virtual-reg: '%0' }
+ - { reg: '$x1', virtual-reg: '%1' }
+ - { reg: '$w2', virtual-reg: '%2' }
+ - { reg: '$w3', virtual-reg: '%3' }
+ - { reg: '$w4', virtual-reg: '%4' }
+body: |
+ ; CHECK-LABEL: name: cascade_selector_loop
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.10(0x30000000), %bb.1(0x50000000)
+ ; CHECK-NEXT: liveins: $x0, $x1, $w2, $w3, $w4
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr32 = COPY $w2
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gpr32 = COPY $w3
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:gpr32 = COPY $w4
+ ; CHECK-NEXT: [[COPY5:%[0-9]+]]:gpr64all = COPY $xzr
+ ; CHECK-NEXT: CBZW [[COPY2]], %bb.10
+ ; CHECK-NEXT: B %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.2(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 0 :: (load (s32))
+ ; CHECK-NEXT: [[LDRWui1:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 1 :: (load (s32))
+ ; CHECK-NEXT: [[LDRWui2:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 2 :: (load (s32))
+ ; CHECK-NEXT: [[LDRWui3:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 3 :: (load (s32))
+ ; CHECK-NEXT: [[LDRWui4:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 4 :: (load (s32))
+ ; CHECK-NEXT: [[LDRWui5:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 5 :: (load (s32))
+ ; CHECK-NEXT: [[LDRWui6:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 6 :: (load (s32))
+ ; CHECK-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: successors: %bb.3(0x40000000), %bb.9(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[PHI:%[0-9]+]]:gpr32 = PHI [[COPY2]], %bb.1, %15, %bb.9
+ ; CHECK-NEXT: [[LDRWui7:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load (s32))
+ ; CHECK-NEXT: [[MADDWrrr:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[LDRWui7]], $wzr
+ ; CHECK-NEXT: [[MADDWrrr1:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY4]], [[LDRWui7]], [[MADDWrrr]]
+ ; CHECK-NEXT: [[MADDWrrr2:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[LDRWui7]], [[MADDWrrr1]]
+ ; CHECK-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 7
+ ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; CHECK-NEXT: B %bb.3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.3:
+ ; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.9(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui1]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 6
+ ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; CHECK-NEXT: B %bb.4
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.4:
+ ; CHECK-NEXT: successors: %bb.5(0x40000000), %bb.9(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[SUBSWrr2:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui2]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 5
+ ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; CHECK-NEXT: B %bb.5
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.5:
+ ; CHECK-NEXT: successors: %bb.6(0x40000000), %bb.9(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[SUBSWrr3:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui3]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 4
+ ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; CHECK-NEXT: B %bb.6
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.6:
+ ; CHECK-NEXT: successors: %bb.7(0x40000000), %bb.9(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[SUBSWrr4:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui4]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm4:%[0-9]+]]:gpr64 = MOVi64imm 3
+ ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; CHECK-NEXT: B %bb.7
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.7:
+ ; CHECK-NEXT: successors: %bb.8(0x40000000), %bb.9(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[SUBSWrr5:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui5]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm5:%[0-9]+]]:gpr64 = MOVi64imm 2
+ ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; CHECK-NEXT: B %bb.8
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.8:
+ ; CHECK-NEXT: successors: %bb.9(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[MOVi64imm6:%[0-9]+]]:gpr64 = MOVi64imm 1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.9:
+ ; CHECK-NEXT: successors: %bb.10(0x04000000), %bb.2(0x7c000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[PHI1:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.2, [[MOVi64imm1]], %bb.3, [[MOVi64imm2]], %bb.4, [[MOVi64imm3]], %bb.5, [[MOVi64imm4]], %bb.6, [[MOVi64imm5]], %bb.7, [[MOVi64imm6]], %bb.8
+ ; CHECK-NEXT: [[SUBSWrr6:%[0-9]+]]:gpr32 = SUBSWrr [[PHI]], [[MOVi32imm]], implicit-def $nzcv
+ ; CHECK-NEXT: Bcc 1, %bb.2, implicit $nzcv
+ ; CHECK-NEXT: B %bb.10
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.10:
+ ; CHECK-NEXT: [[PHI2:%[0-9]+]]:gpr64 = PHI [[COPY5]], %bb.0, [[PHI1]], %bb.9
+ ; CHECK-NEXT: $x0 = COPY [[PHI2]]
+ ; CHECK-NEXT: RET_ReallyLR implicit $x0
+ bb.0:
+ successors: %bb.10(0x30000000), %bb.1(0x50000000)
+ liveins: $x0, $x1, $w2, $w3, $w4
+
+ %0:gpr64common = COPY $x0
+ %1:gpr64common = COPY $x1
+ %2:gpr32 = COPY $w2
+ %3:gpr32 = COPY $w3
+ %4:gpr32 = COPY $w4
+ %99:gpr64all = COPY $xzr
+ CBZW %2, %bb.10
+ B %bb.1
+
+ bb.1:
+ successors: %bb.2(0x80000000)
+
+ %10:gpr32 = LDRWui %1, 0 :: (load (s32))
+ %11:gpr32 = LDRWui %1, 1 :: (load (s32))
+ %12:gpr32 = LDRWui %1, 2 :: (load (s32))
+ %13:gpr32 = LDRWui %1, 3 :: (load (s32))
+ %14:gpr32 = LDRWui %1, 4 :: (load (s32))
+ %15:gpr32 = LDRWui %1, 5 :: (load (s32))
+ %16:gpr32 = LDRWui %1, 6 :: (load (s32))
+ %17:gpr32 = MOVi32imm 1
+
+ bb.2:
+ successors: %bb.3(0x40000000), %bb.9(0x40000000)
+
+ %50:gpr32 = PHI %2, %bb.1, %51, %bb.9
+ %20:gpr32 = LDRWui %0, 0 :: (load (s32))
+ %31:gpr32 = nsw MADDWrrr %3, %20, $wzr
+ %32:gpr32 = nsw MADDWrrr %4, %20, %31
+ %30:gpr32 = nsw MADDWrrr %3, %20, %32
+ %21:gpr32 = SUBSWrr %30, %10, implicit-def $nzcv
+ %40:gpr64 = MOVi64imm 7
+ Bcc 10, %bb.9, implicit $nzcv
+ B %bb.3
+
+ bb.3:
+ successors: %bb.4(0x40000000), %bb.9(0x40000000)
+
+ %22:gpr32 = SUBSWrr %30, %11, implicit-def $nzcv
+ %41:gpr64 = MOVi64imm 6
+ Bcc 10, %bb.9, implicit $nzcv
+ B %bb.4
+
+ bb.4:
+ successors: %bb.5(0x40000000), %bb.9(0x40000000)
+
+ %23:gpr32 = SUBSWrr %30, %12, implicit-def $nzcv
+ %42:gpr64 = MOVi64imm 5
+ Bcc 10, %bb.9, implicit $nzcv
+ B %bb.5
+
+ bb.5:
+ successors: %bb.6(0x40000000), %bb.9(0x40000000)
+
+ %24:gpr32 = SUBSWrr %30, %13, implicit-def $nzcv
+ %43:gpr64 = MOVi64imm 4
+ Bcc 10, %bb.9, implicit $nzcv
+ B %bb.6
+
+ bb.6:
+ successors: %bb.7(0x40000000), %bb.9(0x40000000)
+
+ %25:gpr32 = SUBSWrr %30, %14, implicit-def $nzcv
+ %44:gpr64 = MOVi64imm 3
+ Bcc 10, %bb.9, implicit $nzcv
+ B %bb.7
+
+ bb.7:
+ successors: %bb.8(0x40000000), %bb.9(0x40000000)
+
+ %26:gpr32 = SUBSWrr %30, %15, implicit-def $nzcv
+ %45:gpr64 = MOVi64imm 2
+ Bcc 10, %bb.9, implicit $nzcv
+ B %bb.8
+
+ bb.8:
+ successors: %bb.9(0x80000000)
+
+ %47:gpr64 = MOVi64imm 1
+
+ bb.9:
+ successors: %bb.10(0x04000000), %bb.2(0x7c000000)
+
+ %90:gpr64 = PHI %40, %bb.2, %41, %bb.3, %42, %bb.4, %43, %bb.5, %44, %bb.6, %45, %bb.7, %47, %bb.8
+ %51:gpr32 = SUBSWrr %50, %17, implicit-def $nzcv
+ Bcc 1, %bb.2, implicit $nzcv
+ B %bb.10
+
+ bb.10:
+ %95:gpr64 = PHI %99, %bb.0, %90, %bb.9
+ $x0 = COPY %95
+ RET_ReallyLR implicit $x0
+...
+---
+name: cascade_branches_from_args_not_dd
+tracksRegLiveness: true
+liveins:
+ - { reg: '$x1', virtual-reg: '%1' }
+ - { reg: '$w2', virtual-reg: '%2' }
+ - { reg: '$w3', virtual-reg: '%3' }
+ - { reg: '$w4', virtual-reg: '%4' }
+ - { reg: '$w5', virtual-reg: '%5' }
+body: |
+ ; CHECK-LABEL: name: cascade_branches_from_args_not_dd
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.10(0x30000000), %bb.1(0x50000000)
+ ; CHECK-NEXT: liveins: $x1, $w2, $w3, $w4, $w5
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x1
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr32 = COPY $w2
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr32 = COPY $w3
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gpr32 = COPY $w4
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:gpr32 = COPY $w5
+ ; CHECK-NEXT: [[COPY5:%[0-9]+]]:gpr64all = COPY $xzr
+ ; CHECK-NEXT: CBZW [[COPY1]], %bb.10
+ ; CHECK-NEXT: B %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.2(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY6:%[0-9]+]]:gpr32 = COPY [[COPY2]]
+ ; CHECK-NEXT: [[COPY7:%[0-9]+]]:gpr32 = COPY [[COPY3]]
+ ; CHECK-NEXT: [[COPY8:%[0-9]+]]:gpr32 = COPY [[COPY4]]
+ ; CHECK-NEXT: [[COPY9:%[0-9]+]]:gpr32 = COPY [[COPY2]]
+ ; CHECK-NEXT: [[COPY10:%[0-9]+]]:gpr32 = COPY [[COPY3]]
+ ; CHECK-NEXT: [[COPY11:%[0-9]+]]:gpr32 = COPY [[COPY4]]
+ ; CHECK-NEXT: [[COPY12:%[0-9]+]]:gpr32 = COPY [[COPY2]]
+ ; CHECK-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: successors: %bb.3(0x40000000), %bb.9(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[PHI:%[0-9]+]]:gpr32 = PHI [[COPY1]], %bb.1, %15, %bb.9
+ ; CHECK-NEXT: [[MADDWrrr:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY2]], [[COPY4]], $wzr
+ ; CHECK-NEXT: [[MADDWrrr1:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[COPY4]], [[MADDWrrr]]
+ ; CHECK-NEXT: [[MADDWrrr2:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY2]], [[COPY3]], [[MADDWrrr1]]
+ ; CHECK-NEXT: [[MADDWrrr3:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[COPY2]], [[MADDWrrr2]]
+ ; CHECK-NEXT: [[MADDWrrr4:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY4]], [[COPY3]], [[MADDWrrr3]]
+ ; CHECK-NEXT: [[MADDWrrr5:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY2]], [[COPY4]], [[MADDWrrr4]]
+ ; CHECK-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr5]], [[COPY6]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 7
+ ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; CHECK-NEXT: B %bb.3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.3:
+ ; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.9(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr5]], [[COPY7]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 6
+ ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; CHECK-NEXT: B %bb.4
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.4:
+ ; CHECK-NEXT: successors: %bb.5(0x40000000), %bb.9(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[SUBSWrr2:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr5]], [[COPY8]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 5
+ ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; CHECK-NEXT: B %bb.5
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.5:
+ ; CHECK-NEXT: successors: %bb.6(0x40000000), %bb.9(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[SUBSWrr3:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr5]], [[COPY9]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 4
+ ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; CHECK-NEXT: B %bb.6
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.6:
+ ; CHECK-NEXT: successors: %bb.7(0x40000000), %bb.9(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[SUBSWrr4:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr5]], [[COPY10]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm4:%[0-9]+]]:gpr64 = MOVi64imm 3
+ ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; CHECK-NEXT: B %bb.7
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.7:
+ ; CHECK-NEXT: successors: %bb.8(0x40000000), %bb.9(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[SUBSWrr5:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr5]], [[COPY11]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm5:%[0-9]+]]:gpr64 = MOVi64imm 2
+ ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; CHECK-NEXT: B %bb.8
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.8:
+ ; CHECK-NEXT: successors: %bb.9(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[MOVi64imm6:%[0-9]+]]:gpr64 = MOVi64imm 1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.9:
+ ; CHECK-NEXT: successors: %bb.10(0x04000000), %bb.2(0x7c000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[PHI1:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.2, [[MOVi64imm1]], %bb.3, [[MOVi64imm2]], %bb.4, [[MOVi64imm3]], %bb.5, [[MOVi64imm4]], %bb.6, [[MOVi64imm5]], %bb.7, [[MOVi64imm6]], %bb.8
+ ; CHECK-NEXT: [[SUBSWrr6:%[0-9]+]]:gpr32 = SUBSWrr [[PHI]], [[MOVi32imm]], implicit-def $nzcv
+ ; CHECK-NEXT: Bcc 1, %bb.2, implicit $nzcv
+ ; CHECK-NEXT: B %bb.10
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.10:
+ ; CHECK-NEXT: [[PHI2:%[0-9]+]]:gpr64 = PHI [[COPY5]], %bb.0, [[PHI1]], %bb.9
+ ; CHECK-NEXT: $x0 = COPY [[PHI2]]
+ ; CHECK-NEXT: RET_ReallyLR implicit $x0
+ bb.0:
+ successors: %bb.10(0x30000000), %bb.1(0x50000000)
+ liveins: $x1, $w2, $w3, $w4, $w5
+
+ %1:gpr64common = COPY $x1
+ %2:gpr32 = COPY $w2
+ %3:gpr32 = COPY $w3
+ %4:gpr32 = COPY $w4
+ %5:gpr32 = COPY $w5
+ %99:gpr64all = COPY $xzr
+ CBZW %2, %bb.10
+ B %bb.1
+
+ bb.1:
+ successors: %bb.2(0x80000000)
+
+ ; Thresholds from registers, not memory loads
+ %10:gpr32 = COPY %3
+ %11:gpr32 = COPY %4
+ %12:gpr32 = COPY %5
+ %13:gpr32 = COPY %3
+ %14:gpr32 = COPY %4
+ %15:gpr32 = COPY %5
+ %16:gpr32 = COPY %3
+ %17:gpr32 = MOVi32imm 1
+
+ ; Cascade head: NOT data-dependent (computed from register args, no loads)
+ bb.2:
+ successors: %bb.3(0x40000000), %bb.9(0x40000000)
+
+ %50:gpr32 = PHI %2, %bb.1, %51, %bb.9
+ %31:gpr32 = nsw MADDWrrr %3, %5, $wzr
+ %32:gpr32 = nsw MADDWrrr %4, %5, %31
+ %33:gpr32 = nsw MADDWrrr %3, %4, %32
+ %34:gpr32 = nsw MADDWrrr %4, %3, %33
+ %35:gpr32 = nsw MADDWrrr %5, %4, %34
+ %30:gpr32 = nsw MADDWrrr %3, %5, %35
+ %21:gpr32 = SUBSWrr %30, %10, implicit-def $nzcv
+ %40:gpr64 = MOVi64imm 7
+ Bcc 10, %bb.9, implicit $nzcv
+ B %bb.3
+
+ bb.3:
+ successors: %bb.4(0x40000000), %bb.9(0x40000000)
+
+ %22:gpr32 = SUBSWrr %30, %11, implicit-def $nzcv
+ %41:gpr64 = MOVi64imm 6
+ Bcc 10, %bb.9, implicit $nzcv
+ B %bb.4
+
+ bb.4:
+ successors: %bb.5(0x40000000), %bb.9(0x40000000)
+
+ %23:gpr32 = SUBSWrr %30, %12, implicit-def $nzcv
+ %42:gpr64 = MOVi64imm 5
+ Bcc 10, %bb.9, implicit $nzcv
+ B %bb.5
+
+ bb.5:
+ successors: %bb.6(0x40000000), %bb.9(0x40000000)
+
+ %24:gpr32 = SUBSWrr %30, %13, implicit-def $nzcv
+ %43:gpr64 = MOVi64imm 4
+ Bcc 10, %bb.9, implicit $nzcv
+ B %bb.6
+
+ bb.6:
+ successors: %bb.7(0x40000000), %bb.9(0x40000000)
+
+ %25:gpr32 = SUBSWrr %30, %14, implicit-def $nzcv
+ %44:gpr64 = MOVi64imm 3
+ Bcc 10, %bb.9, implicit $nzcv
+ B %bb.7
+
+ bb.7:
+ successors: %bb.8(0x40000000), %bb.9(0x40000000)
+
+ %26:gpr32 = SUBSWrr %30, %15, implicit-def $nzcv
+ %45:gpr64 = MOVi64imm 2
+ Bcc 10, %bb.9, implicit $nzcv
+ B %bb.8
+
+ bb.8:
+ successors: %bb.9(0x80000000)
+
+ %47:gpr64 = MOVi64imm 1
+
+ bb.9:
+ successors: %bb.10(0x04000000), %bb.2(0x7c000000)
+
+ %90:gpr64 = PHI %40, %bb.2, %41, %bb.3, %42, %bb.4, %43, %bb.5, %44, %bb.6, %45, %bb.7, %47, %bb.8
+ %51:gpr32 = SUBSWrr %50, %17, implicit-def $nzcv
+ Bcc 1, %bb.2, implicit $nzcv
+ B %bb.10
+
+ bb.10:
+ %95:gpr64 = PHI %99, %bb.0, %90, %bb.9
+ $x0 = COPY %95
+ RET_ReallyLR implicit $x0
+...
+---
+name: cascade_exceeds_crit_limit
+tracksRegLiveness: true
+liveins:
+ - { reg: '$x0', virtual-reg: '%0' }
+ - { reg: '$x1', virtual-reg: '%1' }
+ - { reg: '$w2', virtual-reg: '%2' }
+ - { reg: '$w3', virtual-reg: '%3' }
+ - { reg: '$w4', virtual-reg: '%4' }
+body: |
+ ; CHECK-LABEL: name: cascade_exceeds_crit_limit
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.11(0x30000000), %bb.1(0x50000000)
+ ; CHECK-NEXT: liveins: $x0, $x1, $w2, $w3, $w4
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr32 = COPY $w2
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gpr32 = COPY $w3
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:gpr32 = COPY $w4
+ ; CHECK-NEXT: [[COPY5:%[0-9]+]]:gpr64all = COPY $xzr
+ ; CHECK-NEXT: CBZW [[COPY2]], %bb.11
+ ; CHECK-NEXT: B %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.2(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 0 :: (load (s32))
+ ; CHECK-NEXT: [[LDRWui1:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 1 :: (load (s32))
+ ; CHECK-NEXT: [[LDRWui2:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 2 :: (load (s32))
+ ; CHECK-NEXT: [[LDRWui3:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 3 :: (load (s32))
+ ; CHECK-NEXT: [[LDRWui4:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 4 :: (load (s32))
+ ; CHECK-NEXT: [[LDRWui5:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 5 :: (load (s32))
+ ; CHECK-NEXT: [[LDRWui6:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 6 :: (load (s32))
+ ; CHECK-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: successors: %bb.3(0x40000000), %bb.10(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[PHI:%[0-9]+]]:gpr32 = PHI [[COPY2]], %bb.1, %15, %bb.10
+ ; CHECK-NEXT: [[LDRWui7:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load (s32))
+ ; CHECK-NEXT: [[MADDWrrr:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[LDRWui7]], $wzr
+ ; CHECK-NEXT: [[MADDWrrr1:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY4]], [[LDRWui7]], [[MADDWrrr]]
+ ; CHECK-NEXT: [[MADDWrrr2:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[LDRWui7]], [[MADDWrrr1]]
+ ; CHECK-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 7
+ ; CHECK-NEXT: Bcc 10, %bb.10, implicit $nzcv
+ ; CHECK-NEXT: B %bb.3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.3:
+ ; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.10(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui1]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 6
+ ; CHECK-NEXT: Bcc 10, %bb.10, implicit $nzcv
+ ; CHECK-NEXT: B %bb.4
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.4:
+ ; CHECK-NEXT: successors: %bb.6(0x40000000), %bb.5(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[SUBSWrr2:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui2]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 5
+ ; CHECK-NEXT: Bcc 10, %bb.5, implicit $nzcv
+ ; CHECK-NEXT: B %bb.6
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.5:
+ ; CHECK-NEXT: successors: %bb.10(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: B %bb.10
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.6:
+ ; CHECK-NEXT: successors: %bb.7(0x40000000), %bb.10(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[SUBSWrr3:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui3]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 4
+ ; CHECK-NEXT: Bcc 10, %bb.10, implicit $nzcv
+ ; CHECK-NEXT: B %bb.7
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.7:
+ ; CHECK-NEXT: successors: %bb.8(0x40000000), %bb.10(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[SUBSWrr4:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui4]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm4:%[0-9]+]]:gpr64 = MOVi64imm 3
+ ; CHECK-NEXT: Bcc 10, %bb.10, implicit $nzcv
+ ; CHECK-NEXT: B %bb.8
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.8:
+ ; CHECK-NEXT: successors: %bb.9(0x40000000), %bb.10(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[SUBSWrr5:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui5]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm5:%[0-9]+]]:gpr64 = MOVi64imm 2
+ ; CHECK-NEXT: Bcc 10, %bb.10, implicit $nzcv
+ ; CHECK-NEXT: B %bb.9
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.9:
+ ; CHECK-NEXT: successors: %bb.10(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[MOVi64imm6:%[0-9]+]]:gpr64 = MOVi64imm 1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.10:
+ ; CHECK-NEXT: successors: %bb.11(0x04000000), %bb.2(0x7c000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[PHI1:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.2, [[MOVi64imm1]], %bb.3, [[MOVi64imm2]], %bb.5, [[MOVi64imm3]], %bb.6, [[MOVi64imm4]], %bb.7, [[MOVi64imm5]], %bb.8, [[MOVi64imm6]], %bb.9
+ ; CHECK-NEXT: [[SUBSWrr6:%[0-9]+]]:gpr32 = SUBSWrr [[PHI]], [[MOVi32imm]], implicit-def $nzcv
+ ; CHECK-NEXT: Bcc 1, %bb.2, implicit $nzcv
+ ; CHECK-NEXT: B %bb.11
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.11:
+ ; CHECK-NEXT: [[PHI2:%[0-9]+]]:gpr64 = PHI [[COPY5]], %bb.0, [[PHI1]], %bb.10
+ ; CHECK-NEXT: $x0 = COPY [[PHI2]]
+ ; CHECK-NEXT: RET_ReallyLR implicit $x0
+ bb.0:
+ successors: %bb.11(0x30000000), %bb.1(0x50000000)
+ liveins: $x0, $x1, $w2, $w3, $w4
+
+ %0:gpr64common = COPY $x0
+ %1:gpr64common = COPY $x1
+ %2:gpr32 = COPY $w2
+ %3:gpr32 = COPY $w3
+ %4:gpr32 = COPY $w4
+ %99:gpr64all = COPY $xzr
+ CBZW %2, %bb.11
+ B %bb.1
+
+ bb.1:
+ successors: %bb.2(0x80000000)
+
+ %10:gpr32 = LDRWui %1, 0 :: (load (s32))
+ %11:gpr32 = LDRWui %1, 1 :: (load (s32))
+ %12:gpr32 = LDRWui %1, 2 :: (load (s32))
+ %13:gpr32 = LDRWui %1, 3 :: (load (s32))
+ %14:gpr32 = LDRWui %1, 4 :: (load (s32))
+ %15:gpr32 = LDRWui %1, 5 :: (load (s32))
+ %16:gpr32 = LDRWui %1, 6 :: (load (s32))
+ %17:gpr32 = MOVi32imm 1
+
+ ; Cascade head: data-dependent
+ bb.2:
+ successors: %bb.3(0x40000000), %bb.10(0x40000000)
+
+ %50:gpr32 = PHI %2, %bb.1, %51, %bb.10
+ %20:gpr32 = LDRWui %0, 0 :: (load (s32))
+ %31:gpr32 = nsw MADDWrrr %3, %20, $wzr
+ %32:gpr32 = nsw MADDWrrr %4, %20, %31
+ %30:gpr32 = nsw MADDWrrr %3, %20, %32
+ %21:gpr32 = SUBSWrr %30, %10, implicit-def $nzcv
+ %40:gpr64 = MOVi64imm 7
+ Bcc 10, %bb.10, implicit $nzcv
+ B %bb.3
+
+ bb.3:
+ successors: %bb.4(0x40000000), %bb.10(0x40000000)
+
+ %22:gpr32 = SUBSWrr %30, %11, implicit-def $nzcv
+ %41:gpr64 = MOVi64imm 6
+ Bcc 10, %bb.10, implicit $nzcv
+ B %bb.4
+
+ ; bb.4 branches to bb.11 (not directly to tail) - breaks cascade
+ bb.4:
+ successors: %bb.6(0x40000000), %bb.5(0x40000000)
+
+ %23:gpr32 = SUBSWrr %30, %12, implicit-def $nzcv
+ %42:gpr64 = MOVi64imm 5
+ Bcc 10, %bb.5, implicit $nzcv
+ B %bb.6
+
+ ; Extra block that breaks the cascade chain
+ bb.5:
+ successors: %bb.10(0x80000000)
+
+ B %bb.10
+
+ bb.6:
+ successors: %bb.7(0x40000000), %bb.10(0x40000000)
+
+ %24:gpr32 = SUBSWrr %30, %13, implicit-def $nzcv
+ %43:gpr64 = MOVi64imm 4
+ Bcc 10, %bb.10, implicit $nzcv
+ B %bb.7
+
+ bb.7:
+ successors: %bb.8(0x40000000), %bb.10(0x40000000)
+
+ %25:gpr32 = SUBSWrr %30, %14, implicit-def $nzcv
+ %44:gpr64 = MOVi64imm 3
+ Bcc 10, %bb.10, implicit $nzcv
+ B %bb.8
+
+ bb.8:
+ successors: %bb.9(0x40000000), %bb.10(0x40000000)
+
+ %26:gpr32 = SUBSWrr %30, %15, implicit-def $nzcv
+ %45:gpr64 = MOVi64imm 2
+ Bcc 10, %bb.10, implicit $nzcv
+ B %bb.9
+
+ bb.9:
+ successors: %bb.10(0x80000000)
+
+ %47:gpr64 = MOVi64imm 1
+
+ bb.10:
+ successors: %bb.11(0x04000000), %bb.2(0x7c000000)
+
+ %90:gpr64 = PHI %40, %bb.2, %41, %bb.3, %42, %bb.5, %43, %bb.6, %44, %bb.7, %45, %bb.8, %47, %bb.9
+ %51:gpr32 = SUBSWrr %50, %17, implicit-def $nzcv
+ Bcc 1, %bb.2, implicit $nzcv
+ B %bb.11
+
+ bb.11:
+ %95:gpr64 = PHI %99, %bb.0, %90, %bb.10
+ $x0 = COPY %95
+ RET_ReallyLR implicit $x0
+...
+---
+name: cascade_backedge_to_earlier_block
+alignment: 4
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: cascade_backedge_to_earlier_block
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: liveins: $x0, $x1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.2(0x40000000), %bb.5(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load (s32))
+ ; CHECK-NEXT: [[LDRWui1:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 0 :: (load (s32))
+ ; CHECK-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui1]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 100
+ ; CHECK-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 200
+ ; CHECK-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 400
+ ; CHECK-NEXT: Bcc 10, %bb.5, implicit $nzcv
+ ; CHECK-NEXT: B %bb.2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: successors: %bb.3(0x40000000), %bb.5(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[LDRWui2:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 1 :: (load (s32))
+ ; CHECK-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui2]], implicit-def $nzcv
+ ; CHECK-NEXT: Bcc 10, %bb.5, implicit $nzcv
+ ; CHECK-NEXT: B %bb.3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.3:
+ ; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[LDRWui3:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 2 :: (load (s32))
+ ; CHECK-NEXT: [[SUBSWrr2:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui3]], implicit-def $nzcv
+ ; CHECK-NEXT: Bcc 10, %bb.2, implicit $nzcv
+ ; CHECK-NEXT: B %bb.4
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.4:
+ ; CHECK-NEXT: successors: %bb.5(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.5:
+ ; CHECK-NEXT: [[PHI:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.1, [[MOVi64imm1]], %bb.2, [[MOVi64imm2]], %bb.4
+ ; CHECK-NEXT: $x0 = COPY [[PHI]]
+ ; CHECK-NEXT: RET_ReallyLR implicit $x0
+ bb.0:
+ successors: %bb.1(0x80000000)
+ liveins: $x0, $x1
+
+ %0:gpr64common = COPY $x0
+ %1:gpr64common = COPY $x1
+ bb.1:
+ successors: %bb.2(0x40000000), %bb.5(0x40000000)
+
+ %10:gpr32 = LDRWui %0, 0 :: (load (s32))
+ %11:gpr32 = LDRWui %1, 0 :: (load (s32))
+ %12:gpr32 = SUBSWrr %10, %11, implicit-def $nzcv
+ %40:gpr64 = MOVi64imm 100
+ %41:gpr64 = MOVi64imm 200
+ %43:gpr64 = MOVi64imm 400
+ Bcc 10, %bb.5, implicit $nzcv
+ B %bb.2
+ bb.2:
+ successors: %bb.3(0x40000000), %bb.5(0x40000000)
+
+ %20:gpr32 = LDRWui %1, 1 :: (load (s32))
+ %21:gpr32 = SUBSWrr %10, %20, implicit-def $nzcv
+ Bcc 10, %bb.5, implicit $nzcv
+ B %bb.3
+
+ bb.3:
+ successors: %bb.4(0x40000000), %bb.2(0x40000000)
+
+ %30:gpr32 = LDRWui %1, 2 :: (load (s32))
+ %31:gpr32 = SUBSWrr %10, %30, implicit-def $nzcv
+ ; Branches to bb.2 instead of bb.5 (Tail) - creates back-edge
+ Bcc 10, %bb.2, implicit $nzcv
+ B %bb.4
+
+ bb.4:
+ successors: %bb.5(0x80000000)
+
+ bb.5:
+ %90:gpr64 = PHI %40, %bb.1, %41, %bb.2, %43, %bb.4
+ $x0 = COPY %90
+ RET_ReallyLR implicit $x0
+...
+---
+name: cascade_forward_skip_to_later_block
+alignment: 4
+tracksRegLiveness: true
+liveins:
+ - { reg: '$x0', virtual-reg: '%0' }
+ - { reg: '$x1', virtual-reg: '%1' }
+body: |
+ ; CHECK-LABEL: name: cascade_forward_skip_to_later_block
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: liveins: $x0, $x1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.2(0x40000000), %bb.6(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load (s32))
+ ; CHECK-NEXT: [[LDRWui1:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 0 :: (load (s32))
+ ; CHECK-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui1]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 100
+ ; CHECK-NEXT: Bcc 10, %bb.6, implicit $nzcv
+ ; CHECK-NEXT: B %bb.2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: successors: %bb.3(0x40000000), %bb.6(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[LDRWui2:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 1 :: (load (s32))
+ ; CHECK-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui2]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 200
+ ; CHECK-NEXT: Bcc 10, %bb.6, implicit $nzcv
+ ; CHECK-NEXT: B %bb.3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.3:
+ ; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.5(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[LDRWui3:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 2 :: (load (s32))
+ ; CHECK-NEXT: [[SUBSWrr2:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui3]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 300
+ ; CHECK-NEXT: Bcc 10, %bb.5, implicit $nzcv
+ ; CHECK-NEXT: B %bb.4
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.4:
+ ; CHECK-NEXT: successors: %bb.5(0x40000000), %bb.6(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[LDRWui4:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 3 :: (load (s32))
+ ; CHECK-NEXT: [[SUBSWrr3:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui4]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 400
+ ; CHECK-NEXT: Bcc 10, %bb.6, implicit $nzcv
+ ; CHECK-NEXT: B %bb.5
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.5:
+ ; CHECK-NEXT: successors: %bb.6(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[MOVi64imm4:%[0-9]+]]:gpr64 = MOVi64imm 500
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.6:
+ ; CHECK-NEXT: [[PHI:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.1, [[MOVi64imm1]], %bb.2, [[MOVi64imm3]], %bb.4, [[MOVi64imm4]], %bb.5
+ ; CHECK-NEXT: $x0 = COPY [[PHI]]
+ ; CHECK-NEXT: RET_ReallyLR implicit $x0
+ bb.0:
+ successors: %bb.1(0x80000000)
+ liveins: $x0, $x1
+
+ %0:gpr64common = COPY $x0
+ %1:gpr64common = COPY $x1
+
+ bb.1:
+ successors: %bb.2(0x40000000), %bb.6(0x40000000)
+
+ %10:gpr32 = LDRWui %0, 0 :: (load (s32))
+ %11:gpr32 = LDRWui %1, 0 :: (load (s32))
+ %12:gpr32 = SUBSWrr %10, %11, implicit-def $nzcv
+ %40:gpr64 = MOVi64imm 100
+ Bcc 10, %bb.6, implicit $nzcv
+ B %bb.2
+
+ bb.2:
+ successors: %bb.3(0x40000000), %bb.6(0x40000000)
+
+ %20:gpr32 = LDRWui %1, 1 :: (load (s32))
+ %21:gpr32 = SUBSWrr %10, %20, implicit-def $nzcv
+ %41:gpr64 = MOVi64imm 200
+ Bcc 10, %bb.6, implicit $nzcv
+ B %bb.3
+
+ bb.3:
+ successors: %bb.4(0x40000000), %bb.5(0x40000000)
+
+ %30:gpr32 = LDRWui %1, 2 :: (load (s32))
+ %31:gpr32 = SUBSWrr %10, %30, implicit-def $nzcv
+ %42:gpr64 = MOVi64imm 300
+ ; Branches to bb.5 instead of bb.6 (Tail) - forward skip
+ Bcc 10, %bb.5, implicit $nzcv
+ B %bb.4
+
+ bb.4:
+ successors: %bb.5(0x40000000), %bb.6(0x40000000)
+
+ %32:gpr32 = LDRWui %1, 3 :: (load (s32))
+ %33:gpr32 = SUBSWrr %10, %32, implicit-def $nzcv
+ %43:gpr64 = MOVi64imm 400
+ Bcc 10, %bb.6, implicit $nzcv
+ B %bb.5
+
+ bb.5:
+ successors: %bb.6(0x80000000)
+
+ %44:gpr64 = MOVi64imm 500
+
+ bb.6:
+ %90:gpr64 = PHI %40, %bb.1, %41, %bb.2, %43, %bb.4, %44, %bb.5
+ $x0 = COPY %90
+ RET_ReallyLR implicit $x0
+...
+
diff --git a/llvm/test/CodeGen/AArch64/early-ifcvt-remarks.mir b/llvm/test/CodeGen/AArch64/early-ifcvt-remarks.mir
index 93460d5822ab4..6f01d48601fa2 100644
--- a/llvm/test/CodeGen/AArch64/early-ifcvt-remarks.mir
+++ b/llvm/test/CodeGen/AArch64/early-ifcvt-remarks.mir
@@ -11,6 +11,12 @@
define i32 @data_dependent_branch(ptr %p, i32 %x, i32 %y) {
ret i32 0
}
+ define i64 @cascade_success(ptr %p, ptr %q, i32 %n) {
+ ret i64 0
+ }
+ define i64 @cascade_increasing_cond_depth(ptr %p) {
+ ret i64 0
+ }
...
---
name: data_dependent_branch
@@ -43,3 +49,156 @@ body: |
$w0 = COPY %100
RET_ReallyLR implicit $w0
...
+---
+name: cascade_success
+alignment: 4
+tracksRegLiveness: true
+liveins:
+ - { reg: '$x0', virtual-reg: '%0' }
+ - { reg: '$x1', virtual-reg: '%1' }
+ - { reg: '$w2', virtual-reg: '%2' }
+body: |
+ ; Entry - check loop count
+ bb.0:
+ successors: %bb.6(0x30000000), %bb.1(0x50000000)
+ liveins: $x0, $x1, $w2
+
+ %0:gpr64common = COPY $x0
+ %1:gpr64common = COPY $x1
+ %2:gpr32 = COPY $w2
+ %50:gpr64all = COPY $xzr
+ %51:gpr64all = COPY %50
+ CBZW %2, %bb.6
+ B %bb.1
+
+ ; Loop header / cascade head - ALL loads here, branches to bb.2 or bb.5
+ bb.1:
+ successors: %bb.2(0x40000000), %bb.5(0x40000000)
+
+ %60:gpr64 = PHI %51, %bb.0, %81, %bb.5
+ %61:gpr32 = PHI %2, %bb.0, %80, %bb.5
+ ; Load ALL data in head - makes branch data-dependent
+ %62:gpr32 = LDRWui %0, 0 :: (load (s32))
+ %63:gpr32 = LDRWui %1, 0 :: (load (s32))
+ %66:gpr32 = LDRWui %1, 1 :: (load (s32))
+ %69:gpr32 = LDRWui %1, 2 :: (load (s32))
+ %90:gpr32 = LDRWui %1, 3 :: (load (s32))
+ %64:gpr64 = MOVi64imm 4
+ %65:gpr32 = SUBSWrr %62, %63, implicit-def $nzcv
+ Bcc 10, %bb.5, implicit $nzcv
+ B %bb.2
+
+ ; First cascade block (branches to bb.3 or bb.5)
+ bb.2:
+ successors: %bb.3(0x40000000), %bb.5(0x40000000)
+
+ %67:gpr32 = SUBSWrr %62, %66, implicit-def $nzcv
+ %68:gpr64 = MOVi64imm 3
+ Bcc 10, %bb.5, implicit $nzcv
+ B %bb.3
+
+ ; Second cascade block (branches to bb.4 or bb.5)
+ bb.3:
+ successors: %bb.4(0x40000000), %bb.5(0x40000000)
+
+ %70:gpr32 = SUBSWrr %62, %69, implicit-def $nzcv
+ %91:gpr64 = MOVi64imm 2
+ Bcc 10, %bb.5, implicit $nzcv
+ B %bb.4
+
+ ; Third cascade block (only branches to bb.5)
+ bb.4:
+ successors: %bb.5(0x80000000)
+
+ %92:gpr32 = SUBSWrr %62, %90, implicit-def $nzcv
+ %71:gpr32 = CSINCWr $wzr, $wzr, 11, implicit $nzcv
+ %72:gpr64 = SUBREG_TO_REG %71, %subreg.sub_32
+
+ ; Loop latch / cascade tail - PHI merge and back edge
+ bb.5:
+ successors: %bb.6(0x04000000), %bb.1(0x7c000000)
+
+ %73:gpr64 = PHI %64, %bb.1, %68, %bb.2, %91, %bb.3, %72, %bb.4
+ %74:gpr64 = ADDXrr %60, %73
+ %75:gpr32 = MOVi32imm 1
+ %80:gpr32 = SUBSWrr %61, %75, implicit-def $nzcv
+ %81:gpr64all = COPY %74
+ Bcc 1, %bb.1, implicit $nzcv
+ B %bb.6
+
+ ; Exit
+ bb.6:
+ %82:gpr64 = PHI %51, %bb.0, %81, %bb.5
+ $x0 = COPY %82
+ RET_ReallyLR implicit $x0
+...
+---
+name: cascade_increasing_cond_depth
+alignment: 4
+tracksRegLiveness: true
+liveins:
+ - { reg: '$x0', virtual-reg: '%0' }
+body: |
+ bb.0:
+ successors: %bb.1(0x80000000)
+ liveins: $x0
+
+ %0:gpr64common = COPY $x0
+
+ bb.1:
+ successors: %bb.2(0x40000000), %bb.7(0x40000000)
+
+ %10:gpr32 = LDRWui %0, 0 :: (load (s32))
+ %11:gpr32 = MADDWrrr %10, %10, $wzr
+ %12:gpr32 = MADDWrrr %11, %11, $wzr
+ %20:gpr32 = ADDWrr %12, %12
+ %21:gpr32 = ADDWrr %20, %20
+ %22:gpr32 = ADDWrr %21, %21
+ %23:gpr32 = ADDWrr %22, %22
+ %40:gpr64 = MOVi64imm 100
+ %30:gpr32 = SUBSWrr %12, %12, implicit-def $nzcv
+ Bcc 10, %bb.7, implicit $nzcv
+ B %bb.2
+
+ bb.2:
+ successors: %bb.3(0x40000000), %bb.7(0x40000000)
+
+ %41:gpr64 = MOVi64imm 200
+ %31:gpr32 = SUBSWrr %20, %20, implicit-def $nzcv
+ Bcc 10, %bb.7, implicit $nzcv
+ B %bb.3
+
+ bb.3:
+ successors: %bb.4(0x40000000), %bb.7(0x40000000)
+
+ %42:gpr64 = MOVi64imm 300
+ %32:gpr32 = SUBSWrr %21, %21, implicit-def $nzcv
+ Bcc 10, %bb.7, implicit $nzcv
+ B %bb.4
+
+ bb.4:
+ successors: %bb.5(0x40000000), %bb.7(0x40000000)
+
+ %43:gpr64 = MOVi64imm 400
+ %33:gpr32 = SUBSWrr %22, %22, implicit-def $nzcv
+ Bcc 10, %bb.7, implicit $nzcv
+ B %bb.5
+
+ bb.5:
+ successors: %bb.6(0x40000000), %bb.7(0x40000000)
+
+ %44:gpr64 = MOVi64imm 500
+ %34:gpr32 = SUBSWrr %23, %23, implicit-def $nzcv
+ Bcc 10, %bb.7, implicit $nzcv
+ B %bb.6
+
+ bb.6:
+ successors: %bb.7(0x80000000)
+
+ %45:gpr64 = MOVi64imm 600
+
+ bb.7:
+ %90:gpr64 = PHI %40, %bb.1, %41, %bb.2, %42, %bb.3, %43, %bb.4, %44, %bb.5, %45, %bb.6
+ $x0 = COPY %90
+ RET_ReallyLR implicit $x0
+...
>From cb341e4fb84001e5a91d9e879471918854f68adb Mon Sep 17 00:00:00 2001
From: Jonathan Cohen <joncoh at apple.com>
Date: Thu, 16 Apr 2026 16:21:31 +0300
Subject: [PATCH 2/2] [EarlyIfConvert] Add heuristic to convert cascades of
conditional branches
Recognize and convert cascade patterns: chains of conditional branches where each block branches to either the next block or a shared tail. These patterns arise from if-else-if chains and switch-like constructs, and may suffer from serial misprediction when the branch conditions do not follow any predictable pattern.
When enabled via -enable-early-ifcvt-cascade, the pass walks up from single-successor blocks to identify cascades, validates each block for speculative execution, and converts the entire chain if profitable.
The profitability model requires all cascade branches to be data-dependent (conditions which have recently been loaded from memory and therefore may not be correlated to other branches, see PR #174457), checks that sufficient ILP exists to hide the speculated instructions, and bounds the CSEL chain depth against a scaled misprediction penalty.
---
llvm/lib/CodeGen/EarlyIfConversion.cpp | 658 +++++++-
.../CodeGen/AArch64/early-ifcvt-cascade.mir | 1320 ++++++++++++-----
.../early-ifcvt-load-to-cond-br-limit.mir | 577 +++++++
.../CodeGen/AArch64/early-ifcvt-remarks.mir | 97 ++
4 files changed, 2184 insertions(+), 468 deletions(-)
diff --git a/llvm/lib/CodeGen/EarlyIfConversion.cpp b/llvm/lib/CodeGen/EarlyIfConversion.cpp
index bf2664fd4225c..24fc9a90d88cc 100644
--- a/llvm/lib/CodeGen/EarlyIfConversion.cpp
+++ b/llvm/lib/CodeGen/EarlyIfConversion.cpp
@@ -16,11 +16,16 @@
//===----------------------------------------------------------------------===//
#include "llvm/CodeGen/EarlyIfConversion.h"
+#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/BitVector.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/DepthFirstIterator.h"
#include "llvm/ADT/PostOrderIterator.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/SparseSet.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
@@ -65,6 +70,16 @@ static cl::opt<bool> EnableDataDependentBranchAnalysis(
"enable-early-ifcvt-data-dependent", cl::Hidden, cl::init(false),
cl::desc("Enable hard-to-predict branch analysis for if-conversion"));
+// Enable recognition of cascade if-else patterns (chains of triangles).
+static cl::opt<bool> EnableCascadeIfConv(
+ "enable-early-ifcvt-cascade", cl::Hidden, cl::init(false),
+ cl::desc("Enable recognition of cascade if-else patterns"));
+
+// Maximum depth for cascade patterns.
+static cl::opt<unsigned>
+ MaxCascadeDepth("early-ifcvt-max-cascade-depth", cl::Hidden, cl::init(8),
+ cl::desc("Maximum cascade depth for early if-conversion"));
+
// Limit the number steps we take when searching conditions that depend on
// values recently loaded from memory.
static cl::opt<unsigned>
@@ -86,6 +101,8 @@ STATISTIC(NumTrianglesConv, "Number of triangles converted");
STATISTIC(NumDataDependant,
"Number of data dependent conditional branches encountered");
STATISTIC(NumLikelyBiased, "Number of branches with a hot path encountered");
+STATISTIC(NumCascadesSeen, "Number of cascade patterns detected");
+STATISTIC(NumCascadesConv, "Number of cascade patterns converted");
//===----------------------------------------------------------------------===//
// SSAIfConv
@@ -109,6 +126,20 @@ STATISTIC(NumLikelyBiased, "Number of branches with a hot path encountered");
// Head block, and phis in the Tail block are converted to select instructions.
//
namespace {
+
+/// What one cascade collapse step would do to one Tail phi.
+struct CascadeSelectInfo {
+ int CondCycles = 0;
+ bool NeedsSelect = false;
+};
+
+struct CascadeResult {
+ MachineBasicBlock *Head = nullptr;
+ SmallVector<MachineBasicBlock *> Blocks;
+ SmallVector<SmallVector<CascadeSelectInfo, 4>> Selects;
+ explicit operator bool() const { return Head != nullptr; }
+};
+
class SSAIfConv {
const TargetInstrInfo *TII;
const TargetRegisterInfo *TRI;
@@ -213,12 +244,36 @@ class SSAIfConv {
/// initialize the internal state, and return true.
/// If predicate is set try to predicate the block otherwise try to
/// speculatively execute it.
- bool canConvertIf(MachineBasicBlock *MBB, bool Predicate = false);
+ bool canConvertIf(MachineBasicBlock *MBB, bool Predicate = false,
+ bool AllowMultiSuccTBB = false,
+ MachineBasicBlock *ExplicitTail = nullptr);
/// convertIf - If-convert the last block passed to canConvertIf(), assuming
/// it is possible. Add any blocks that are to be erased to RemoveBlocks.
void convertIf(SmallVectorImpl<MachineBasicBlock *> &RemoveBlocks,
bool Predicate = false);
+
+ /// matchCascade - match a cascade of conditional branches by walking up
+ /// from MBB as a potential cascade end. A cascade is a chain of triangles
+ /// where each block has 2 successors (next cascade block + Tail) and the
+ /// last block has 1 successor (Tail). Returns the discovered Head and list
+ /// of cascade blocks [BB1, ..., BBn] if valid, empty result otherwise.
+ ///
+ /// Head --------+
+ /// | |
+ /// v |
+ /// BB1 ----------+
+ /// | |
+ /// v |
+ /// BB2 ----------+
+ /// | |
+ /// ...
+ /// ...
+ /// | |
+ /// v v
+ /// BBn (MBB) -> Tail
+ ///
+ CascadeResult matchCascade(MachineBasicBlock *MBB);
};
} // end anonymous namespace
@@ -469,7 +524,9 @@ bool SSAIfConv::findInsertionPoint() {
/// canConvertIf - analyze the sub-cfg rooted in MBB, and return true if it is
/// a potential candidate for if-conversion. Fill out the internal state.
///
-bool SSAIfConv::canConvertIf(MachineBasicBlock *MBB, bool Predicate) {
+bool SSAIfConv::canConvertIf(MachineBasicBlock *MBB, bool Predicate,
+ bool AllowMultiSuccTBB,
+ MachineBasicBlock *ExplicitTail) {
Head = MBB;
TBB = FBB = Tail = nullptr;
@@ -482,31 +539,40 @@ bool SSAIfConv::canConvertIf(MachineBasicBlock *MBB, bool Predicate) {
if (Succ0->pred_size() != 1)
std::swap(Succ0, Succ1);
- if (Succ0->pred_size() != 1 || Succ0->succ_size() != 1)
+ if (Succ0->pred_size() != 1 ||
+ (!AllowMultiSuccTBB && Succ0->succ_size() != 1))
return false;
- Tail = Succ0->succ_begin()[0];
-
- // This is not a triangle.
- if (Tail != Succ1) {
- // Check for a diamond. We won't deal with any critical edges.
- if (Succ1->pred_size() != 1 || Succ1->succ_size() != 1 ||
- Succ1->succ_begin()[0] != Tail)
- return false;
- LLVM_DEBUG(dbgs() << "\nDiamond: " << printMBBReference(*Head) << " -> "
- << printMBBReference(*Succ0) << "/"
- << printMBBReference(*Succ1) << " -> "
+ // Use explicit tail if provided (for cascade validation), otherwise compute.
+ if (ExplicitTail) {
+ Tail = ExplicitTail;
+ LLVM_DEBUG(dbgs() << "\nCascade triangle: " << printMBBReference(*Head)
+ << " -> " << printMBBReference(*Succ0) << " -> "
<< printMBBReference(*Tail) << '\n');
-
- // Live-in physregs are tricky to get right when speculating code.
- if (!Tail->livein_empty()) {
- LLVM_DEBUG(dbgs() << "Tail has live-ins.\n");
- return false;
- }
} else {
- LLVM_DEBUG(dbgs() << "\nTriangle: " << printMBBReference(*Head) << " -> "
- << printMBBReference(*Succ0) << " -> "
- << printMBBReference(*Tail) << '\n');
+ Tail = Succ0->succ_begin()[0];
+
+ // This is not a triangle.
+ if (Tail != Succ1) {
+ // Check for a diamond. We won't deal with any critical edges.
+ if (Succ1->pred_size() != 1 || Succ1->succ_size() != 1 ||
+ Succ1->succ_begin()[0] != Tail)
+ return false;
+ LLVM_DEBUG(dbgs() << "\nDiamond: " << printMBBReference(*Head) << " -> "
+ << printMBBReference(*Succ0) << "/"
+ << printMBBReference(*Succ1) << " -> "
+ << printMBBReference(*Tail) << '\n');
+
+ // Live-in physregs are tricky to get right when speculating code.
+ if (!Tail->livein_empty()) {
+ LLVM_DEBUG(dbgs() << "Tail has live-ins.\n");
+ return false;
+ }
+ } else {
+ LLVM_DEBUG(dbgs() << "\nTriangle: " << printMBBReference(*Head) << " -> "
+ << printMBBReference(*Succ0) << " -> "
+ << printMBBReference(*Tail) << '\n');
+ }
}
// This is a triangle or a diamond.
@@ -588,10 +654,15 @@ bool SSAIfConv::canConvertIf(MachineBasicBlock *MBB, bool Predicate) {
if (!findInsertionPoint())
return false;
- if (isTriangle())
- ++NumTrianglesSeen;
- else
- ++NumDiamondsSeen;
+ // ExplicitTail means this is a validation call. Don't count towards the
+ // triangle / diamonds seen stat - that should happen only when we consider
+ // them for conversion seperately.
+ if (!ExplicitTail) {
+ if (isTriangle())
+ ++NumTrianglesSeen;
+ else
+ ++NumDiamondsSeen;
+ }
return true;
}
@@ -641,6 +712,121 @@ static bool hasSameValue(const MachineRegisterInfo &MRI,
return TIdx == FIdx;
}
+/// matchCascade - match a cascade pattern by walking up from MBB.
+/// MBB is a potential cascade end (last block before Tail).
+/// A cascade is a chain of triangles where:
+/// - Head has 2 successors: one to first cascade block, one to Tail
+/// - Each cascade block has 2 successors: next cascade + Tail (or 1 for last)
+/// - Last cascade block (MBB) has 1 successor: Tail
+/// Returns the discovered Head and cascade blocks if valid, empty otherwise.
+///
+/// Head --------+
+/// | |
+/// v |
+/// BB1 ----------+
+/// | |
+/// v |
+/// BB2 ----------+
+/// | |
+/// ...
+/// ...
+/// | |
+/// v v
+/// BBn (MBB) -> Tail
+///
+CascadeResult SSAIfConv::matchCascade(MachineBasicBlock *MBB) {
+ // Cascade end must have exactly 1 successor (to Tail) and 1 predecessor
+ if (MBB->succ_size() != 1 || MBB->pred_size() != 1)
+ return {};
+
+ MachineBasicBlock *Tail = MBB->succ_begin()[0];
+
+ // Collect cascade blocks from bottom to top: [BBn, BB(n-1), ..., BB1, Head]
+ SmallVector<MachineBasicBlock *> Blocks;
+ Blocks.push_back(MBB);
+ MachineBasicBlock *Current = MBB;
+
+ // Walk up the cascade chain
+ while (true) {
+ MachineBasicBlock *Pred = Current->pred_begin()[0];
+
+ // Predecessor must have 2 successors: Current and Tail
+ if (Pred->succ_size() != 2)
+ break;
+
+ MachineBasicBlock *S0 = Pred->succ_begin()[0];
+ MachineBasicBlock *S1 = Pred->succ_begin()[1];
+
+ // One successor must be Current, the other must be Tail
+ if (!((S0 == Current && S1 == Tail) || (S1 == Current && S0 == Tail)))
+ break;
+
+ Blocks.push_back(Pred);
+
+ // If Pred has multiple predecessors, it's the Head - stop here
+ if (Pred->pred_size() != 1)
+ break;
+
+ // Pred has 1 predecessor, so the cascade continues. If we've reached
+ // max depth, bail out entirely - we don't want to convert a partial cascade.
+ if (Blocks.size() > MaxCascadeDepth) {
+ LLVM_DEBUG(dbgs() << "Cascade extends beyond max depth "
+ << MaxCascadeDepth << ", not converting.\n");
+ return {};
+ }
+
+ Current = Pred;
+ }
+
+ // Need at least 3 blocks: [BBn, BB1, Head] for a minimal 2-block cascade
+ if (Blocks.size() < 3)
+ return {};
+
+ // Reverse to get [HEAD, BB1, BB2, ..., BBn] order
+ std::reverse(Blocks.begin(), Blocks.end());
+
+ // Check that we can actually convert all the blocks in the cascade.
+ // We skip the last cascade block (BBn) since it has a single successor
+ // (to Tail), and is only used as a TBB (not a HEAD) - its speculatability
+ // is validated when we check its predecessor.
+ SmallVector<SmallVector<CascadeSelectInfo, 4>> CollectedSelects;
+ size_t NumBlocksToValidate = Blocks.size() - 1;
+ for (size_t I = 0; I < NumBlocksToValidate; ++I) {
+ if (!canConvertIf(Blocks[I], /*Predicate=*/false,
+ /*AllowMultiSuccTBB=*/true,
+ /*ExplicitTail=*/Tail)) {
+ LLVM_DEBUG(dbgs() << "Cannot convert cascade, block "
+ << printMBBReference(*Blocks[I])
+ << " is not if-convertible.\n");
+ return {};
+ }
+ assert((CollectedSelects.empty() ||
+ CollectedSelects.back().size() == PHIs.size()) &&
+ "Tail phi list changed between cascade blocks");
+
+ // Capture PHI info from canConvertIf's canInsertSelect call.
+ auto &StepSelects = CollectedSelects.emplace_back();
+ for (const PHIInfo &PI : PHIs)
+ StepSelects.push_back(
+ {PI.CondCycles, !hasSameValue(*MRI, TII, PI.TReg, PI.FReg)});
+ }
+
+ // The first block in Blocks is the cascade Head
+ MachineBasicBlock *Head = Blocks.front();
+
+ // Remove Head from the cascade blocks to convert
+ Blocks.erase(Blocks.begin());
+
+ LLVM_DEBUG({
+ dbgs() << "\nCascade found: " << printMBBReference(*Head) << " -> [";
+ for (auto *BB : Blocks)
+ dbgs() << printMBBReference(*BB) << ", ";
+ dbgs() << "] -> " << printMBBReference(*Tail) << "\n";
+ });
+
+ return {Head, std::move(Blocks), std::move(CollectedSelects)};
+}
+
/// replacePHIInstrs - Completely replace PHI instructions with selects.
/// This is possible when the only Tail predecessors are the if-converted
/// blocks.
@@ -843,6 +1029,10 @@ class EarlyIfConverter {
/// each block to the number of instructions scanned in it.
DenseMap<const MachineBasicBlock *, unsigned> NoCallBlocksCache;
+ /// Set of blocks that must be converted (part of a cascade).
+ /// These blocks bypass normal profitability checks in shouldConvertIf().
+ SmallPtrSet<MachineBasicBlock *, 16> MustConvertBlocks;
+
public:
EarlyIfConverter(MachineDominatorTree &DT, MachineLoopInfo &LI,
MachineTraceMetrics &MTM, MachineBranchProbabilityInfo *MBPI)
@@ -853,11 +1043,20 @@ class EarlyIfConverter {
private:
bool tryConvertIf(MachineBasicBlock *);
+ void detectCascades(MachineBasicBlock *);
+ void convertIf();
void invalidateTraces();
bool shouldConvertIf();
- bool isConditionDataDependent();
- bool doOperandsComeFromMemory(const MachineInstr *ConditionDef);
+ bool isConditionDataDependent(MachineBasicBlock *BB, bool RecordStats = true);
+ bool isCascadeDataDependent(MachineBasicBlock *Head,
+ ArrayRef<MachineBasicBlock *> CascadeBlocks);
+ bool doOperandsComeFromMemory(const MachineInstr *ConditionDef,
+ MachineBasicBlock *BB);
bool hasCallOrLoopInRange(const MachineInstr *From, const MachineInstr *To);
+ bool shouldConvertCascade(CascadeResult &Cascade, MachineBasicBlock *Tail);
+ bool hasEnoughILP(MachineBasicBlock *TraceBlock,
+ SmallVectorImpl<MachineBasicBlock *> &ExtraBlocks,
+ unsigned CritLimit);
};
class EarlyIfConverterLegacy : public MachineFunctionPass {
@@ -1047,9 +1246,10 @@ bool EarlyIfConverter::hasCallOrLoopInRange(const MachineInstr *From,
/// depend on values loaded from memory (unless they are loop invariant,
/// or come from a constant pool). The walk starts from the definition of
/// ConditionDef's first operand, which is not ConditionDef itself for
-/// instructions such as FCMPSrr, where that operand is a use.
+/// instructions such as FCMPSrr, where that operand is a use. BB is the block
+/// whose terminator consumes the condition.
bool EarlyIfConverter::doOperandsComeFromMemory(
- const MachineInstr *ConditionDef) {
+ const MachineInstr *ConditionDef, MachineBasicBlock *BB) {
Register Reg = ConditionDef->getOperand(0).getReg();
if (!Reg.isVirtual())
return false;
@@ -1057,10 +1257,10 @@ bool EarlyIfConverter::doOperandsComeFromMemory(
LLVM_DEBUG(dbgs() << " doOperandsComeFromMemory starting from reg "
<< printReg(Reg) << "\n");
- // The condition is consumed by the branch terminating Head, so this is the
+ // The condition is consumed by the branch terminating BB, so this is the
// end of the interval a load has to survive without a call in between.
- const MachineInstr *Br = &*IfConv.Head->getFirstTerminator();
- MachineLoop *IfConvLoop = Loops->getLoopFor(IfConv.Head);
+ const MachineInstr *Br = &*BB->getFirstTerminator();
+ MachineLoop *IfConvLoop = Loops->getLoopFor(BB);
// Walk the def-use chain.
SmallPtrSet<const MachineInstr *, 8> VisitedInstrs;
@@ -1127,9 +1327,12 @@ bool EarlyIfConverter::doOperandsComeFromMemory(
}
/// Check if the branch condition is data-dependent (comes from memory loads).
-bool EarlyIfConverter::isConditionDataDependent() {
+/// RecordStats should be false when the same branch may be examined again
+/// later, so that it is only counted once.
+bool EarlyIfConverter::isConditionDataDependent(MachineBasicBlock *BB,
+ bool RecordStats) {
TargetInstrInfo::MachineBranchPredicate MBP;
- if (TII->analyzeBranchPredicate(*IfConv.Head, MBP, /*AllowModify=*/false))
+ if (TII->analyzeBranchPredicate(*BB, MBP, /*AllowModify=*/false))
return false;
if (!MBP.ConditionDef)
@@ -1138,29 +1341,52 @@ bool EarlyIfConverter::isConditionDataDependent() {
// If the branch is biased (not 50/50), don't consider it data dependent.
// This is to prevent converting unprofitable checks such as
// `x[i] != 0;`
- auto TBBProb = MBPI->getEdgeProbability(IfConv.Head, IfConv.TBB);
- auto FBBProb = MBPI->getEdgeProbability(IfConv.Head, IfConv.FBB);
- if (TBBProb != FBBProb) {
- ++NumLikelyBiased;
- return false;
+ if (MBP.TrueDest && MBP.FalseDest && MBPI) {
+ auto TBBProb = MBPI->getEdgeProbability(BB, MBP.TrueDest);
+ auto FBBProb = MBPI->getEdgeProbability(BB, MBP.FalseDest);
+ if (TBBProb != FBBProb) {
+ if (RecordStats)
+ ++NumLikelyBiased;
+ return false;
+ }
}
// Check if operands used to compute the branch condition were loaded recently
// from memory, starting by the ConditionDef itself and walking up the use-def
// chain.
- if (doOperandsComeFromMemory(MBP.ConditionDef)) {
- ++NumDataDependant;
+ if (doOperandsComeFromMemory(MBP.ConditionDef, BB)) {
+ if (RecordStats)
+ ++NumDataDependant;
return true;
}
return false;
}
-// Adjust cycles with downward saturation.
-static unsigned adjCycles(unsigned Cyc, int Delta) {
- if (Delta < 0 && Cyc + Delta > Cyc)
- return 0;
- return Cyc + Delta;
+/// Check if ALL branches in a cascade are data-dependent (come from loads).
+/// Cascade conversion always requires data-dependent branches, so this check
+/// is enabled whenever cascade conversion is enabled.
+bool EarlyIfConverter::isCascadeDataDependent(
+ MachineBasicBlock *Head, ArrayRef<MachineBasicBlock *> CascadeBlocks) {
+
+ // Check Head block
+ if (!isConditionDataDependent(Head, /*RecordStats=*/false)) {
+ LLVM_DEBUG(dbgs() << "Cascade: Head not data-dependent\n");
+ return false;
+ }
+
+ // Check all cascade blocks except the last one (which has only 1 successor).
+ // Don't record stats because we are not converting anything yet.
+ for (auto *CascadeBlock : CascadeBlocks.drop_back()) {
+ if (!isConditionDataDependent(CascadeBlock, /*RecordStats=*/false)) {
+ LLVM_DEBUG(dbgs() << "Cascade: " << printMBBReference(*CascadeBlock)
+ << " not data-dependent\n");
+ return false;
+ }
+ }
+
+ LLVM_DEBUG(dbgs() << "Cascade: all branches are data-dependent\n");
+ return true;
}
namespace {
@@ -1168,20 +1394,251 @@ namespace {
struct Cycles {
const char *Key;
unsigned Value;
+ Cycles(const char *K, unsigned V) : Key(K), Value(V) {}
};
template <typename Remark> Remark &operator<<(Remark &R, Cycles C) {
return R << ore::NV(C.Key, C.Value) << (C.Value == 1 ? " cycle" : " cycles");
}
} // anonymous namespace
+// Adjust cycles with downward saturation.
+static unsigned adjCycles(unsigned Cyc, int Delta) {
+ if (Delta < 0 && Cyc + Delta > Cyc)
+ return 0;
+ return Cyc + Delta;
+}
+
+/// Count the instructions of MBB that a predecessor would have to speculate to
+/// absorb it. This matches what canSpeculateInstrs() checks against
+/// BlockInstrLimit: the non-debug instructions ahead of the terminators.
+/// If we don't perform this check we could bail out of a cascade conversion
+/// midway by exceeding `BlockInstrLimit`.
+static unsigned countSpeculatedInstrs(const MachineBasicBlock &MBB) {
+ return count_if(make_range(MBB.begin(), MBB.getFirstTerminator()),
+ [](const MachineInstr &MI) { return !MI.isDebugInstr(); });
+}
+
+/// Apply profitability check for cascade conversion.
+bool EarlyIfConverter::shouldConvertCascade(CascadeResult &Cascade,
+ MachineBasicBlock *Tail) {
+ MachineBasicBlock *Head = Cascade.Head;
+ auto &CascadeBlocks = Cascade.Blocks;
+
+ if (!isCascadeDataDependent(Head, CascadeBlocks)) {
+ LLVM_DEBUG(dbgs() << "Cascade: not all branches are data-dependent\n");
+ return false;
+ }
+
+ // Calculate CritLimit using cascade formula.
+ unsigned CascadeSize = CascadeBlocks.size();
+ unsigned MispredictPenalty = STI->getMispredictionPenalty();
+ unsigned CritLimit =
+ std::min(MispredictPenalty * 25 * CascadeSize / 100,
+ 2 * MispredictPenalty);
+
+ LLVM_DEBUG(dbgs() << "Cascade: size=" << CascadeSize
+ << ", CritLimit=" << CritLimit << " (MispredictPenalty="
+ << MispredictPenalty << ")\n");
+
+ if (CritLimit == 0) {
+ LLVM_DEBUG(dbgs() << "Cascade: CritLimit is 0, skipping\n");
+ return false;
+ }
+
+ MachineOptimizationRemarkEmitter MORE(*Head->getParent(), nullptr);
+ SmallVector<MachineBasicBlock *, 8> CondBlocks;
+ CondBlocks.push_back(Head);
+ append_range(CondBlocks, ArrayRef(CascadeBlocks).drop_back());
+ assert(CondBlocks.size() == Cascade.Selects.size() &&
+ "Mismatched cascade condition info");
+
+ unsigned NumPHIs = Cascade.Selects.front().size();
+ assert(NumPHIs == range_size(Tail->phis()) &&
+ "Tail phi list changed since matchCascade()");
+
+ // A step only needs a select if the phi's two incoming values differ. But
+ // once some deeper step has produced a select, every shallower step takes
+ // that select's result as an incoming value and needs one too.
+ SmallVector<int, 4> DeepestSelectStep(NumPHIs, -1);
+ for (auto [I, Step] : enumerate(Cascade.Selects))
+ for (auto [J, Sel] : enumerate(Step))
+ if (Sel.NeedsSelect)
+ DeepestSelectStep[J] = I;
+
+ if (!Stress) {
+ unsigned NumSelects = 0;
+ for (int Deepest : DeepestSelectStep)
+ NumSelects += Deepest + 1;
+ unsigned TotalInstrs = NumSelects;
+ for (MachineBasicBlock *BB : CascadeBlocks)
+ TotalInstrs += countSpeculatedInstrs(*BB);
+
+ LLVM_DEBUG(dbgs() << "Cascade: collapsed cascade holds " << TotalInstrs
+ << " instructions (" << NumSelects
+ << " of them selects), limit is " << BlockInstrLimit
+ << '\n');
+
+ if (TotalInstrs > BlockInstrLimit) {
+ LLVM_DEBUG(dbgs() << "Cascade: collapsed cascade exceeds instruction "
+ "limit, skipping\n");
+ MORE.emit([&]() {
+ return MachineOptimizationRemarkMissed(DEBUG_TYPE,
+ "CascadeIfConversion",
+ Head->back().getDebugLoc(), Head)
+ << "did not if-convert cascade with "
+ << ore::NV("CascadeSize", CascadeSize)
+ << " blocks: the speculated blocks would hold "
+ << ore::NV("TotalInstrs", TotalInstrs)
+ << " instructions, exceeding the limit of "
+ << ore::NV("BlockInstrLimit", BlockInstrLimit) << ".";
+ });
+ return false;
+ }
+ }
+
+ // Check if there's enough ILP to hide the speculated cascade blocks.
+ if (!hasEnoughILP(Tail, CascadeBlocks, CritLimit)) {
+ LLVM_DEBUG(dbgs() << "Cascade: not enough ILP, skipping\n");
+ MORE.emit([&]() {
+ return MachineOptimizationRemarkMissed(DEBUG_TYPE, "CascadeIfConversion",
+ Head->back().getDebugLoc(), Head)
+ << "did not if-convert cascade with "
+ << ore::NV("CascadeSize", CascadeSize)
+ << " blocks: not enough ILP to hide speculated instructions.";
+ });
+ return false;
+ }
+
+ // Compute CSEL chain critical path depth.
+ // After conversion, we get a chain of CSELs where each CSEL depends on:
+ // 1. The condition from that block's branch
+ // 2. The previous CSEL's output (chain dependency)
+ // The chain is rooted at the deepest cascade block and ends at Head, which is
+ // the order convertIf() creates the selects in.
+ if (!MinInstr)
+ MinInstr = Traces->getEnsemble(MachineTraceStrategy::TS_MinInstrCount);
+
+ MachineTraceMetrics::Trace TailTrace = MinInstr->getTrace(Tail);
+
+ SmallVector<unsigned, 8> BranchDepths;
+ for (MachineBasicBlock *CondBlock : CondBlocks) {
+ MachineTraceMetrics::Trace BlockTrace = MinInstr->getTrace(CondBlock);
+ BranchDepths.push_back(
+ BlockTrace.getInstrCycles(*CondBlock->getFirstTerminator()).Depth);
+ }
+
+ // Each phi gets its own budget and its own chain: select latency depends on
+ // the phi's register class, and a phi with a lot of slack must not raise the
+ // bar for a phi with none. All of them have to be profitable.
+ unsigned MaxExtension = 0;
+ bool ExceedsLimit = false;
+ for (auto [J, PHI] : enumerate(Tail->phis())) {
+ unsigned Slack = TailTrace.getInstrSlack(PHI);
+ unsigned MaxDepth = Slack + TailTrace.getInstrCycles(PHI).Depth;
+
+ // convertIf() collapses the cascade bottom-up, so the deepest block's
+ // condition roots the CSEL chain and Head's condition feeds the last
+ // select.
+ LLVM_DEBUG(dbgs() << "CSEL chain depth computation for " << PHI);
+ unsigned CSELChainDepth = 0;
+ for (int I = DeepestSelectStep[J]; I >= 0; --I) {
+ int CondCycles = Cascade.Selects[I][J].CondCycles;
+ CSELChainDepth =
+ adjCycles(std::max(BranchDepths[I], CSELChainDepth), CondCycles);
+
+ LLVM_DEBUG(dbgs() << " " << printMBBReference(*CondBlocks[I])
+ << ": branch depth=" << BranchDepths[I]
+ << ", CondCycles=" << CondCycles
+ << ", new CSEL depth=" << CSELChainDepth << "\n");
+ }
+
+ unsigned Extension =
+ CSELChainDepth > MaxDepth ? CSELChainDepth - MaxDepth : 0;
+ MaxExtension = std::max(MaxExtension, Extension);
+
+ LLVM_DEBUG(dbgs() << "Final CSEL chain depth: " << CSELChainDepth
+ << ", MaxDepth: " << MaxDepth << ", Extension: "
+ << Extension << ", CritLimit: " << CritLimit << '\n');
+
+ if (Extension > CritLimit) {
+ LLVM_DEBUG(dbgs() << "Cascade: critical path extension exceeds "
+ "limit, skipping\n");
+ ExceedsLimit = true;
+ }
+ }
+
+ if (ExceedsLimit) {
+ MORE.emit([&]() {
+ MachineOptimizationRemarkMissed R(DEBUG_TYPE, "CascadeIfConversion",
+ Head->back().getDebugLoc(), Head);
+ R << "did not if-convert cascade with "
+ << ore::NV("CascadeSize", CascadeSize)
+ << " blocks: critical path extension of "
+ << Cycles("CritPathExtension", MaxExtension)
+ << " exceeds the threshold of " << Cycles("CritLimit", CritLimit)
+ << ".";
+ return R;
+ });
+ return false;
+ }
+
+ MORE.emit([&]() {
+ MachineOptimizationRemark R(DEBUG_TYPE, "CascadeIfConversion",
+ Head->back().getDebugLoc(), Head);
+ R << "performing cascade if-conversion on "
+ << ore::NV("CascadeSize", CascadeSize)
+ << " blocks: critical path extension is "
+ << Cycles("CritPathExtension", MaxExtension)
+ << ", staying under the threshold of " << Cycles("CritLimit", CritLimit)
+ << ".";
+ return R;
+ });
+
+ return true;
+}
+
+/// Check if there is enough ILP to hide the latency of speculatively executing
+/// the extra blocks
+bool EarlyIfConverter::hasEnoughILP(
+ MachineBasicBlock *TraceBlock,
+ SmallVectorImpl<MachineBasicBlock *> &ExtraBlocks, unsigned CritLimit) {
+ if (!MinInstr)
+ MinInstr = Traces->getEnsemble(MachineTraceStrategy::TS_MinInstrCount);
+
+ MachineTraceMetrics::Trace Trace = MinInstr->getTrace(TraceBlock);
+ unsigned CritPath = Trace.getCriticalPath();
+ SmallVector<const MachineBasicBlock *, 8> ConstExtraBlocks(
+ ExtraBlocks.begin(), ExtraBlocks.end());
+ unsigned ResLength = Trace.getResourceLength(ConstExtraBlocks);
+
+ LLVM_DEBUG(dbgs() << "ILP check: CriticalPath=" << CritPath
+ << ", ResourceLength=" << ResLength
+ << ", CritLimit=" << CritLimit << "\n");
+
+ if (ResLength > CritPath + CritLimit) {
+ LLVM_DEBUG(dbgs() << "Not enough ILP: resource length " << ResLength
+ << " exceeds critical path " << CritPath << " + limit "
+ << CritLimit << "\n");
+ return false;
+ }
+ return true;
+}
+
/// Apply cost model and heuristics to the if-conversion in IfConv.
/// Return true if the conversion is a good idea.
///
bool EarlyIfConverter::shouldConvertIf() {
+ bool InCascade = MustConvertBlocks.erase(IfConv.Head);
+
// Stress testing mode disables all cost considerations.
if (Stress)
return true;
+ if (InCascade) {
+ LLVM_DEBUG(dbgs() << "Block is part of cascade, skipping profitability\n");
+ return true;
+ }
+
// Do not try to if-convert if the condition has a high chance of being
// predictable.
MachineLoop *CurrentLoop = Loops->getLoopFor(IfConv.Head);
@@ -1228,7 +1685,7 @@ bool EarlyIfConverter::shouldConvertIf() {
// hard-to-predict branches, half for others. Otherwise use half for all.
bool DataDependent = false;
if (EnableDataDependentBranchAnalysis)
- DataDependent = isConditionDataDependent();
+ DataDependent = isConditionDataDependent(IfConv.Head);
unsigned CritLimit = DataDependent ? STI->getMispredictionPenalty()
: STI->getMispredictionPenalty() / 2;
@@ -1263,10 +1720,10 @@ bool EarlyIfConverter::shouldConvertIf() {
MachineOptimizationRemarkMissed R(DEBUG_TYPE, "IfConversion",
MBB.findDebugLoc(MBB.back()), &MBB);
R << "did not if-convert branch: the resulting critical path ("
- << Cycles{"ResLength", ResLength}
+ << Cycles("ResLength", ResLength)
<< ") would extend the shorter leg's critical path ("
- << Cycles{"MinCrit", MinCrit} << ") by more than the threshold of "
- << Cycles{"CritLimit", CritLimit}
+ << Cycles("MinCrit", MinCrit) << ") by more than the threshold of "
+ << Cycles("CritLimit", CritLimit)
<< ", which cannot be hidden by available ILP.";
return R;
});
@@ -1348,15 +1805,15 @@ bool EarlyIfConverter::shouldConvertIf() {
MachineOptimizationRemark R(DEBUG_TYPE, "IfConversion",
MBB.back().getDebugLoc(), &MBB);
R << "performing if-conversion on branch: the condition adds "
- << Cycles{"CondCycles", Cond.Extra} << " to the critical path";
+ << Cycles("CondCycles", Cond.Extra) << " to the critical path";
if (Short.Extra > 0)
R << ", and the short leg adds another "
- << Cycles{"ShortCycles", Short.Extra};
+ << Cycles("ShortCycles", Short.Extra);
if (Long.Extra > 0)
R << ", and the long leg adds another "
- << Cycles{"LongCycles", Long.Extra};
+ << Cycles("LongCycles", Long.Extra);
R << ", each staying under the threshold of "
- << Cycles{"CritLimit", CritLimit} << ".";
+ << Cycles("CritLimit", CritLimit) << ".";
return R;
});
} else {
@@ -1364,20 +1821,20 @@ bool EarlyIfConverter::shouldConvertIf() {
MachineOptimizationRemarkMissed R(DEBUG_TYPE, "IfConversion",
MBB.back().getDebugLoc(), &MBB);
R << "did not if-convert branch: the condition would add "
- << Cycles{"CondCycles", Cond.Extra} << " to the critical path";
+ << Cycles("CondCycles", Cond.Extra) << " to the critical path";
if (Cond.Extra > CritLimit)
- R << " exceeding the limit of " << Cycles{"CritLimit", CritLimit};
+ R << " exceeding the limit of " << Cycles("CritLimit", CritLimit);
if (Short.Extra > 0) {
R << ", and the short leg would add another "
- << Cycles{"ShortCycles", Short.Extra};
+ << Cycles("ShortCycles", Short.Extra);
if (Short.Extra > CritLimit)
- R << " exceeding the limit of " << Cycles{"CritLimit", CritLimit};
+ R << " exceeding the limit of " << Cycles("CritLimit", CritLimit);
}
if (Long.Extra > 0) {
R << ", and the long leg would add another "
- << Cycles{"LongCycles", Long.Extra};
+ << Cycles("LongCycles", Long.Extra);
if (Long.Extra > CritLimit)
- R << " exceeding the limit of " << Cycles{"CritLimit", CritLimit};
+ R << " exceeding the limit of " << Cycles("CritLimit", CritLimit);
}
R << ".";
return R;
@@ -1387,26 +1844,73 @@ bool EarlyIfConverter::shouldConvertIf() {
return ShouldConvert;
}
+/// Perform the actual if-conversion and update analyses.
+void EarlyIfConverter::convertIf() {
+ SmallVector<MachineBasicBlock *, 4> RemoveBlocks;
+ invalidateTraces();
+ IfConv.convertIf(RemoveBlocks);
+ updateDomTree(DomTree, IfConv, RemoveBlocks);
+ updateLoops(Loops, RemoveBlocks);
+ // Head absorbs the instructions of the removed blocks, including any calls,
+ // so a Head cached as call-free may no longer be.
+ NoCallBlocksCache.erase(IfConv.Head);
+ for (MachineBasicBlock *MBB : RemoveBlocks) {
+ NoCallBlocksCache.erase(MBB);
+ MustConvertBlocks.erase(MBB);
+ MBB->eraseFromParent();
+ }
+}
+
+/// Try to detect a cascade ending at MBB and mark blocks for conversion.
+/// Since we visit blocks in postorder, the cascade end is visited before
+/// its head, allowing us to mark cascade blocks before they are visited.
+void EarlyIfConverter::detectCascades(MachineBasicBlock *MBB) {
+ if (!EnableCascadeIfConv || !EnableDataDependentBranchAnalysis ||
+ MustConvertBlocks.count(MBB))
+ return;
+
+ auto Cascade = IfConv.matchCascade(MBB);
+ if (!Cascade)
+ return;
+
+ ++NumCascadesSeen;
+ MachineBasicBlock *Tail = MBB->succ_begin()[0];
+ LLVM_DEBUG(dbgs() << "Found cascade with " << Cascade.Blocks.size()
+ << " blocks, Head=" << printMBBReference(*Cascade.Head)
+ << ", Tail=" << printMBBReference(*Tail) << "\n");
+
+ if (!shouldConvertCascade(Cascade, Tail))
+ return;
+
+ ++NumCascadesConv;
+ NumDataDependant += Cascade.Blocks.size();
+
+ // Mark Head and all cascade blocks except the last one.
+ // The last cascade block (BBn) is never a head - it's always the TBB.
+ MustConvertBlocks.insert(Cascade.Head);
+ for (size_t I = 0; I + 1 < Cascade.Blocks.size(); ++I)
+ MustConvertBlocks.insert(Cascade.Blocks[I]);
+ LLVM_DEBUG(dbgs() << "Marked " << (1 + Cascade.Blocks.size() - 1)
+ << " blocks for cascade conversion\n");
+}
+
/// Attempt repeated if-conversion on MBB, return true if successful.
///
bool EarlyIfConverter::tryConvertIf(MachineBasicBlock *MBB) {
+ detectCascades(MBB);
+
bool Changed = false;
while (IfConv.canConvertIf(MBB) && shouldConvertIf()) {
- // If-convert MBB and update analyses.
- invalidateTraces();
- SmallVector<MachineBasicBlock *, 4> RemoveBlocks;
- IfConv.convertIf(RemoveBlocks);
+ convertIf();
Changed = true;
- updateDomTree(DomTree, IfConv, RemoveBlocks);
- updateLoops(Loops, RemoveBlocks);
- // Head absorbs the instructions of the removed blocks, including any calls,
- // so a Head cached as call-free may no longer be.
- NoCallBlocksCache.erase(IfConv.Head);
- for (MachineBasicBlock *MBB : RemoveBlocks) {
- NoCallBlocksCache.erase(MBB);
- MBB->eraseFromParent();
- }
}
+
+ // A mark that survives means a cascade block was not converted. We only
+ // convert cascades if the entire cascade conversion is profitable, so
+ // something went wrong.
+ assert(!MustConvertBlocks.contains(MBB) &&
+ "cascade block failed to if-convert");
+
return Changed;
}
@@ -1427,6 +1931,8 @@ bool EarlyIfConverter::run(MachineFunction &MF) {
bool Changed = false;
IfConv.init(MF);
+ MustConvertBlocks.clear();
+
// Visit blocks in dominator tree post-order. The post-order enables nested
// if-conversion in a single pass. The tryConvertIf() function may erase
// blocks, but only blocks dominated by the head block. This makes it safe to
@@ -1435,6 +1941,8 @@ bool EarlyIfConverter::run(MachineFunction &MF) {
if (tryConvertIf(DomNode->getBlock()))
Changed = true;
+ assert(MustConvertBlocks.empty() && "cascade block failed to if-convert");
+
return Changed;
}
diff --git a/llvm/test/CodeGen/AArch64/early-ifcvt-cascade.mir b/llvm/test/CodeGen/AArch64/early-ifcvt-cascade.mir
index 99c0ea79267a1..8939c2cca2235 100644
--- a/llvm/test/CodeGen/AArch64/early-ifcvt-cascade.mir
+++ b/llvm/test/CodeGen/AArch64/early-ifcvt-cascade.mir
@@ -1,5 +1,7 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
-# RUN: llc -mtriple=aarch64 -mcpu=apple-m1 -run-pass=early-ifcvt -enable-early-ifcvt-data-dependent %s -o - | FileCheck %s
+# RUN: llc -mtriple=aarch64 -mcpu=apple-m1 -run-pass=early-ifcvt -enable-early-ifcvt-cascade -enable-early-ifcvt-data-dependent %s -o - | FileCheck %s --check-prefixes=COMMON,CASCADE-WITH-DD
+# RUN: llc -mtriple=aarch64 -mcpu=apple-m1 -run-pass=early-ifcvt -enable-early-ifcvt-cascade -enable-early-ifcvt-data-dependent -early-ifcvt-max-cascade-depth=3 %s -o - | FileCheck %s --check-prefixes=COMMON,DEPTH3
+# RUN: llc -mtriple=aarch64 -mcpu=apple-m1 -run-pass=early-ifcvt -enable-early-ifcvt-cascade -enable-early-ifcvt-data-dependent -early-ifcvt-limit=3 %s -o - | FileCheck %s --check-prefixes=COMMON,LIMIT3
--- |
define void @cascade_selector_loop() { ret void }
@@ -7,6 +9,8 @@
define void @cascade_exceeds_crit_limit() { ret void }
define void @cascade_backedge_to_earlier_block() { ret void }
define void @cascade_forward_skip_to_later_block() { ret void }
+ define void @cascade_per_phi_slack() { ret void }
+ define void @cascade_phi_subset() { ret void }
...
---
name: cascade_selector_loop
@@ -18,102 +22,261 @@ liveins:
- { reg: '$w3', virtual-reg: '%3' }
- { reg: '$w4', virtual-reg: '%4' }
body: |
- ; CHECK-LABEL: name: cascade_selector_loop
- ; CHECK: bb.0:
- ; CHECK-NEXT: successors: %bb.10(0x30000000), %bb.1(0x50000000)
- ; CHECK-NEXT: liveins: $x0, $x1, $w2, $w3, $w4
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr32 = COPY $w2
- ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gpr32 = COPY $w3
- ; CHECK-NEXT: [[COPY4:%[0-9]+]]:gpr32 = COPY $w4
- ; CHECK-NEXT: [[COPY5:%[0-9]+]]:gpr64all = COPY $xzr
- ; CHECK-NEXT: CBZW [[COPY2]], %bb.10
- ; CHECK-NEXT: B %bb.1
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.1:
- ; CHECK-NEXT: successors: %bb.2(0x80000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 0 :: (load (s32))
- ; CHECK-NEXT: [[LDRWui1:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 1 :: (load (s32))
- ; CHECK-NEXT: [[LDRWui2:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 2 :: (load (s32))
- ; CHECK-NEXT: [[LDRWui3:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 3 :: (load (s32))
- ; CHECK-NEXT: [[LDRWui4:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 4 :: (load (s32))
- ; CHECK-NEXT: [[LDRWui5:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 5 :: (load (s32))
- ; CHECK-NEXT: [[LDRWui6:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 6 :: (load (s32))
- ; CHECK-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 1
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.2:
- ; CHECK-NEXT: successors: %bb.3(0x40000000), %bb.9(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[PHI:%[0-9]+]]:gpr32 = PHI [[COPY2]], %bb.1, %15, %bb.9
- ; CHECK-NEXT: [[LDRWui7:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load (s32))
- ; CHECK-NEXT: [[MADDWrrr:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[LDRWui7]], $wzr
- ; CHECK-NEXT: [[MADDWrrr1:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY4]], [[LDRWui7]], [[MADDWrrr]]
- ; CHECK-NEXT: [[MADDWrrr2:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[LDRWui7]], [[MADDWrrr1]]
- ; CHECK-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 7
- ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
- ; CHECK-NEXT: B %bb.3
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.3:
- ; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.9(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui1]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 6
- ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
- ; CHECK-NEXT: B %bb.4
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.4:
- ; CHECK-NEXT: successors: %bb.5(0x40000000), %bb.9(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[SUBSWrr2:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui2]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 5
- ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
- ; CHECK-NEXT: B %bb.5
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.5:
- ; CHECK-NEXT: successors: %bb.6(0x40000000), %bb.9(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[SUBSWrr3:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui3]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 4
- ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
- ; CHECK-NEXT: B %bb.6
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.6:
- ; CHECK-NEXT: successors: %bb.7(0x40000000), %bb.9(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[SUBSWrr4:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui4]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm4:%[0-9]+]]:gpr64 = MOVi64imm 3
- ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
- ; CHECK-NEXT: B %bb.7
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.7:
- ; CHECK-NEXT: successors: %bb.8(0x40000000), %bb.9(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[SUBSWrr5:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui5]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm5:%[0-9]+]]:gpr64 = MOVi64imm 2
- ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
- ; CHECK-NEXT: B %bb.8
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.8:
- ; CHECK-NEXT: successors: %bb.9(0x80000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[MOVi64imm6:%[0-9]+]]:gpr64 = MOVi64imm 1
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.9:
- ; CHECK-NEXT: successors: %bb.10(0x04000000), %bb.2(0x7c000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[PHI1:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.2, [[MOVi64imm1]], %bb.3, [[MOVi64imm2]], %bb.4, [[MOVi64imm3]], %bb.5, [[MOVi64imm4]], %bb.6, [[MOVi64imm5]], %bb.7, [[MOVi64imm6]], %bb.8
- ; CHECK-NEXT: [[SUBSWrr6:%[0-9]+]]:gpr32 = SUBSWrr [[PHI]], [[MOVi32imm]], implicit-def $nzcv
- ; CHECK-NEXT: Bcc 1, %bb.2, implicit $nzcv
- ; CHECK-NEXT: B %bb.10
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.10:
- ; CHECK-NEXT: [[PHI2:%[0-9]+]]:gpr64 = PHI [[COPY5]], %bb.0, [[PHI1]], %bb.9
- ; CHECK-NEXT: $x0 = COPY [[PHI2]]
- ; CHECK-NEXT: RET_ReallyLR implicit $x0
+ ; CASCADE-WITH-DD-LABEL: name: cascade_selector_loop
+ ; CASCADE-WITH-DD: bb.0:
+ ; CASCADE-WITH-DD-NEXT: successors: %bb.10(0x30000000), %bb.1(0x50000000)
+ ; CASCADE-WITH-DD-NEXT: liveins: $x0, $x1, $w2, $w3, $w4
+ ; CASCADE-WITH-DD-NEXT: {{ $}}
+ ; CASCADE-WITH-DD-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; CASCADE-WITH-DD-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1
+ ; CASCADE-WITH-DD-NEXT: [[COPY2:%[0-9]+]]:gpr32 = COPY $w2
+ ; CASCADE-WITH-DD-NEXT: [[COPY3:%[0-9]+]]:gpr32 = COPY $w3
+ ; CASCADE-WITH-DD-NEXT: [[COPY4:%[0-9]+]]:gpr32 = COPY $w4
+ ; CASCADE-WITH-DD-NEXT: [[COPY5:%[0-9]+]]:gpr64all = COPY $xzr
+ ; CASCADE-WITH-DD-NEXT: CBZW [[COPY2]], %bb.10
+ ; CASCADE-WITH-DD-NEXT: B %bb.1
+ ; CASCADE-WITH-DD-NEXT: {{ $}}
+ ; CASCADE-WITH-DD-NEXT: bb.1:
+ ; CASCADE-WITH-DD-NEXT: successors: %bb.2(0x80000000)
+ ; CASCADE-WITH-DD-NEXT: {{ $}}
+ ; CASCADE-WITH-DD-NEXT: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 0 :: (load (s32))
+ ; CASCADE-WITH-DD-NEXT: [[LDRWui1:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 1 :: (load (s32))
+ ; CASCADE-WITH-DD-NEXT: [[LDRWui2:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 2 :: (load (s32))
+ ; CASCADE-WITH-DD-NEXT: [[LDRWui3:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 3 :: (load (s32))
+ ; CASCADE-WITH-DD-NEXT: [[LDRWui4:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 4 :: (load (s32))
+ ; CASCADE-WITH-DD-NEXT: [[LDRWui5:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 5 :: (load (s32))
+ ; CASCADE-WITH-DD-NEXT: [[LDRWui6:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 6 :: (load (s32))
+ ; CASCADE-WITH-DD-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 1
+ ; CASCADE-WITH-DD-NEXT: {{ $}}
+ ; CASCADE-WITH-DD-NEXT: bb.2:
+ ; CASCADE-WITH-DD-NEXT: successors: %bb.10(0x04000000), %bb.2(0x7c000000)
+ ; CASCADE-WITH-DD-NEXT: {{ $}}
+ ; CASCADE-WITH-DD-NEXT: [[PHI:%[0-9]+]]:gpr32 = PHI [[COPY2]], %bb.1, %15, %bb.2
+ ; CASCADE-WITH-DD-NEXT: [[LDRWui7:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load (s32))
+ ; CASCADE-WITH-DD-NEXT: [[MADDWrrr:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[LDRWui7]], $wzr
+ ; CASCADE-WITH-DD-NEXT: [[MADDWrrr1:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY4]], [[LDRWui7]], [[MADDWrrr]]
+ ; CASCADE-WITH-DD-NEXT: [[MADDWrrr2:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[LDRWui7]], [[MADDWrrr1]]
+ ; CASCADE-WITH-DD-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui5]], implicit-def $nzcv
+ ; CASCADE-WITH-DD-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 2
+ ; CASCADE-WITH-DD-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 1
+ ; CASCADE-WITH-DD-NEXT: [[CSINCXr:%[0-9]+]]:gpr64 = CSINCXr [[MOVi64imm]], $xzr, 10, implicit $nzcv
+ ; CASCADE-WITH-DD-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui4]], implicit-def $nzcv
+ ; CASCADE-WITH-DD-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 3
+ ; CASCADE-WITH-DD-NEXT: [[CSELXr:%[0-9]+]]:gpr64 = CSELXr [[MOVi64imm2]], [[CSINCXr]], 10, implicit $nzcv
+ ; CASCADE-WITH-DD-NEXT: [[SUBSWrr2:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui3]], implicit-def $nzcv
+ ; CASCADE-WITH-DD-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 4
+ ; CASCADE-WITH-DD-NEXT: [[CSELXr1:%[0-9]+]]:gpr64 = CSELXr [[MOVi64imm3]], [[CSELXr]], 10, implicit $nzcv
+ ; CASCADE-WITH-DD-NEXT: [[SUBSWrr3:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui2]], implicit-def $nzcv
+ ; CASCADE-WITH-DD-NEXT: [[MOVi64imm4:%[0-9]+]]:gpr64 = MOVi64imm 5
+ ; CASCADE-WITH-DD-NEXT: [[CSELXr2:%[0-9]+]]:gpr64 = CSELXr [[MOVi64imm4]], [[CSELXr1]], 10, implicit $nzcv
+ ; CASCADE-WITH-DD-NEXT: [[SUBSWrr4:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui1]], implicit-def $nzcv
+ ; CASCADE-WITH-DD-NEXT: [[MOVi64imm5:%[0-9]+]]:gpr64 = MOVi64imm 6
+ ; CASCADE-WITH-DD-NEXT: [[CSELXr3:%[0-9]+]]:gpr64 = CSELXr [[MOVi64imm5]], [[CSELXr2]], 10, implicit $nzcv
+ ; CASCADE-WITH-DD-NEXT: [[SUBSWrr5:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui]], implicit-def $nzcv
+ ; CASCADE-WITH-DD-NEXT: [[MOVi64imm6:%[0-9]+]]:gpr64 = MOVi64imm 7
+ ; CASCADE-WITH-DD-NEXT: [[CSELXr4:%[0-9]+]]:gpr64 = CSELXr [[MOVi64imm6]], [[CSELXr3]], 10, implicit $nzcv
+ ; CASCADE-WITH-DD-NEXT: [[SUBSWrr6:%[0-9]+]]:gpr32 = SUBSWrr [[PHI]], [[MOVi32imm]], implicit-def $nzcv
+ ; CASCADE-WITH-DD-NEXT: Bcc 1, %bb.2, implicit $nzcv
+ ; CASCADE-WITH-DD-NEXT: B %bb.10
+ ; CASCADE-WITH-DD-NEXT: {{ $}}
+ ; CASCADE-WITH-DD-NEXT: bb.10:
+ ; CASCADE-WITH-DD-NEXT: [[PHI1:%[0-9]+]]:gpr64 = PHI [[COPY5]], %bb.0, [[CSELXr4]], %bb.2
+ ; CASCADE-WITH-DD-NEXT: $x0 = COPY [[PHI1]]
+ ; CASCADE-WITH-DD-NEXT: RET_ReallyLR implicit $x0
+ ;
+ ; DEPTH3-LABEL: name: cascade_selector_loop
+ ; DEPTH3: bb.0:
+ ; DEPTH3-NEXT: successors: %bb.10(0x30000000), %bb.1(0x50000000)
+ ; DEPTH3-NEXT: liveins: $x0, $x1, $w2, $w3, $w4
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; DEPTH3-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1
+ ; DEPTH3-NEXT: [[COPY2:%[0-9]+]]:gpr32 = COPY $w2
+ ; DEPTH3-NEXT: [[COPY3:%[0-9]+]]:gpr32 = COPY $w3
+ ; DEPTH3-NEXT: [[COPY4:%[0-9]+]]:gpr32 = COPY $w4
+ ; DEPTH3-NEXT: [[COPY5:%[0-9]+]]:gpr64all = COPY $xzr
+ ; DEPTH3-NEXT: CBZW [[COPY2]], %bb.10
+ ; DEPTH3-NEXT: B %bb.1
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: bb.1:
+ ; DEPTH3-NEXT: successors: %bb.2(0x80000000)
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 0 :: (load (s32))
+ ; DEPTH3-NEXT: [[LDRWui1:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 1 :: (load (s32))
+ ; DEPTH3-NEXT: [[LDRWui2:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 2 :: (load (s32))
+ ; DEPTH3-NEXT: [[LDRWui3:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 3 :: (load (s32))
+ ; DEPTH3-NEXT: [[LDRWui4:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 4 :: (load (s32))
+ ; DEPTH3-NEXT: [[LDRWui5:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 5 :: (load (s32))
+ ; DEPTH3-NEXT: [[LDRWui6:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 6 :: (load (s32))
+ ; DEPTH3-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 1
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: bb.2:
+ ; DEPTH3-NEXT: successors: %bb.3(0x40000000), %bb.9(0x40000000)
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: [[PHI:%[0-9]+]]:gpr32 = PHI [[COPY2]], %bb.1, %15, %bb.9
+ ; DEPTH3-NEXT: [[LDRWui7:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load (s32))
+ ; DEPTH3-NEXT: [[MADDWrrr:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[LDRWui7]], $wzr
+ ; DEPTH3-NEXT: [[MADDWrrr1:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY4]], [[LDRWui7]], [[MADDWrrr]]
+ ; DEPTH3-NEXT: [[MADDWrrr2:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[LDRWui7]], [[MADDWrrr1]]
+ ; DEPTH3-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui]], implicit-def $nzcv
+ ; DEPTH3-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 7
+ ; DEPTH3-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; DEPTH3-NEXT: B %bb.3
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: bb.3:
+ ; DEPTH3-NEXT: successors: %bb.4(0x40000000), %bb.9(0x40000000)
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui1]], implicit-def $nzcv
+ ; DEPTH3-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 6
+ ; DEPTH3-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; DEPTH3-NEXT: B %bb.4
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: bb.4:
+ ; DEPTH3-NEXT: successors: %bb.5(0x40000000), %bb.9(0x40000000)
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: [[SUBSWrr2:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui2]], implicit-def $nzcv
+ ; DEPTH3-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 5
+ ; DEPTH3-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; DEPTH3-NEXT: B %bb.5
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: bb.5:
+ ; DEPTH3-NEXT: successors: %bb.6(0x40000000), %bb.9(0x40000000)
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: [[SUBSWrr3:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui3]], implicit-def $nzcv
+ ; DEPTH3-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 4
+ ; DEPTH3-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; DEPTH3-NEXT: B %bb.6
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: bb.6:
+ ; DEPTH3-NEXT: successors: %bb.7(0x40000000), %bb.9(0x40000000)
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: [[SUBSWrr4:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui4]], implicit-def $nzcv
+ ; DEPTH3-NEXT: [[MOVi64imm4:%[0-9]+]]:gpr64 = MOVi64imm 3
+ ; DEPTH3-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; DEPTH3-NEXT: B %bb.7
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: bb.7:
+ ; DEPTH3-NEXT: successors: %bb.8(0x40000000), %bb.9(0x40000000)
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: [[SUBSWrr5:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui5]], implicit-def $nzcv
+ ; DEPTH3-NEXT: [[MOVi64imm5:%[0-9]+]]:gpr64 = MOVi64imm 2
+ ; DEPTH3-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; DEPTH3-NEXT: B %bb.8
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: bb.8:
+ ; DEPTH3-NEXT: successors: %bb.9(0x80000000)
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: [[MOVi64imm6:%[0-9]+]]:gpr64 = MOVi64imm 1
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: bb.9:
+ ; DEPTH3-NEXT: successors: %bb.10(0x04000000), %bb.2(0x7c000000)
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: [[PHI1:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.2, [[MOVi64imm1]], %bb.3, [[MOVi64imm2]], %bb.4, [[MOVi64imm3]], %bb.5, [[MOVi64imm4]], %bb.6, [[MOVi64imm5]], %bb.7, [[MOVi64imm6]], %bb.8
+ ; DEPTH3-NEXT: [[SUBSWrr6:%[0-9]+]]:gpr32 = SUBSWrr [[PHI]], [[MOVi32imm]], implicit-def $nzcv
+ ; DEPTH3-NEXT: Bcc 1, %bb.2, implicit $nzcv
+ ; DEPTH3-NEXT: B %bb.10
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: bb.10:
+ ; DEPTH3-NEXT: [[PHI2:%[0-9]+]]:gpr64 = PHI [[COPY5]], %bb.0, [[PHI1]], %bb.9
+ ; DEPTH3-NEXT: $x0 = COPY [[PHI2]]
+ ; DEPTH3-NEXT: RET_ReallyLR implicit $x0
+ ;
+ ; LIMIT3-LABEL: name: cascade_selector_loop
+ ; LIMIT3: bb.0:
+ ; LIMIT3-NEXT: successors: %bb.10(0x30000000), %bb.1(0x50000000)
+ ; LIMIT3-NEXT: liveins: $x0, $x1, $w2, $w3, $w4
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; LIMIT3-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1
+ ; LIMIT3-NEXT: [[COPY2:%[0-9]+]]:gpr32 = COPY $w2
+ ; LIMIT3-NEXT: [[COPY3:%[0-9]+]]:gpr32 = COPY $w3
+ ; LIMIT3-NEXT: [[COPY4:%[0-9]+]]:gpr32 = COPY $w4
+ ; LIMIT3-NEXT: [[COPY5:%[0-9]+]]:gpr64all = COPY $xzr
+ ; LIMIT3-NEXT: CBZW [[COPY2]], %bb.10
+ ; LIMIT3-NEXT: B %bb.1
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: bb.1:
+ ; LIMIT3-NEXT: successors: %bb.2(0x80000000)
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 0 :: (load (s32))
+ ; LIMIT3-NEXT: [[LDRWui1:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 1 :: (load (s32))
+ ; LIMIT3-NEXT: [[LDRWui2:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 2 :: (load (s32))
+ ; LIMIT3-NEXT: [[LDRWui3:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 3 :: (load (s32))
+ ; LIMIT3-NEXT: [[LDRWui4:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 4 :: (load (s32))
+ ; LIMIT3-NEXT: [[LDRWui5:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 5 :: (load (s32))
+ ; LIMIT3-NEXT: [[LDRWui6:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 6 :: (load (s32))
+ ; LIMIT3-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 1
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: bb.2:
+ ; LIMIT3-NEXT: successors: %bb.3(0x40000000), %bb.9(0x40000000)
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: [[PHI:%[0-9]+]]:gpr32 = PHI [[COPY2]], %bb.1, %15, %bb.9
+ ; LIMIT3-NEXT: [[LDRWui7:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load (s32))
+ ; LIMIT3-NEXT: [[MADDWrrr:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[LDRWui7]], $wzr
+ ; LIMIT3-NEXT: [[MADDWrrr1:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY4]], [[LDRWui7]], [[MADDWrrr]]
+ ; LIMIT3-NEXT: [[MADDWrrr2:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[LDRWui7]], [[MADDWrrr1]]
+ ; LIMIT3-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui]], implicit-def $nzcv
+ ; LIMIT3-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 7
+ ; LIMIT3-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; LIMIT3-NEXT: B %bb.3
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: bb.3:
+ ; LIMIT3-NEXT: successors: %bb.4(0x40000000), %bb.9(0x40000000)
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui1]], implicit-def $nzcv
+ ; LIMIT3-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 6
+ ; LIMIT3-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; LIMIT3-NEXT: B %bb.4
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: bb.4:
+ ; LIMIT3-NEXT: successors: %bb.5(0x40000000), %bb.9(0x40000000)
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: [[SUBSWrr2:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui2]], implicit-def $nzcv
+ ; LIMIT3-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 5
+ ; LIMIT3-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; LIMIT3-NEXT: B %bb.5
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: bb.5:
+ ; LIMIT3-NEXT: successors: %bb.6(0x40000000), %bb.9(0x40000000)
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: [[SUBSWrr3:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui3]], implicit-def $nzcv
+ ; LIMIT3-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 4
+ ; LIMIT3-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; LIMIT3-NEXT: B %bb.6
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: bb.6:
+ ; LIMIT3-NEXT: successors: %bb.7(0x40000000), %bb.9(0x40000000)
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: [[SUBSWrr4:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui4]], implicit-def $nzcv
+ ; LIMIT3-NEXT: [[MOVi64imm4:%[0-9]+]]:gpr64 = MOVi64imm 3
+ ; LIMIT3-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; LIMIT3-NEXT: B %bb.7
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: bb.7:
+ ; LIMIT3-NEXT: successors: %bb.8(0x40000000), %bb.9(0x40000000)
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: [[SUBSWrr5:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui5]], implicit-def $nzcv
+ ; LIMIT3-NEXT: [[MOVi64imm5:%[0-9]+]]:gpr64 = MOVi64imm 2
+ ; LIMIT3-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; LIMIT3-NEXT: B %bb.8
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: bb.8:
+ ; LIMIT3-NEXT: successors: %bb.9(0x80000000)
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: [[MOVi64imm6:%[0-9]+]]:gpr64 = MOVi64imm 1
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: bb.9:
+ ; LIMIT3-NEXT: successors: %bb.10(0x04000000), %bb.2(0x7c000000)
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: [[PHI1:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.2, [[MOVi64imm1]], %bb.3, [[MOVi64imm2]], %bb.4, [[MOVi64imm3]], %bb.5, [[MOVi64imm4]], %bb.6, [[MOVi64imm5]], %bb.7, [[MOVi64imm6]], %bb.8
+ ; LIMIT3-NEXT: [[SUBSWrr6:%[0-9]+]]:gpr32 = SUBSWrr [[PHI]], [[MOVi32imm]], implicit-def $nzcv
+ ; LIMIT3-NEXT: Bcc 1, %bb.2, implicit $nzcv
+ ; LIMIT3-NEXT: B %bb.10
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: bb.10:
+ ; LIMIT3-NEXT: [[PHI2:%[0-9]+]]:gpr64 = PHI [[COPY5]], %bb.0, [[PHI1]], %bb.9
+ ; LIMIT3-NEXT: $x0 = COPY [[PHI2]]
+ ; LIMIT3-NEXT: RET_ReallyLR implicit $x0
bb.0:
successors: %bb.10(0x30000000), %bb.1(0x50000000)
liveins: $x0, $x1, $w2, $w3, $w4
@@ -139,6 +302,7 @@ body: |
%16:gpr32 = LDRWui %1, 6 :: (load (s32))
%17:gpr32 = MOVi32imm 1
+ ; Cascade head: data-dependent condition (derived from load + compute chain)
bb.2:
successors: %bb.3(0x40000000), %bb.9(0x40000000)
@@ -220,104 +384,104 @@ liveins:
- { reg: '$w4', virtual-reg: '%4' }
- { reg: '$w5', virtual-reg: '%5' }
body: |
- ; CHECK-LABEL: name: cascade_branches_from_args_not_dd
- ; CHECK: bb.0:
- ; CHECK-NEXT: successors: %bb.10(0x30000000), %bb.1(0x50000000)
- ; CHECK-NEXT: liveins: $x1, $w2, $w3, $w4, $w5
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x1
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr32 = COPY $w2
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr32 = COPY $w3
- ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gpr32 = COPY $w4
- ; CHECK-NEXT: [[COPY4:%[0-9]+]]:gpr32 = COPY $w5
- ; CHECK-NEXT: [[COPY5:%[0-9]+]]:gpr64all = COPY $xzr
- ; CHECK-NEXT: CBZW [[COPY1]], %bb.10
- ; CHECK-NEXT: B %bb.1
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.1:
- ; CHECK-NEXT: successors: %bb.2(0x80000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[COPY6:%[0-9]+]]:gpr32 = COPY [[COPY2]]
- ; CHECK-NEXT: [[COPY7:%[0-9]+]]:gpr32 = COPY [[COPY3]]
- ; CHECK-NEXT: [[COPY8:%[0-9]+]]:gpr32 = COPY [[COPY4]]
- ; CHECK-NEXT: [[COPY9:%[0-9]+]]:gpr32 = COPY [[COPY2]]
- ; CHECK-NEXT: [[COPY10:%[0-9]+]]:gpr32 = COPY [[COPY3]]
- ; CHECK-NEXT: [[COPY11:%[0-9]+]]:gpr32 = COPY [[COPY4]]
- ; CHECK-NEXT: [[COPY12:%[0-9]+]]:gpr32 = COPY [[COPY2]]
- ; CHECK-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 1
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.2:
- ; CHECK-NEXT: successors: %bb.3(0x40000000), %bb.9(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[PHI:%[0-9]+]]:gpr32 = PHI [[COPY1]], %bb.1, %15, %bb.9
- ; CHECK-NEXT: [[MADDWrrr:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY2]], [[COPY4]], $wzr
- ; CHECK-NEXT: [[MADDWrrr1:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[COPY4]], [[MADDWrrr]]
- ; CHECK-NEXT: [[MADDWrrr2:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY2]], [[COPY3]], [[MADDWrrr1]]
- ; CHECK-NEXT: [[MADDWrrr3:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[COPY2]], [[MADDWrrr2]]
- ; CHECK-NEXT: [[MADDWrrr4:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY4]], [[COPY3]], [[MADDWrrr3]]
- ; CHECK-NEXT: [[MADDWrrr5:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY2]], [[COPY4]], [[MADDWrrr4]]
- ; CHECK-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr5]], [[COPY6]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 7
- ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
- ; CHECK-NEXT: B %bb.3
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.3:
- ; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.9(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr5]], [[COPY7]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 6
- ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
- ; CHECK-NEXT: B %bb.4
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.4:
- ; CHECK-NEXT: successors: %bb.5(0x40000000), %bb.9(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[SUBSWrr2:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr5]], [[COPY8]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 5
- ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
- ; CHECK-NEXT: B %bb.5
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.5:
- ; CHECK-NEXT: successors: %bb.6(0x40000000), %bb.9(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[SUBSWrr3:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr5]], [[COPY9]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 4
- ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
- ; CHECK-NEXT: B %bb.6
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.6:
- ; CHECK-NEXT: successors: %bb.7(0x40000000), %bb.9(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[SUBSWrr4:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr5]], [[COPY10]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm4:%[0-9]+]]:gpr64 = MOVi64imm 3
- ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
- ; CHECK-NEXT: B %bb.7
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.7:
- ; CHECK-NEXT: successors: %bb.8(0x40000000), %bb.9(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[SUBSWrr5:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr5]], [[COPY11]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm5:%[0-9]+]]:gpr64 = MOVi64imm 2
- ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
- ; CHECK-NEXT: B %bb.8
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.8:
- ; CHECK-NEXT: successors: %bb.9(0x80000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[MOVi64imm6:%[0-9]+]]:gpr64 = MOVi64imm 1
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.9:
- ; CHECK-NEXT: successors: %bb.10(0x04000000), %bb.2(0x7c000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[PHI1:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.2, [[MOVi64imm1]], %bb.3, [[MOVi64imm2]], %bb.4, [[MOVi64imm3]], %bb.5, [[MOVi64imm4]], %bb.6, [[MOVi64imm5]], %bb.7, [[MOVi64imm6]], %bb.8
- ; CHECK-NEXT: [[SUBSWrr6:%[0-9]+]]:gpr32 = SUBSWrr [[PHI]], [[MOVi32imm]], implicit-def $nzcv
- ; CHECK-NEXT: Bcc 1, %bb.2, implicit $nzcv
- ; CHECK-NEXT: B %bb.10
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.10:
- ; CHECK-NEXT: [[PHI2:%[0-9]+]]:gpr64 = PHI [[COPY5]], %bb.0, [[PHI1]], %bb.9
- ; CHECK-NEXT: $x0 = COPY [[PHI2]]
- ; CHECK-NEXT: RET_ReallyLR implicit $x0
+ ; COMMON-LABEL: name: cascade_branches_from_args_not_dd
+ ; COMMON: bb.0:
+ ; COMMON-NEXT: successors: %bb.10(0x30000000), %bb.1(0x50000000)
+ ; COMMON-NEXT: liveins: $x1, $w2, $w3, $w4, $w5
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x1
+ ; COMMON-NEXT: [[COPY1:%[0-9]+]]:gpr32 = COPY $w2
+ ; COMMON-NEXT: [[COPY2:%[0-9]+]]:gpr32 = COPY $w3
+ ; COMMON-NEXT: [[COPY3:%[0-9]+]]:gpr32 = COPY $w4
+ ; COMMON-NEXT: [[COPY4:%[0-9]+]]:gpr32 = COPY $w5
+ ; COMMON-NEXT: [[COPY5:%[0-9]+]]:gpr64all = COPY $xzr
+ ; COMMON-NEXT: CBZW [[COPY1]], %bb.10
+ ; COMMON-NEXT: B %bb.1
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.1:
+ ; COMMON-NEXT: successors: %bb.2(0x80000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[COPY6:%[0-9]+]]:gpr32 = COPY [[COPY2]]
+ ; COMMON-NEXT: [[COPY7:%[0-9]+]]:gpr32 = COPY [[COPY3]]
+ ; COMMON-NEXT: [[COPY8:%[0-9]+]]:gpr32 = COPY [[COPY4]]
+ ; COMMON-NEXT: [[COPY9:%[0-9]+]]:gpr32 = COPY [[COPY2]]
+ ; COMMON-NEXT: [[COPY10:%[0-9]+]]:gpr32 = COPY [[COPY3]]
+ ; COMMON-NEXT: [[COPY11:%[0-9]+]]:gpr32 = COPY [[COPY4]]
+ ; COMMON-NEXT: [[COPY12:%[0-9]+]]:gpr32 = COPY [[COPY2]]
+ ; COMMON-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 1
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.2:
+ ; COMMON-NEXT: successors: %bb.3(0x40000000), %bb.9(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[PHI:%[0-9]+]]:gpr32 = PHI [[COPY1]], %bb.1, %15, %bb.9
+ ; COMMON-NEXT: [[MADDWrrr:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY2]], [[COPY4]], $wzr
+ ; COMMON-NEXT: [[MADDWrrr1:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[COPY4]], [[MADDWrrr]]
+ ; COMMON-NEXT: [[MADDWrrr2:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY2]], [[COPY3]], [[MADDWrrr1]]
+ ; COMMON-NEXT: [[MADDWrrr3:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[COPY2]], [[MADDWrrr2]]
+ ; COMMON-NEXT: [[MADDWrrr4:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY4]], [[COPY3]], [[MADDWrrr3]]
+ ; COMMON-NEXT: [[MADDWrrr5:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY2]], [[COPY4]], [[MADDWrrr4]]
+ ; COMMON-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr5]], [[COPY6]], implicit-def $nzcv
+ ; COMMON-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 7
+ ; COMMON-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; COMMON-NEXT: B %bb.3
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.3:
+ ; COMMON-NEXT: successors: %bb.4(0x40000000), %bb.9(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr5]], [[COPY7]], implicit-def $nzcv
+ ; COMMON-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 6
+ ; COMMON-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; COMMON-NEXT: B %bb.4
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.4:
+ ; COMMON-NEXT: successors: %bb.5(0x40000000), %bb.9(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[SUBSWrr2:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr5]], [[COPY8]], implicit-def $nzcv
+ ; COMMON-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 5
+ ; COMMON-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; COMMON-NEXT: B %bb.5
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.5:
+ ; COMMON-NEXT: successors: %bb.6(0x40000000), %bb.9(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[SUBSWrr3:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr5]], [[COPY9]], implicit-def $nzcv
+ ; COMMON-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 4
+ ; COMMON-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; COMMON-NEXT: B %bb.6
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.6:
+ ; COMMON-NEXT: successors: %bb.7(0x40000000), %bb.9(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[SUBSWrr4:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr5]], [[COPY10]], implicit-def $nzcv
+ ; COMMON-NEXT: [[MOVi64imm4:%[0-9]+]]:gpr64 = MOVi64imm 3
+ ; COMMON-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; COMMON-NEXT: B %bb.7
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.7:
+ ; COMMON-NEXT: successors: %bb.8(0x40000000), %bb.9(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[SUBSWrr5:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr5]], [[COPY11]], implicit-def $nzcv
+ ; COMMON-NEXT: [[MOVi64imm5:%[0-9]+]]:gpr64 = MOVi64imm 2
+ ; COMMON-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; COMMON-NEXT: B %bb.8
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.8:
+ ; COMMON-NEXT: successors: %bb.9(0x80000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[MOVi64imm6:%[0-9]+]]:gpr64 = MOVi64imm 1
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.9:
+ ; COMMON-NEXT: successors: %bb.10(0x04000000), %bb.2(0x7c000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[PHI1:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.2, [[MOVi64imm1]], %bb.3, [[MOVi64imm2]], %bb.4, [[MOVi64imm3]], %bb.5, [[MOVi64imm4]], %bb.6, [[MOVi64imm5]], %bb.7, [[MOVi64imm6]], %bb.8
+ ; COMMON-NEXT: [[SUBSWrr6:%[0-9]+]]:gpr32 = SUBSWrr [[PHI]], [[MOVi32imm]], implicit-def $nzcv
+ ; COMMON-NEXT: Bcc 1, %bb.2, implicit $nzcv
+ ; COMMON-NEXT: B %bb.10
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.10:
+ ; COMMON-NEXT: [[PHI2:%[0-9]+]]:gpr64 = PHI [[COPY5]], %bb.0, [[PHI1]], %bb.9
+ ; COMMON-NEXT: $x0 = COPY [[PHI2]]
+ ; COMMON-NEXT: RET_ReallyLR implicit $x0
bb.0:
successors: %bb.10(0x30000000), %bb.1(0x50000000)
liveins: $x1, $w2, $w3, $w4, $w5
@@ -428,107 +592,107 @@ liveins:
- { reg: '$w3', virtual-reg: '%3' }
- { reg: '$w4', virtual-reg: '%4' }
body: |
- ; CHECK-LABEL: name: cascade_exceeds_crit_limit
- ; CHECK: bb.0:
- ; CHECK-NEXT: successors: %bb.11(0x30000000), %bb.1(0x50000000)
- ; CHECK-NEXT: liveins: $x0, $x1, $w2, $w3, $w4
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr32 = COPY $w2
- ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gpr32 = COPY $w3
- ; CHECK-NEXT: [[COPY4:%[0-9]+]]:gpr32 = COPY $w4
- ; CHECK-NEXT: [[COPY5:%[0-9]+]]:gpr64all = COPY $xzr
- ; CHECK-NEXT: CBZW [[COPY2]], %bb.11
- ; CHECK-NEXT: B %bb.1
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.1:
- ; CHECK-NEXT: successors: %bb.2(0x80000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 0 :: (load (s32))
- ; CHECK-NEXT: [[LDRWui1:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 1 :: (load (s32))
- ; CHECK-NEXT: [[LDRWui2:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 2 :: (load (s32))
- ; CHECK-NEXT: [[LDRWui3:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 3 :: (load (s32))
- ; CHECK-NEXT: [[LDRWui4:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 4 :: (load (s32))
- ; CHECK-NEXT: [[LDRWui5:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 5 :: (load (s32))
- ; CHECK-NEXT: [[LDRWui6:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 6 :: (load (s32))
- ; CHECK-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 1
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.2:
- ; CHECK-NEXT: successors: %bb.3(0x40000000), %bb.10(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[PHI:%[0-9]+]]:gpr32 = PHI [[COPY2]], %bb.1, %15, %bb.10
- ; CHECK-NEXT: [[LDRWui7:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load (s32))
- ; CHECK-NEXT: [[MADDWrrr:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[LDRWui7]], $wzr
- ; CHECK-NEXT: [[MADDWrrr1:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY4]], [[LDRWui7]], [[MADDWrrr]]
- ; CHECK-NEXT: [[MADDWrrr2:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[LDRWui7]], [[MADDWrrr1]]
- ; CHECK-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 7
- ; CHECK-NEXT: Bcc 10, %bb.10, implicit $nzcv
- ; CHECK-NEXT: B %bb.3
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.3:
- ; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.10(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui1]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 6
- ; CHECK-NEXT: Bcc 10, %bb.10, implicit $nzcv
- ; CHECK-NEXT: B %bb.4
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.4:
- ; CHECK-NEXT: successors: %bb.6(0x40000000), %bb.5(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[SUBSWrr2:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui2]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 5
- ; CHECK-NEXT: Bcc 10, %bb.5, implicit $nzcv
- ; CHECK-NEXT: B %bb.6
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.5:
- ; CHECK-NEXT: successors: %bb.10(0x80000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: B %bb.10
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.6:
- ; CHECK-NEXT: successors: %bb.7(0x40000000), %bb.10(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[SUBSWrr3:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui3]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 4
- ; CHECK-NEXT: Bcc 10, %bb.10, implicit $nzcv
- ; CHECK-NEXT: B %bb.7
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.7:
- ; CHECK-NEXT: successors: %bb.8(0x40000000), %bb.10(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[SUBSWrr4:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui4]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm4:%[0-9]+]]:gpr64 = MOVi64imm 3
- ; CHECK-NEXT: Bcc 10, %bb.10, implicit $nzcv
- ; CHECK-NEXT: B %bb.8
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.8:
- ; CHECK-NEXT: successors: %bb.9(0x40000000), %bb.10(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[SUBSWrr5:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui5]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm5:%[0-9]+]]:gpr64 = MOVi64imm 2
- ; CHECK-NEXT: Bcc 10, %bb.10, implicit $nzcv
- ; CHECK-NEXT: B %bb.9
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.9:
- ; CHECK-NEXT: successors: %bb.10(0x80000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[MOVi64imm6:%[0-9]+]]:gpr64 = MOVi64imm 1
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.10:
- ; CHECK-NEXT: successors: %bb.11(0x04000000), %bb.2(0x7c000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[PHI1:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.2, [[MOVi64imm1]], %bb.3, [[MOVi64imm2]], %bb.5, [[MOVi64imm3]], %bb.6, [[MOVi64imm4]], %bb.7, [[MOVi64imm5]], %bb.8, [[MOVi64imm6]], %bb.9
- ; CHECK-NEXT: [[SUBSWrr6:%[0-9]+]]:gpr32 = SUBSWrr [[PHI]], [[MOVi32imm]], implicit-def $nzcv
- ; CHECK-NEXT: Bcc 1, %bb.2, implicit $nzcv
- ; CHECK-NEXT: B %bb.11
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.11:
- ; CHECK-NEXT: [[PHI2:%[0-9]+]]:gpr64 = PHI [[COPY5]], %bb.0, [[PHI1]], %bb.10
- ; CHECK-NEXT: $x0 = COPY [[PHI2]]
- ; CHECK-NEXT: RET_ReallyLR implicit $x0
+ ; COMMON-LABEL: name: cascade_exceeds_crit_limit
+ ; COMMON: bb.0:
+ ; COMMON-NEXT: successors: %bb.11(0x30000000), %bb.1(0x50000000)
+ ; COMMON-NEXT: liveins: $x0, $x1, $w2, $w3, $w4
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; COMMON-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1
+ ; COMMON-NEXT: [[COPY2:%[0-9]+]]:gpr32 = COPY $w2
+ ; COMMON-NEXT: [[COPY3:%[0-9]+]]:gpr32 = COPY $w3
+ ; COMMON-NEXT: [[COPY4:%[0-9]+]]:gpr32 = COPY $w4
+ ; COMMON-NEXT: [[COPY5:%[0-9]+]]:gpr64all = COPY $xzr
+ ; COMMON-NEXT: CBZW [[COPY2]], %bb.11
+ ; COMMON-NEXT: B %bb.1
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.1:
+ ; COMMON-NEXT: successors: %bb.2(0x80000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 0 :: (load (s32))
+ ; COMMON-NEXT: [[LDRWui1:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 1 :: (load (s32))
+ ; COMMON-NEXT: [[LDRWui2:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 2 :: (load (s32))
+ ; COMMON-NEXT: [[LDRWui3:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 3 :: (load (s32))
+ ; COMMON-NEXT: [[LDRWui4:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 4 :: (load (s32))
+ ; COMMON-NEXT: [[LDRWui5:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 5 :: (load (s32))
+ ; COMMON-NEXT: [[LDRWui6:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 6 :: (load (s32))
+ ; COMMON-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 1
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.2:
+ ; COMMON-NEXT: successors: %bb.3(0x40000000), %bb.10(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[PHI:%[0-9]+]]:gpr32 = PHI [[COPY2]], %bb.1, %15, %bb.10
+ ; COMMON-NEXT: [[LDRWui7:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load (s32))
+ ; COMMON-NEXT: [[MADDWrrr:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[LDRWui7]], $wzr
+ ; COMMON-NEXT: [[MADDWrrr1:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY4]], [[LDRWui7]], [[MADDWrrr]]
+ ; COMMON-NEXT: [[MADDWrrr2:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[LDRWui7]], [[MADDWrrr1]]
+ ; COMMON-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui]], implicit-def $nzcv
+ ; COMMON-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 7
+ ; COMMON-NEXT: Bcc 10, %bb.10, implicit $nzcv
+ ; COMMON-NEXT: B %bb.3
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.3:
+ ; COMMON-NEXT: successors: %bb.4(0x40000000), %bb.10(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui1]], implicit-def $nzcv
+ ; COMMON-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 6
+ ; COMMON-NEXT: Bcc 10, %bb.10, implicit $nzcv
+ ; COMMON-NEXT: B %bb.4
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.4:
+ ; COMMON-NEXT: successors: %bb.6(0x40000000), %bb.5(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[SUBSWrr2:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui2]], implicit-def $nzcv
+ ; COMMON-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 5
+ ; COMMON-NEXT: Bcc 10, %bb.5, implicit $nzcv
+ ; COMMON-NEXT: B %bb.6
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.5:
+ ; COMMON-NEXT: successors: %bb.10(0x80000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: B %bb.10
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.6:
+ ; COMMON-NEXT: successors: %bb.7(0x40000000), %bb.10(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[SUBSWrr3:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui3]], implicit-def $nzcv
+ ; COMMON-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 4
+ ; COMMON-NEXT: Bcc 10, %bb.10, implicit $nzcv
+ ; COMMON-NEXT: B %bb.7
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.7:
+ ; COMMON-NEXT: successors: %bb.8(0x40000000), %bb.10(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[SUBSWrr4:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui4]], implicit-def $nzcv
+ ; COMMON-NEXT: [[MOVi64imm4:%[0-9]+]]:gpr64 = MOVi64imm 3
+ ; COMMON-NEXT: Bcc 10, %bb.10, implicit $nzcv
+ ; COMMON-NEXT: B %bb.8
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.8:
+ ; COMMON-NEXT: successors: %bb.9(0x40000000), %bb.10(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[SUBSWrr5:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui5]], implicit-def $nzcv
+ ; COMMON-NEXT: [[MOVi64imm5:%[0-9]+]]:gpr64 = MOVi64imm 2
+ ; COMMON-NEXT: Bcc 10, %bb.10, implicit $nzcv
+ ; COMMON-NEXT: B %bb.9
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.9:
+ ; COMMON-NEXT: successors: %bb.10(0x80000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[MOVi64imm6:%[0-9]+]]:gpr64 = MOVi64imm 1
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.10:
+ ; COMMON-NEXT: successors: %bb.11(0x04000000), %bb.2(0x7c000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[PHI1:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.2, [[MOVi64imm1]], %bb.3, [[MOVi64imm2]], %bb.5, [[MOVi64imm3]], %bb.6, [[MOVi64imm4]], %bb.7, [[MOVi64imm5]], %bb.8, [[MOVi64imm6]], %bb.9
+ ; COMMON-NEXT: [[SUBSWrr6:%[0-9]+]]:gpr32 = SUBSWrr [[PHI]], [[MOVi32imm]], implicit-def $nzcv
+ ; COMMON-NEXT: Bcc 1, %bb.2, implicit $nzcv
+ ; COMMON-NEXT: B %bb.11
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.11:
+ ; COMMON-NEXT: [[PHI2:%[0-9]+]]:gpr64 = PHI [[COPY5]], %bb.0, [[PHI1]], %bb.10
+ ; COMMON-NEXT: $x0 = COPY [[PHI2]]
+ ; COMMON-NEXT: RET_ReallyLR implicit $x0
bb.0:
successors: %bb.11(0x30000000), %bb.1(0x50000000)
liveins: $x0, $x1, $w2, $w3, $w4
@@ -638,49 +802,49 @@ name: cascade_backedge_to_earlier_block
alignment: 4
tracksRegLiveness: true
body: |
- ; CHECK-LABEL: name: cascade_backedge_to_earlier_block
- ; CHECK: bb.0:
- ; CHECK-NEXT: successors: %bb.1(0x80000000)
- ; CHECK-NEXT: liveins: $x0, $x1
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.1:
- ; CHECK-NEXT: successors: %bb.2(0x40000000), %bb.5(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load (s32))
- ; CHECK-NEXT: [[LDRWui1:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 0 :: (load (s32))
- ; CHECK-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui1]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 100
- ; CHECK-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 200
- ; CHECK-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 400
- ; CHECK-NEXT: Bcc 10, %bb.5, implicit $nzcv
- ; CHECK-NEXT: B %bb.2
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.2:
- ; CHECK-NEXT: successors: %bb.3(0x40000000), %bb.5(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[LDRWui2:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 1 :: (load (s32))
- ; CHECK-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui2]], implicit-def $nzcv
- ; CHECK-NEXT: Bcc 10, %bb.5, implicit $nzcv
- ; CHECK-NEXT: B %bb.3
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.3:
- ; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[LDRWui3:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 2 :: (load (s32))
- ; CHECK-NEXT: [[SUBSWrr2:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui3]], implicit-def $nzcv
- ; CHECK-NEXT: Bcc 10, %bb.2, implicit $nzcv
- ; CHECK-NEXT: B %bb.4
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.4:
- ; CHECK-NEXT: successors: %bb.5(0x80000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.5:
- ; CHECK-NEXT: [[PHI:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.1, [[MOVi64imm1]], %bb.2, [[MOVi64imm2]], %bb.4
- ; CHECK-NEXT: $x0 = COPY [[PHI]]
- ; CHECK-NEXT: RET_ReallyLR implicit $x0
+ ; COMMON-LABEL: name: cascade_backedge_to_earlier_block
+ ; COMMON: bb.0:
+ ; COMMON-NEXT: successors: %bb.1(0x80000000)
+ ; COMMON-NEXT: liveins: $x0, $x1
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; COMMON-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.1:
+ ; COMMON-NEXT: successors: %bb.2(0x40000000), %bb.5(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load (s32))
+ ; COMMON-NEXT: [[LDRWui1:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 0 :: (load (s32))
+ ; COMMON-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui1]], implicit-def $nzcv
+ ; COMMON-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 100
+ ; COMMON-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 200
+ ; COMMON-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 400
+ ; COMMON-NEXT: Bcc 10, %bb.5, implicit $nzcv
+ ; COMMON-NEXT: B %bb.2
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.2:
+ ; COMMON-NEXT: successors: %bb.3(0x40000000), %bb.5(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[LDRWui2:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 1 :: (load (s32))
+ ; COMMON-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui2]], implicit-def $nzcv
+ ; COMMON-NEXT: Bcc 10, %bb.5, implicit $nzcv
+ ; COMMON-NEXT: B %bb.3
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.3:
+ ; COMMON-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[LDRWui3:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 2 :: (load (s32))
+ ; COMMON-NEXT: [[SUBSWrr2:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui3]], implicit-def $nzcv
+ ; COMMON-NEXT: Bcc 10, %bb.2, implicit $nzcv
+ ; COMMON-NEXT: B %bb.4
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.4:
+ ; COMMON-NEXT: successors: %bb.5(0x80000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.5:
+ ; COMMON-NEXT: [[PHI:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.1, [[MOVi64imm1]], %bb.2, [[MOVi64imm2]], %bb.4
+ ; COMMON-NEXT: $x0 = COPY [[PHI]]
+ ; COMMON-NEXT: RET_ReallyLR implicit $x0
bb.0:
successors: %bb.1(0x80000000)
liveins: $x0, $x1
@@ -731,60 +895,60 @@ liveins:
- { reg: '$x0', virtual-reg: '%0' }
- { reg: '$x1', virtual-reg: '%1' }
body: |
- ; CHECK-LABEL: name: cascade_forward_skip_to_later_block
- ; CHECK: bb.0:
- ; CHECK-NEXT: successors: %bb.1(0x80000000)
- ; CHECK-NEXT: liveins: $x0, $x1
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.1:
- ; CHECK-NEXT: successors: %bb.2(0x40000000), %bb.6(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load (s32))
- ; CHECK-NEXT: [[LDRWui1:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 0 :: (load (s32))
- ; CHECK-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui1]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 100
- ; CHECK-NEXT: Bcc 10, %bb.6, implicit $nzcv
- ; CHECK-NEXT: B %bb.2
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.2:
- ; CHECK-NEXT: successors: %bb.3(0x40000000), %bb.6(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[LDRWui2:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 1 :: (load (s32))
- ; CHECK-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui2]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 200
- ; CHECK-NEXT: Bcc 10, %bb.6, implicit $nzcv
- ; CHECK-NEXT: B %bb.3
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.3:
- ; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.5(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[LDRWui3:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 2 :: (load (s32))
- ; CHECK-NEXT: [[SUBSWrr2:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui3]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 300
- ; CHECK-NEXT: Bcc 10, %bb.5, implicit $nzcv
- ; CHECK-NEXT: B %bb.4
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.4:
- ; CHECK-NEXT: successors: %bb.5(0x40000000), %bb.6(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[LDRWui4:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 3 :: (load (s32))
- ; CHECK-NEXT: [[SUBSWrr3:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui4]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 400
- ; CHECK-NEXT: Bcc 10, %bb.6, implicit $nzcv
- ; CHECK-NEXT: B %bb.5
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.5:
- ; CHECK-NEXT: successors: %bb.6(0x80000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[MOVi64imm4:%[0-9]+]]:gpr64 = MOVi64imm 500
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.6:
- ; CHECK-NEXT: [[PHI:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.1, [[MOVi64imm1]], %bb.2, [[MOVi64imm3]], %bb.4, [[MOVi64imm4]], %bb.5
- ; CHECK-NEXT: $x0 = COPY [[PHI]]
- ; CHECK-NEXT: RET_ReallyLR implicit $x0
+ ; COMMON-LABEL: name: cascade_forward_skip_to_later_block
+ ; COMMON: bb.0:
+ ; COMMON-NEXT: successors: %bb.1(0x80000000)
+ ; COMMON-NEXT: liveins: $x0, $x1
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; COMMON-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.1:
+ ; COMMON-NEXT: successors: %bb.2(0x40000000), %bb.6(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load (s32))
+ ; COMMON-NEXT: [[LDRWui1:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 0 :: (load (s32))
+ ; COMMON-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui1]], implicit-def $nzcv
+ ; COMMON-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 100
+ ; COMMON-NEXT: Bcc 10, %bb.6, implicit $nzcv
+ ; COMMON-NEXT: B %bb.2
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.2:
+ ; COMMON-NEXT: successors: %bb.3(0x40000000), %bb.6(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[LDRWui2:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 1 :: (load (s32))
+ ; COMMON-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui2]], implicit-def $nzcv
+ ; COMMON-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 200
+ ; COMMON-NEXT: Bcc 10, %bb.6, implicit $nzcv
+ ; COMMON-NEXT: B %bb.3
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.3:
+ ; COMMON-NEXT: successors: %bb.4(0x40000000), %bb.5(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[LDRWui3:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 2 :: (load (s32))
+ ; COMMON-NEXT: [[SUBSWrr2:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui3]], implicit-def $nzcv
+ ; COMMON-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 300
+ ; COMMON-NEXT: Bcc 10, %bb.5, implicit $nzcv
+ ; COMMON-NEXT: B %bb.4
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.4:
+ ; COMMON-NEXT: successors: %bb.5(0x40000000), %bb.6(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[LDRWui4:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 3 :: (load (s32))
+ ; COMMON-NEXT: [[SUBSWrr3:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui4]], implicit-def $nzcv
+ ; COMMON-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 400
+ ; COMMON-NEXT: Bcc 10, %bb.6, implicit $nzcv
+ ; COMMON-NEXT: B %bb.5
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.5:
+ ; COMMON-NEXT: successors: %bb.6(0x80000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[MOVi64imm4:%[0-9]+]]:gpr64 = MOVi64imm 500
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.6:
+ ; COMMON-NEXT: [[PHI:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.1, [[MOVi64imm1]], %bb.2, [[MOVi64imm3]], %bb.4, [[MOVi64imm4]], %bb.5
+ ; COMMON-NEXT: $x0 = COPY [[PHI]]
+ ; COMMON-NEXT: RET_ReallyLR implicit $x0
bb.0:
successors: %bb.1(0x80000000)
liveins: $x0, $x1
@@ -840,4 +1004,374 @@ body: |
$x0 = COPY %90
RET_ReallyLR implicit $x0
...
+---
+name: cascade_per_phi_slack
+alignment: 4
+tracksRegLiveness: true
+body: |
+ ; COMMON-LABEL: name: cascade_per_phi_slack
+ ; COMMON: bb.0:
+ ; COMMON-NEXT: successors: %bb.1(0x80000000)
+ ; COMMON-NEXT: liveins: $x0
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.1:
+ ; COMMON-NEXT: successors: %bb.2(0x40000000), %bb.7(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load (s32))
+ ; COMMON-NEXT: [[MADDWrrr:%[0-9]+]]:gpr32 = MADDWrrr [[LDRWui]], [[LDRWui]], $wzr
+ ; COMMON-NEXT: [[MADDWrrr1:%[0-9]+]]:gpr32 = MADDWrrr [[MADDWrrr]], [[MADDWrrr]], $wzr
+ ; COMMON-NEXT: [[ADDWrr:%[0-9]+]]:gpr32 = ADDWrr [[MADDWrrr1]], [[MADDWrrr1]]
+ ; COMMON-NEXT: [[ADDWrr1:%[0-9]+]]:gpr32 = ADDWrr [[ADDWrr]], [[ADDWrr]]
+ ; COMMON-NEXT: [[ADDWrr2:%[0-9]+]]:gpr32 = ADDWrr [[ADDWrr1]], [[ADDWrr1]]
+ ; COMMON-NEXT: [[ADDWrr3:%[0-9]+]]:gpr32 = ADDWrr [[ADDWrr2]], [[ADDWrr2]]
+ ; COMMON-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 100
+ ; COMMON-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 1100
+ ; COMMON-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr1]], [[MADDWrrr1]], implicit-def $nzcv
+ ; COMMON-NEXT: Bcc 10, %bb.7, implicit $nzcv
+ ; COMMON-NEXT: B %bb.2
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.2:
+ ; COMMON-NEXT: successors: %bb.3(0x40000000), %bb.7(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 200
+ ; COMMON-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 1200
+ ; COMMON-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[ADDWrr]], [[ADDWrr]], implicit-def $nzcv
+ ; COMMON-NEXT: Bcc 10, %bb.7, implicit $nzcv
+ ; COMMON-NEXT: B %bb.3
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.3:
+ ; COMMON-NEXT: successors: %bb.4(0x40000000), %bb.7(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[MOVi64imm4:%[0-9]+]]:gpr64 = MOVi64imm 300
+ ; COMMON-NEXT: [[MOVi64imm5:%[0-9]+]]:gpr64 = MOVi64imm 1300
+ ; COMMON-NEXT: [[SUBSWrr2:%[0-9]+]]:gpr32 = SUBSWrr [[ADDWrr1]], [[ADDWrr1]], implicit-def $nzcv
+ ; COMMON-NEXT: Bcc 10, %bb.7, implicit $nzcv
+ ; COMMON-NEXT: B %bb.4
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.4:
+ ; COMMON-NEXT: successors: %bb.5(0x40000000), %bb.7(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[MOVi64imm6:%[0-9]+]]:gpr64 = MOVi64imm 400
+ ; COMMON-NEXT: [[MOVi64imm7:%[0-9]+]]:gpr64 = MOVi64imm 1400
+ ; COMMON-NEXT: [[SUBSWrr3:%[0-9]+]]:gpr32 = SUBSWrr [[ADDWrr2]], [[ADDWrr2]], implicit-def $nzcv
+ ; COMMON-NEXT: Bcc 10, %bb.7, implicit $nzcv
+ ; COMMON-NEXT: B %bb.5
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.5:
+ ; COMMON-NEXT: successors: %bb.6(0x40000000), %bb.7(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[MOVi64imm8:%[0-9]+]]:gpr64 = MOVi64imm 500
+ ; COMMON-NEXT: [[MOVi64imm9:%[0-9]+]]:gpr64 = MOVi64imm 1500
+ ; COMMON-NEXT: [[SUBSWrr4:%[0-9]+]]:gpr32 = SUBSWrr [[ADDWrr3]], [[ADDWrr3]], implicit-def $nzcv
+ ; COMMON-NEXT: Bcc 10, %bb.7, implicit $nzcv
+ ; COMMON-NEXT: B %bb.6
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.6:
+ ; COMMON-NEXT: successors: %bb.7(0x80000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[MOVi64imm10:%[0-9]+]]:gpr64 = MOVi64imm 600
+ ; COMMON-NEXT: [[MOVi64imm11:%[0-9]+]]:gpr64 = MOVi64imm 1600
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.7:
+ ; COMMON-NEXT: [[PHI:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.1, [[MOVi64imm2]], %bb.2, [[MOVi64imm4]], %bb.3, [[MOVi64imm6]], %bb.4, [[MOVi64imm8]], %bb.5, [[MOVi64imm10]], %bb.6
+ ; COMMON-NEXT: [[PHI1:%[0-9]+]]:gpr64 = PHI [[MOVi64imm1]], %bb.1, [[MOVi64imm3]], %bb.2, [[MOVi64imm5]], %bb.3, [[MOVi64imm7]], %bb.4, [[MOVi64imm9]], %bb.5, [[MOVi64imm11]], %bb.6
+ ; COMMON-NEXT: [[MADDXrrr:%[0-9]+]]:gpr64 = MADDXrrr [[PHI]], [[PHI]], $xzr
+ ; COMMON-NEXT: [[MADDXrrr1:%[0-9]+]]:gpr64 = MADDXrrr [[MADDXrrr]], [[MADDXrrr]], $xzr
+ ; COMMON-NEXT: [[MADDXrrr2:%[0-9]+]]:gpr64 = MADDXrrr [[MADDXrrr1]], [[MADDXrrr1]], $xzr
+ ; COMMON-NEXT: [[MADDXrrr3:%[0-9]+]]:gpr64 = MADDXrrr [[MADDXrrr2]], [[MADDXrrr2]], $xzr
+ ; COMMON-NEXT: [[ADDXrr:%[0-9]+]]:gpr64 = ADDXrr [[MADDXrrr3]], [[PHI1]]
+ ; COMMON-NEXT: $x0 = COPY [[ADDXrr]]
+ ; COMMON-NEXT: RET_ReallyLR implicit $x0
+ bb.0:
+ successors: %bb.1(0x80000000)
+ liveins: $x0
+
+ %0:gpr64common = COPY $x0
+
+ bb.1:
+ successors: %bb.2(0x40000000), %bb.7(0x40000000)
+
+ %10:gpr32 = LDRWui %0, 0 :: (load (s32))
+ %11:gpr32 = MADDWrrr %10, %10, $wzr
+ %12:gpr32 = MADDWrrr %11, %11, $wzr
+ %20:gpr32 = ADDWrr %12, %12
+ %21:gpr32 = ADDWrr %20, %20
+ %22:gpr32 = ADDWrr %21, %21
+ %23:gpr32 = ADDWrr %22, %22
+ %40:gpr64 = MOVi64imm 100
+ %50:gpr64 = MOVi64imm 1100
+ %30:gpr32 = SUBSWrr %12, %12, implicit-def $nzcv
+ Bcc 10, %bb.7, implicit $nzcv
+ B %bb.2
+
+ bb.2:
+ successors: %bb.3(0x40000000), %bb.7(0x40000000)
+
+ %41:gpr64 = MOVi64imm 200
+ %51:gpr64 = MOVi64imm 1200
+ %31:gpr32 = SUBSWrr %20, %20, implicit-def $nzcv
+ Bcc 10, %bb.7, implicit $nzcv
+ B %bb.3
+
+ bb.3:
+ successors: %bb.4(0x40000000), %bb.7(0x40000000)
+
+ %42:gpr64 = MOVi64imm 300
+ %52:gpr64 = MOVi64imm 1300
+ %32:gpr32 = SUBSWrr %21, %21, implicit-def $nzcv
+ Bcc 10, %bb.7, implicit $nzcv
+ B %bb.4
+
+ bb.4:
+ successors: %bb.5(0x40000000), %bb.7(0x40000000)
+
+ %43:gpr64 = MOVi64imm 400
+ %53:gpr64 = MOVi64imm 1400
+ %33:gpr32 = SUBSWrr %22, %22, implicit-def $nzcv
+ Bcc 10, %bb.7, implicit $nzcv
+ B %bb.5
+
+ bb.5:
+ successors: %bb.6(0x40000000), %bb.7(0x40000000)
+
+ %44:gpr64 = MOVi64imm 500
+ %54:gpr64 = MOVi64imm 1500
+ %34:gpr32 = SUBSWrr %23, %23, implicit-def $nzcv
+ Bcc 10, %bb.7, implicit $nzcv
+ B %bb.6
+
+ bb.6:
+ successors: %bb.7(0x80000000)
+
+ %45:gpr64 = MOVi64imm 600
+ %55:gpr64 = MOVi64imm 1600
+
+ bb.7:
+ %90:gpr64 = PHI %40, %bb.1, %41, %bb.2, %42, %bb.3, %43, %bb.4, %44, %bb.5, %45, %bb.6
+ %91:gpr64 = PHI %50, %bb.1, %51, %bb.2, %52, %bb.3, %53, %bb.4, %54, %bb.5, %55, %bb.6
+ %60:gpr64 = MADDXrrr %90, %90, $xzr
+ %61:gpr64 = MADDXrrr %60, %60, $xzr
+ %62:gpr64 = MADDXrrr %61, %61, $xzr
+ %63:gpr64 = MADDXrrr %62, %62, $xzr
+ %64:gpr64 = ADDXrr %63, %91
+ $x0 = COPY %64
+ RET_ReallyLR implicit $x0
+...
+---
+name: cascade_phi_subset
+alignment: 4
+tracksRegLiveness: true
+body: |
+ ; CASCADE-WITH-DD-LABEL: name: cascade_phi_subset
+ ; CASCADE-WITH-DD: bb.0:
+ ; CASCADE-WITH-DD-NEXT: successors: %bb.1(0x80000000)
+ ; CASCADE-WITH-DD-NEXT: liveins: $x0
+ ; CASCADE-WITH-DD-NEXT: {{ $}}
+ ; CASCADE-WITH-DD-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; CASCADE-WITH-DD-NEXT: {{ $}}
+ ; CASCADE-WITH-DD-NEXT: bb.1:
+ ; CASCADE-WITH-DD-NEXT: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load (s32))
+ ; CASCADE-WITH-DD-NEXT: [[MADDWrrr:%[0-9]+]]:gpr32 = MADDWrrr [[LDRWui]], [[LDRWui]], $wzr
+ ; CASCADE-WITH-DD-NEXT: [[MADDWrrr1:%[0-9]+]]:gpr32 = MADDWrrr [[MADDWrrr]], [[MADDWrrr]], $wzr
+ ; CASCADE-WITH-DD-NEXT: [[ADDWrr:%[0-9]+]]:gpr32 = ADDWrr [[MADDWrrr1]], [[MADDWrrr1]]
+ ; CASCADE-WITH-DD-NEXT: [[ADDWrr1:%[0-9]+]]:gpr32 = ADDWrr [[ADDWrr]], [[ADDWrr]]
+ ; CASCADE-WITH-DD-NEXT: [[ADDWrr2:%[0-9]+]]:gpr32 = ADDWrr [[ADDWrr1]], [[ADDWrr1]]
+ ; CASCADE-WITH-DD-NEXT: [[ADDWrr3:%[0-9]+]]:gpr32 = ADDWrr [[ADDWrr2]], [[ADDWrr2]]
+ ; CASCADE-WITH-DD-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 100
+ ; CASCADE-WITH-DD-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 200
+ ; CASCADE-WITH-DD-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 300
+ ; CASCADE-WITH-DD-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 400
+ ; CASCADE-WITH-DD-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[ADDWrr3]], [[ADDWrr3]], implicit-def $nzcv
+ ; CASCADE-WITH-DD-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[ADDWrr2]], [[ADDWrr2]], implicit-def $nzcv
+ ; CASCADE-WITH-DD-NEXT: [[SUBSWrr2:%[0-9]+]]:gpr32 = SUBSWrr [[ADDWrr1]], [[ADDWrr1]], implicit-def $nzcv
+ ; CASCADE-WITH-DD-NEXT: [[CSELXr:%[0-9]+]]:gpr64 = CSELXr [[MOVi64imm2]], [[MOVi64imm3]], 10, implicit $nzcv
+ ; CASCADE-WITH-DD-NEXT: [[SUBSWrr3:%[0-9]+]]:gpr32 = SUBSWrr [[ADDWrr]], [[ADDWrr]], implicit-def $nzcv
+ ; CASCADE-WITH-DD-NEXT: [[CSELXr1:%[0-9]+]]:gpr64 = CSELXr [[MOVi64imm1]], [[CSELXr]], 10, implicit $nzcv
+ ; CASCADE-WITH-DD-NEXT: [[SUBSWrr4:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr1]], [[MADDWrrr1]], implicit-def $nzcv
+ ; CASCADE-WITH-DD-NEXT: [[CSELXr2:%[0-9]+]]:gpr64 = CSELXr [[MOVi64imm]], [[CSELXr1]], 10, implicit $nzcv
+ ; CASCADE-WITH-DD-NEXT: $x0 = COPY [[CSELXr2]]
+ ; CASCADE-WITH-DD-NEXT: RET_ReallyLR implicit $x0
+ ;
+ ; DEPTH3-LABEL: name: cascade_phi_subset
+ ; DEPTH3: bb.0:
+ ; DEPTH3-NEXT: successors: %bb.1(0x80000000)
+ ; DEPTH3-NEXT: liveins: $x0
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: bb.1:
+ ; DEPTH3-NEXT: successors: %bb.2(0x40000000), %bb.7(0x40000000)
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load (s32))
+ ; DEPTH3-NEXT: [[MADDWrrr:%[0-9]+]]:gpr32 = MADDWrrr [[LDRWui]], [[LDRWui]], $wzr
+ ; DEPTH3-NEXT: [[MADDWrrr1:%[0-9]+]]:gpr32 = MADDWrrr [[MADDWrrr]], [[MADDWrrr]], $wzr
+ ; DEPTH3-NEXT: [[ADDWrr:%[0-9]+]]:gpr32 = ADDWrr [[MADDWrrr1]], [[MADDWrrr1]]
+ ; DEPTH3-NEXT: [[ADDWrr1:%[0-9]+]]:gpr32 = ADDWrr [[ADDWrr]], [[ADDWrr]]
+ ; DEPTH3-NEXT: [[ADDWrr2:%[0-9]+]]:gpr32 = ADDWrr [[ADDWrr1]], [[ADDWrr1]]
+ ; DEPTH3-NEXT: [[ADDWrr3:%[0-9]+]]:gpr32 = ADDWrr [[ADDWrr2]], [[ADDWrr2]]
+ ; DEPTH3-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 100
+ ; DEPTH3-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr1]], [[MADDWrrr1]], implicit-def $nzcv
+ ; DEPTH3-NEXT: Bcc 10, %bb.7, implicit $nzcv
+ ; DEPTH3-NEXT: B %bb.2
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: bb.2:
+ ; DEPTH3-NEXT: successors: %bb.3(0x40000000), %bb.7(0x40000000)
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 200
+ ; DEPTH3-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[ADDWrr]], [[ADDWrr]], implicit-def $nzcv
+ ; DEPTH3-NEXT: Bcc 10, %bb.7, implicit $nzcv
+ ; DEPTH3-NEXT: B %bb.3
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: bb.3:
+ ; DEPTH3-NEXT: successors: %bb.4(0x40000000), %bb.7(0x40000000)
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 300
+ ; DEPTH3-NEXT: [[SUBSWrr2:%[0-9]+]]:gpr32 = SUBSWrr [[ADDWrr1]], [[ADDWrr1]], implicit-def $nzcv
+ ; DEPTH3-NEXT: Bcc 10, %bb.7, implicit $nzcv
+ ; DEPTH3-NEXT: B %bb.4
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: bb.4:
+ ; DEPTH3-NEXT: successors: %bb.5(0x40000000), %bb.7(0x40000000)
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 400
+ ; DEPTH3-NEXT: [[SUBSWrr3:%[0-9]+]]:gpr32 = SUBSWrr [[ADDWrr2]], [[ADDWrr2]], implicit-def $nzcv
+ ; DEPTH3-NEXT: Bcc 10, %bb.7, implicit $nzcv
+ ; DEPTH3-NEXT: B %bb.5
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: bb.5:
+ ; DEPTH3-NEXT: successors: %bb.6(0x40000000), %bb.7(0x40000000)
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: [[SUBSWrr4:%[0-9]+]]:gpr32 = SUBSWrr [[ADDWrr3]], [[ADDWrr3]], implicit-def $nzcv
+ ; DEPTH3-NEXT: Bcc 10, %bb.7, implicit $nzcv
+ ; DEPTH3-NEXT: B %bb.6
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: bb.6:
+ ; DEPTH3-NEXT: successors: %bb.7(0x80000000)
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: bb.7:
+ ; DEPTH3-NEXT: [[PHI:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.1, [[MOVi64imm1]], %bb.2, [[MOVi64imm2]], %bb.3, [[MOVi64imm3]], %bb.4, [[MOVi64imm3]], %bb.5, [[MOVi64imm3]], %bb.6
+ ; DEPTH3-NEXT: $x0 = COPY [[PHI]]
+ ; DEPTH3-NEXT: RET_ReallyLR implicit $x0
+ ;
+ ; LIMIT3-LABEL: name: cascade_phi_subset
+ ; LIMIT3: bb.0:
+ ; LIMIT3-NEXT: successors: %bb.1(0x80000000)
+ ; LIMIT3-NEXT: liveins: $x0
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: bb.1:
+ ; LIMIT3-NEXT: successors: %bb.2(0x40000000), %bb.7(0x40000000)
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load (s32))
+ ; LIMIT3-NEXT: [[MADDWrrr:%[0-9]+]]:gpr32 = MADDWrrr [[LDRWui]], [[LDRWui]], $wzr
+ ; LIMIT3-NEXT: [[MADDWrrr1:%[0-9]+]]:gpr32 = MADDWrrr [[MADDWrrr]], [[MADDWrrr]], $wzr
+ ; LIMIT3-NEXT: [[ADDWrr:%[0-9]+]]:gpr32 = ADDWrr [[MADDWrrr1]], [[MADDWrrr1]]
+ ; LIMIT3-NEXT: [[ADDWrr1:%[0-9]+]]:gpr32 = ADDWrr [[ADDWrr]], [[ADDWrr]]
+ ; LIMIT3-NEXT: [[ADDWrr2:%[0-9]+]]:gpr32 = ADDWrr [[ADDWrr1]], [[ADDWrr1]]
+ ; LIMIT3-NEXT: [[ADDWrr3:%[0-9]+]]:gpr32 = ADDWrr [[ADDWrr2]], [[ADDWrr2]]
+ ; LIMIT3-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 100
+ ; LIMIT3-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr1]], [[MADDWrrr1]], implicit-def $nzcv
+ ; LIMIT3-NEXT: Bcc 10, %bb.7, implicit $nzcv
+ ; LIMIT3-NEXT: B %bb.2
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: bb.2:
+ ; LIMIT3-NEXT: successors: %bb.3(0x40000000), %bb.7(0x40000000)
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 200
+ ; LIMIT3-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[ADDWrr]], [[ADDWrr]], implicit-def $nzcv
+ ; LIMIT3-NEXT: Bcc 10, %bb.7, implicit $nzcv
+ ; LIMIT3-NEXT: B %bb.3
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: bb.3:
+ ; LIMIT3-NEXT: successors: %bb.4(0x40000000), %bb.7(0x40000000)
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 300
+ ; LIMIT3-NEXT: [[SUBSWrr2:%[0-9]+]]:gpr32 = SUBSWrr [[ADDWrr1]], [[ADDWrr1]], implicit-def $nzcv
+ ; LIMIT3-NEXT: Bcc 10, %bb.7, implicit $nzcv
+ ; LIMIT3-NEXT: B %bb.4
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: bb.4:
+ ; LIMIT3-NEXT: successors: %bb.5(0x40000000), %bb.7(0x40000000)
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 400
+ ; LIMIT3-NEXT: [[SUBSWrr3:%[0-9]+]]:gpr32 = SUBSWrr [[ADDWrr2]], [[ADDWrr2]], implicit-def $nzcv
+ ; LIMIT3-NEXT: Bcc 10, %bb.7, implicit $nzcv
+ ; LIMIT3-NEXT: B %bb.5
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: bb.5:
+ ; LIMIT3-NEXT: successors: %bb.6(0x40000000), %bb.7(0x40000000)
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: [[SUBSWrr4:%[0-9]+]]:gpr32 = SUBSWrr [[ADDWrr3]], [[ADDWrr3]], implicit-def $nzcv
+ ; LIMIT3-NEXT: Bcc 10, %bb.7, implicit $nzcv
+ ; LIMIT3-NEXT: B %bb.6
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: bb.6:
+ ; LIMIT3-NEXT: successors: %bb.7(0x80000000)
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: bb.7:
+ ; LIMIT3-NEXT: [[PHI:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.1, [[MOVi64imm1]], %bb.2, [[MOVi64imm2]], %bb.3, [[MOVi64imm3]], %bb.4, [[MOVi64imm3]], %bb.5, [[MOVi64imm3]], %bb.6
+ ; LIMIT3-NEXT: $x0 = COPY [[PHI]]
+ ; LIMIT3-NEXT: RET_ReallyLR implicit $x0
+ bb.0:
+ successors: %bb.1(0x80000000)
+ liveins: $x0
+
+ %0:gpr64common = COPY $x0
+
+ bb.1:
+ successors: %bb.2(0x40000000), %bb.7(0x40000000)
+
+ %10:gpr32 = LDRWui %0, 0 :: (load (s32))
+ %11:gpr32 = MADDWrrr %10, %10, $wzr
+ %12:gpr32 = MADDWrrr %11, %11, $wzr
+ %20:gpr32 = ADDWrr %12, %12
+ %21:gpr32 = ADDWrr %20, %20
+ %22:gpr32 = ADDWrr %21, %21
+ %23:gpr32 = ADDWrr %22, %22
+ %40:gpr64 = MOVi64imm 100
+ %30:gpr32 = SUBSWrr %12, %12, implicit-def $nzcv
+ Bcc 10, %bb.7, implicit $nzcv
+ B %bb.2
+
+ bb.2:
+ successors: %bb.3(0x40000000), %bb.7(0x40000000)
+
+ %41:gpr64 = MOVi64imm 200
+ %31:gpr32 = SUBSWrr %20, %20, implicit-def $nzcv
+ Bcc 10, %bb.7, implicit $nzcv
+ B %bb.3
+ bb.3:
+ successors: %bb.4(0x40000000), %bb.7(0x40000000)
+
+ %42:gpr64 = MOVi64imm 300
+ %32:gpr32 = SUBSWrr %21, %21, implicit-def $nzcv
+ Bcc 10, %bb.7, implicit $nzcv
+ B %bb.4
+
+ bb.4:
+ successors: %bb.5(0x40000000), %bb.7(0x40000000)
+
+ %43:gpr64 = MOVi64imm 400
+ %33:gpr32 = SUBSWrr %22, %22, implicit-def $nzcv
+ Bcc 10, %bb.7, implicit $nzcv
+ B %bb.5
+
+ bb.5:
+ successors: %bb.6(0x40000000), %bb.7(0x40000000)
+
+ %34:gpr32 = SUBSWrr %23, %23, implicit-def $nzcv
+ Bcc 10, %bb.7, implicit $nzcv
+ B %bb.6
+
+ bb.6:
+ successors: %bb.7(0x80000000)
+
+ bb.7:
+ %90:gpr64 = PHI %40, %bb.1, %41, %bb.2, %42, %bb.3, %43, %bb.4, %43, %bb.5, %43, %bb.6
+ $x0 = COPY %90
+ RET_ReallyLR implicit $x0
+...
diff --git a/llvm/test/CodeGen/AArch64/early-ifcvt-load-to-cond-br-limit.mir b/llvm/test/CodeGen/AArch64/early-ifcvt-load-to-cond-br-limit.mir
index 12170733e2332..c8ab9bb5da684 100644
--- a/llvm/test/CodeGen/AArch64/early-ifcvt-load-to-cond-br-limit.mir
+++ b/llvm/test/CodeGen/AArch64/early-ifcvt-load-to-cond-br-limit.mir
@@ -2,6 +2,7 @@
# RUN: llc -mtriple=aarch64 -mcpu=apple-m1 -run-pass=early-ifcvt -verify-machine-dom-info -o - %s | FileCheck %s --check-prefix=CHECK-DEFAULT
# RUN: llc -mtriple=aarch64 -mcpu=apple-m1 -run-pass=early-ifcvt -enable-early-ifcvt-data-dependent -early-ifcvt-max-region-instrs=3 -verify-machine-dom-info -o - %s | FileCheck %s --check-prefix=CHECK-SMALL-BUDGET
# RUN: llc -mtriple=aarch64 -mcpu=apple-m1 -run-pass=early-ifcvt -enable-early-ifcvt-data-dependent -verify-machine-dom-info -o - %s | FileCheck %s --check-prefix=CHECK-LARGE-BUDGET
+# RUN: llc -mtriple=aarch64 -mcpu=apple-m1 -run-pass=early-ifcvt -enable-early-ifcvt-cascade -enable-early-ifcvt-data-dependent -early-ifcvt-limit=50 -early-ifcvt-max-region-instrs=65 -verify-machine-dom-info -o - %s | FileCheck %s --check-prefix=CHECK-CASCADE-BUDGET
---
name: load_in_pred_region_over_instruction_budget
@@ -138,6 +139,43 @@ body: |
; CHECK-LARGE-BUDGET-NEXT: [[CSELWr:%[0-9]+]]:gpr32 = CSELWr [[COPY1]], [[MADDWrrr1]], 12, implicit $nzcv
; CHECK-LARGE-BUDGET-NEXT: $w0 = COPY [[CSELWr]]
; CHECK-LARGE-BUDGET-NEXT: RET_ReallyLR implicit $w0
+ ;
+ ; CHECK-CASCADE-BUDGET-LABEL: name: load_in_pred_region_over_instruction_budget
+ ; CHECK-CASCADE-BUDGET: bb.0:
+ ; CHECK-CASCADE-BUDGET-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-CASCADE-BUDGET-NEXT: liveins: $x0, $w1, $w2
+ ; CHECK-CASCADE-BUDGET-NEXT: {{ $}}
+ ; CHECK-CASCADE-BUDGET-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[COPY1:%[0-9]+]]:gpr32 = COPY $w1
+ ; CHECK-CASCADE-BUDGET-NEXT: [[COPY2:%[0-9]+]]:gpr32 = COPY $w2
+ ; CHECK-CASCADE-BUDGET-NEXT: {{ $}}
+ ; CHECK-CASCADE-BUDGET-NEXT: bb.1:
+ ; CHECK-CASCADE-BUDGET-NEXT: successors: %bb.2(0x40000000), %bb.3(0x40000000)
+ ; CHECK-CASCADE-BUDGET-NEXT: {{ $}}
+ ; CHECK-CASCADE-BUDGET-NEXT: [[LDRWui:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 0 :: (load (s32))
+ ; CHECK-CASCADE-BUDGET-NEXT: [[SUBSWri:%[0-9]+]]:gpr32 = SUBSWri [[LDRWui]], 0, 0, implicit-def $nzcv
+ ; CHECK-CASCADE-BUDGET-NEXT: Bcc 0, %bb.2, implicit $nzcv
+ ; CHECK-CASCADE-BUDGET-NEXT: B %bb.3
+ ; CHECK-CASCADE-BUDGET-NEXT: {{ $}}
+ ; CHECK-CASCADE-BUDGET-NEXT: bb.2:
+ ; CHECK-CASCADE-BUDGET-NEXT: successors: %bb.3(0x80000000)
+ ; CHECK-CASCADE-BUDGET-NEXT: {{ $}}
+ ; CHECK-CASCADE-BUDGET-NEXT: STRWui $wzr, [[COPY]], 0 :: (store (s32))
+ ; CHECK-CASCADE-BUDGET-NEXT: STRWui $wzr, [[COPY]], 1 :: (store (s32))
+ ; CHECK-CASCADE-BUDGET-NEXT: STRWui $wzr, [[COPY]], 2 :: (store (s32))
+ ; CHECK-CASCADE-BUDGET-NEXT: STRWui $wzr, [[COPY]], 3 :: (store (s32))
+ ; CHECK-CASCADE-BUDGET-NEXT: STRWui $wzr, [[COPY]], 4 :: (store (s32))
+ ; CHECK-CASCADE-BUDGET-NEXT: STRWui $wzr, [[COPY]], 5 :: (store (s32))
+ ; CHECK-CASCADE-BUDGET-NEXT: B %bb.3
+ ; CHECK-CASCADE-BUDGET-NEXT: {{ $}}
+ ; CHECK-CASCADE-BUDGET-NEXT: bb.3:
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri:%[0-9]+]]:gpr32common = ADDWri [[LDRWui]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[SUBSWri1:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri]], 100, 0, implicit-def $nzcv
+ ; CHECK-CASCADE-BUDGET-NEXT: [[MADDWrrr:%[0-9]+]]:gpr32 = MADDWrrr [[COPY1]], [[COPY2]], $wzr
+ ; CHECK-CASCADE-BUDGET-NEXT: [[MADDWrrr1:%[0-9]+]]:gpr32 = MADDWrrr [[MADDWrrr]], [[COPY1]], $wzr
+ ; CHECK-CASCADE-BUDGET-NEXT: [[CSELWr:%[0-9]+]]:gpr32 = CSELWr [[COPY1]], [[MADDWrrr1]], 12, implicit $nzcv
+ ; CHECK-CASCADE-BUDGET-NEXT: $w0 = COPY [[CSELWr]]
+ ; CHECK-CASCADE-BUDGET-NEXT: RET_ReallyLR implicit $w0
bb.0:
successors: %bb.1(0x80000000)
liveins: $x0, $w1, $w2
@@ -184,3 +222,542 @@ body: |
$w0 = COPY %8
RET_ReallyLR implicit $w0
...
+---
+name: cascade_load_to_cond_br_scan_budget
+alignment: 4
+tracksRegLiveness: true
+body: |
+ ; CHECK-DEFAULT-LABEL: name: cascade_load_to_cond_br_scan_budget
+ ; CHECK-DEFAULT: bb.0:
+ ; CHECK-DEFAULT-NEXT: successors: %bb.8(0x40000000), %bb.1(0x40000000)
+ ; CHECK-DEFAULT-NEXT: liveins: $x0
+ ; CHECK-DEFAULT-NEXT: {{ $}}
+ ; CHECK-DEFAULT-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; CHECK-DEFAULT-NEXT: [[LDRWui:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 0 :: (load (s32))
+ ; CHECK-DEFAULT-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 1
+ ; CHECK-DEFAULT-NEXT: [[MOVi32imm1:%[0-9]+]]:gpr32 = MOVi32imm 2
+ ; CHECK-DEFAULT-NEXT: [[MOVi32imm2:%[0-9]+]]:gpr32 = MOVi32imm 3
+ ; CHECK-DEFAULT-NEXT: [[MOVi32imm3:%[0-9]+]]:gpr32 = MOVi32imm 4
+ ; CHECK-DEFAULT-NEXT: [[LDRWui1:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 1 :: (load (s32))
+ ; CHECK-DEFAULT-NEXT: [[LDRWui2:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 2 :: (load (s32))
+ ; CHECK-DEFAULT-NEXT: [[LDRWui3:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 3 :: (load (s32))
+ ; CHECK-DEFAULT-NEXT: [[LDRWui4:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 4 :: (load (s32))
+ ; CHECK-DEFAULT-NEXT: [[LDRWui5:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 5 :: (load (s32))
+ ; CHECK-DEFAULT-NEXT: [[LDRWui6:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 6 :: (load (s32))
+ ; CHECK-DEFAULT-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 100
+ ; CHECK-DEFAULT-NEXT: [[ADDWri:%[0-9]+]]:gpr32common = ADDWri [[LDRWui]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri1:%[0-9]+]]:gpr32common = ADDWri [[ADDWri]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri2:%[0-9]+]]:gpr32common = ADDWri [[ADDWri1]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri3:%[0-9]+]]:gpr32common = ADDWri [[ADDWri2]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri4:%[0-9]+]]:gpr32common = ADDWri [[ADDWri3]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[SUBSWri:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri4]], 1, 0, implicit-def $nzcv
+ ; CHECK-DEFAULT-NEXT: Bcc 10, %bb.8, implicit $nzcv
+ ; CHECK-DEFAULT-NEXT: B %bb.1
+ ; CHECK-DEFAULT-NEXT: {{ $}}
+ ; CHECK-DEFAULT-NEXT: bb.1:
+ ; CHECK-DEFAULT-NEXT: successors: %bb.8(0x40000000), %bb.2(0x40000000)
+ ; CHECK-DEFAULT-NEXT: {{ $}}
+ ; CHECK-DEFAULT-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 101
+ ; CHECK-DEFAULT-NEXT: [[ADDWri5:%[0-9]+]]:gpr32common = ADDWri [[LDRWui1]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri6:%[0-9]+]]:gpr32common = ADDWri [[ADDWri5]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri7:%[0-9]+]]:gpr32common = ADDWri [[ADDWri6]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri8:%[0-9]+]]:gpr32common = ADDWri [[ADDWri7]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri9:%[0-9]+]]:gpr32common = ADDWri [[ADDWri8]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[SUBSWri1:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri9]], 2, 0, implicit-def $nzcv
+ ; CHECK-DEFAULT-NEXT: Bcc 11, %bb.8, implicit $nzcv
+ ; CHECK-DEFAULT-NEXT: B %bb.2
+ ; CHECK-DEFAULT-NEXT: {{ $}}
+ ; CHECK-DEFAULT-NEXT: bb.2:
+ ; CHECK-DEFAULT-NEXT: successors: %bb.8(0x40000000), %bb.3(0x40000000)
+ ; CHECK-DEFAULT-NEXT: {{ $}}
+ ; CHECK-DEFAULT-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 102
+ ; CHECK-DEFAULT-NEXT: [[ADDWri10:%[0-9]+]]:gpr32common = ADDWri [[LDRWui2]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri11:%[0-9]+]]:gpr32common = ADDWri [[ADDWri10]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri12:%[0-9]+]]:gpr32common = ADDWri [[ADDWri11]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri13:%[0-9]+]]:gpr32common = ADDWri [[ADDWri12]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri14:%[0-9]+]]:gpr32common = ADDWri [[ADDWri13]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[SUBSWri2:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri14]], 3, 0, implicit-def $nzcv
+ ; CHECK-DEFAULT-NEXT: Bcc 12, %bb.8, implicit $nzcv
+ ; CHECK-DEFAULT-NEXT: B %bb.3
+ ; CHECK-DEFAULT-NEXT: {{ $}}
+ ; CHECK-DEFAULT-NEXT: bb.3:
+ ; CHECK-DEFAULT-NEXT: successors: %bb.8(0x40000000), %bb.4(0x40000000)
+ ; CHECK-DEFAULT-NEXT: {{ $}}
+ ; CHECK-DEFAULT-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 103
+ ; CHECK-DEFAULT-NEXT: [[ADDWri15:%[0-9]+]]:gpr32common = ADDWri [[LDRWui3]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri16:%[0-9]+]]:gpr32common = ADDWri [[ADDWri15]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri17:%[0-9]+]]:gpr32common = ADDWri [[ADDWri16]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri18:%[0-9]+]]:gpr32common = ADDWri [[ADDWri17]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri19:%[0-9]+]]:gpr32common = ADDWri [[ADDWri18]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[SUBSWri3:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri19]], 4, 0, implicit-def $nzcv
+ ; CHECK-DEFAULT-NEXT: Bcc 13, %bb.8, implicit $nzcv
+ ; CHECK-DEFAULT-NEXT: B %bb.4
+ ; CHECK-DEFAULT-NEXT: {{ $}}
+ ; CHECK-DEFAULT-NEXT: bb.4:
+ ; CHECK-DEFAULT-NEXT: successors: %bb.8(0x40000000), %bb.5(0x40000000)
+ ; CHECK-DEFAULT-NEXT: {{ $}}
+ ; CHECK-DEFAULT-NEXT: [[MOVi64imm4:%[0-9]+]]:gpr64 = MOVi64imm 104
+ ; CHECK-DEFAULT-NEXT: [[ADDWri20:%[0-9]+]]:gpr32common = ADDWri [[LDRWui4]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri21:%[0-9]+]]:gpr32common = ADDWri [[ADDWri20]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri22:%[0-9]+]]:gpr32common = ADDWri [[ADDWri21]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri23:%[0-9]+]]:gpr32common = ADDWri [[ADDWri22]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri24:%[0-9]+]]:gpr32common = ADDWri [[ADDWri23]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[SUBSWri4:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri24]], 5, 0, implicit-def $nzcv
+ ; CHECK-DEFAULT-NEXT: Bcc 14, %bb.8, implicit $nzcv
+ ; CHECK-DEFAULT-NEXT: B %bb.5
+ ; CHECK-DEFAULT-NEXT: {{ $}}
+ ; CHECK-DEFAULT-NEXT: bb.5:
+ ; CHECK-DEFAULT-NEXT: successors: %bb.8(0x40000000), %bb.6(0x40000000)
+ ; CHECK-DEFAULT-NEXT: {{ $}}
+ ; CHECK-DEFAULT-NEXT: [[MOVi64imm5:%[0-9]+]]:gpr64 = MOVi64imm 105
+ ; CHECK-DEFAULT-NEXT: [[ADDWri25:%[0-9]+]]:gpr32common = ADDWri [[LDRWui5]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri26:%[0-9]+]]:gpr32common = ADDWri [[ADDWri25]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri27:%[0-9]+]]:gpr32common = ADDWri [[ADDWri26]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri28:%[0-9]+]]:gpr32common = ADDWri [[ADDWri27]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri29:%[0-9]+]]:gpr32common = ADDWri [[ADDWri28]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[SUBSWri5:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri29]], 6, 0, implicit-def $nzcv
+ ; CHECK-DEFAULT-NEXT: Bcc 15, %bb.8, implicit $nzcv
+ ; CHECK-DEFAULT-NEXT: B %bb.6
+ ; CHECK-DEFAULT-NEXT: {{ $}}
+ ; CHECK-DEFAULT-NEXT: bb.6:
+ ; CHECK-DEFAULT-NEXT: successors: %bb.8(0x40000000), %bb.7(0x40000000)
+ ; CHECK-DEFAULT-NEXT: {{ $}}
+ ; CHECK-DEFAULT-NEXT: [[MOVi64imm6:%[0-9]+]]:gpr64 = MOVi64imm 106
+ ; CHECK-DEFAULT-NEXT: [[ADDWri30:%[0-9]+]]:gpr32common = ADDWri [[LDRWui6]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri31:%[0-9]+]]:gpr32common = ADDWri [[ADDWri30]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri32:%[0-9]+]]:gpr32common = ADDWri [[ADDWri31]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri33:%[0-9]+]]:gpr32common = ADDWri [[ADDWri32]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri34:%[0-9]+]]:gpr32common = ADDWri [[ADDWri33]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[SUBSWri6:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri34]], 7, 0, implicit-def $nzcv
+ ; CHECK-DEFAULT-NEXT: Bcc 16, %bb.8, implicit $nzcv
+ ; CHECK-DEFAULT-NEXT: B %bb.7
+ ; CHECK-DEFAULT-NEXT: {{ $}}
+ ; CHECK-DEFAULT-NEXT: bb.7:
+ ; CHECK-DEFAULT-NEXT: successors: %bb.8(0x80000000)
+ ; CHECK-DEFAULT-NEXT: {{ $}}
+ ; CHECK-DEFAULT-NEXT: [[MOVi64imm7:%[0-9]+]]:gpr64 = MOVi64imm 107
+ ; CHECK-DEFAULT-NEXT: {{ $}}
+ ; CHECK-DEFAULT-NEXT: bb.8:
+ ; CHECK-DEFAULT-NEXT: [[PHI:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.0, [[MOVi64imm1]], %bb.1, [[MOVi64imm2]], %bb.2, [[MOVi64imm3]], %bb.3, [[MOVi64imm4]], %bb.4, [[MOVi64imm5]], %bb.5, [[MOVi64imm6]], %bb.6, [[MOVi64imm7]], %bb.7
+ ; CHECK-DEFAULT-NEXT: $x0 = COPY [[PHI]]
+ ; CHECK-DEFAULT-NEXT: RET_ReallyLR implicit $x0
+ ;
+ ; CHECK-SMALL-BUDGET-LABEL: name: cascade_load_to_cond_br_scan_budget
+ ; CHECK-SMALL-BUDGET: bb.0:
+ ; CHECK-SMALL-BUDGET-NEXT: successors: %bb.8(0x40000000), %bb.1(0x40000000)
+ ; CHECK-SMALL-BUDGET-NEXT: liveins: $x0
+ ; CHECK-SMALL-BUDGET-NEXT: {{ $}}
+ ; CHECK-SMALL-BUDGET-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; CHECK-SMALL-BUDGET-NEXT: [[LDRWui:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 0 :: (load (s32))
+ ; CHECK-SMALL-BUDGET-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 1
+ ; CHECK-SMALL-BUDGET-NEXT: [[MOVi32imm1:%[0-9]+]]:gpr32 = MOVi32imm 2
+ ; CHECK-SMALL-BUDGET-NEXT: [[MOVi32imm2:%[0-9]+]]:gpr32 = MOVi32imm 3
+ ; CHECK-SMALL-BUDGET-NEXT: [[MOVi32imm3:%[0-9]+]]:gpr32 = MOVi32imm 4
+ ; CHECK-SMALL-BUDGET-NEXT: [[LDRWui1:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 1 :: (load (s32))
+ ; CHECK-SMALL-BUDGET-NEXT: [[LDRWui2:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 2 :: (load (s32))
+ ; CHECK-SMALL-BUDGET-NEXT: [[LDRWui3:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 3 :: (load (s32))
+ ; CHECK-SMALL-BUDGET-NEXT: [[LDRWui4:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 4 :: (load (s32))
+ ; CHECK-SMALL-BUDGET-NEXT: [[LDRWui5:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 5 :: (load (s32))
+ ; CHECK-SMALL-BUDGET-NEXT: [[LDRWui6:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 6 :: (load (s32))
+ ; CHECK-SMALL-BUDGET-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 100
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri:%[0-9]+]]:gpr32common = ADDWri [[LDRWui]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri1:%[0-9]+]]:gpr32common = ADDWri [[ADDWri]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri2:%[0-9]+]]:gpr32common = ADDWri [[ADDWri1]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri3:%[0-9]+]]:gpr32common = ADDWri [[ADDWri2]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri4:%[0-9]+]]:gpr32common = ADDWri [[ADDWri3]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[SUBSWri:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri4]], 1, 0, implicit-def $nzcv
+ ; CHECK-SMALL-BUDGET-NEXT: Bcc 10, %bb.8, implicit $nzcv
+ ; CHECK-SMALL-BUDGET-NEXT: B %bb.1
+ ; CHECK-SMALL-BUDGET-NEXT: {{ $}}
+ ; CHECK-SMALL-BUDGET-NEXT: bb.1:
+ ; CHECK-SMALL-BUDGET-NEXT: successors: %bb.8(0x40000000), %bb.2(0x40000000)
+ ; CHECK-SMALL-BUDGET-NEXT: {{ $}}
+ ; CHECK-SMALL-BUDGET-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 101
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri5:%[0-9]+]]:gpr32common = ADDWri [[LDRWui1]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri6:%[0-9]+]]:gpr32common = ADDWri [[ADDWri5]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri7:%[0-9]+]]:gpr32common = ADDWri [[ADDWri6]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri8:%[0-9]+]]:gpr32common = ADDWri [[ADDWri7]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri9:%[0-9]+]]:gpr32common = ADDWri [[ADDWri8]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[SUBSWri1:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri9]], 2, 0, implicit-def $nzcv
+ ; CHECK-SMALL-BUDGET-NEXT: Bcc 11, %bb.8, implicit $nzcv
+ ; CHECK-SMALL-BUDGET-NEXT: B %bb.2
+ ; CHECK-SMALL-BUDGET-NEXT: {{ $}}
+ ; CHECK-SMALL-BUDGET-NEXT: bb.2:
+ ; CHECK-SMALL-BUDGET-NEXT: successors: %bb.8(0x40000000), %bb.3(0x40000000)
+ ; CHECK-SMALL-BUDGET-NEXT: {{ $}}
+ ; CHECK-SMALL-BUDGET-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 102
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri10:%[0-9]+]]:gpr32common = ADDWri [[LDRWui2]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri11:%[0-9]+]]:gpr32common = ADDWri [[ADDWri10]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri12:%[0-9]+]]:gpr32common = ADDWri [[ADDWri11]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri13:%[0-9]+]]:gpr32common = ADDWri [[ADDWri12]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri14:%[0-9]+]]:gpr32common = ADDWri [[ADDWri13]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[SUBSWri2:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri14]], 3, 0, implicit-def $nzcv
+ ; CHECK-SMALL-BUDGET-NEXT: Bcc 12, %bb.8, implicit $nzcv
+ ; CHECK-SMALL-BUDGET-NEXT: B %bb.3
+ ; CHECK-SMALL-BUDGET-NEXT: {{ $}}
+ ; CHECK-SMALL-BUDGET-NEXT: bb.3:
+ ; CHECK-SMALL-BUDGET-NEXT: successors: %bb.8(0x40000000), %bb.4(0x40000000)
+ ; CHECK-SMALL-BUDGET-NEXT: {{ $}}
+ ; CHECK-SMALL-BUDGET-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 103
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri15:%[0-9]+]]:gpr32common = ADDWri [[LDRWui3]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri16:%[0-9]+]]:gpr32common = ADDWri [[ADDWri15]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri17:%[0-9]+]]:gpr32common = ADDWri [[ADDWri16]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri18:%[0-9]+]]:gpr32common = ADDWri [[ADDWri17]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri19:%[0-9]+]]:gpr32common = ADDWri [[ADDWri18]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[SUBSWri3:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri19]], 4, 0, implicit-def $nzcv
+ ; CHECK-SMALL-BUDGET-NEXT: Bcc 13, %bb.8, implicit $nzcv
+ ; CHECK-SMALL-BUDGET-NEXT: B %bb.4
+ ; CHECK-SMALL-BUDGET-NEXT: {{ $}}
+ ; CHECK-SMALL-BUDGET-NEXT: bb.4:
+ ; CHECK-SMALL-BUDGET-NEXT: successors: %bb.8(0x40000000), %bb.5(0x40000000)
+ ; CHECK-SMALL-BUDGET-NEXT: {{ $}}
+ ; CHECK-SMALL-BUDGET-NEXT: [[MOVi64imm4:%[0-9]+]]:gpr64 = MOVi64imm 104
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri20:%[0-9]+]]:gpr32common = ADDWri [[LDRWui4]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri21:%[0-9]+]]:gpr32common = ADDWri [[ADDWri20]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri22:%[0-9]+]]:gpr32common = ADDWri [[ADDWri21]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri23:%[0-9]+]]:gpr32common = ADDWri [[ADDWri22]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri24:%[0-9]+]]:gpr32common = ADDWri [[ADDWri23]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[SUBSWri4:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri24]], 5, 0, implicit-def $nzcv
+ ; CHECK-SMALL-BUDGET-NEXT: Bcc 14, %bb.8, implicit $nzcv
+ ; CHECK-SMALL-BUDGET-NEXT: B %bb.5
+ ; CHECK-SMALL-BUDGET-NEXT: {{ $}}
+ ; CHECK-SMALL-BUDGET-NEXT: bb.5:
+ ; CHECK-SMALL-BUDGET-NEXT: successors: %bb.8(0x40000000), %bb.6(0x40000000)
+ ; CHECK-SMALL-BUDGET-NEXT: {{ $}}
+ ; CHECK-SMALL-BUDGET-NEXT: [[MOVi64imm5:%[0-9]+]]:gpr64 = MOVi64imm 105
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri25:%[0-9]+]]:gpr32common = ADDWri [[LDRWui5]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri26:%[0-9]+]]:gpr32common = ADDWri [[ADDWri25]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri27:%[0-9]+]]:gpr32common = ADDWri [[ADDWri26]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri28:%[0-9]+]]:gpr32common = ADDWri [[ADDWri27]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri29:%[0-9]+]]:gpr32common = ADDWri [[ADDWri28]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[SUBSWri5:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri29]], 6, 0, implicit-def $nzcv
+ ; CHECK-SMALL-BUDGET-NEXT: Bcc 15, %bb.8, implicit $nzcv
+ ; CHECK-SMALL-BUDGET-NEXT: B %bb.6
+ ; CHECK-SMALL-BUDGET-NEXT: {{ $}}
+ ; CHECK-SMALL-BUDGET-NEXT: bb.6:
+ ; CHECK-SMALL-BUDGET-NEXT: successors: %bb.8(0x40000000), %bb.7(0x40000000)
+ ; CHECK-SMALL-BUDGET-NEXT: {{ $}}
+ ; CHECK-SMALL-BUDGET-NEXT: [[MOVi64imm6:%[0-9]+]]:gpr64 = MOVi64imm 106
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri30:%[0-9]+]]:gpr32common = ADDWri [[LDRWui6]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri31:%[0-9]+]]:gpr32common = ADDWri [[ADDWri30]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri32:%[0-9]+]]:gpr32common = ADDWri [[ADDWri31]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri33:%[0-9]+]]:gpr32common = ADDWri [[ADDWri32]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri34:%[0-9]+]]:gpr32common = ADDWri [[ADDWri33]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[SUBSWri6:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri34]], 7, 0, implicit-def $nzcv
+ ; CHECK-SMALL-BUDGET-NEXT: Bcc 16, %bb.8, implicit $nzcv
+ ; CHECK-SMALL-BUDGET-NEXT: B %bb.7
+ ; CHECK-SMALL-BUDGET-NEXT: {{ $}}
+ ; CHECK-SMALL-BUDGET-NEXT: bb.7:
+ ; CHECK-SMALL-BUDGET-NEXT: successors: %bb.8(0x80000000)
+ ; CHECK-SMALL-BUDGET-NEXT: {{ $}}
+ ; CHECK-SMALL-BUDGET-NEXT: [[MOVi64imm7:%[0-9]+]]:gpr64 = MOVi64imm 107
+ ; CHECK-SMALL-BUDGET-NEXT: {{ $}}
+ ; CHECK-SMALL-BUDGET-NEXT: bb.8:
+ ; CHECK-SMALL-BUDGET-NEXT: [[PHI:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.0, [[MOVi64imm1]], %bb.1, [[MOVi64imm2]], %bb.2, [[MOVi64imm3]], %bb.3, [[MOVi64imm4]], %bb.4, [[MOVi64imm5]], %bb.5, [[MOVi64imm6]], %bb.6, [[MOVi64imm7]], %bb.7
+ ; CHECK-SMALL-BUDGET-NEXT: $x0 = COPY [[PHI]]
+ ; CHECK-SMALL-BUDGET-NEXT: RET_ReallyLR implicit $x0
+ ;
+ ; CHECK-LARGE-BUDGET-LABEL: name: cascade_load_to_cond_br_scan_budget
+ ; CHECK-LARGE-BUDGET: bb.0:
+ ; CHECK-LARGE-BUDGET-NEXT: successors: %bb.8(0x40000000), %bb.1(0x40000000)
+ ; CHECK-LARGE-BUDGET-NEXT: liveins: $x0
+ ; CHECK-LARGE-BUDGET-NEXT: {{ $}}
+ ; CHECK-LARGE-BUDGET-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; CHECK-LARGE-BUDGET-NEXT: [[LDRWui:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 0 :: (load (s32))
+ ; CHECK-LARGE-BUDGET-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 1
+ ; CHECK-LARGE-BUDGET-NEXT: [[MOVi32imm1:%[0-9]+]]:gpr32 = MOVi32imm 2
+ ; CHECK-LARGE-BUDGET-NEXT: [[MOVi32imm2:%[0-9]+]]:gpr32 = MOVi32imm 3
+ ; CHECK-LARGE-BUDGET-NEXT: [[MOVi32imm3:%[0-9]+]]:gpr32 = MOVi32imm 4
+ ; CHECK-LARGE-BUDGET-NEXT: [[LDRWui1:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 1 :: (load (s32))
+ ; CHECK-LARGE-BUDGET-NEXT: [[LDRWui2:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 2 :: (load (s32))
+ ; CHECK-LARGE-BUDGET-NEXT: [[LDRWui3:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 3 :: (load (s32))
+ ; CHECK-LARGE-BUDGET-NEXT: [[LDRWui4:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 4 :: (load (s32))
+ ; CHECK-LARGE-BUDGET-NEXT: [[LDRWui5:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 5 :: (load (s32))
+ ; CHECK-LARGE-BUDGET-NEXT: [[LDRWui6:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 6 :: (load (s32))
+ ; CHECK-LARGE-BUDGET-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 100
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri:%[0-9]+]]:gpr32common = ADDWri [[LDRWui]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri1:%[0-9]+]]:gpr32common = ADDWri [[ADDWri]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri2:%[0-9]+]]:gpr32common = ADDWri [[ADDWri1]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri3:%[0-9]+]]:gpr32common = ADDWri [[ADDWri2]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri4:%[0-9]+]]:gpr32common = ADDWri [[ADDWri3]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[SUBSWri:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri4]], 1, 0, implicit-def $nzcv
+ ; CHECK-LARGE-BUDGET-NEXT: Bcc 10, %bb.8, implicit $nzcv
+ ; CHECK-LARGE-BUDGET-NEXT: B %bb.1
+ ; CHECK-LARGE-BUDGET-NEXT: {{ $}}
+ ; CHECK-LARGE-BUDGET-NEXT: bb.1:
+ ; CHECK-LARGE-BUDGET-NEXT: successors: %bb.8(0x40000000), %bb.2(0x40000000)
+ ; CHECK-LARGE-BUDGET-NEXT: {{ $}}
+ ; CHECK-LARGE-BUDGET-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 101
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri5:%[0-9]+]]:gpr32common = ADDWri [[LDRWui1]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri6:%[0-9]+]]:gpr32common = ADDWri [[ADDWri5]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri7:%[0-9]+]]:gpr32common = ADDWri [[ADDWri6]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri8:%[0-9]+]]:gpr32common = ADDWri [[ADDWri7]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri9:%[0-9]+]]:gpr32common = ADDWri [[ADDWri8]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[SUBSWri1:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri9]], 2, 0, implicit-def $nzcv
+ ; CHECK-LARGE-BUDGET-NEXT: Bcc 11, %bb.8, implicit $nzcv
+ ; CHECK-LARGE-BUDGET-NEXT: B %bb.2
+ ; CHECK-LARGE-BUDGET-NEXT: {{ $}}
+ ; CHECK-LARGE-BUDGET-NEXT: bb.2:
+ ; CHECK-LARGE-BUDGET-NEXT: successors: %bb.8(0x40000000), %bb.3(0x40000000)
+ ; CHECK-LARGE-BUDGET-NEXT: {{ $}}
+ ; CHECK-LARGE-BUDGET-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 102
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri10:%[0-9]+]]:gpr32common = ADDWri [[LDRWui2]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri11:%[0-9]+]]:gpr32common = ADDWri [[ADDWri10]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri12:%[0-9]+]]:gpr32common = ADDWri [[ADDWri11]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri13:%[0-9]+]]:gpr32common = ADDWri [[ADDWri12]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri14:%[0-9]+]]:gpr32common = ADDWri [[ADDWri13]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[SUBSWri2:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri14]], 3, 0, implicit-def $nzcv
+ ; CHECK-LARGE-BUDGET-NEXT: Bcc 12, %bb.8, implicit $nzcv
+ ; CHECK-LARGE-BUDGET-NEXT: B %bb.3
+ ; CHECK-LARGE-BUDGET-NEXT: {{ $}}
+ ; CHECK-LARGE-BUDGET-NEXT: bb.3:
+ ; CHECK-LARGE-BUDGET-NEXT: successors: %bb.8(0x40000000), %bb.4(0x40000000)
+ ; CHECK-LARGE-BUDGET-NEXT: {{ $}}
+ ; CHECK-LARGE-BUDGET-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 103
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri15:%[0-9]+]]:gpr32common = ADDWri [[LDRWui3]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri16:%[0-9]+]]:gpr32common = ADDWri [[ADDWri15]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri17:%[0-9]+]]:gpr32common = ADDWri [[ADDWri16]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri18:%[0-9]+]]:gpr32common = ADDWri [[ADDWri17]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri19:%[0-9]+]]:gpr32common = ADDWri [[ADDWri18]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[SUBSWri3:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri19]], 4, 0, implicit-def $nzcv
+ ; CHECK-LARGE-BUDGET-NEXT: Bcc 13, %bb.8, implicit $nzcv
+ ; CHECK-LARGE-BUDGET-NEXT: B %bb.4
+ ; CHECK-LARGE-BUDGET-NEXT: {{ $}}
+ ; CHECK-LARGE-BUDGET-NEXT: bb.4:
+ ; CHECK-LARGE-BUDGET-NEXT: successors: %bb.8(0x40000000), %bb.5(0x40000000)
+ ; CHECK-LARGE-BUDGET-NEXT: {{ $}}
+ ; CHECK-LARGE-BUDGET-NEXT: [[MOVi64imm4:%[0-9]+]]:gpr64 = MOVi64imm 104
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri20:%[0-9]+]]:gpr32common = ADDWri [[LDRWui4]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri21:%[0-9]+]]:gpr32common = ADDWri [[ADDWri20]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri22:%[0-9]+]]:gpr32common = ADDWri [[ADDWri21]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri23:%[0-9]+]]:gpr32common = ADDWri [[ADDWri22]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri24:%[0-9]+]]:gpr32common = ADDWri [[ADDWri23]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[SUBSWri4:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri24]], 5, 0, implicit-def $nzcv
+ ; CHECK-LARGE-BUDGET-NEXT: Bcc 14, %bb.8, implicit $nzcv
+ ; CHECK-LARGE-BUDGET-NEXT: B %bb.5
+ ; CHECK-LARGE-BUDGET-NEXT: {{ $}}
+ ; CHECK-LARGE-BUDGET-NEXT: bb.5:
+ ; CHECK-LARGE-BUDGET-NEXT: successors: %bb.8(0x40000000), %bb.6(0x40000000)
+ ; CHECK-LARGE-BUDGET-NEXT: {{ $}}
+ ; CHECK-LARGE-BUDGET-NEXT: [[MOVi64imm5:%[0-9]+]]:gpr64 = MOVi64imm 105
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri25:%[0-9]+]]:gpr32common = ADDWri [[LDRWui5]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri26:%[0-9]+]]:gpr32common = ADDWri [[ADDWri25]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri27:%[0-9]+]]:gpr32common = ADDWri [[ADDWri26]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri28:%[0-9]+]]:gpr32common = ADDWri [[ADDWri27]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri29:%[0-9]+]]:gpr32common = ADDWri [[ADDWri28]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[SUBSWri5:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri29]], 6, 0, implicit-def $nzcv
+ ; CHECK-LARGE-BUDGET-NEXT: Bcc 15, %bb.8, implicit $nzcv
+ ; CHECK-LARGE-BUDGET-NEXT: B %bb.6
+ ; CHECK-LARGE-BUDGET-NEXT: {{ $}}
+ ; CHECK-LARGE-BUDGET-NEXT: bb.6:
+ ; CHECK-LARGE-BUDGET-NEXT: successors: %bb.8(0x40000000), %bb.7(0x40000000)
+ ; CHECK-LARGE-BUDGET-NEXT: {{ $}}
+ ; CHECK-LARGE-BUDGET-NEXT: [[MOVi64imm6:%[0-9]+]]:gpr64 = MOVi64imm 106
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri30:%[0-9]+]]:gpr32common = ADDWri [[LDRWui6]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri31:%[0-9]+]]:gpr32common = ADDWri [[ADDWri30]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri32:%[0-9]+]]:gpr32common = ADDWri [[ADDWri31]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri33:%[0-9]+]]:gpr32common = ADDWri [[ADDWri32]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri34:%[0-9]+]]:gpr32common = ADDWri [[ADDWri33]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[SUBSWri6:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri34]], 7, 0, implicit-def $nzcv
+ ; CHECK-LARGE-BUDGET-NEXT: Bcc 16, %bb.8, implicit $nzcv
+ ; CHECK-LARGE-BUDGET-NEXT: B %bb.7
+ ; CHECK-LARGE-BUDGET-NEXT: {{ $}}
+ ; CHECK-LARGE-BUDGET-NEXT: bb.7:
+ ; CHECK-LARGE-BUDGET-NEXT: successors: %bb.8(0x80000000)
+ ; CHECK-LARGE-BUDGET-NEXT: {{ $}}
+ ; CHECK-LARGE-BUDGET-NEXT: [[MOVi64imm7:%[0-9]+]]:gpr64 = MOVi64imm 107
+ ; CHECK-LARGE-BUDGET-NEXT: {{ $}}
+ ; CHECK-LARGE-BUDGET-NEXT: bb.8:
+ ; CHECK-LARGE-BUDGET-NEXT: [[PHI:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.0, [[MOVi64imm1]], %bb.1, [[MOVi64imm2]], %bb.2, [[MOVi64imm3]], %bb.3, [[MOVi64imm4]], %bb.4, [[MOVi64imm5]], %bb.5, [[MOVi64imm6]], %bb.6, [[MOVi64imm7]], %bb.7
+ ; CHECK-LARGE-BUDGET-NEXT: $x0 = COPY [[PHI]]
+ ; CHECK-LARGE-BUDGET-NEXT: RET_ReallyLR implicit $x0
+ ;
+ ; CHECK-CASCADE-BUDGET-LABEL: name: cascade_load_to_cond_br_scan_budget
+ ; CHECK-CASCADE-BUDGET: bb.0:
+ ; CHECK-CASCADE-BUDGET-NEXT: liveins: $x0
+ ; CHECK-CASCADE-BUDGET-NEXT: {{ $}}
+ ; CHECK-CASCADE-BUDGET-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[LDRWui:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 0 :: (load (s32))
+ ; CHECK-CASCADE-BUDGET-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 1
+ ; CHECK-CASCADE-BUDGET-NEXT: [[MOVi32imm1:%[0-9]+]]:gpr32 = MOVi32imm 2
+ ; CHECK-CASCADE-BUDGET-NEXT: [[MOVi32imm2:%[0-9]+]]:gpr32 = MOVi32imm 3
+ ; CHECK-CASCADE-BUDGET-NEXT: [[MOVi32imm3:%[0-9]+]]:gpr32 = MOVi32imm 4
+ ; CHECK-CASCADE-BUDGET-NEXT: [[LDRWui1:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 1 :: (load (s32))
+ ; CHECK-CASCADE-BUDGET-NEXT: [[LDRWui2:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 2 :: (load (s32))
+ ; CHECK-CASCADE-BUDGET-NEXT: [[LDRWui3:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 3 :: (load (s32))
+ ; CHECK-CASCADE-BUDGET-NEXT: [[LDRWui4:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 4 :: (load (s32))
+ ; CHECK-CASCADE-BUDGET-NEXT: [[LDRWui5:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 5 :: (load (s32))
+ ; CHECK-CASCADE-BUDGET-NEXT: [[LDRWui6:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 6 :: (load (s32))
+ ; CHECK-CASCADE-BUDGET-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 100
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri:%[0-9]+]]:gpr32common = ADDWri [[LDRWui]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri1:%[0-9]+]]:gpr32common = ADDWri [[ADDWri]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri2:%[0-9]+]]:gpr32common = ADDWri [[ADDWri1]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri3:%[0-9]+]]:gpr32common = ADDWri [[ADDWri2]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri4:%[0-9]+]]:gpr32common = ADDWri [[ADDWri3]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 101
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri5:%[0-9]+]]:gpr32common = ADDWri [[LDRWui1]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri6:%[0-9]+]]:gpr32common = ADDWri [[ADDWri5]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri7:%[0-9]+]]:gpr32common = ADDWri [[ADDWri6]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri8:%[0-9]+]]:gpr32common = ADDWri [[ADDWri7]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri9:%[0-9]+]]:gpr32common = ADDWri [[ADDWri8]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 102
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri10:%[0-9]+]]:gpr32common = ADDWri [[LDRWui2]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri11:%[0-9]+]]:gpr32common = ADDWri [[ADDWri10]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri12:%[0-9]+]]:gpr32common = ADDWri [[ADDWri11]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri13:%[0-9]+]]:gpr32common = ADDWri [[ADDWri12]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri14:%[0-9]+]]:gpr32common = ADDWri [[ADDWri13]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 103
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri15:%[0-9]+]]:gpr32common = ADDWri [[LDRWui3]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri16:%[0-9]+]]:gpr32common = ADDWri [[ADDWri15]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri17:%[0-9]+]]:gpr32common = ADDWri [[ADDWri16]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri18:%[0-9]+]]:gpr32common = ADDWri [[ADDWri17]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri19:%[0-9]+]]:gpr32common = ADDWri [[ADDWri18]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[MOVi64imm4:%[0-9]+]]:gpr64 = MOVi64imm 104
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri20:%[0-9]+]]:gpr32common = ADDWri [[LDRWui4]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri21:%[0-9]+]]:gpr32common = ADDWri [[ADDWri20]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri22:%[0-9]+]]:gpr32common = ADDWri [[ADDWri21]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri23:%[0-9]+]]:gpr32common = ADDWri [[ADDWri22]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri24:%[0-9]+]]:gpr32common = ADDWri [[ADDWri23]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[MOVi64imm5:%[0-9]+]]:gpr64 = MOVi64imm 105
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri25:%[0-9]+]]:gpr32common = ADDWri [[LDRWui5]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri26:%[0-9]+]]:gpr32common = ADDWri [[ADDWri25]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri27:%[0-9]+]]:gpr32common = ADDWri [[ADDWri26]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri28:%[0-9]+]]:gpr32common = ADDWri [[ADDWri27]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri29:%[0-9]+]]:gpr32common = ADDWri [[ADDWri28]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[MOVi64imm6:%[0-9]+]]:gpr64 = MOVi64imm 106
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri30:%[0-9]+]]:gpr32common = ADDWri [[LDRWui6]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri31:%[0-9]+]]:gpr32common = ADDWri [[ADDWri30]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri32:%[0-9]+]]:gpr32common = ADDWri [[ADDWri31]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri33:%[0-9]+]]:gpr32common = ADDWri [[ADDWri32]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri34:%[0-9]+]]:gpr32common = ADDWri [[ADDWri33]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[SUBSWri:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri34]], 7, 0, implicit-def $nzcv
+ ; CHECK-CASCADE-BUDGET-NEXT: [[MOVi64imm7:%[0-9]+]]:gpr64 = MOVi64imm 107
+ ; CHECK-CASCADE-BUDGET-NEXT: [[CSELXr:%[0-9]+]]:gpr64 = CSELXr [[MOVi64imm6]], [[MOVi64imm7]], 16, implicit $nzcv
+ ; CHECK-CASCADE-BUDGET-NEXT: [[SUBSWri1:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri29]], 6, 0, implicit-def $nzcv
+ ; CHECK-CASCADE-BUDGET-NEXT: [[CSELXr1:%[0-9]+]]:gpr64 = CSELXr [[MOVi64imm5]], [[CSELXr]], 15, implicit $nzcv
+ ; CHECK-CASCADE-BUDGET-NEXT: [[SUBSWri2:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri24]], 5, 0, implicit-def $nzcv
+ ; CHECK-CASCADE-BUDGET-NEXT: [[CSELXr2:%[0-9]+]]:gpr64 = CSELXr [[MOVi64imm4]], [[CSELXr1]], 14, implicit $nzcv
+ ; CHECK-CASCADE-BUDGET-NEXT: [[SUBSWri3:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri19]], 4, 0, implicit-def $nzcv
+ ; CHECK-CASCADE-BUDGET-NEXT: [[CSELXr3:%[0-9]+]]:gpr64 = CSELXr [[MOVi64imm3]], [[CSELXr2]], 13, implicit $nzcv
+ ; CHECK-CASCADE-BUDGET-NEXT: [[SUBSWri4:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri14]], 3, 0, implicit-def $nzcv
+ ; CHECK-CASCADE-BUDGET-NEXT: [[CSELXr4:%[0-9]+]]:gpr64 = CSELXr [[MOVi64imm2]], [[CSELXr3]], 12, implicit $nzcv
+ ; CHECK-CASCADE-BUDGET-NEXT: [[SUBSWri5:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri9]], 2, 0, implicit-def $nzcv
+ ; CHECK-CASCADE-BUDGET-NEXT: [[CSELXr5:%[0-9]+]]:gpr64 = CSELXr [[MOVi64imm1]], [[CSELXr4]], 11, implicit $nzcv
+ ; CHECK-CASCADE-BUDGET-NEXT: [[SUBSWri6:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri4]], 1, 0, implicit-def $nzcv
+ ; CHECK-CASCADE-BUDGET-NEXT: [[CSELXr6:%[0-9]+]]:gpr64 = CSELXr [[MOVi64imm]], [[CSELXr5]], 10, implicit $nzcv
+ ; CHECK-CASCADE-BUDGET-NEXT: $x0 = COPY [[CSELXr6]]
+ ; CHECK-CASCADE-BUDGET-NEXT: RET_ReallyLR implicit $x0
+ bb.0:
+ successors: %bb.8(0x40000000), %bb.1(0x40000000)
+ liveins: $x0
+
+ %0:gpr64common = COPY $x0
+ %1:gpr32common = LDRWui %0, 0 :: (load (s32))
+ %200:gpr32 = MOVi32imm 1
+ %201:gpr32 = MOVi32imm 2
+ %202:gpr32 = MOVi32imm 3
+ %203:gpr32 = MOVi32imm 4
+ %2:gpr32common = LDRWui %0, 1 :: (load (s32))
+ %3:gpr32common = LDRWui %0, 2 :: (load (s32))
+ %4:gpr32common = LDRWui %0, 3 :: (load (s32))
+ %5:gpr32common = LDRWui %0, 4 :: (load (s32))
+ %6:gpr32common = LDRWui %0, 5 :: (load (s32))
+ %7:gpr32common = LDRWui %0, 6 :: (load (s32))
+ %50:gpr64 = MOVi64imm 100
+ %204:gpr32common = ADDWri %1, 1, 0
+ %205:gpr32common = ADDWri %204, 1, 0
+ %206:gpr32common = ADDWri %205, 1, 0
+ %207:gpr32common = ADDWri %206, 1, 0
+ %208:gpr32common = ADDWri %207, 1, 0
+ %209:gpr32 = SUBSWri %208, 1, 0, implicit-def $nzcv
+ Bcc 10, %bb.8, implicit $nzcv
+ B %bb.1
+
+ bb.1:
+ successors: %bb.8(0x40000000), %bb.2(0x40000000)
+
+ %51:gpr64 = MOVi64imm 101
+ %210:gpr32common = ADDWri %2, 1, 0
+ %211:gpr32common = ADDWri %210, 1, 0
+ %212:gpr32common = ADDWri %211, 1, 0
+ %213:gpr32common = ADDWri %212, 1, 0
+ %214:gpr32common = ADDWri %213, 1, 0
+ %215:gpr32 = SUBSWri %214, 2, 0, implicit-def $nzcv
+ Bcc 11, %bb.8, implicit $nzcv
+ B %bb.2
+
+ bb.2:
+ successors: %bb.8(0x40000000), %bb.3(0x40000000)
+
+ %52:gpr64 = MOVi64imm 102
+ %216:gpr32common = ADDWri %3, 1, 0
+ %217:gpr32common = ADDWri %216, 1, 0
+ %218:gpr32common = ADDWri %217, 1, 0
+ %219:gpr32common = ADDWri %218, 1, 0
+ %220:gpr32common = ADDWri %219, 1, 0
+ %221:gpr32 = SUBSWri %220, 3, 0, implicit-def $nzcv
+ Bcc 12, %bb.8, implicit $nzcv
+ B %bb.3
+
+ bb.3:
+ successors: %bb.8(0x40000000), %bb.4(0x40000000)
+
+ %53:gpr64 = MOVi64imm 103
+ %222:gpr32common = ADDWri %4, 1, 0
+ %223:gpr32common = ADDWri %222, 1, 0
+ %224:gpr32common = ADDWri %223, 1, 0
+ %225:gpr32common = ADDWri %224, 1, 0
+ %226:gpr32common = ADDWri %225, 1, 0
+ %227:gpr32 = SUBSWri %226, 4, 0, implicit-def $nzcv
+ Bcc 13, %bb.8, implicit $nzcv
+ B %bb.4
+
+ bb.4:
+ successors: %bb.8(0x40000000), %bb.5(0x40000000)
+
+ %54:gpr64 = MOVi64imm 104
+ %228:gpr32common = ADDWri %5, 1, 0
+ %229:gpr32common = ADDWri %228, 1, 0
+ %230:gpr32common = ADDWri %229, 1, 0
+ %231:gpr32common = ADDWri %230, 1, 0
+ %232:gpr32common = ADDWri %231, 1, 0
+ %233:gpr32 = SUBSWri %232, 5, 0, implicit-def $nzcv
+ Bcc 14, %bb.8, implicit $nzcv
+ B %bb.5
+
+ bb.5:
+ successors: %bb.8(0x40000000), %bb.6(0x40000000)
+
+ %55:gpr64 = MOVi64imm 105
+ %234:gpr32common = ADDWri %6, 1, 0
+ %235:gpr32common = ADDWri %234, 1, 0
+ %236:gpr32common = ADDWri %235, 1, 0
+ %237:gpr32common = ADDWri %236, 1, 0
+ %238:gpr32common = ADDWri %237, 1, 0
+ %239:gpr32 = SUBSWri %238, 6, 0, implicit-def $nzcv
+ Bcc 15, %bb.8, implicit $nzcv
+ B %bb.6
+
+ bb.6:
+ successors: %bb.8(0x40000000), %bb.7(0x40000000)
+
+ %56:gpr64 = MOVi64imm 106
+ %240:gpr32common = ADDWri %7, 1, 0
+ %241:gpr32common = ADDWri %240, 1, 0
+ %242:gpr32common = ADDWri %241, 1, 0
+ %243:gpr32common = ADDWri %242, 1, 0
+ %244:gpr32common = ADDWri %243, 1, 0
+ %245:gpr32 = SUBSWri %244, 7, 0, implicit-def $nzcv
+ Bcc 16, %bb.8, implicit $nzcv
+ B %bb.7
+
+ bb.7:
+ successors: %bb.8(0x80000000)
+
+ %57:gpr64 = MOVi64imm 107
+
+ bb.8:
+ %90:gpr64 = PHI %50, %bb.0, %51, %bb.1, %52, %bb.2, %53, %bb.3, %54, %bb.4, %55, %bb.5, %56, %bb.6, %57, %bb.7
+ $x0 = COPY %90
+ RET_ReallyLR implicit $x0
+...
diff --git a/llvm/test/CodeGen/AArch64/early-ifcvt-remarks.mir b/llvm/test/CodeGen/AArch64/early-ifcvt-remarks.mir
index 6f01d48601fa2..7d4dd2e9f0af6 100644
--- a/llvm/test/CodeGen/AArch64/early-ifcvt-remarks.mir
+++ b/llvm/test/CodeGen/AArch64/early-ifcvt-remarks.mir
@@ -2,11 +2,44 @@
# RUN: -enable-early-ifcvt-data-dependent \
# RUN: -pass-remarks-analysis='early-ifcvt' -o /dev/null %s 2>&1 | FileCheck %s
+# RUN: llc -mtriple=aarch64 -mcpu=apple-m1 -run-pass=early-ifcvt \
+# RUN: -enable-early-ifcvt-cascade -enable-early-ifcvt-data-dependent \
+# RUN: -pass-remarks='early-ifcvt' -o /dev/null %s 2>&1 | FileCheck %s --check-prefix=CASCADE-SUCCESS \
+# RUN: --implicit-check-not="performing cascade if-conversion"
+
+# RUN: llc -mtriple=aarch64 -mcpu=apple-m1 -run-pass=early-ifcvt \
+# RUN: -enable-early-ifcvt-cascade -enable-early-ifcvt-data-dependent \
+# RUN: -pass-remarks-missed='early-ifcvt' -o /dev/null %s 2>&1 | FileCheck %s --check-prefix=CASCADE-MISSED
+
+# RUN: llc -mtriple=aarch64 -mcpu=apple-m1 -run-pass=early-ifcvt \
+# RUN: -enable-early-ifcvt-cascade \
+# RUN: -pass-remarks='early-ifcvt' -o /dev/null %s 2>&1 \
+# RUN: | FileCheck %s --check-prefix=CASCADE-NO-DD --implicit-check-not="cascade if-conversion"
+
+# RUN: llc -mtriple=aarch64 -mcpu=apple-m1 -run-pass=early-ifcvt \
+# RUN: -enable-early-ifcvt-cascade -enable-early-ifcvt-data-dependent \
+# RUN: -early-ifcvt-limit=3 -pass-remarks='early-ifcvt' \
+# RUN: -pass-remarks-missed='early-ifcvt' -o /dev/null %s 2>&1 \
+# RUN: | FileCheck %s --check-prefix=CASCADE-OVER-LIMIT \
+# RUN: --implicit-check-not="performing cascade if-conversion"
+
# Test that we emit an analysis remark when the branch condition is data-dependent
# (derived from a memory load).
# CHECK: remark: <unknown>:0:0: branch condition is data-dependent (from memory load), using higher CritLimit of {{[0-9]+}} cycles
+# Test cascade conversion success remark.
+# CASCADE-SUCCESS: remark: <unknown>:0:0: performing cascade if-conversion on 3 blocks: critical path extension is {{[0-9]+}} cycle
+
+# CASCADE-MISSED: remark: <unknown>:0:0: did not if-convert cascade with 5 blocks: critical path extension of 21 cycles exceeds the threshold of 20 cycles.
+# CASCADE-MISSED: remark: <unknown>:0:0: did not if-convert cascade with 3 blocks: critical path extension of 30 cycles exceeds the threshold of 12 cycles.
+
+# CASCADE-NO-DD: remark: <unknown>:0:0: performing if-conversion on branch: {{.*}} staying under the threshold of 8 cycles.
+
+# CASCADE-OVER-LIMIT: remark: <unknown>:0:0: did not if-convert cascade with 3 blocks: the speculated blocks would hold 10 instructions, exceeding the limit of 3.
+# CASCADE-OVER-LIMIT: remark: <unknown>:0:0: did not if-convert cascade with 5 blocks: the speculated blocks would hold 14 instructions, exceeding the limit of 3.
+# CASCADE-OVER-LIMIT: remark: <unknown>:0:0: did not if-convert cascade with 3 blocks: the speculated blocks would hold 11 instructions, exceeding the limit of 3.
+
--- |
define i32 @data_dependent_branch(ptr %p, i32 %x, i32 %y) {
ret i32 0
@@ -17,6 +50,9 @@
define i64 @cascade_increasing_cond_depth(ptr %p) {
ret i64 0
}
+ define i64 @cascade_mixed_reg_classes(ptr %p) {
+ ret i64 0
+ }
...
---
name: data_dependent_branch
@@ -202,3 +238,64 @@ body: |
$x0 = COPY %90
RET_ReallyLR implicit $x0
...
+---
+name: cascade_mixed_reg_classes
+alignment: 4
+tracksRegLiveness: true
+body: |
+ bb.0:
+ successors: %bb.1(0x80000000)
+ liveins: $x0, $d0, $d1, $d2, $d3
+
+ %0:gpr64common = COPY $x0
+ %5:fpr64 = COPY $d0
+ %6:fpr64 = COPY $d1
+ %7:fpr64 = COPY $d2
+ %8:fpr64 = COPY $d3
+
+ bb.1:
+ successors: %bb.2(0x40000000), %bb.5(0x40000000)
+
+ %10:gpr32 = LDRWui %0, 0 :: (load (s32))
+ %11:gpr32 = MADDWrrr %10, %10, $wzr
+ %12:gpr32 = MADDWrrr %11, %11, $wzr
+ %20:gpr32 = ADDWrr %12, %12
+ %21:gpr32 = ADDWrr %20, %20
+ %40:gpr64 = MOVi64imm 100
+ %30:gpr32 = SUBSWrr %12, %12, implicit-def $nzcv
+ Bcc 10, %bb.5, implicit $nzcv
+ B %bb.2
+
+ bb.2:
+ successors: %bb.3(0x40000000), %bb.5(0x40000000)
+
+ %41:gpr64 = MOVi64imm 200
+ %31:gpr32 = SUBSWrr %20, %20, implicit-def $nzcv
+ Bcc 10, %bb.5, implicit $nzcv
+ B %bb.3
+
+ bb.3:
+ successors: %bb.4(0x40000000), %bb.5(0x40000000)
+
+ %42:gpr64 = MOVi64imm 300
+ %32:gpr32 = SUBSWrr %21, %21, implicit-def $nzcv
+ Bcc 10, %bb.5, implicit $nzcv
+ B %bb.4
+
+ bb.4:
+ successors: %bb.5(0x80000000)
+
+ %43:gpr64 = MOVi64imm 400
+
+ bb.5:
+ %90:gpr64 = PHI %40, %bb.1, %41, %bb.2, %42, %bb.3, %43, %bb.4
+ %91:fpr64 = PHI %5, %bb.1, %6, %bb.2, %7, %bb.3, %8, %bb.4
+ %60:fpr64 = FMULDrr %91, %91, implicit $fpcr
+ %61:fpr64 = FMULDrr %60, %60, implicit $fpcr
+ %62:fpr64 = FMULDrr %61, %61, implicit $fpcr
+ %63:fpr64 = FMULDrr %62, %62, implicit $fpcr
+ %64:fpr64 = FMULDrr %63, %63, implicit $fpcr
+ $x0 = COPY %90
+ $d0 = COPY %64
+ RET_ReallyLR implicit $x0, implicit $d0
+...
More information about the llvm-commits
mailing list