[llvm] [EarlyIfConversion] Add cascading conditional branches as a conversion pattern (PR #218415)
Jonathan Cohen via llvm-commits
llvm-commits at lists.llvm.org
Thu Sep 24 00:48:18 PDT 2026
https://github.com/jcohen-apple updated https://github.com/llvm/llvm-project/pull/218415
>From c812eb67af02afc2b0f957853d5f7e4dff956313 Mon Sep 17 00:00:00 2001
From: Jonathan Cohen <joncoh at apple.com>
Date: Thu, 16 Apr 2026 15:29:47 +0300
Subject: [PATCH 1/3] [EarlyIfConversion] Unit tests documenting current state
before cascading conversion logic
---
.../CodeGen/AArch64/early-ifcvt-cascade.mir | 843 ++++++++++++++++++
.../CodeGen/AArch64/early-ifcvt-remarks.mir | 159 ++++
2 files changed, 1002 insertions(+)
create mode 100644 llvm/test/CodeGen/AArch64/early-ifcvt-cascade.mir
diff --git a/llvm/test/CodeGen/AArch64/early-ifcvt-cascade.mir b/llvm/test/CodeGen/AArch64/early-ifcvt-cascade.mir
new file mode 100644
index 0000000000000..99c0ea79267a1
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/early-ifcvt-cascade.mir
@@ -0,0 +1,843 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=aarch64 -mcpu=apple-m1 -run-pass=early-ifcvt -enable-early-ifcvt-data-dependent %s -o - | FileCheck %s
+
+--- |
+ define void @cascade_selector_loop() { ret void }
+ define void @cascade_branches_from_args_not_dd() { ret void }
+ define void @cascade_exceeds_crit_limit() { ret void }
+ define void @cascade_backedge_to_earlier_block() { ret void }
+ define void @cascade_forward_skip_to_later_block() { ret void }
+...
+---
+name: cascade_selector_loop
+tracksRegLiveness: true
+liveins:
+ - { reg: '$x0', virtual-reg: '%0' }
+ - { reg: '$x1', virtual-reg: '%1' }
+ - { reg: '$w2', virtual-reg: '%2' }
+ - { reg: '$w3', virtual-reg: '%3' }
+ - { reg: '$w4', virtual-reg: '%4' }
+body: |
+ ; CHECK-LABEL: name: cascade_selector_loop
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.10(0x30000000), %bb.1(0x50000000)
+ ; CHECK-NEXT: liveins: $x0, $x1, $w2, $w3, $w4
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr32 = COPY $w2
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gpr32 = COPY $w3
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:gpr32 = COPY $w4
+ ; CHECK-NEXT: [[COPY5:%[0-9]+]]:gpr64all = COPY $xzr
+ ; CHECK-NEXT: CBZW [[COPY2]], %bb.10
+ ; CHECK-NEXT: B %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.2(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 0 :: (load (s32))
+ ; CHECK-NEXT: [[LDRWui1:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 1 :: (load (s32))
+ ; CHECK-NEXT: [[LDRWui2:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 2 :: (load (s32))
+ ; CHECK-NEXT: [[LDRWui3:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 3 :: (load (s32))
+ ; CHECK-NEXT: [[LDRWui4:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 4 :: (load (s32))
+ ; CHECK-NEXT: [[LDRWui5:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 5 :: (load (s32))
+ ; CHECK-NEXT: [[LDRWui6:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 6 :: (load (s32))
+ ; CHECK-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: successors: %bb.3(0x40000000), %bb.9(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[PHI:%[0-9]+]]:gpr32 = PHI [[COPY2]], %bb.1, %15, %bb.9
+ ; CHECK-NEXT: [[LDRWui7:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load (s32))
+ ; CHECK-NEXT: [[MADDWrrr:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[LDRWui7]], $wzr
+ ; CHECK-NEXT: [[MADDWrrr1:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY4]], [[LDRWui7]], [[MADDWrrr]]
+ ; CHECK-NEXT: [[MADDWrrr2:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[LDRWui7]], [[MADDWrrr1]]
+ ; CHECK-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 7
+ ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; CHECK-NEXT: B %bb.3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.3:
+ ; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.9(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui1]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 6
+ ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; CHECK-NEXT: B %bb.4
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.4:
+ ; CHECK-NEXT: successors: %bb.5(0x40000000), %bb.9(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[SUBSWrr2:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui2]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 5
+ ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; CHECK-NEXT: B %bb.5
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.5:
+ ; CHECK-NEXT: successors: %bb.6(0x40000000), %bb.9(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[SUBSWrr3:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui3]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 4
+ ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; CHECK-NEXT: B %bb.6
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.6:
+ ; CHECK-NEXT: successors: %bb.7(0x40000000), %bb.9(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[SUBSWrr4:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui4]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm4:%[0-9]+]]:gpr64 = MOVi64imm 3
+ ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; CHECK-NEXT: B %bb.7
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.7:
+ ; CHECK-NEXT: successors: %bb.8(0x40000000), %bb.9(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[SUBSWrr5:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui5]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm5:%[0-9]+]]:gpr64 = MOVi64imm 2
+ ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; CHECK-NEXT: B %bb.8
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.8:
+ ; CHECK-NEXT: successors: %bb.9(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[MOVi64imm6:%[0-9]+]]:gpr64 = MOVi64imm 1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.9:
+ ; CHECK-NEXT: successors: %bb.10(0x04000000), %bb.2(0x7c000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[PHI1:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.2, [[MOVi64imm1]], %bb.3, [[MOVi64imm2]], %bb.4, [[MOVi64imm3]], %bb.5, [[MOVi64imm4]], %bb.6, [[MOVi64imm5]], %bb.7, [[MOVi64imm6]], %bb.8
+ ; CHECK-NEXT: [[SUBSWrr6:%[0-9]+]]:gpr32 = SUBSWrr [[PHI]], [[MOVi32imm]], implicit-def $nzcv
+ ; CHECK-NEXT: Bcc 1, %bb.2, implicit $nzcv
+ ; CHECK-NEXT: B %bb.10
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.10:
+ ; CHECK-NEXT: [[PHI2:%[0-9]+]]:gpr64 = PHI [[COPY5]], %bb.0, [[PHI1]], %bb.9
+ ; CHECK-NEXT: $x0 = COPY [[PHI2]]
+ ; CHECK-NEXT: RET_ReallyLR implicit $x0
+ bb.0:
+ successors: %bb.10(0x30000000), %bb.1(0x50000000)
+ liveins: $x0, $x1, $w2, $w3, $w4
+
+ %0:gpr64common = COPY $x0
+ %1:gpr64common = COPY $x1
+ %2:gpr32 = COPY $w2
+ %3:gpr32 = COPY $w3
+ %4:gpr32 = COPY $w4
+ %99:gpr64all = COPY $xzr
+ CBZW %2, %bb.10
+ B %bb.1
+
+ bb.1:
+ successors: %bb.2(0x80000000)
+
+ %10:gpr32 = LDRWui %1, 0 :: (load (s32))
+ %11:gpr32 = LDRWui %1, 1 :: (load (s32))
+ %12:gpr32 = LDRWui %1, 2 :: (load (s32))
+ %13:gpr32 = LDRWui %1, 3 :: (load (s32))
+ %14:gpr32 = LDRWui %1, 4 :: (load (s32))
+ %15:gpr32 = LDRWui %1, 5 :: (load (s32))
+ %16:gpr32 = LDRWui %1, 6 :: (load (s32))
+ %17:gpr32 = MOVi32imm 1
+
+ bb.2:
+ successors: %bb.3(0x40000000), %bb.9(0x40000000)
+
+ %50:gpr32 = PHI %2, %bb.1, %51, %bb.9
+ %20:gpr32 = LDRWui %0, 0 :: (load (s32))
+ %31:gpr32 = nsw MADDWrrr %3, %20, $wzr
+ %32:gpr32 = nsw MADDWrrr %4, %20, %31
+ %30:gpr32 = nsw MADDWrrr %3, %20, %32
+ %21:gpr32 = SUBSWrr %30, %10, implicit-def $nzcv
+ %40:gpr64 = MOVi64imm 7
+ Bcc 10, %bb.9, implicit $nzcv
+ B %bb.3
+
+ bb.3:
+ successors: %bb.4(0x40000000), %bb.9(0x40000000)
+
+ %22:gpr32 = SUBSWrr %30, %11, implicit-def $nzcv
+ %41:gpr64 = MOVi64imm 6
+ Bcc 10, %bb.9, implicit $nzcv
+ B %bb.4
+
+ bb.4:
+ successors: %bb.5(0x40000000), %bb.9(0x40000000)
+
+ %23:gpr32 = SUBSWrr %30, %12, implicit-def $nzcv
+ %42:gpr64 = MOVi64imm 5
+ Bcc 10, %bb.9, implicit $nzcv
+ B %bb.5
+
+ bb.5:
+ successors: %bb.6(0x40000000), %bb.9(0x40000000)
+
+ %24:gpr32 = SUBSWrr %30, %13, implicit-def $nzcv
+ %43:gpr64 = MOVi64imm 4
+ Bcc 10, %bb.9, implicit $nzcv
+ B %bb.6
+
+ bb.6:
+ successors: %bb.7(0x40000000), %bb.9(0x40000000)
+
+ %25:gpr32 = SUBSWrr %30, %14, implicit-def $nzcv
+ %44:gpr64 = MOVi64imm 3
+ Bcc 10, %bb.9, implicit $nzcv
+ B %bb.7
+
+ bb.7:
+ successors: %bb.8(0x40000000), %bb.9(0x40000000)
+
+ %26:gpr32 = SUBSWrr %30, %15, implicit-def $nzcv
+ %45:gpr64 = MOVi64imm 2
+ Bcc 10, %bb.9, implicit $nzcv
+ B %bb.8
+
+ bb.8:
+ successors: %bb.9(0x80000000)
+
+ %47:gpr64 = MOVi64imm 1
+
+ bb.9:
+ successors: %bb.10(0x04000000), %bb.2(0x7c000000)
+
+ %90:gpr64 = PHI %40, %bb.2, %41, %bb.3, %42, %bb.4, %43, %bb.5, %44, %bb.6, %45, %bb.7, %47, %bb.8
+ %51:gpr32 = SUBSWrr %50, %17, implicit-def $nzcv
+ Bcc 1, %bb.2, implicit $nzcv
+ B %bb.10
+
+ bb.10:
+ %95:gpr64 = PHI %99, %bb.0, %90, %bb.9
+ $x0 = COPY %95
+ RET_ReallyLR implicit $x0
+...
+---
+name: cascade_branches_from_args_not_dd
+tracksRegLiveness: true
+liveins:
+ - { reg: '$x1', virtual-reg: '%1' }
+ - { reg: '$w2', virtual-reg: '%2' }
+ - { reg: '$w3', virtual-reg: '%3' }
+ - { reg: '$w4', virtual-reg: '%4' }
+ - { reg: '$w5', virtual-reg: '%5' }
+body: |
+ ; CHECK-LABEL: name: cascade_branches_from_args_not_dd
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.10(0x30000000), %bb.1(0x50000000)
+ ; CHECK-NEXT: liveins: $x1, $w2, $w3, $w4, $w5
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x1
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr32 = COPY $w2
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr32 = COPY $w3
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gpr32 = COPY $w4
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:gpr32 = COPY $w5
+ ; CHECK-NEXT: [[COPY5:%[0-9]+]]:gpr64all = COPY $xzr
+ ; CHECK-NEXT: CBZW [[COPY1]], %bb.10
+ ; CHECK-NEXT: B %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.2(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY6:%[0-9]+]]:gpr32 = COPY [[COPY2]]
+ ; CHECK-NEXT: [[COPY7:%[0-9]+]]:gpr32 = COPY [[COPY3]]
+ ; CHECK-NEXT: [[COPY8:%[0-9]+]]:gpr32 = COPY [[COPY4]]
+ ; CHECK-NEXT: [[COPY9:%[0-9]+]]:gpr32 = COPY [[COPY2]]
+ ; CHECK-NEXT: [[COPY10:%[0-9]+]]:gpr32 = COPY [[COPY3]]
+ ; CHECK-NEXT: [[COPY11:%[0-9]+]]:gpr32 = COPY [[COPY4]]
+ ; CHECK-NEXT: [[COPY12:%[0-9]+]]:gpr32 = COPY [[COPY2]]
+ ; CHECK-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: successors: %bb.3(0x40000000), %bb.9(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[PHI:%[0-9]+]]:gpr32 = PHI [[COPY1]], %bb.1, %15, %bb.9
+ ; CHECK-NEXT: [[MADDWrrr:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY2]], [[COPY4]], $wzr
+ ; CHECK-NEXT: [[MADDWrrr1:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[COPY4]], [[MADDWrrr]]
+ ; CHECK-NEXT: [[MADDWrrr2:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY2]], [[COPY3]], [[MADDWrrr1]]
+ ; CHECK-NEXT: [[MADDWrrr3:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[COPY2]], [[MADDWrrr2]]
+ ; CHECK-NEXT: [[MADDWrrr4:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY4]], [[COPY3]], [[MADDWrrr3]]
+ ; CHECK-NEXT: [[MADDWrrr5:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY2]], [[COPY4]], [[MADDWrrr4]]
+ ; CHECK-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr5]], [[COPY6]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 7
+ ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; CHECK-NEXT: B %bb.3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.3:
+ ; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.9(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr5]], [[COPY7]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 6
+ ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; CHECK-NEXT: B %bb.4
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.4:
+ ; CHECK-NEXT: successors: %bb.5(0x40000000), %bb.9(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[SUBSWrr2:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr5]], [[COPY8]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 5
+ ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; CHECK-NEXT: B %bb.5
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.5:
+ ; CHECK-NEXT: successors: %bb.6(0x40000000), %bb.9(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[SUBSWrr3:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr5]], [[COPY9]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 4
+ ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; CHECK-NEXT: B %bb.6
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.6:
+ ; CHECK-NEXT: successors: %bb.7(0x40000000), %bb.9(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[SUBSWrr4:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr5]], [[COPY10]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm4:%[0-9]+]]:gpr64 = MOVi64imm 3
+ ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; CHECK-NEXT: B %bb.7
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.7:
+ ; CHECK-NEXT: successors: %bb.8(0x40000000), %bb.9(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[SUBSWrr5:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr5]], [[COPY11]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm5:%[0-9]+]]:gpr64 = MOVi64imm 2
+ ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; CHECK-NEXT: B %bb.8
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.8:
+ ; CHECK-NEXT: successors: %bb.9(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[MOVi64imm6:%[0-9]+]]:gpr64 = MOVi64imm 1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.9:
+ ; CHECK-NEXT: successors: %bb.10(0x04000000), %bb.2(0x7c000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[PHI1:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.2, [[MOVi64imm1]], %bb.3, [[MOVi64imm2]], %bb.4, [[MOVi64imm3]], %bb.5, [[MOVi64imm4]], %bb.6, [[MOVi64imm5]], %bb.7, [[MOVi64imm6]], %bb.8
+ ; CHECK-NEXT: [[SUBSWrr6:%[0-9]+]]:gpr32 = SUBSWrr [[PHI]], [[MOVi32imm]], implicit-def $nzcv
+ ; CHECK-NEXT: Bcc 1, %bb.2, implicit $nzcv
+ ; CHECK-NEXT: B %bb.10
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.10:
+ ; CHECK-NEXT: [[PHI2:%[0-9]+]]:gpr64 = PHI [[COPY5]], %bb.0, [[PHI1]], %bb.9
+ ; CHECK-NEXT: $x0 = COPY [[PHI2]]
+ ; CHECK-NEXT: RET_ReallyLR implicit $x0
+ bb.0:
+ successors: %bb.10(0x30000000), %bb.1(0x50000000)
+ liveins: $x1, $w2, $w3, $w4, $w5
+
+ %1:gpr64common = COPY $x1
+ %2:gpr32 = COPY $w2
+ %3:gpr32 = COPY $w3
+ %4:gpr32 = COPY $w4
+ %5:gpr32 = COPY $w5
+ %99:gpr64all = COPY $xzr
+ CBZW %2, %bb.10
+ B %bb.1
+
+ bb.1:
+ successors: %bb.2(0x80000000)
+
+ ; Thresholds from registers, not memory loads
+ %10:gpr32 = COPY %3
+ %11:gpr32 = COPY %4
+ %12:gpr32 = COPY %5
+ %13:gpr32 = COPY %3
+ %14:gpr32 = COPY %4
+ %15:gpr32 = COPY %5
+ %16:gpr32 = COPY %3
+ %17:gpr32 = MOVi32imm 1
+
+ ; Cascade head: NOT data-dependent (computed from register args, no loads)
+ bb.2:
+ successors: %bb.3(0x40000000), %bb.9(0x40000000)
+
+ %50:gpr32 = PHI %2, %bb.1, %51, %bb.9
+ %31:gpr32 = nsw MADDWrrr %3, %5, $wzr
+ %32:gpr32 = nsw MADDWrrr %4, %5, %31
+ %33:gpr32 = nsw MADDWrrr %3, %4, %32
+ %34:gpr32 = nsw MADDWrrr %4, %3, %33
+ %35:gpr32 = nsw MADDWrrr %5, %4, %34
+ %30:gpr32 = nsw MADDWrrr %3, %5, %35
+ %21:gpr32 = SUBSWrr %30, %10, implicit-def $nzcv
+ %40:gpr64 = MOVi64imm 7
+ Bcc 10, %bb.9, implicit $nzcv
+ B %bb.3
+
+ bb.3:
+ successors: %bb.4(0x40000000), %bb.9(0x40000000)
+
+ %22:gpr32 = SUBSWrr %30, %11, implicit-def $nzcv
+ %41:gpr64 = MOVi64imm 6
+ Bcc 10, %bb.9, implicit $nzcv
+ B %bb.4
+
+ bb.4:
+ successors: %bb.5(0x40000000), %bb.9(0x40000000)
+
+ %23:gpr32 = SUBSWrr %30, %12, implicit-def $nzcv
+ %42:gpr64 = MOVi64imm 5
+ Bcc 10, %bb.9, implicit $nzcv
+ B %bb.5
+
+ bb.5:
+ successors: %bb.6(0x40000000), %bb.9(0x40000000)
+
+ %24:gpr32 = SUBSWrr %30, %13, implicit-def $nzcv
+ %43:gpr64 = MOVi64imm 4
+ Bcc 10, %bb.9, implicit $nzcv
+ B %bb.6
+
+ bb.6:
+ successors: %bb.7(0x40000000), %bb.9(0x40000000)
+
+ %25:gpr32 = SUBSWrr %30, %14, implicit-def $nzcv
+ %44:gpr64 = MOVi64imm 3
+ Bcc 10, %bb.9, implicit $nzcv
+ B %bb.7
+
+ bb.7:
+ successors: %bb.8(0x40000000), %bb.9(0x40000000)
+
+ %26:gpr32 = SUBSWrr %30, %15, implicit-def $nzcv
+ %45:gpr64 = MOVi64imm 2
+ Bcc 10, %bb.9, implicit $nzcv
+ B %bb.8
+
+ bb.8:
+ successors: %bb.9(0x80000000)
+
+ %47:gpr64 = MOVi64imm 1
+
+ bb.9:
+ successors: %bb.10(0x04000000), %bb.2(0x7c000000)
+
+ %90:gpr64 = PHI %40, %bb.2, %41, %bb.3, %42, %bb.4, %43, %bb.5, %44, %bb.6, %45, %bb.7, %47, %bb.8
+ %51:gpr32 = SUBSWrr %50, %17, implicit-def $nzcv
+ Bcc 1, %bb.2, implicit $nzcv
+ B %bb.10
+
+ bb.10:
+ %95:gpr64 = PHI %99, %bb.0, %90, %bb.9
+ $x0 = COPY %95
+ RET_ReallyLR implicit $x0
+...
+---
+name: cascade_exceeds_crit_limit
+tracksRegLiveness: true
+liveins:
+ - { reg: '$x0', virtual-reg: '%0' }
+ - { reg: '$x1', virtual-reg: '%1' }
+ - { reg: '$w2', virtual-reg: '%2' }
+ - { reg: '$w3', virtual-reg: '%3' }
+ - { reg: '$w4', virtual-reg: '%4' }
+body: |
+ ; CHECK-LABEL: name: cascade_exceeds_crit_limit
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.11(0x30000000), %bb.1(0x50000000)
+ ; CHECK-NEXT: liveins: $x0, $x1, $w2, $w3, $w4
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr32 = COPY $w2
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gpr32 = COPY $w3
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:gpr32 = COPY $w4
+ ; CHECK-NEXT: [[COPY5:%[0-9]+]]:gpr64all = COPY $xzr
+ ; CHECK-NEXT: CBZW [[COPY2]], %bb.11
+ ; CHECK-NEXT: B %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.2(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 0 :: (load (s32))
+ ; CHECK-NEXT: [[LDRWui1:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 1 :: (load (s32))
+ ; CHECK-NEXT: [[LDRWui2:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 2 :: (load (s32))
+ ; CHECK-NEXT: [[LDRWui3:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 3 :: (load (s32))
+ ; CHECK-NEXT: [[LDRWui4:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 4 :: (load (s32))
+ ; CHECK-NEXT: [[LDRWui5:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 5 :: (load (s32))
+ ; CHECK-NEXT: [[LDRWui6:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 6 :: (load (s32))
+ ; CHECK-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: successors: %bb.3(0x40000000), %bb.10(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[PHI:%[0-9]+]]:gpr32 = PHI [[COPY2]], %bb.1, %15, %bb.10
+ ; CHECK-NEXT: [[LDRWui7:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load (s32))
+ ; CHECK-NEXT: [[MADDWrrr:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[LDRWui7]], $wzr
+ ; CHECK-NEXT: [[MADDWrrr1:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY4]], [[LDRWui7]], [[MADDWrrr]]
+ ; CHECK-NEXT: [[MADDWrrr2:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[LDRWui7]], [[MADDWrrr1]]
+ ; CHECK-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 7
+ ; CHECK-NEXT: Bcc 10, %bb.10, implicit $nzcv
+ ; CHECK-NEXT: B %bb.3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.3:
+ ; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.10(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui1]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 6
+ ; CHECK-NEXT: Bcc 10, %bb.10, implicit $nzcv
+ ; CHECK-NEXT: B %bb.4
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.4:
+ ; CHECK-NEXT: successors: %bb.6(0x40000000), %bb.5(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[SUBSWrr2:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui2]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 5
+ ; CHECK-NEXT: Bcc 10, %bb.5, implicit $nzcv
+ ; CHECK-NEXT: B %bb.6
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.5:
+ ; CHECK-NEXT: successors: %bb.10(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: B %bb.10
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.6:
+ ; CHECK-NEXT: successors: %bb.7(0x40000000), %bb.10(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[SUBSWrr3:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui3]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 4
+ ; CHECK-NEXT: Bcc 10, %bb.10, implicit $nzcv
+ ; CHECK-NEXT: B %bb.7
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.7:
+ ; CHECK-NEXT: successors: %bb.8(0x40000000), %bb.10(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[SUBSWrr4:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui4]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm4:%[0-9]+]]:gpr64 = MOVi64imm 3
+ ; CHECK-NEXT: Bcc 10, %bb.10, implicit $nzcv
+ ; CHECK-NEXT: B %bb.8
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.8:
+ ; CHECK-NEXT: successors: %bb.9(0x40000000), %bb.10(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[SUBSWrr5:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui5]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm5:%[0-9]+]]:gpr64 = MOVi64imm 2
+ ; CHECK-NEXT: Bcc 10, %bb.10, implicit $nzcv
+ ; CHECK-NEXT: B %bb.9
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.9:
+ ; CHECK-NEXT: successors: %bb.10(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[MOVi64imm6:%[0-9]+]]:gpr64 = MOVi64imm 1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.10:
+ ; CHECK-NEXT: successors: %bb.11(0x04000000), %bb.2(0x7c000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[PHI1:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.2, [[MOVi64imm1]], %bb.3, [[MOVi64imm2]], %bb.5, [[MOVi64imm3]], %bb.6, [[MOVi64imm4]], %bb.7, [[MOVi64imm5]], %bb.8, [[MOVi64imm6]], %bb.9
+ ; CHECK-NEXT: [[SUBSWrr6:%[0-9]+]]:gpr32 = SUBSWrr [[PHI]], [[MOVi32imm]], implicit-def $nzcv
+ ; CHECK-NEXT: Bcc 1, %bb.2, implicit $nzcv
+ ; CHECK-NEXT: B %bb.11
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.11:
+ ; CHECK-NEXT: [[PHI2:%[0-9]+]]:gpr64 = PHI [[COPY5]], %bb.0, [[PHI1]], %bb.10
+ ; CHECK-NEXT: $x0 = COPY [[PHI2]]
+ ; CHECK-NEXT: RET_ReallyLR implicit $x0
+ bb.0:
+ successors: %bb.11(0x30000000), %bb.1(0x50000000)
+ liveins: $x0, $x1, $w2, $w3, $w4
+
+ %0:gpr64common = COPY $x0
+ %1:gpr64common = COPY $x1
+ %2:gpr32 = COPY $w2
+ %3:gpr32 = COPY $w3
+ %4:gpr32 = COPY $w4
+ %99:gpr64all = COPY $xzr
+ CBZW %2, %bb.11
+ B %bb.1
+
+ bb.1:
+ successors: %bb.2(0x80000000)
+
+ %10:gpr32 = LDRWui %1, 0 :: (load (s32))
+ %11:gpr32 = LDRWui %1, 1 :: (load (s32))
+ %12:gpr32 = LDRWui %1, 2 :: (load (s32))
+ %13:gpr32 = LDRWui %1, 3 :: (load (s32))
+ %14:gpr32 = LDRWui %1, 4 :: (load (s32))
+ %15:gpr32 = LDRWui %1, 5 :: (load (s32))
+ %16:gpr32 = LDRWui %1, 6 :: (load (s32))
+ %17:gpr32 = MOVi32imm 1
+
+ ; Cascade head: data-dependent
+ bb.2:
+ successors: %bb.3(0x40000000), %bb.10(0x40000000)
+
+ %50:gpr32 = PHI %2, %bb.1, %51, %bb.10
+ %20:gpr32 = LDRWui %0, 0 :: (load (s32))
+ %31:gpr32 = nsw MADDWrrr %3, %20, $wzr
+ %32:gpr32 = nsw MADDWrrr %4, %20, %31
+ %30:gpr32 = nsw MADDWrrr %3, %20, %32
+ %21:gpr32 = SUBSWrr %30, %10, implicit-def $nzcv
+ %40:gpr64 = MOVi64imm 7
+ Bcc 10, %bb.10, implicit $nzcv
+ B %bb.3
+
+ bb.3:
+ successors: %bb.4(0x40000000), %bb.10(0x40000000)
+
+ %22:gpr32 = SUBSWrr %30, %11, implicit-def $nzcv
+ %41:gpr64 = MOVi64imm 6
+ Bcc 10, %bb.10, implicit $nzcv
+ B %bb.4
+
+ ; bb.4 branches to bb.11 (not directly to tail) - breaks cascade
+ bb.4:
+ successors: %bb.6(0x40000000), %bb.5(0x40000000)
+
+ %23:gpr32 = SUBSWrr %30, %12, implicit-def $nzcv
+ %42:gpr64 = MOVi64imm 5
+ Bcc 10, %bb.5, implicit $nzcv
+ B %bb.6
+
+ ; Extra block that breaks the cascade chain
+ bb.5:
+ successors: %bb.10(0x80000000)
+
+ B %bb.10
+
+ bb.6:
+ successors: %bb.7(0x40000000), %bb.10(0x40000000)
+
+ %24:gpr32 = SUBSWrr %30, %13, implicit-def $nzcv
+ %43:gpr64 = MOVi64imm 4
+ Bcc 10, %bb.10, implicit $nzcv
+ B %bb.7
+
+ bb.7:
+ successors: %bb.8(0x40000000), %bb.10(0x40000000)
+
+ %25:gpr32 = SUBSWrr %30, %14, implicit-def $nzcv
+ %44:gpr64 = MOVi64imm 3
+ Bcc 10, %bb.10, implicit $nzcv
+ B %bb.8
+
+ bb.8:
+ successors: %bb.9(0x40000000), %bb.10(0x40000000)
+
+ %26:gpr32 = SUBSWrr %30, %15, implicit-def $nzcv
+ %45:gpr64 = MOVi64imm 2
+ Bcc 10, %bb.10, implicit $nzcv
+ B %bb.9
+
+ bb.9:
+ successors: %bb.10(0x80000000)
+
+ %47:gpr64 = MOVi64imm 1
+
+ bb.10:
+ successors: %bb.11(0x04000000), %bb.2(0x7c000000)
+
+ %90:gpr64 = PHI %40, %bb.2, %41, %bb.3, %42, %bb.5, %43, %bb.6, %44, %bb.7, %45, %bb.8, %47, %bb.9
+ %51:gpr32 = SUBSWrr %50, %17, implicit-def $nzcv
+ Bcc 1, %bb.2, implicit $nzcv
+ B %bb.11
+
+ bb.11:
+ %95:gpr64 = PHI %99, %bb.0, %90, %bb.10
+ $x0 = COPY %95
+ RET_ReallyLR implicit $x0
+...
+---
+name: cascade_backedge_to_earlier_block
+alignment: 4
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: cascade_backedge_to_earlier_block
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: liveins: $x0, $x1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.2(0x40000000), %bb.5(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load (s32))
+ ; CHECK-NEXT: [[LDRWui1:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 0 :: (load (s32))
+ ; CHECK-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui1]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 100
+ ; CHECK-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 200
+ ; CHECK-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 400
+ ; CHECK-NEXT: Bcc 10, %bb.5, implicit $nzcv
+ ; CHECK-NEXT: B %bb.2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: successors: %bb.3(0x40000000), %bb.5(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[LDRWui2:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 1 :: (load (s32))
+ ; CHECK-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui2]], implicit-def $nzcv
+ ; CHECK-NEXT: Bcc 10, %bb.5, implicit $nzcv
+ ; CHECK-NEXT: B %bb.3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.3:
+ ; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[LDRWui3:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 2 :: (load (s32))
+ ; CHECK-NEXT: [[SUBSWrr2:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui3]], implicit-def $nzcv
+ ; CHECK-NEXT: Bcc 10, %bb.2, implicit $nzcv
+ ; CHECK-NEXT: B %bb.4
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.4:
+ ; CHECK-NEXT: successors: %bb.5(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.5:
+ ; CHECK-NEXT: [[PHI:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.1, [[MOVi64imm1]], %bb.2, [[MOVi64imm2]], %bb.4
+ ; CHECK-NEXT: $x0 = COPY [[PHI]]
+ ; CHECK-NEXT: RET_ReallyLR implicit $x0
+ bb.0:
+ successors: %bb.1(0x80000000)
+ liveins: $x0, $x1
+
+ %0:gpr64common = COPY $x0
+ %1:gpr64common = COPY $x1
+ bb.1:
+ successors: %bb.2(0x40000000), %bb.5(0x40000000)
+
+ %10:gpr32 = LDRWui %0, 0 :: (load (s32))
+ %11:gpr32 = LDRWui %1, 0 :: (load (s32))
+ %12:gpr32 = SUBSWrr %10, %11, implicit-def $nzcv
+ %40:gpr64 = MOVi64imm 100
+ %41:gpr64 = MOVi64imm 200
+ %43:gpr64 = MOVi64imm 400
+ Bcc 10, %bb.5, implicit $nzcv
+ B %bb.2
+ bb.2:
+ successors: %bb.3(0x40000000), %bb.5(0x40000000)
+
+ %20:gpr32 = LDRWui %1, 1 :: (load (s32))
+ %21:gpr32 = SUBSWrr %10, %20, implicit-def $nzcv
+ Bcc 10, %bb.5, implicit $nzcv
+ B %bb.3
+
+ bb.3:
+ successors: %bb.4(0x40000000), %bb.2(0x40000000)
+
+ %30:gpr32 = LDRWui %1, 2 :: (load (s32))
+ %31:gpr32 = SUBSWrr %10, %30, implicit-def $nzcv
+ ; Branches to bb.2 instead of bb.5 (Tail) - creates back-edge
+ Bcc 10, %bb.2, implicit $nzcv
+ B %bb.4
+
+ bb.4:
+ successors: %bb.5(0x80000000)
+
+ bb.5:
+ %90:gpr64 = PHI %40, %bb.1, %41, %bb.2, %43, %bb.4
+ $x0 = COPY %90
+ RET_ReallyLR implicit $x0
+...
+---
+name: cascade_forward_skip_to_later_block
+alignment: 4
+tracksRegLiveness: true
+liveins:
+ - { reg: '$x0', virtual-reg: '%0' }
+ - { reg: '$x1', virtual-reg: '%1' }
+body: |
+ ; CHECK-LABEL: name: cascade_forward_skip_to_later_block
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: liveins: $x0, $x1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.2(0x40000000), %bb.6(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load (s32))
+ ; CHECK-NEXT: [[LDRWui1:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 0 :: (load (s32))
+ ; CHECK-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui1]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 100
+ ; CHECK-NEXT: Bcc 10, %bb.6, implicit $nzcv
+ ; CHECK-NEXT: B %bb.2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: successors: %bb.3(0x40000000), %bb.6(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[LDRWui2:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 1 :: (load (s32))
+ ; CHECK-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui2]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 200
+ ; CHECK-NEXT: Bcc 10, %bb.6, implicit $nzcv
+ ; CHECK-NEXT: B %bb.3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.3:
+ ; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.5(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[LDRWui3:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 2 :: (load (s32))
+ ; CHECK-NEXT: [[SUBSWrr2:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui3]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 300
+ ; CHECK-NEXT: Bcc 10, %bb.5, implicit $nzcv
+ ; CHECK-NEXT: B %bb.4
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.4:
+ ; CHECK-NEXT: successors: %bb.5(0x40000000), %bb.6(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[LDRWui4:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 3 :: (load (s32))
+ ; CHECK-NEXT: [[SUBSWrr3:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui4]], implicit-def $nzcv
+ ; CHECK-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 400
+ ; CHECK-NEXT: Bcc 10, %bb.6, implicit $nzcv
+ ; CHECK-NEXT: B %bb.5
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.5:
+ ; CHECK-NEXT: successors: %bb.6(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[MOVi64imm4:%[0-9]+]]:gpr64 = MOVi64imm 500
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.6:
+ ; CHECK-NEXT: [[PHI:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.1, [[MOVi64imm1]], %bb.2, [[MOVi64imm3]], %bb.4, [[MOVi64imm4]], %bb.5
+ ; CHECK-NEXT: $x0 = COPY [[PHI]]
+ ; CHECK-NEXT: RET_ReallyLR implicit $x0
+ bb.0:
+ successors: %bb.1(0x80000000)
+ liveins: $x0, $x1
+
+ %0:gpr64common = COPY $x0
+ %1:gpr64common = COPY $x1
+
+ bb.1:
+ successors: %bb.2(0x40000000), %bb.6(0x40000000)
+
+ %10:gpr32 = LDRWui %0, 0 :: (load (s32))
+ %11:gpr32 = LDRWui %1, 0 :: (load (s32))
+ %12:gpr32 = SUBSWrr %10, %11, implicit-def $nzcv
+ %40:gpr64 = MOVi64imm 100
+ Bcc 10, %bb.6, implicit $nzcv
+ B %bb.2
+
+ bb.2:
+ successors: %bb.3(0x40000000), %bb.6(0x40000000)
+
+ %20:gpr32 = LDRWui %1, 1 :: (load (s32))
+ %21:gpr32 = SUBSWrr %10, %20, implicit-def $nzcv
+ %41:gpr64 = MOVi64imm 200
+ Bcc 10, %bb.6, implicit $nzcv
+ B %bb.3
+
+ bb.3:
+ successors: %bb.4(0x40000000), %bb.5(0x40000000)
+
+ %30:gpr32 = LDRWui %1, 2 :: (load (s32))
+ %31:gpr32 = SUBSWrr %10, %30, implicit-def $nzcv
+ %42:gpr64 = MOVi64imm 300
+ ; Branches to bb.5 instead of bb.6 (Tail) - forward skip
+ Bcc 10, %bb.5, implicit $nzcv
+ B %bb.4
+
+ bb.4:
+ successors: %bb.5(0x40000000), %bb.6(0x40000000)
+
+ %32:gpr32 = LDRWui %1, 3 :: (load (s32))
+ %33:gpr32 = SUBSWrr %10, %32, implicit-def $nzcv
+ %43:gpr64 = MOVi64imm 400
+ Bcc 10, %bb.6, implicit $nzcv
+ B %bb.5
+
+ bb.5:
+ successors: %bb.6(0x80000000)
+
+ %44:gpr64 = MOVi64imm 500
+
+ bb.6:
+ %90:gpr64 = PHI %40, %bb.1, %41, %bb.2, %43, %bb.4, %44, %bb.5
+ $x0 = COPY %90
+ RET_ReallyLR implicit $x0
+...
+
diff --git a/llvm/test/CodeGen/AArch64/early-ifcvt-remarks.mir b/llvm/test/CodeGen/AArch64/early-ifcvt-remarks.mir
index 93460d5822ab4..6f01d48601fa2 100644
--- a/llvm/test/CodeGen/AArch64/early-ifcvt-remarks.mir
+++ b/llvm/test/CodeGen/AArch64/early-ifcvt-remarks.mir
@@ -11,6 +11,12 @@
define i32 @data_dependent_branch(ptr %p, i32 %x, i32 %y) {
ret i32 0
}
+ define i64 @cascade_success(ptr %p, ptr %q, i32 %n) {
+ ret i64 0
+ }
+ define i64 @cascade_increasing_cond_depth(ptr %p) {
+ ret i64 0
+ }
...
---
name: data_dependent_branch
@@ -43,3 +49,156 @@ body: |
$w0 = COPY %100
RET_ReallyLR implicit $w0
...
+---
+name: cascade_success
+alignment: 4
+tracksRegLiveness: true
+liveins:
+ - { reg: '$x0', virtual-reg: '%0' }
+ - { reg: '$x1', virtual-reg: '%1' }
+ - { reg: '$w2', virtual-reg: '%2' }
+body: |
+ ; Entry - check loop count
+ bb.0:
+ successors: %bb.6(0x30000000), %bb.1(0x50000000)
+ liveins: $x0, $x1, $w2
+
+ %0:gpr64common = COPY $x0
+ %1:gpr64common = COPY $x1
+ %2:gpr32 = COPY $w2
+ %50:gpr64all = COPY $xzr
+ %51:gpr64all = COPY %50
+ CBZW %2, %bb.6
+ B %bb.1
+
+ ; Loop header / cascade head - ALL loads here, branches to bb.2 or bb.5
+ bb.1:
+ successors: %bb.2(0x40000000), %bb.5(0x40000000)
+
+ %60:gpr64 = PHI %51, %bb.0, %81, %bb.5
+ %61:gpr32 = PHI %2, %bb.0, %80, %bb.5
+ ; Load ALL data in head - makes branch data-dependent
+ %62:gpr32 = LDRWui %0, 0 :: (load (s32))
+ %63:gpr32 = LDRWui %1, 0 :: (load (s32))
+ %66:gpr32 = LDRWui %1, 1 :: (load (s32))
+ %69:gpr32 = LDRWui %1, 2 :: (load (s32))
+ %90:gpr32 = LDRWui %1, 3 :: (load (s32))
+ %64:gpr64 = MOVi64imm 4
+ %65:gpr32 = SUBSWrr %62, %63, implicit-def $nzcv
+ Bcc 10, %bb.5, implicit $nzcv
+ B %bb.2
+
+ ; First cascade block (branches to bb.3 or bb.5)
+ bb.2:
+ successors: %bb.3(0x40000000), %bb.5(0x40000000)
+
+ %67:gpr32 = SUBSWrr %62, %66, implicit-def $nzcv
+ %68:gpr64 = MOVi64imm 3
+ Bcc 10, %bb.5, implicit $nzcv
+ B %bb.3
+
+ ; Second cascade block (branches to bb.4 or bb.5)
+ bb.3:
+ successors: %bb.4(0x40000000), %bb.5(0x40000000)
+
+ %70:gpr32 = SUBSWrr %62, %69, implicit-def $nzcv
+ %91:gpr64 = MOVi64imm 2
+ Bcc 10, %bb.5, implicit $nzcv
+ B %bb.4
+
+ ; Third cascade block (only branches to bb.5)
+ bb.4:
+ successors: %bb.5(0x80000000)
+
+ %92:gpr32 = SUBSWrr %62, %90, implicit-def $nzcv
+ %71:gpr32 = CSINCWr $wzr, $wzr, 11, implicit $nzcv
+ %72:gpr64 = SUBREG_TO_REG %71, %subreg.sub_32
+
+ ; Loop latch / cascade tail - PHI merge and back edge
+ bb.5:
+ successors: %bb.6(0x04000000), %bb.1(0x7c000000)
+
+ %73:gpr64 = PHI %64, %bb.1, %68, %bb.2, %91, %bb.3, %72, %bb.4
+ %74:gpr64 = ADDXrr %60, %73
+ %75:gpr32 = MOVi32imm 1
+ %80:gpr32 = SUBSWrr %61, %75, implicit-def $nzcv
+ %81:gpr64all = COPY %74
+ Bcc 1, %bb.1, implicit $nzcv
+ B %bb.6
+
+ ; Exit
+ bb.6:
+ %82:gpr64 = PHI %51, %bb.0, %81, %bb.5
+ $x0 = COPY %82
+ RET_ReallyLR implicit $x0
+...
+---
+name: cascade_increasing_cond_depth
+alignment: 4
+tracksRegLiveness: true
+liveins:
+ - { reg: '$x0', virtual-reg: '%0' }
+body: |
+ bb.0:
+ successors: %bb.1(0x80000000)
+ liveins: $x0
+
+ %0:gpr64common = COPY $x0
+
+ bb.1:
+ successors: %bb.2(0x40000000), %bb.7(0x40000000)
+
+ %10:gpr32 = LDRWui %0, 0 :: (load (s32))
+ %11:gpr32 = MADDWrrr %10, %10, $wzr
+ %12:gpr32 = MADDWrrr %11, %11, $wzr
+ %20:gpr32 = ADDWrr %12, %12
+ %21:gpr32 = ADDWrr %20, %20
+ %22:gpr32 = ADDWrr %21, %21
+ %23:gpr32 = ADDWrr %22, %22
+ %40:gpr64 = MOVi64imm 100
+ %30:gpr32 = SUBSWrr %12, %12, implicit-def $nzcv
+ Bcc 10, %bb.7, implicit $nzcv
+ B %bb.2
+
+ bb.2:
+ successors: %bb.3(0x40000000), %bb.7(0x40000000)
+
+ %41:gpr64 = MOVi64imm 200
+ %31:gpr32 = SUBSWrr %20, %20, implicit-def $nzcv
+ Bcc 10, %bb.7, implicit $nzcv
+ B %bb.3
+
+ bb.3:
+ successors: %bb.4(0x40000000), %bb.7(0x40000000)
+
+ %42:gpr64 = MOVi64imm 300
+ %32:gpr32 = SUBSWrr %21, %21, implicit-def $nzcv
+ Bcc 10, %bb.7, implicit $nzcv
+ B %bb.4
+
+ bb.4:
+ successors: %bb.5(0x40000000), %bb.7(0x40000000)
+
+ %43:gpr64 = MOVi64imm 400
+ %33:gpr32 = SUBSWrr %22, %22, implicit-def $nzcv
+ Bcc 10, %bb.7, implicit $nzcv
+ B %bb.5
+
+ bb.5:
+ successors: %bb.6(0x40000000), %bb.7(0x40000000)
+
+ %44:gpr64 = MOVi64imm 500
+ %34:gpr32 = SUBSWrr %23, %23, implicit-def $nzcv
+ Bcc 10, %bb.7, implicit $nzcv
+ B %bb.6
+
+ bb.6:
+ successors: %bb.7(0x80000000)
+
+ %45:gpr64 = MOVi64imm 600
+
+ bb.7:
+ %90:gpr64 = PHI %40, %bb.1, %41, %bb.2, %42, %bb.3, %43, %bb.4, %44, %bb.5, %45, %bb.6
+ $x0 = COPY %90
+ RET_ReallyLR implicit $x0
+...
>From 0dfe7fe93b8e0e8ed876a2628c1ea2111a222ba9 Mon Sep 17 00:00:00 2001
From: Jonathan Cohen <joncoh at apple.com>
Date: Mon, 7 Sep 2026 20:41:06 +0300
Subject: [PATCH 2/3] [CodeGen] Let canInsertSelect() describe the select
expansion (NFC)
`canInsertSelect()` only reports latencies, but insertSelect() may emit more
than one instruction and may clobber physregs. For example, on AArch64 a
cbz/tbz condition needs a flag-setting SUBS/ANDS before the csel, which
clobbers NZCV.
Replace the out-parameters with a new SelectExpansion struct that holds
the latencies plus NumInsts and ClobberedPhysRegs.
These new fields are not used yet in this commit, but will be wired in
a follow-up commit to Early If Conversion.
Note that these fields are only implemented for AArch64 for now. Targets
which do not support the new struct will return `0` in NumInsts and default
to the existing behavior.
---
llvm/include/llvm/CodeGen/TargetInstrInfo.h | 45 ++++++++++++--------
llvm/lib/CodeGen/EarlyIfConversion.cpp | 15 ++++---
llvm/lib/Target/AArch64/AArch64InstrInfo.cpp | 28 ++++++------
llvm/lib/Target/AArch64/AArch64InstrInfo.h | 4 +-
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 8 ++--
llvm/lib/Target/AMDGPU/SIInstrInfo.h | 4 +-
llvm/lib/Target/PowerPC/PPCInstrInfo.cpp | 10 ++---
llvm/lib/Target/PowerPC/PPCInstrInfo.h | 4 +-
llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp | 11 +++--
llvm/lib/Target/SystemZ/SystemZInstrInfo.h | 4 +-
llvm/lib/Target/X86/X86InstrInfo.cpp | 10 ++---
llvm/lib/Target/X86/X86InstrInfo.h | 4 +-
12 files changed, 80 insertions(+), 67 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/TargetInstrInfo.h b/llvm/include/llvm/CodeGen/TargetInstrInfo.h
index b013511d33313..55efcbb9eff6b 100644
--- a/llvm/include/llvm/CodeGen/TargetInstrInfo.h
+++ b/llvm/include/llvm/CodeGen/TargetInstrInfo.h
@@ -1022,29 +1022,38 @@ class LLVM_ABI TargetInstrInfo : public MCInstrInfo {
return false;
}
+ /// Describes how the select will be materialized:
+ /// CondCycles - number of cycles for compare and branch
+ /// TrueCycles - number of cycles to the True register
+ /// FalseCycles - same for False
+ /// NumInsts - number of instructions required for the select, 0 if
+ /// this field is not supported by a given backend
+ /// ClobberedPhysRegs - physical registers which will be defined by the
+ /// newly inserted select. Useful for finding a legal
+ /// insertion point
+ struct SelectExpansion {
+ int CondCycles = 1;
+ int TrueCycles = 1;
+ int FalseCycles = 1;
+ unsigned NumInsts = 0;
+ SmallVector<MCRegister, 2> ClobberedPhysRegs;
+ };
+
/// Return true if it is possible to insert a select
/// instruction that chooses between TrueReg and FalseReg based on the
- /// condition code in Cond.
- ///
- /// When successful, also return the latency in cycles from TrueReg,
- /// FalseReg, and Cond to the destination register. In most cases, a select
- /// instruction will be 1 cycle, so CondCycles = TrueCycles = FalseCycles = 1
- ///
- /// Some x86 implementations have 2-cycle cmov instructions.
- ///
- /// @param MBB Block where select instruction would be inserted.
- /// @param Cond Condition returned by analyzeBranch.
- /// @param DstReg Virtual dest register that the result should write to.
- /// @param TrueReg Virtual register to select when Cond is true.
- /// @param FalseReg Virtual register to select when Cond is false.
- /// @param CondCycles Latency from Cond+Branch to select output.
- /// @param TrueCycles Latency from TrueReg to select output.
- /// @param FalseCycles Latency from FalseReg to select output.
+ /// condition code in Cond. Exp provides information on how the select
+ /// will be materialized.
+ ///
+ /// @param MBB Block where select instruction would be inserted.
+ /// @param Cond Condition returned by analyzeBranch.
+ /// @param DstReg Virtual dest register that the result should write to.
+ /// @param TrueReg Virtual register to select when Cond is true.
+ /// @param FalseReg Virtual register to select when Cond is false.
+ /// @param Exp Latency and number of instructions added by the select.
virtual bool canInsertSelect(const MachineBasicBlock &MBB,
ArrayRef<MachineOperand> Cond, Register DstReg,
Register TrueReg, Register FalseReg,
- int &CondCycles, int &TrueCycles,
- int &FalseCycles) const {
+ SelectExpansion &Exp) const {
return false;
}
diff --git a/llvm/lib/CodeGen/EarlyIfConversion.cpp b/llvm/lib/CodeGen/EarlyIfConversion.cpp
index bf2664fd4225c..71fe114bace81 100644
--- a/llvm/lib/CodeGen/EarlyIfConversion.cpp
+++ b/llvm/lib/CodeGen/EarlyIfConversion.cpp
@@ -141,8 +141,8 @@ class SSAIfConv {
struct PHIInfo {
MachineInstr *PHI;
Register TReg, FReg;
- // Latencies from Cond+Branch, TReg, and FReg to DstReg.
- int CondCycles = 0, TCycles = 0, FCycles = 0;
+ /// Description of the select that will replace this phi.
+ TargetInstrInfo::SelectExpansion Select;
PHIInfo(MachineInstr *phi) : PHI(phi) {}
};
@@ -561,8 +561,7 @@ bool SSAIfConv::canConvertIf(MachineBasicBlock *MBB, bool Predicate) {
// Get target information.
if (!TII->canInsertSelect(*Head, Cond, PI.PHI->getOperand(0).getReg(),
- PI.TReg, PI.FReg, PI.CondCycles, PI.TCycles,
- PI.FCycles)) {
+ PI.TReg, PI.FReg, PI.Select)) {
LLVM_DEBUG(dbgs() << "Can't convert: " << *PI.PHI);
return false;
}
@@ -1298,7 +1297,7 @@ bool EarlyIfConverter::shouldConvertIf() {
LLVM_DEBUG(dbgs() << "Slack " << Slack << ":\t" << *PI.PHI);
// The condition is pulled into the critical path.
- unsigned CondDepth = adjCycles(BranchDepth, PI.CondCycles);
+ unsigned CondDepth = adjCycles(BranchDepth, PI.Select.CondCycles);
if (CondDepth > MaxDepth) {
unsigned Extra = CondDepth - MaxDepth;
LLVM_DEBUG(dbgs() << "Condition adds " << Extra << " cycles.\n");
@@ -1311,7 +1310,8 @@ bool EarlyIfConverter::shouldConvertIf() {
}
// The TBB value is pulled into the critical path.
- unsigned TDepth = adjCycles(TBBTrace.getPHIDepth(*PI.PHI), PI.TCycles);
+ unsigned TDepth =
+ adjCycles(TBBTrace.getPHIDepth(*PI.PHI), PI.Select.TrueCycles);
if (TDepth > MaxDepth) {
unsigned Extra = TDepth - MaxDepth;
LLVM_DEBUG(dbgs() << "TBB data adds " << Extra << " cycles.\n");
@@ -1324,7 +1324,8 @@ bool EarlyIfConverter::shouldConvertIf() {
}
// The FBB value is pulled into the critical path.
- unsigned FDepth = adjCycles(FBBTrace.getPHIDepth(*PI.PHI), PI.FCycles);
+ unsigned FDepth =
+ adjCycles(FBBTrace.getPHIDepth(*PI.PHI), PI.Select.FalseCycles);
if (FDepth > MaxDepth) {
unsigned Extra = FDepth - MaxDepth;
LLVM_DEBUG(dbgs() << "FBB data adds " << Extra << " cycles.\n");
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index b8e74f8ad496d..b0959c92090c9 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -1238,9 +1238,8 @@ static unsigned canFoldIntoCSel(const MachineRegisterInfo &MRI, unsigned VReg,
bool AArch64InstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
ArrayRef<MachineOperand> Cond,
Register DstReg, Register TrueReg,
- Register FalseReg, int &CondCycles,
- int &TrueCycles,
- int &FalseCycles) const {
+ Register FalseReg,
+ SelectExpansion &Exp) const {
// Check register classes.
const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
const TargetRegisterClass *RC =
@@ -1254,20 +1253,24 @@ bool AArch64InstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
if (!RI.getCommonSubClass(RC, MRI.getRegClass(DstReg)))
return false;
- // Expanding cbz/tbz requires an extra cycle of latency on the condition.
+ // Expanding cbz/tbz requires a flag setting instruction in addition to the
+ // select, adding an extra cycle of latency on the condition.
unsigned ExtraCondLat = Cond.size() != 1;
+ Exp.NumInsts = 1 + ExtraCondLat;
+ if (ExtraCondLat)
+ Exp.ClobberedPhysRegs.push_back(AArch64::NZCV);
// GPRs are handled by csel.
// FIXME: Fold in x+1, -x, and ~x when applicable.
if (AArch64::GPR64allRegClass.hasSubClassEq(RC) ||
AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
// Single-cycle csel, csinc, csinv, and csneg.
- CondCycles = 1 + ExtraCondLat;
- TrueCycles = FalseCycles = 1;
+ Exp.CondCycles = 1 + ExtraCondLat;
+ Exp.TrueCycles = Exp.FalseCycles = 1;
if (canFoldIntoCSel(MRI, TrueReg))
- TrueCycles = 0;
+ Exp.TrueCycles = 0;
else if (canFoldIntoCSel(MRI, FalseReg))
- FalseCycles = 0;
+ Exp.FalseCycles = 0;
return true;
}
@@ -1275,8 +1278,8 @@ bool AArch64InstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
// FIXME: Form fabs, fmin, and fmax when applicable.
if (AArch64::FPR64RegClass.hasSubClassEq(RC) ||
AArch64::FPR32RegClass.hasSubClassEq(RC)) {
- CondCycles = 5 + ExtraCondLat;
- TrueCycles = FalseCycles = 2;
+ Exp.CondCycles = 5 + ExtraCondLat;
+ Exp.TrueCycles = Exp.FalseCycles = 2;
return true;
}
@@ -1285,8 +1288,9 @@ bool AArch64InstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
if (AArch64::FPR128RegClass.hasSubClassEq(RC) &&
Subtarget.isNeonAvailable() &&
!MBB.getParent()->getFunction().hasMinSize()) {
- CondCycles = 8 + ExtraCondLat;
- TrueCycles = FalseCycles = 2;
+ Exp.CondCycles = 8 + ExtraCondLat;
+ Exp.TrueCycles = Exp.FalseCycles = 2;
+ Exp.NumInsts = 3 + ExtraCondLat;
return true;
}
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.h b/llvm/lib/Target/AArch64/AArch64InstrInfo.h
index 99e0142824d43..88d19a72ab3c4 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.h
@@ -437,8 +437,8 @@ class AArch64InstrInfo final : public AArch64GenInstrInfo {
bool
reverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const override;
bool canInsertSelect(const MachineBasicBlock &, ArrayRef<MachineOperand> Cond,
- Register, Register, Register, int &, int &,
- int &) const override;
+ Register, Register, Register,
+ SelectExpansion &) const override;
void insertSelect(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
const DebugLoc &DL, Register DstReg,
ArrayRef<MachineOperand> Cond, Register TrueReg,
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index add5d1df83d8e..794302ab9cd14 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -3553,8 +3553,8 @@ SIInstrInfo::analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const {
bool SIInstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
ArrayRef<MachineOperand> Cond,
Register DstReg, Register TrueReg,
- Register FalseReg, int &CondCycles,
- int &TrueCycles, int &FalseCycles) const {
+ Register FalseReg,
+ SelectExpansion &Exp) const {
switch (Cond[0].getImm()) {
case VCCNZ:
case VCCZ: {
@@ -3564,7 +3564,7 @@ bool SIInstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
return false;
int NumInsts = AMDGPU::getRegBitWidth(*RC) / 32;
- CondCycles = TrueCycles = FalseCycles = NumInsts; // ???
+ Exp.CondCycles = Exp.TrueCycles = Exp.FalseCycles = NumInsts; // ???
// Limit to equal cost for branch vs. N v_cndmask_b32s.
return RI.hasVGPRs(RC) && NumInsts <= 6;
@@ -3584,7 +3584,7 @@ bool SIInstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
if (NumInsts % 2 == 0)
NumInsts /= 2;
- CondCycles = TrueCycles = FalseCycles = NumInsts; // ???
+ Exp.CondCycles = Exp.TrueCycles = Exp.FalseCycles = NumInsts; // ???
return RI.isSGPRClass(RC);
}
default:
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
index 48eba6b2a567d..4de2c12c17473 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
@@ -444,8 +444,8 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
bool canInsertSelect(const MachineBasicBlock &MBB,
ArrayRef<MachineOperand> Cond, Register DstReg,
- Register TrueReg, Register FalseReg, int &CondCycles,
- int &TrueCycles, int &FalseCycles) const override;
+ Register TrueReg, Register FalseReg,
+ SelectExpansion &Exp) const override;
void insertSelect(MachineBasicBlock &MBB,
MachineBasicBlock::iterator I, const DebugLoc &DL,
diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp b/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp
index bf7f3df6035d8..cfc3cb17b6f36 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp
+++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp
@@ -1520,8 +1520,8 @@ unsigned PPCInstrInfo::insertBranch(MachineBasicBlock &MBB,
bool PPCInstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
ArrayRef<MachineOperand> Cond,
Register DstReg, Register TrueReg,
- Register FalseReg, int &CondCycles,
- int &TrueCycles, int &FalseCycles) const {
+ Register FalseReg,
+ SelectExpansion &Exp) const {
if (!Subtarget.hasISEL())
return false;
@@ -1556,9 +1556,9 @@ bool PPCInstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
// an open question. On the A2, the isel instruction has a 2-cycle latency
// but single-cycle throughput. These numbers are used in combination with
// the MispredictPenalty setting from the active SchedMachineModel.
- CondCycles = 1;
- TrueCycles = 1;
- FalseCycles = 1;
+ Exp.CondCycles = 1;
+ Exp.TrueCycles = 1;
+ Exp.FalseCycles = 1;
return true;
}
diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.h b/llvm/lib/Target/PowerPC/PPCInstrInfo.h
index 542e0b0701506..dcd5182150323 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrInfo.h
+++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.h
@@ -558,8 +558,8 @@ class PPCInstrInfo : public PPCGenInstrInfo {
// Select analysis.
bool canInsertSelect(const MachineBasicBlock &, ArrayRef<MachineOperand> Cond,
- Register, Register, Register, int &, int &,
- int &) const override;
+ Register, Register, Register,
+ SelectExpansion &) const override;
void insertSelect(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
const DebugLoc &DL, Register DstReg,
ArrayRef<MachineOperand> Cond, Register TrueReg,
diff --git a/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp b/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp
index bfffa39117443..38e9a7b80e7a5 100644
--- a/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp
@@ -571,9 +571,8 @@ bool SystemZInstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg,
bool SystemZInstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
ArrayRef<MachineOperand> Pred,
Register DstReg, Register TrueReg,
- Register FalseReg, int &CondCycles,
- int &TrueCycles,
- int &FalseCycles) const {
+ Register FalseReg,
+ SelectExpansion &Exp) const {
// Not all subtargets have LOCR instructions.
if (!STI.hasLoadStoreOnCond())
return false;
@@ -592,9 +591,9 @@ bool SystemZInstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
SystemZ::GRX32BitRegClass.hasSubClassEq(RC)) ||
SystemZ::GR32BitRegClass.hasSubClassEq(RC) ||
SystemZ::GR64BitRegClass.hasSubClassEq(RC)) {
- CondCycles = 2;
- TrueCycles = 2;
- FalseCycles = 2;
+ Exp.CondCycles = 2;
+ Exp.TrueCycles = 2;
+ Exp.FalseCycles = 2;
return true;
}
diff --git a/llvm/lib/Target/SystemZ/SystemZInstrInfo.h b/llvm/lib/Target/SystemZ/SystemZInstrInfo.h
index 452671443bb12..ba4b96f215061 100644
--- a/llvm/lib/Target/SystemZ/SystemZInstrInfo.h
+++ b/llvm/lib/Target/SystemZ/SystemZInstrInfo.h
@@ -246,8 +246,8 @@ class SystemZInstrInfo : public SystemZGenInstrInfo {
Register &SrcReg2, int64_t &Mask,
int64_t &Value) const override;
bool canInsertSelect(const MachineBasicBlock &, ArrayRef<MachineOperand> Cond,
- Register, Register, Register, int &, int &,
- int &) const override;
+ Register, Register, Register,
+ SelectExpansion &) const override;
void insertSelect(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
const DebugLoc &DL, Register DstReg,
ArrayRef<MachineOperand> Cond, Register TrueReg,
diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp
index 6fb5a4866dc13..fe43c749414b0 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.cpp
+++ b/llvm/lib/Target/X86/X86InstrInfo.cpp
@@ -4204,8 +4204,8 @@ unsigned X86InstrInfo::insertBranch(MachineBasicBlock &MBB,
bool X86InstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
ArrayRef<MachineOperand> Cond,
Register DstReg, Register TrueReg,
- Register FalseReg, int &CondCycles,
- int &TrueCycles, int &FalseCycles) const {
+ Register FalseReg,
+ SelectExpansion &Exp) const {
// Not all subtargets have cmov instructions.
if (!Subtarget.canUseCMOV())
return false;
@@ -4228,9 +4228,9 @@ bool X86InstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
X86::GR64RegClass.hasSubClassEq(RC)) {
// This latency applies to Pentium M, Merom, Wolfdale, Nehalem, and Sandy
// Bridge. Probably Ivy Bridge as well.
- CondCycles = 2;
- TrueCycles = 2;
- FalseCycles = 2;
+ Exp.CondCycles = 2;
+ Exp.TrueCycles = 2;
+ Exp.FalseCycles = 2;
return true;
}
diff --git a/llvm/lib/Target/X86/X86InstrInfo.h b/llvm/lib/Target/X86/X86InstrInfo.h
index c018fcb124e70..afdf279f9d592 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.h
+++ b/llvm/lib/Target/X86/X86InstrInfo.h
@@ -471,8 +471,8 @@ class X86InstrInfo final : public X86GenInstrInfo {
const DebugLoc &DL,
int *BytesAdded = nullptr) const override;
bool canInsertSelect(const MachineBasicBlock &, ArrayRef<MachineOperand> Cond,
- Register, Register, Register, int &, int &,
- int &) const override;
+ Register, Register, Register,
+ SelectExpansion &) const override;
void insertSelect(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
const DebugLoc &DL, Register DstReg,
ArrayRef<MachineOperand> Cond, Register TrueReg,
>From 2a037cf3441b1f44b4c743892a1102042d8e9853 Mon Sep 17 00:00:00 2001
From: Jonathan Cohen <joncoh at apple.com>
Date: Thu, 16 Apr 2026 16:21:31 +0300
Subject: [PATCH 3/3] [EarlyIfConvert] Add heuristic to convert cascades of
conditional branches
Recognize and convert cascade patterns: chains of conditional branches where
each block branches to either the next block or a shared tail. These patterns
arise from if-else-if chains and switch-like constructs, and may suffer from
serial misprediction when the branch conditions do not follow any predictable
pattern.
When enabled via -enable-early-ifcvt-cascade, the pass walks up from
single-successor blocks to identify cascades, validates each block for
speculative execution, and converts the entire chain if profitable.
The profitability model requires all cascade branches to be data-dependent
(conditions which have recently been loaded from memory and therefore may not
be correlated to other branches, see PR #174457), checks that sufficient ILP
exists to hide the speculated instructions, and bounds the CSEL chain depth
against a scaled misprediction penalty.
---
llvm/lib/CodeGen/EarlyIfConversion.cpp | 815 +++++++-
.../CodeGen/AArch64/early-ifcvt-cascade.mir | 1840 +++++++++++++----
.../early-ifcvt-load-to-cond-br-limit.mir | 577 ++++++
.../CodeGen/AArch64/early-ifcvt-remarks.mir | 97 +
4 files changed, 2851 insertions(+), 478 deletions(-)
diff --git a/llvm/lib/CodeGen/EarlyIfConversion.cpp b/llvm/lib/CodeGen/EarlyIfConversion.cpp
index 71fe114bace81..d441afd227aba 100644
--- a/llvm/lib/CodeGen/EarlyIfConversion.cpp
+++ b/llvm/lib/CodeGen/EarlyIfConversion.cpp
@@ -16,11 +16,16 @@
//===----------------------------------------------------------------------===//
#include "llvm/CodeGen/EarlyIfConversion.h"
+#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/BitVector.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/DepthFirstIterator.h"
#include "llvm/ADT/PostOrderIterator.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/SparseSet.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
@@ -65,6 +70,16 @@ static cl::opt<bool> EnableDataDependentBranchAnalysis(
"enable-early-ifcvt-data-dependent", cl::Hidden, cl::init(false),
cl::desc("Enable hard-to-predict branch analysis for if-conversion"));
+// Enable recognition of cascade if-else patterns (chains of triangles).
+static cl::opt<bool> EnableCascadeIfConv(
+ "enable-early-ifcvt-cascade", cl::Hidden, cl::init(false),
+ cl::desc("Enable recognition of cascade if-else patterns"));
+
+// Maximum depth for cascade patterns.
+static cl::opt<unsigned>
+ MaxCascadeDepth("early-ifcvt-max-cascade-depth", cl::Hidden, cl::init(8),
+ cl::desc("Maximum cascade depth for early if-conversion"));
+
// Limit the number steps we take when searching conditions that depend on
// values recently loaded from memory.
static cl::opt<unsigned>
@@ -86,6 +101,9 @@ STATISTIC(NumTrianglesConv, "Number of triangles converted");
STATISTIC(NumDataDependant,
"Number of data dependent conditional branches encountered");
STATISTIC(NumLikelyBiased, "Number of branches with a hot path encountered");
+STATISTIC(NumCascadesSeen, "Number of cascade patterns detected");
+STATISTIC(NumCascadesConv, "Number of cascade patterns converted");
+STATISTIC(NumCascadesRejected, "Number of cascade patterns rejected");
//===----------------------------------------------------------------------===//
// SSAIfConv
@@ -109,6 +127,28 @@ STATISTIC(NumLikelyBiased, "Number of branches with a hot path encountered");
// Head block, and phis in the Tail block are converted to select instructions.
//
namespace {
+
+/// What one cascade collapse step would do to one Tail phi.
+struct CascadeSelectInfo {
+ int CondCycles = 0;
+ /// Number of instructions the select expands to, as reported by
+ /// TII::canInsertSelect(). Zero when the target does not report it.
+ unsigned NumInsts = 0;
+ bool NeedsSelect = false;
+};
+
+struct CascadeResult {
+ MachineBasicBlock *Head = nullptr;
+ SmallVector<MachineBasicBlock *> Blocks;
+ SmallVector<SmallVector<CascadeSelectInfo, 4>> Selects;
+ /// Whether TII::canInsertSelect() reported the expansion size of every
+ /// select. Without it the collapsed cascade cannot be sized.
+ bool SelectSizesKnown = true;
+ /// Whether any select would define a physreg, such as the flags register.
+ bool SelectsClobberPhysRegs = false;
+ explicit operator bool() const { return Head != nullptr; }
+};
+
class SSAIfConv {
const TargetInstrInfo *TII;
const TargetRegisterInfo *TRI;
@@ -179,6 +219,11 @@ class SSAIfConv {
/// Return false if any dependency is incompatible with if conversion.
bool InstrDependenciesAllowIfConv(MachineInstr *I);
+ /// If Reg is defined in Head, add its defining instruction to InsertAfter.
+ /// Returns false if the definition is a terminator, which cannot be
+ /// speculated below.
+ bool regDependencyAllowsIfConv(Register Reg, const MachineBasicBlock *User);
+
/// Predicate all instructions of the basic block with current condition
/// except for terminators. Reverse the condition if ReversePredicate is set.
void PredicateBlock(MachineBasicBlock *MBB, bool ReversePredicate);
@@ -213,12 +258,40 @@ class SSAIfConv {
/// initialize the internal state, and return true.
/// If predicate is set try to predicate the block otherwise try to
/// speculatively execute it.
- bool canConvertIf(MachineBasicBlock *MBB, bool Predicate = false);
+ /// ExtraSpeculatedBlocks names blocks that a later conversion step will also
+ /// speculate into MBB. We track dependencies and register clobbers to find an
+ /// insertion point to satisfy all constraints (if possible).
+ bool canConvertIf(MachineBasicBlock *MBB, bool Predicate = false,
+ bool AllowMultiSuccTBB = false,
+ MachineBasicBlock *ExplicitTail = nullptr,
+ ArrayRef<MachineBasicBlock *> ExtraSpeculatedBlocks = {});
/// convertIf - If-convert the last block passed to canConvertIf(), assuming
/// it is possible. Add any blocks that are to be erased to RemoveBlocks.
void convertIf(SmallVectorImpl<MachineBasicBlock *> &RemoveBlocks,
bool Predicate = false);
+
+ /// matchCascade - match a cascade of conditional branches by walking up
+ /// from MBB as a potential cascade end. A cascade is a chain of triangles
+ /// where each block has 2 successors (next cascade block + Tail) and the
+ /// last block has 1 successor (Tail). Returns the discovered Head and list
+ /// of cascade blocks [BB1, ..., BBn] if valid, empty result otherwise.
+ ///
+ /// Head --------+
+ /// | |
+ /// v |
+ /// BB1 ----------+
+ /// | |
+ /// v |
+ /// BB2 ----------+
+ /// | |
+ /// ...
+ /// ...
+ /// | |
+ /// v v
+ /// BBn (MBB) -> Tail
+ ///
+ CascadeResult matchCascade(MachineBasicBlock *MBB);
};
} // end anonymous namespace
@@ -281,6 +354,22 @@ bool SSAIfConv::canSpeculateInstrs(MachineBasicBlock *MBB) {
return true;
}
+bool SSAIfConv::regDependencyAllowsIfConv(Register Reg,
+ const MachineBasicBlock *User) {
+ if (!Reg.isVirtual())
+ return true;
+ MachineInstr *DefMI = MRI->getVRegDef(Reg);
+ if (!DefMI || DefMI->getParent() != Head)
+ return true;
+ if (InsertAfter.insert(DefMI).second)
+ LLVM_DEBUG(dbgs() << printMBBReference(*User) << " depends on " << *DefMI);
+ if (DefMI->isTerminator()) {
+ LLVM_DEBUG(dbgs() << "Can't insert instructions below terminator.\n");
+ return false;
+ }
+ return true;
+}
+
/// Check that there is no dependencies preventing if conversion.
///
/// If instruction uses any values that are defined in the head basic block,
@@ -300,18 +389,10 @@ bool SSAIfConv::InstrDependenciesAllowIfConv(MachineInstr *I) {
for (MCRegUnit Unit : TRI->regunits(Reg.asMCReg()))
ClobberedRegUnits.set(static_cast<unsigned>(Unit));
- if (!MO.readsReg() || !Reg.isVirtual())
- continue;
- MachineInstr *DefMI = MRI->getVRegDef(Reg);
- if (!DefMI || DefMI->getParent() != Head)
+ if (!MO.readsReg())
continue;
- if (InsertAfter.insert(DefMI).second)
- LLVM_DEBUG(dbgs() << printMBBReference(*I->getParent()) << " depends on "
- << *DefMI);
- if (DefMI->isTerminator()) {
- LLVM_DEBUG(dbgs() << "Can't insert instructions below terminator.\n");
+ if (!regDependencyAllowsIfConv(Reg, I->getParent()))
return false;
- }
}
return true;
}
@@ -469,7 +550,10 @@ bool SSAIfConv::findInsertionPoint() {
/// canConvertIf - analyze the sub-cfg rooted in MBB, and return true if it is
/// a potential candidate for if-conversion. Fill out the internal state.
///
-bool SSAIfConv::canConvertIf(MachineBasicBlock *MBB, bool Predicate) {
+bool SSAIfConv::canConvertIf(
+ MachineBasicBlock *MBB, bool Predicate, bool AllowMultiSuccTBB,
+ MachineBasicBlock *ExplicitTail,
+ ArrayRef<MachineBasicBlock *> ExtraSpeculatedBlocks) {
Head = MBB;
TBB = FBB = Tail = nullptr;
@@ -482,31 +566,40 @@ bool SSAIfConv::canConvertIf(MachineBasicBlock *MBB, bool Predicate) {
if (Succ0->pred_size() != 1)
std::swap(Succ0, Succ1);
- if (Succ0->pred_size() != 1 || Succ0->succ_size() != 1)
+ if (Succ0->pred_size() != 1 ||
+ (!AllowMultiSuccTBB && Succ0->succ_size() != 1))
return false;
- Tail = Succ0->succ_begin()[0];
-
- // This is not a triangle.
- if (Tail != Succ1) {
- // Check for a diamond. We won't deal with any critical edges.
- if (Succ1->pred_size() != 1 || Succ1->succ_size() != 1 ||
- Succ1->succ_begin()[0] != Tail)
- return false;
- LLVM_DEBUG(dbgs() << "\nDiamond: " << printMBBReference(*Head) << " -> "
- << printMBBReference(*Succ0) << "/"
- << printMBBReference(*Succ1) << " -> "
+ // Use explicit tail if provided (for cascade validation), otherwise compute.
+ if (ExplicitTail) {
+ Tail = ExplicitTail;
+ LLVM_DEBUG(dbgs() << "\nCascade triangle: " << printMBBReference(*Head)
+ << " -> " << printMBBReference(*Succ0) << " -> "
<< printMBBReference(*Tail) << '\n');
-
- // Live-in physregs are tricky to get right when speculating code.
- if (!Tail->livein_empty()) {
- LLVM_DEBUG(dbgs() << "Tail has live-ins.\n");
- return false;
- }
} else {
- LLVM_DEBUG(dbgs() << "\nTriangle: " << printMBBReference(*Head) << " -> "
- << printMBBReference(*Succ0) << " -> "
- << printMBBReference(*Tail) << '\n');
+ Tail = Succ0->succ_begin()[0];
+
+ // This is not a triangle.
+ if (Tail != Succ1) {
+ // Check for a diamond. We won't deal with any critical edges.
+ if (Succ1->pred_size() != 1 || Succ1->succ_size() != 1 ||
+ Succ1->succ_begin()[0] != Tail)
+ return false;
+ LLVM_DEBUG(dbgs() << "\nDiamond: " << printMBBReference(*Head) << " -> "
+ << printMBBReference(*Succ0) << "/"
+ << printMBBReference(*Succ1) << " -> "
+ << printMBBReference(*Tail) << '\n');
+
+ // Live-in physregs are tricky to get right when speculating code.
+ if (!Tail->livein_empty()) {
+ LLVM_DEBUG(dbgs() << "Tail has live-ins.\n");
+ return false;
+ }
+ } else {
+ LLVM_DEBUG(dbgs() << "\nTriangle: " << printMBBReference(*Head) << " -> "
+ << printMBBReference(*Succ0) << " -> "
+ << printMBBReference(*Tail) << '\n');
+ }
}
// This is a triangle or a diamond.
@@ -582,15 +675,54 @@ bool SSAIfConv::canConvertIf(MachineBasicBlock *MBB, bool Predicate) {
return false;
}
+ // Perform additional legality checks for cascading conditions. We check
+ // legality of the whole transformation upfront to prevent partial
+ // conversions.
+ for (MachineBasicBlock *Extra : ExtraSpeculatedBlocks)
+ if (!canSpeculateInstrs(Extra))
+ return false;
+
+ // For each of the PHIs we are attempting to convert - check that
+ // all operands defined in the cascade allow if-conversion.
+ if (!ExtraSpeculatedBlocks.empty()) {
+ MachineBasicBlock *SpecBB = TBB != Tail ? TBB : FBB;
+ for (PHIInfo &PI : PHIs) {
+ for (const MachineOperand &MO : PI.PHI->uses()) {
+ if (!MO.isReg())
+ continue;
+ MachineBasicBlock *Pred =
+ PI.PHI->getOperand(MO.getOperandNo() + 1).getMBB();
+ if (Pred != SpecBB && !is_contained(ExtraSpeculatedBlocks, Pred))
+ continue;
+ if (!regDependencyAllowsIfConv(MO.getReg(), Pred))
+ return false;
+ }
+ // At conversion time TReg is the result of the previous step's select,
+ // not the original incoming value, so validate against a register of the
+ // phi destination's class instead.
+ Register DstReg = PI.PHI->getOperand(0).getReg();
+ TargetInstrInfo::SelectExpansion Exp;
+ if (!TII->canInsertSelect(*Head, Cond, DstReg, DstReg, PI.FReg, Exp)) {
+ LLVM_DEBUG(dbgs() << "Can't convert deeper cascade step: " << *PI.PHI);
+ return false;
+ }
+ }
+ }
+
// Try to find a valid insertion point for the speculated instructions in the
// head basic block.
if (!findInsertionPoint())
return false;
- if (isTriangle())
- ++NumTrianglesSeen;
- else
- ++NumDiamondsSeen;
+ // ExplicitTail means this is a validation call. Don't count towards the
+ // triangle / diamonds seen stat - that should happen only when we consider
+ // them for conversion seperately.
+ if (!ExplicitTail) {
+ if (isTriangle())
+ ++NumTrianglesSeen;
+ else
+ ++NumDiamondsSeen;
+ }
return true;
}
@@ -640,6 +772,134 @@ static bool hasSameValue(const MachineRegisterInfo &MRI,
return TIdx == FIdx;
}
+/// matchCascade - match a cascade pattern by walking up from MBB.
+/// MBB is a potential cascade end (last block before Tail).
+/// A cascade is a chain of triangles where:
+/// - Head has 2 successors: one to first cascade block, one to Tail
+/// - Each cascade block has 2 successors: next cascade + Tail (or 1 for last)
+/// - Last cascade block (MBB) has 1 successor: Tail
+/// Returns the discovered Head and cascade blocks if valid, empty otherwise.
+///
+/// Head --------+
+/// | |
+/// v |
+/// BB1 ----------+
+/// | |
+/// v |
+/// BB2 ----------+
+/// | |
+/// ...
+/// ...
+/// | |
+/// v v
+/// BBn (MBB) -> Tail
+///
+CascadeResult SSAIfConv::matchCascade(MachineBasicBlock *MBB) {
+ // Cascade end must have exactly 1 successor (to Tail) and 1 predecessor
+ if (MBB->succ_size() != 1 || MBB->pred_size() != 1)
+ return {};
+
+ MachineBasicBlock *Tail = MBB->succ_begin()[0];
+
+ // Collect cascade blocks from bottom to top: [BBn, BB(n-1), ..., BB1, Head]
+ SmallVector<MachineBasicBlock *> Blocks;
+ Blocks.push_back(MBB);
+ MachineBasicBlock *Current = MBB;
+
+ // Walk up the cascade chain
+ while (true) {
+ MachineBasicBlock *Pred = Current->pred_begin()[0];
+
+ // Predecessor must have 2 successors: Current and Tail
+ if (Pred->succ_size() != 2)
+ break;
+
+ MachineBasicBlock *S0 = Pred->succ_begin()[0];
+ MachineBasicBlock *S1 = Pred->succ_begin()[1];
+
+ // One successor must be Current, the other must be Tail
+ if (!((S0 == Current && S1 == Tail) || (S1 == Current && S0 == Tail)))
+ break;
+
+ Blocks.push_back(Pred);
+
+ // If Pred has multiple predecessors, it's the Head - stop here
+ if (Pred->pred_size() != 1)
+ break;
+
+ // Pred has 1 predecessor, so the cascade continues. If we've reached
+ // max depth, bail out entirely - we don't want to convert a partial
+ // cascade.
+ if (Blocks.size() > MaxCascadeDepth) {
+ LLVM_DEBUG(dbgs() << "Cascade extends beyond max depth "
+ << MaxCascadeDepth << ", not converting.\n");
+ return {};
+ }
+
+ Current = Pred;
+ }
+
+ // Need at least 3 blocks: [BBn, BB1, Head] for a minimal 2-block cascade
+ if (Blocks.size() < 3)
+ return {};
+
+ // Reverse to get [HEAD, BB1, BB2, ..., BBn] order
+ std::reverse(Blocks.begin(), Blocks.end());
+
+ // Check that we can actually convert all the blocks in the cascade.
+ // We skip the last cascade block (BBn) since it has a single successor
+ // (to Tail), and is only used as a TBB (not a HEAD) - its speculatability
+ // is validated when we check its predecessor.
+ SmallVector<SmallVector<CascadeSelectInfo, 4>> CollectedSelects;
+ bool SelectSizesKnown = true;
+ bool SelectsClobberPhysRegs = false;
+ size_t NumBlocksToValidate = Blocks.size() - 1;
+ for (size_t I = 0; I < NumBlocksToValidate; ++I) {
+ // We validate against Blocks[I] because that is the block
+ // that all conditions below will be spliced into.
+ if (!canConvertIf(
+ Blocks[I], /*Predicate=*/false,
+ /*AllowMultiSuccTBB=*/true,
+ /*ExplicitTail=*/Tail,
+ // Blocks[I + 1] is implicitly checked
+ /*ExtraSpeculatedBlocks=*/ArrayRef(Blocks).drop_front(I + 2))) {
+ LLVM_DEBUG(dbgs() << "Cannot convert cascade, block "
+ << printMBBReference(*Blocks[I])
+ << " is not if-convertible.\n");
+ return {};
+ }
+ assert((CollectedSelects.empty() ||
+ CollectedSelects.back().size() == PHIs.size()) &&
+ "Tail phi list changed between cascade blocks");
+
+ auto &StepSelects = CollectedSelects.emplace_back();
+ for (const PHIInfo &PI : PHIs) {
+ StepSelects.push_back({PI.Select.CondCycles, PI.Select.NumInsts,
+ !hasSameValue(*MRI, TII, PI.TReg, PI.FReg)});
+ if (PI.Select.NumInsts == 0)
+ SelectSizesKnown = false;
+ if (!PI.Select.ClobberedPhysRegs.empty())
+ SelectsClobberPhysRegs = true;
+ }
+ }
+
+ // The first block in Blocks is the cascade Head
+ MachineBasicBlock *Head = Blocks.front();
+
+ // Remove Head from the cascade blocks to convert
+ Blocks.erase(Blocks.begin());
+
+ LLVM_DEBUG({
+ dbgs() << "\nCascade found: " << printMBBReference(*Head) << " -> [";
+ for (auto *BB : Blocks)
+ dbgs() << printMBBReference(*BB) << ", ";
+ dbgs() << "] -> " << printMBBReference(*Tail) << "\n";
+ });
+
+ return {Head, std::move(Blocks), std::move(CollectedSelects),
+ SelectSizesKnown, SelectsClobberPhysRegs};
+}
+
/// replacePHIInstrs - Completely replace PHI instructions with selects.
/// This is possible when the only Tail predecessors are the if-converted
/// blocks.
@@ -842,6 +1102,10 @@ class EarlyIfConverter {
/// each block to the number of instructions scanned in it.
DenseMap<const MachineBasicBlock *, unsigned> NoCallBlocksCache;
+ /// Set of blocks that must be converted (part of a cascade).
+ /// These blocks bypass normal profitability checks in shouldConvertIf().
+ SmallPtrSet<MachineBasicBlock *, 16> MustConvertBlocks;
+
public:
EarlyIfConverter(MachineDominatorTree &DT, MachineLoopInfo &LI,
MachineTraceMetrics &MTM, MachineBranchProbabilityInfo *MBPI)
@@ -852,11 +1116,20 @@ class EarlyIfConverter {
private:
bool tryConvertIf(MachineBasicBlock *);
+ void detectCascades(MachineBasicBlock *);
+ void convertIf();
void invalidateTraces();
bool shouldConvertIf();
- bool isConditionDataDependent();
- bool doOperandsComeFromMemory(const MachineInstr *ConditionDef);
+ bool isConditionDataDependent(MachineBasicBlock *BB, bool RecordStats = true);
+ bool isCascadeDataDependent(MachineBasicBlock *Head,
+ ArrayRef<MachineBasicBlock *> CascadeBlocks);
+ bool doOperandsComeFromMemory(const MachineInstr *ConditionDef,
+ MachineBasicBlock *BB);
bool hasCallOrLoopInRange(const MachineInstr *From, const MachineInstr *To);
+ bool shouldConvertCascade(CascadeResult &Cascade, MachineBasicBlock *Tail);
+ bool hasEnoughILP(MachineBasicBlock *TraceBlock,
+ SmallVectorImpl<MachineBasicBlock *> &ExtraBlocks,
+ unsigned CritLimit);
};
class EarlyIfConverterLegacy : public MachineFunctionPass {
@@ -1046,9 +1319,10 @@ bool EarlyIfConverter::hasCallOrLoopInRange(const MachineInstr *From,
/// depend on values loaded from memory (unless they are loop invariant,
/// or come from a constant pool). The walk starts from the definition of
/// ConditionDef's first operand, which is not ConditionDef itself for
-/// instructions such as FCMPSrr, where that operand is a use.
+/// instructions such as FCMPSrr, where that operand is a use. BB is the block
+/// whose terminator consumes the condition.
bool EarlyIfConverter::doOperandsComeFromMemory(
- const MachineInstr *ConditionDef) {
+ const MachineInstr *ConditionDef, MachineBasicBlock *BB) {
Register Reg = ConditionDef->getOperand(0).getReg();
if (!Reg.isVirtual())
return false;
@@ -1056,10 +1330,10 @@ bool EarlyIfConverter::doOperandsComeFromMemory(
LLVM_DEBUG(dbgs() << " doOperandsComeFromMemory starting from reg "
<< printReg(Reg) << "\n");
- // The condition is consumed by the branch terminating Head, so this is the
+ // The condition is consumed by the branch terminating BB, so this is the
// end of the interval a load has to survive without a call in between.
- const MachineInstr *Br = &*IfConv.Head->getFirstTerminator();
- MachineLoop *IfConvLoop = Loops->getLoopFor(IfConv.Head);
+ const MachineInstr *Br = &*BB->getFirstTerminator();
+ MachineLoop *IfConvLoop = Loops->getLoopFor(BB);
// Walk the def-use chain.
SmallPtrSet<const MachineInstr *, 8> VisitedInstrs;
@@ -1126,9 +1400,12 @@ bool EarlyIfConverter::doOperandsComeFromMemory(
}
/// Check if the branch condition is data-dependent (comes from memory loads).
-bool EarlyIfConverter::isConditionDataDependent() {
+/// RecordStats should be false when the same branch may be examined again
+/// later, so that it is only counted once.
+bool EarlyIfConverter::isConditionDataDependent(MachineBasicBlock *BB,
+ bool RecordStats) {
TargetInstrInfo::MachineBranchPredicate MBP;
- if (TII->analyzeBranchPredicate(*IfConv.Head, MBP, /*AllowModify=*/false))
+ if (TII->analyzeBranchPredicate(*BB, MBP, /*AllowModify=*/false))
return false;
if (!MBP.ConditionDef)
@@ -1137,29 +1414,52 @@ bool EarlyIfConverter::isConditionDataDependent() {
// If the branch is biased (not 50/50), don't consider it data dependent.
// This is to prevent converting unprofitable checks such as
// `x[i] != 0;`
- auto TBBProb = MBPI->getEdgeProbability(IfConv.Head, IfConv.TBB);
- auto FBBProb = MBPI->getEdgeProbability(IfConv.Head, IfConv.FBB);
- if (TBBProb != FBBProb) {
- ++NumLikelyBiased;
- return false;
+ if (MBP.TrueDest && MBP.FalseDest && MBPI) {
+ auto TBBProb = MBPI->getEdgeProbability(BB, MBP.TrueDest);
+ auto FBBProb = MBPI->getEdgeProbability(BB, MBP.FalseDest);
+ if (TBBProb != FBBProb) {
+ if (RecordStats)
+ ++NumLikelyBiased;
+ return false;
+ }
}
// Check if operands used to compute the branch condition were loaded recently
// from memory, starting by the ConditionDef itself and walking up the use-def
// chain.
- if (doOperandsComeFromMemory(MBP.ConditionDef)) {
- ++NumDataDependant;
+ if (doOperandsComeFromMemory(MBP.ConditionDef, BB)) {
+ if (RecordStats)
+ ++NumDataDependant;
return true;
}
return false;
}
-// Adjust cycles with downward saturation.
-static unsigned adjCycles(unsigned Cyc, int Delta) {
- if (Delta < 0 && Cyc + Delta > Cyc)
- return 0;
- return Cyc + Delta;
+/// Check if ALL branches in a cascade are data-dependent (come from loads).
+/// Cascade conversion always requires data-dependent branches, so this check
+/// is enabled whenever cascade conversion is enabled.
+bool EarlyIfConverter::isCascadeDataDependent(
+ MachineBasicBlock *Head, ArrayRef<MachineBasicBlock *> CascadeBlocks) {
+
+ // Check Head block
+ if (!isConditionDataDependent(Head, /*RecordStats=*/false)) {
+ LLVM_DEBUG(dbgs() << "Cascade: Head not data-dependent\n");
+ return false;
+ }
+
+ // Check all cascade blocks except the last one (which has only 1 successor).
+ // Don't record stats because we are not converting anything yet.
+ for (auto *CascadeBlock : CascadeBlocks.drop_back()) {
+ if (!isConditionDataDependent(CascadeBlock, /*RecordStats=*/false)) {
+ LLVM_DEBUG(dbgs() << "Cascade: " << printMBBReference(*CascadeBlock)
+ << " not data-dependent\n");
+ return false;
+ }
+ }
+
+ LLVM_DEBUG(dbgs() << "Cascade: all branches are data-dependent\n");
+ return true;
}
namespace {
@@ -1167,20 +1467,303 @@ namespace {
struct Cycles {
const char *Key;
unsigned Value;
+ Cycles(const char *K, unsigned V) : Key(K), Value(V) {}
};
template <typename Remark> Remark &operator<<(Remark &R, Cycles C) {
return R << ore::NV(C.Key, C.Value) << (C.Value == 1 ? " cycle" : " cycles");
}
} // anonymous namespace
+// Adjust cycles with downward saturation.
+static unsigned adjCycles(unsigned Cyc, int Delta) {
+ if (Delta < 0 && Cyc + Delta > Cyc)
+ return 0;
+ return Cyc + Delta;
+}
+
+/// Count the instructions of MBB that a predecessor would have to speculate to
+/// absorb it. This matches what canSpeculateInstrs() checks against
+/// BlockInstrLimit: the non-debug instructions ahead of the terminators.
+/// If we don't perform this check we could bail out of a cascade conversion
+/// midway by exceeding `BlockInstrLimit`.
+static unsigned countSpeculatedInstrs(const MachineBasicBlock &MBB) {
+ return count_if(make_range(MBB.begin(), MBB.getFirstTerminator()),
+ [](const MachineInstr &MI) { return !MI.isDebugInstr(); });
+}
+
+/// Apply profitability check for cascade conversion.
+bool EarlyIfConverter::shouldConvertCascade(CascadeResult &Cascade,
+ MachineBasicBlock *Tail) {
+ MachineBasicBlock *Head = Cascade.Head;
+ auto &CascadeBlocks = Cascade.Blocks;
+
+ if (!isCascadeDataDependent(Head, CascadeBlocks)) {
+ LLVM_DEBUG(dbgs() << "Cascade: not all branches are data-dependent\n");
+ ++NumCascadesRejected;
+ return false;
+ }
+
+ // Calculate CritLimit using cascade formula.
+ unsigned CascadeSize = CascadeBlocks.size();
+ unsigned MispredictPenalty = STI->getMispredictionPenalty();
+ unsigned CritLimit = std::min(MispredictPenalty * 25 * CascadeSize / 100,
+ 2 * MispredictPenalty);
+
+ LLVM_DEBUG(dbgs() << "Cascade: size=" << CascadeSize
+ << ", CritLimit=" << CritLimit
+ << " (MispredictPenalty=" << MispredictPenalty << ")\n");
+
+ if (CritLimit == 0) {
+ LLVM_DEBUG(dbgs() << "Cascade: CritLimit is 0, skipping\n");
+ ++NumCascadesRejected;
+ return false;
+ }
+
+ MachineOptimizationRemarkEmitter MORE(*Head->getParent(), nullptr);
+
+ // The size estimate below needs to know how big each select is.
+ if (!Cascade.SelectSizesKnown) {
+ LLVM_DEBUG(dbgs() << "Cascade: target does not report select expansion "
+ "sizes, skipping\n");
+ MORE.emit([&]() {
+ return MachineOptimizationRemarkMissed(DEBUG_TYPE, "CascadeIfConversion",
+ Head->back().getDebugLoc(), Head)
+ << "did not if-convert cascade with "
+ << ore::NV("CascadeSize", CascadeSize)
+ << " blocks: the target does not report how many instructions a "
+ "select expands to.";
+ });
+ ++NumCascadesRejected;
+ return false;
+ }
+
+ // A select that defines a physreg lands just before the Head terminators, on
+ // top of whatever the shallower steps still need there.
+ // FIXME: Support these by checking the clobbered registers against what is
+ // live at each step's insertion point.
+ if (Cascade.SelectsClobberPhysRegs) {
+ LLVM_DEBUG(dbgs() << "Cascade: selects clobber physregs, skipping\n");
+ MORE.emit([&]() {
+ return MachineOptimizationRemarkMissed(DEBUG_TYPE, "CascadeIfConversion",
+ Head->back().getDebugLoc(), Head)
+ << "did not if-convert cascade with "
+ << ore::NV("CascadeSize", CascadeSize)
+ << " blocks: the selects would define physical registers.";
+ });
+ ++NumCascadesRejected;
+ return false;
+ }
+
+ SmallVector<MachineBasicBlock *, 8> CondBlocks;
+ CondBlocks.push_back(Head);
+ append_range(CondBlocks, ArrayRef(CascadeBlocks).drop_back());
+ assert(CondBlocks.size() == Cascade.Selects.size() &&
+ "Mismatched cascade condition info");
+
+ unsigned NumPHIs = Cascade.Selects.front().size();
+ assert(NumPHIs == range_size(Tail->phis()) &&
+ "Tail phi list changed since matchCascade()");
+
+ // A step only needs a select if the phi's two incoming values differ. But
+ // once some deeper step has produced a select, every shallower step takes
+ // that select's result as an incoming value and needs one too.
+ SmallVector<int, 8> DeepestSelectStep(NumPHIs, -1);
+ for (auto [I, Step] : enumerate(Cascade.Selects))
+ for (auto [J, Sel] : enumerate(Step))
+ if (Sel.NeedsSelect)
+ DeepestSelectStep[J] = I;
+
+ if (!Stress) {
+ // Charging every step a select for every phi over-estimates the collapsed
+ // block, and that is deliberate: it makes the bound safe for every
+ // intermediate step, not just the final one. At step I the speculated block
+ // holds Sum_{i>I} orig(BBi) plus the selects materialised by the deeper
+ // steps, and both terms are bounded by the totals below. The largest
+ // intermediate block therefore holds at most Sum(orig) plus the select
+ // instructions of every step but the first, so an accepted cascade always
+ // leaves the first step's selects as headroom and the per-step
+ // BlockInstrLimit check in canSpeculateInstrs() can never fire part-way
+ // through the collapse. Each select is charged the size the target reported
+ // rather than a single instruction, so this stays correct if selects that
+ // expand to several instructions are ever allowed here.
+ unsigned NumSelectInstrs = 0;
+ for (const auto &Step : Cascade.Selects)
+ for (const CascadeSelectInfo &Sel : Step)
+ NumSelectInstrs += Sel.NumInsts;
+
+ unsigned TotalInstrs = NumSelectInstrs;
+ for (MachineBasicBlock *BB : CascadeBlocks)
+ TotalInstrs += countSpeculatedInstrs(*BB);
+
+ LLVM_DEBUG(dbgs() << "Cascade: collapsed cascade holds at most "
+ << TotalInstrs << " instructions (" << NumSelectInstrs
+ << " of them from selects), limit is " << BlockInstrLimit
+ << '\n');
+
+ if (TotalInstrs > BlockInstrLimit) {
+ LLVM_DEBUG(dbgs() << "Cascade: collapsed cascade exceeds instruction "
+ "limit, skipping\n");
+ MORE.emit([&]() {
+ return MachineOptimizationRemarkMissed(DEBUG_TYPE,
+ "CascadeIfConversion",
+ Head->back().getDebugLoc(), Head)
+ << "did not if-convert cascade with "
+ << ore::NV("CascadeSize", CascadeSize)
+ << " blocks: the speculated blocks would hold up to "
+ << ore::NV("TotalInstrs", TotalInstrs)
+ << " instructions, exceeding the limit of "
+ << ore::NV("BlockInstrLimit", BlockInstrLimit) << ".";
+ });
+ ++NumCascadesRejected;
+ return false;
+ }
+ }
+
+ // Check if there's enough ILP to hide the speculated cascade blocks.
+ if (!hasEnoughILP(Tail, CascadeBlocks, CritLimit)) {
+ LLVM_DEBUG(dbgs() << "Cascade: not enough ILP, skipping\n");
+ MORE.emit([&]() {
+ return MachineOptimizationRemarkMissed(DEBUG_TYPE, "CascadeIfConversion",
+ Head->back().getDebugLoc(), Head)
+ << "did not if-convert cascade with "
+ << ore::NV("CascadeSize", CascadeSize)
+ << " blocks: not enough ILP to hide speculated instructions.";
+ });
+ ++NumCascadesRejected;
+ return false;
+ }
+
+ // Compute CSEL chain critical path depth.
+ // After conversion, we get a chain of CSELs where each CSEL depends on:
+ // 1. The condition from that block's branch
+ // 2. The previous CSEL's output (chain dependency)
+ // The chain is rooted at the deepest cascade block and ends at Head, which is
+ // the order convertIf() creates the selects in.
+ if (!MinInstr)
+ MinInstr = Traces->getEnsemble(MachineTraceStrategy::TS_MinInstrCount);
+
+ MachineTraceMetrics::Trace TailTrace = MinInstr->getTrace(Tail);
+
+ SmallVector<unsigned, 8> BranchDepths;
+ for (MachineBasicBlock *CondBlock : CondBlocks) {
+ MachineTraceMetrics::Trace BlockTrace = MinInstr->getTrace(CondBlock);
+ BranchDepths.push_back(
+ BlockTrace.getInstrCycles(*CondBlock->getFirstTerminator()).Depth);
+ }
+
+ // Each phi gets its own budget and its own chain: select latency depends on
+ // the phi's register class, and a phi with a lot of slack must not raise the
+ // bar for a phi with none. All of them have to be profitable.
+ unsigned MaxExtension = 0;
+ bool ExceedsLimit = false;
+ for (auto [J, PHI] : enumerate(Tail->phis())) {
+ unsigned Slack = TailTrace.getInstrSlack(PHI);
+ unsigned MaxDepth = Slack + TailTrace.getInstrCycles(PHI).Depth;
+
+ // convertIf() collapses the cascade bottom-up, so the deepest block's
+ // condition roots the CSEL chain and Head's condition feeds the last
+ // select.
+ LLVM_DEBUG(dbgs() << "CSEL chain depth computation for " << PHI);
+ unsigned CSELChainDepth = 0;
+ for (int I = DeepestSelectStep[J]; I >= 0; --I) {
+ int CondCycles = Cascade.Selects[I][J].CondCycles;
+ CSELChainDepth =
+ adjCycles(std::max(BranchDepths[I], CSELChainDepth), CondCycles);
+
+ LLVM_DEBUG(dbgs() << " " << printMBBReference(*CondBlocks[I])
+ << ": branch depth=" << BranchDepths[I]
+ << ", CondCycles=" << CondCycles
+ << ", new CSEL depth=" << CSELChainDepth << "\n");
+ }
+
+ unsigned Extension =
+ CSELChainDepth > MaxDepth ? CSELChainDepth - MaxDepth : 0;
+ MaxExtension = std::max(MaxExtension, Extension);
+
+ LLVM_DEBUG(dbgs() << "Final CSEL chain depth: " << CSELChainDepth
+ << ", MaxDepth: " << MaxDepth << ", Extension: "
+ << Extension << ", CritLimit: " << CritLimit << '\n');
+
+ if (Extension > CritLimit) {
+ LLVM_DEBUG(dbgs() << "Cascade: critical path extension exceeds "
+ "limit, skipping\n");
+ ExceedsLimit = true;
+ }
+ }
+
+ if (ExceedsLimit) {
+ MORE.emit([&]() {
+ MachineOptimizationRemarkMissed R(DEBUG_TYPE, "CascadeIfConversion",
+ Head->back().getDebugLoc(), Head);
+ R << "did not if-convert cascade with "
+ << ore::NV("CascadeSize", CascadeSize)
+ << " blocks: critical path extension of "
+ << Cycles("CritPathExtension", MaxExtension)
+ << " exceeds the threshold of " << Cycles("CritLimit", CritLimit)
+ << ".";
+ return R;
+ });
+ ++NumCascadesRejected;
+ return false;
+ }
+
+ MORE.emit([&]() {
+ MachineOptimizationRemark R(DEBUG_TYPE, "CascadeIfConversion",
+ Head->back().getDebugLoc(), Head);
+ R << "performing cascade if-conversion on "
+ << ore::NV("CascadeSize", CascadeSize)
+ << " blocks: critical path extension is "
+ << Cycles("CritPathExtension", MaxExtension)
+ << ", staying under the threshold of " << Cycles("CritLimit", CritLimit)
+ << ".";
+ return R;
+ });
+
+ return true;
+}
+
+/// Check if there is enough ILP to hide the latency of speculatively executing
+/// the extra blocks
+bool EarlyIfConverter::hasEnoughILP(
+ MachineBasicBlock *TraceBlock,
+ SmallVectorImpl<MachineBasicBlock *> &ExtraBlocks, unsigned CritLimit) {
+ if (!MinInstr)
+ MinInstr = Traces->getEnsemble(MachineTraceStrategy::TS_MinInstrCount);
+
+ MachineTraceMetrics::Trace Trace = MinInstr->getTrace(TraceBlock);
+ unsigned CritPath = Trace.getCriticalPath();
+ SmallVector<const MachineBasicBlock *, 8> ConstExtraBlocks(
+ ExtraBlocks.begin(), ExtraBlocks.end());
+ unsigned ResLength = Trace.getResourceLength(ConstExtraBlocks);
+
+ LLVM_DEBUG(dbgs() << "ILP check: CriticalPath=" << CritPath
+ << ", ResourceLength=" << ResLength
+ << ", CritLimit=" << CritLimit << "\n");
+
+ if (ResLength > CritPath + CritLimit) {
+ LLVM_DEBUG(dbgs() << "Not enough ILP: resource length " << ResLength
+ << " exceeds critical path " << CritPath << " + limit "
+ << CritLimit << "\n");
+ return false;
+ }
+ return true;
+}
+
/// Apply cost model and heuristics to the if-conversion in IfConv.
/// Return true if the conversion is a good idea.
///
bool EarlyIfConverter::shouldConvertIf() {
+ bool InCascade = MustConvertBlocks.erase(IfConv.Head);
+
// Stress testing mode disables all cost considerations.
if (Stress)
return true;
+ if (InCascade) {
+ LLVM_DEBUG(dbgs() << "Block is part of cascade, skipping profitability\n");
+ return true;
+ }
+
// Do not try to if-convert if the condition has a high chance of being
// predictable.
MachineLoop *CurrentLoop = Loops->getLoopFor(IfConv.Head);
@@ -1227,7 +1810,7 @@ bool EarlyIfConverter::shouldConvertIf() {
// hard-to-predict branches, half for others. Otherwise use half for all.
bool DataDependent = false;
if (EnableDataDependentBranchAnalysis)
- DataDependent = isConditionDataDependent();
+ DataDependent = isConditionDataDependent(IfConv.Head);
unsigned CritLimit = DataDependent ? STI->getMispredictionPenalty()
: STI->getMispredictionPenalty() / 2;
@@ -1262,10 +1845,10 @@ bool EarlyIfConverter::shouldConvertIf() {
MachineOptimizationRemarkMissed R(DEBUG_TYPE, "IfConversion",
MBB.findDebugLoc(MBB.back()), &MBB);
R << "did not if-convert branch: the resulting critical path ("
- << Cycles{"ResLength", ResLength}
+ << Cycles("ResLength", ResLength)
<< ") would extend the shorter leg's critical path ("
- << Cycles{"MinCrit", MinCrit} << ") by more than the threshold of "
- << Cycles{"CritLimit", CritLimit}
+ << Cycles("MinCrit", MinCrit) << ") by more than the threshold of "
+ << Cycles("CritLimit", CritLimit)
<< ", which cannot be hidden by available ILP.";
return R;
});
@@ -1349,15 +1932,15 @@ bool EarlyIfConverter::shouldConvertIf() {
MachineOptimizationRemark R(DEBUG_TYPE, "IfConversion",
MBB.back().getDebugLoc(), &MBB);
R << "performing if-conversion on branch: the condition adds "
- << Cycles{"CondCycles", Cond.Extra} << " to the critical path";
+ << Cycles("CondCycles", Cond.Extra) << " to the critical path";
if (Short.Extra > 0)
R << ", and the short leg adds another "
- << Cycles{"ShortCycles", Short.Extra};
+ << Cycles("ShortCycles", Short.Extra);
if (Long.Extra > 0)
R << ", and the long leg adds another "
- << Cycles{"LongCycles", Long.Extra};
+ << Cycles("LongCycles", Long.Extra);
R << ", each staying under the threshold of "
- << Cycles{"CritLimit", CritLimit} << ".";
+ << Cycles("CritLimit", CritLimit) << ".";
return R;
});
} else {
@@ -1365,20 +1948,20 @@ bool EarlyIfConverter::shouldConvertIf() {
MachineOptimizationRemarkMissed R(DEBUG_TYPE, "IfConversion",
MBB.back().getDebugLoc(), &MBB);
R << "did not if-convert branch: the condition would add "
- << Cycles{"CondCycles", Cond.Extra} << " to the critical path";
+ << Cycles("CondCycles", Cond.Extra) << " to the critical path";
if (Cond.Extra > CritLimit)
- R << " exceeding the limit of " << Cycles{"CritLimit", CritLimit};
+ R << " exceeding the limit of " << Cycles("CritLimit", CritLimit);
if (Short.Extra > 0) {
R << ", and the short leg would add another "
- << Cycles{"ShortCycles", Short.Extra};
+ << Cycles("ShortCycles", Short.Extra);
if (Short.Extra > CritLimit)
- R << " exceeding the limit of " << Cycles{"CritLimit", CritLimit};
+ R << " exceeding the limit of " << Cycles("CritLimit", CritLimit);
}
if (Long.Extra > 0) {
R << ", and the long leg would add another "
- << Cycles{"LongCycles", Long.Extra};
+ << Cycles("LongCycles", Long.Extra);
if (Long.Extra > CritLimit)
- R << " exceeding the limit of " << Cycles{"CritLimit", CritLimit};
+ R << " exceeding the limit of " << Cycles("CritLimit", CritLimit);
}
R << ".";
return R;
@@ -1388,26 +1971,84 @@ bool EarlyIfConverter::shouldConvertIf() {
return ShouldConvert;
}
+/// Perform the actual if-conversion and update analyses.
+void EarlyIfConverter::convertIf() {
+ SmallVector<MachineBasicBlock *, 4> RemoveBlocks;
+ invalidateTraces();
+ IfConv.convertIf(RemoveBlocks);
+ updateDomTree(DomTree, IfConv, RemoveBlocks);
+ updateLoops(Loops, RemoveBlocks);
+ // Head absorbs the instructions of the removed blocks, including any calls,
+ // so a Head cached as call-free may no longer be.
+ NoCallBlocksCache.erase(IfConv.Head);
+ for (MachineBasicBlock *MBB : RemoveBlocks) {
+ NoCallBlocksCache.erase(MBB);
+ MustConvertBlocks.erase(MBB);
+ MBB->eraseFromParent();
+ }
+}
+
+/// Try to detect a cascade ending at MBB and mark blocks for conversion.
+/// Since we visit blocks in postorder, the cascade end is visited before
+/// its head, allowing us to mark cascade blocks before they are visited.
+void EarlyIfConverter::detectCascades(MachineBasicBlock *MBB) {
+ if (!EnableCascadeIfConv || !EnableDataDependentBranchAnalysis ||
+ MustConvertBlocks.count(MBB))
+ return;
+
+ auto Cascade = IfConv.matchCascade(MBB);
+ if (!Cascade)
+ return;
+
+ ++NumCascadesSeen;
+ MachineBasicBlock *Tail = MBB->succ_begin()[0];
+ LLVM_DEBUG(dbgs() << "Found cascade with " << Cascade.Blocks.size()
+ << " blocks, Head=" << printMBBReference(*Cascade.Head)
+ << ", Tail=" << printMBBReference(*Tail) << "\n");
+
+ if (!shouldConvertCascade(Cascade, Tail))
+ return;
+
+ ++NumCascadesConv;
+ NumDataDependant += Cascade.Blocks.size();
+
+ // Mark Head and all cascade blocks except the last one.
+ // The last cascade block (BBn) is never a head - it's always the TBB.
+ MustConvertBlocks.insert(Cascade.Head);
+ for (size_t I = 0; I + 1 < Cascade.Blocks.size(); ++I)
+ MustConvertBlocks.insert(Cascade.Blocks[I]);
+ LLVM_DEBUG(dbgs() << "Marked " << (1 + Cascade.Blocks.size() - 1)
+ << " blocks for cascade conversion\n");
+}
+
/// Attempt repeated if-conversion on MBB, return true if successful.
///
bool EarlyIfConverter::tryConvertIf(MachineBasicBlock *MBB) {
+ detectCascades(MBB);
+
bool Changed = false;
while (IfConv.canConvertIf(MBB) && shouldConvertIf()) {
- // If-convert MBB and update analyses.
- invalidateTraces();
- SmallVector<MachineBasicBlock *, 4> RemoveBlocks;
- IfConv.convertIf(RemoveBlocks);
+ convertIf();
Changed = true;
- updateDomTree(DomTree, IfConv, RemoveBlocks);
- updateLoops(Loops, RemoveBlocks);
- // Head absorbs the instructions of the removed blocks, including any calls,
- // so a Head cached as call-free may no longer be.
- NoCallBlocksCache.erase(IfConv.Head);
- for (MachineBasicBlock *MBB : RemoveBlocks) {
- NoCallBlocksCache.erase(MBB);
- MBB->eraseFromParent();
- }
}
+
+ // A mark that survives means a cascade block was not converted. We only
+ // convert cascades if the entire cascade conversion is profitable, so
+ // something went wrong: shouldConvertCascade() accepted a cascade that
+ // canConvertIf() then rejected part-way through the collapse, leaving a
+ // partial cascade behind.
+ LLVM_DEBUG({
+ if (MustConvertBlocks.contains(MBB))
+ dbgs() << "Cascade block " << printMBBReference(*MBB) << " with "
+ << countSpeculatedInstrs(*MBB)
+ << " speculated instructions failed to if-convert:\n"
+ << *MBB;
+ });
+ assert(!MustConvertBlocks.contains(MBB) &&
+ "cascade block failed to if-convert, leaving a partial cascade; "
+ "shouldConvertCascade() accepted a cascade that canConvertIf() "
+ "later rejected");
+
return Changed;
}
@@ -1428,6 +2069,8 @@ bool EarlyIfConverter::run(MachineFunction &MF) {
bool Changed = false;
IfConv.init(MF);
+ MustConvertBlocks.clear();
+
// Visit blocks in dominator tree post-order. The post-order enables nested
// if-conversion in a single pass. The tryConvertIf() function may erase
// blocks, but only blocks dominated by the head block. This makes it safe to
@@ -1436,6 +2079,8 @@ bool EarlyIfConverter::run(MachineFunction &MF) {
if (tryConvertIf(DomNode->getBlock()))
Changed = true;
+ assert(MustConvertBlocks.empty() && "cascade block failed to if-convert");
+
return Changed;
}
diff --git a/llvm/test/CodeGen/AArch64/early-ifcvt-cascade.mir b/llvm/test/CodeGen/AArch64/early-ifcvt-cascade.mir
index 99c0ea79267a1..296f773b0a52f 100644
--- a/llvm/test/CodeGen/AArch64/early-ifcvt-cascade.mir
+++ b/llvm/test/CodeGen/AArch64/early-ifcvt-cascade.mir
@@ -1,5 +1,7 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
-# RUN: llc -mtriple=aarch64 -mcpu=apple-m1 -run-pass=early-ifcvt -enable-early-ifcvt-data-dependent %s -o - | FileCheck %s
+# RUN: llc -mtriple=aarch64 -mcpu=apple-m1 -run-pass=early-ifcvt -enable-early-ifcvt-cascade -enable-early-ifcvt-data-dependent %s -o - | FileCheck %s --check-prefixes=COMMON,CASCADE-WITH-DD
+# RUN: llc -mtriple=aarch64 -mcpu=apple-m1 -run-pass=early-ifcvt -enable-early-ifcvt-cascade -enable-early-ifcvt-data-dependent -early-ifcvt-max-cascade-depth=3 %s -o - | FileCheck %s --check-prefixes=COMMON,DEPTH3
+# RUN: llc -mtriple=aarch64 -mcpu=apple-m1 -run-pass=early-ifcvt -enable-early-ifcvt-cascade -enable-early-ifcvt-data-dependent -early-ifcvt-limit=3 %s -o - | FileCheck %s --check-prefixes=COMMON,LIMIT3
--- |
define void @cascade_selector_loop() { ret void }
@@ -7,6 +9,12 @@
define void @cascade_exceeds_crit_limit() { ret void }
define void @cascade_backedge_to_earlier_block() { ret void }
define void @cascade_forward_skip_to_later_block() { ret void }
+ define void @cascade_per_phi_slack() { ret void }
+ define void @cascade_phi_subset() { ret void }
+ define void @cascade_cbz_condition_not_converted() { ret void }
+ define void @cascade_head_def_after_condition_not_converted() { ret void }
+ define void @cascade_head_def_reaching_select_not_converted() { ret void }
+ define void @cascade_head_def_reaching_select_inverted_branch() { ret void }
...
---
name: cascade_selector_loop
@@ -18,102 +26,261 @@ liveins:
- { reg: '$w3', virtual-reg: '%3' }
- { reg: '$w4', virtual-reg: '%4' }
body: |
- ; CHECK-LABEL: name: cascade_selector_loop
- ; CHECK: bb.0:
- ; CHECK-NEXT: successors: %bb.10(0x30000000), %bb.1(0x50000000)
- ; CHECK-NEXT: liveins: $x0, $x1, $w2, $w3, $w4
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr32 = COPY $w2
- ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gpr32 = COPY $w3
- ; CHECK-NEXT: [[COPY4:%[0-9]+]]:gpr32 = COPY $w4
- ; CHECK-NEXT: [[COPY5:%[0-9]+]]:gpr64all = COPY $xzr
- ; CHECK-NEXT: CBZW [[COPY2]], %bb.10
- ; CHECK-NEXT: B %bb.1
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.1:
- ; CHECK-NEXT: successors: %bb.2(0x80000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 0 :: (load (s32))
- ; CHECK-NEXT: [[LDRWui1:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 1 :: (load (s32))
- ; CHECK-NEXT: [[LDRWui2:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 2 :: (load (s32))
- ; CHECK-NEXT: [[LDRWui3:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 3 :: (load (s32))
- ; CHECK-NEXT: [[LDRWui4:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 4 :: (load (s32))
- ; CHECK-NEXT: [[LDRWui5:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 5 :: (load (s32))
- ; CHECK-NEXT: [[LDRWui6:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 6 :: (load (s32))
- ; CHECK-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 1
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.2:
- ; CHECK-NEXT: successors: %bb.3(0x40000000), %bb.9(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[PHI:%[0-9]+]]:gpr32 = PHI [[COPY2]], %bb.1, %15, %bb.9
- ; CHECK-NEXT: [[LDRWui7:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load (s32))
- ; CHECK-NEXT: [[MADDWrrr:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[LDRWui7]], $wzr
- ; CHECK-NEXT: [[MADDWrrr1:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY4]], [[LDRWui7]], [[MADDWrrr]]
- ; CHECK-NEXT: [[MADDWrrr2:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[LDRWui7]], [[MADDWrrr1]]
- ; CHECK-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 7
- ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
- ; CHECK-NEXT: B %bb.3
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.3:
- ; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.9(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui1]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 6
- ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
- ; CHECK-NEXT: B %bb.4
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.4:
- ; CHECK-NEXT: successors: %bb.5(0x40000000), %bb.9(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[SUBSWrr2:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui2]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 5
- ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
- ; CHECK-NEXT: B %bb.5
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.5:
- ; CHECK-NEXT: successors: %bb.6(0x40000000), %bb.9(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[SUBSWrr3:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui3]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 4
- ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
- ; CHECK-NEXT: B %bb.6
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.6:
- ; CHECK-NEXT: successors: %bb.7(0x40000000), %bb.9(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[SUBSWrr4:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui4]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm4:%[0-9]+]]:gpr64 = MOVi64imm 3
- ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
- ; CHECK-NEXT: B %bb.7
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.7:
- ; CHECK-NEXT: successors: %bb.8(0x40000000), %bb.9(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[SUBSWrr5:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui5]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm5:%[0-9]+]]:gpr64 = MOVi64imm 2
- ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
- ; CHECK-NEXT: B %bb.8
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.8:
- ; CHECK-NEXT: successors: %bb.9(0x80000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[MOVi64imm6:%[0-9]+]]:gpr64 = MOVi64imm 1
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.9:
- ; CHECK-NEXT: successors: %bb.10(0x04000000), %bb.2(0x7c000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[PHI1:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.2, [[MOVi64imm1]], %bb.3, [[MOVi64imm2]], %bb.4, [[MOVi64imm3]], %bb.5, [[MOVi64imm4]], %bb.6, [[MOVi64imm5]], %bb.7, [[MOVi64imm6]], %bb.8
- ; CHECK-NEXT: [[SUBSWrr6:%[0-9]+]]:gpr32 = SUBSWrr [[PHI]], [[MOVi32imm]], implicit-def $nzcv
- ; CHECK-NEXT: Bcc 1, %bb.2, implicit $nzcv
- ; CHECK-NEXT: B %bb.10
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.10:
- ; CHECK-NEXT: [[PHI2:%[0-9]+]]:gpr64 = PHI [[COPY5]], %bb.0, [[PHI1]], %bb.9
- ; CHECK-NEXT: $x0 = COPY [[PHI2]]
- ; CHECK-NEXT: RET_ReallyLR implicit $x0
+ ; CASCADE-WITH-DD-LABEL: name: cascade_selector_loop
+ ; CASCADE-WITH-DD: bb.0:
+ ; CASCADE-WITH-DD-NEXT: successors: %bb.10(0x30000000), %bb.1(0x50000000)
+ ; CASCADE-WITH-DD-NEXT: liveins: $x0, $x1, $w2, $w3, $w4
+ ; CASCADE-WITH-DD-NEXT: {{ $}}
+ ; CASCADE-WITH-DD-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; CASCADE-WITH-DD-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1
+ ; CASCADE-WITH-DD-NEXT: [[COPY2:%[0-9]+]]:gpr32 = COPY $w2
+ ; CASCADE-WITH-DD-NEXT: [[COPY3:%[0-9]+]]:gpr32 = COPY $w3
+ ; CASCADE-WITH-DD-NEXT: [[COPY4:%[0-9]+]]:gpr32 = COPY $w4
+ ; CASCADE-WITH-DD-NEXT: [[COPY5:%[0-9]+]]:gpr64all = COPY $xzr
+ ; CASCADE-WITH-DD-NEXT: CBZW [[COPY2]], %bb.10
+ ; CASCADE-WITH-DD-NEXT: B %bb.1
+ ; CASCADE-WITH-DD-NEXT: {{ $}}
+ ; CASCADE-WITH-DD-NEXT: bb.1:
+ ; CASCADE-WITH-DD-NEXT: successors: %bb.2(0x80000000)
+ ; CASCADE-WITH-DD-NEXT: {{ $}}
+ ; CASCADE-WITH-DD-NEXT: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 0 :: (load (s32))
+ ; CASCADE-WITH-DD-NEXT: [[LDRWui1:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 1 :: (load (s32))
+ ; CASCADE-WITH-DD-NEXT: [[LDRWui2:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 2 :: (load (s32))
+ ; CASCADE-WITH-DD-NEXT: [[LDRWui3:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 3 :: (load (s32))
+ ; CASCADE-WITH-DD-NEXT: [[LDRWui4:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 4 :: (load (s32))
+ ; CASCADE-WITH-DD-NEXT: [[LDRWui5:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 5 :: (load (s32))
+ ; CASCADE-WITH-DD-NEXT: [[LDRWui6:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 6 :: (load (s32))
+ ; CASCADE-WITH-DD-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 1
+ ; CASCADE-WITH-DD-NEXT: {{ $}}
+ ; CASCADE-WITH-DD-NEXT: bb.2:
+ ; CASCADE-WITH-DD-NEXT: successors: %bb.10(0x04000000), %bb.2(0x7c000000)
+ ; CASCADE-WITH-DD-NEXT: {{ $}}
+ ; CASCADE-WITH-DD-NEXT: [[PHI:%[0-9]+]]:gpr32 = PHI [[COPY2]], %bb.1, %15, %bb.2
+ ; CASCADE-WITH-DD-NEXT: [[LDRWui7:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load (s32))
+ ; CASCADE-WITH-DD-NEXT: [[MADDWrrr:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[LDRWui7]], $wzr
+ ; CASCADE-WITH-DD-NEXT: [[MADDWrrr1:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY4]], [[LDRWui7]], [[MADDWrrr]]
+ ; CASCADE-WITH-DD-NEXT: [[MADDWrrr2:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[LDRWui7]], [[MADDWrrr1]]
+ ; CASCADE-WITH-DD-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui5]], implicit-def $nzcv
+ ; CASCADE-WITH-DD-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 2
+ ; CASCADE-WITH-DD-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 1
+ ; CASCADE-WITH-DD-NEXT: [[CSINCXr:%[0-9]+]]:gpr64 = CSINCXr [[MOVi64imm]], $xzr, 10, implicit $nzcv
+ ; CASCADE-WITH-DD-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui4]], implicit-def $nzcv
+ ; CASCADE-WITH-DD-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 3
+ ; CASCADE-WITH-DD-NEXT: [[CSELXr:%[0-9]+]]:gpr64 = CSELXr [[MOVi64imm2]], [[CSINCXr]], 10, implicit $nzcv
+ ; CASCADE-WITH-DD-NEXT: [[SUBSWrr2:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui3]], implicit-def $nzcv
+ ; CASCADE-WITH-DD-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 4
+ ; CASCADE-WITH-DD-NEXT: [[CSELXr1:%[0-9]+]]:gpr64 = CSELXr [[MOVi64imm3]], [[CSELXr]], 10, implicit $nzcv
+ ; CASCADE-WITH-DD-NEXT: [[SUBSWrr3:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui2]], implicit-def $nzcv
+ ; CASCADE-WITH-DD-NEXT: [[MOVi64imm4:%[0-9]+]]:gpr64 = MOVi64imm 5
+ ; CASCADE-WITH-DD-NEXT: [[CSELXr2:%[0-9]+]]:gpr64 = CSELXr [[MOVi64imm4]], [[CSELXr1]], 10, implicit $nzcv
+ ; CASCADE-WITH-DD-NEXT: [[SUBSWrr4:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui1]], implicit-def $nzcv
+ ; CASCADE-WITH-DD-NEXT: [[MOVi64imm5:%[0-9]+]]:gpr64 = MOVi64imm 6
+ ; CASCADE-WITH-DD-NEXT: [[CSELXr3:%[0-9]+]]:gpr64 = CSELXr [[MOVi64imm5]], [[CSELXr2]], 10, implicit $nzcv
+ ; CASCADE-WITH-DD-NEXT: [[SUBSWrr5:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui]], implicit-def $nzcv
+ ; CASCADE-WITH-DD-NEXT: [[MOVi64imm6:%[0-9]+]]:gpr64 = MOVi64imm 7
+ ; CASCADE-WITH-DD-NEXT: [[CSELXr4:%[0-9]+]]:gpr64 = CSELXr [[MOVi64imm6]], [[CSELXr3]], 10, implicit $nzcv
+ ; CASCADE-WITH-DD-NEXT: [[SUBSWrr6:%[0-9]+]]:gpr32 = SUBSWrr [[PHI]], [[MOVi32imm]], implicit-def $nzcv
+ ; CASCADE-WITH-DD-NEXT: Bcc 1, %bb.2, implicit $nzcv
+ ; CASCADE-WITH-DD-NEXT: B %bb.10
+ ; CASCADE-WITH-DD-NEXT: {{ $}}
+ ; CASCADE-WITH-DD-NEXT: bb.10:
+ ; CASCADE-WITH-DD-NEXT: [[PHI1:%[0-9]+]]:gpr64 = PHI [[COPY5]], %bb.0, [[CSELXr4]], %bb.2
+ ; CASCADE-WITH-DD-NEXT: $x0 = COPY [[PHI1]]
+ ; CASCADE-WITH-DD-NEXT: RET_ReallyLR implicit $x0
+ ;
+ ; DEPTH3-LABEL: name: cascade_selector_loop
+ ; DEPTH3: bb.0:
+ ; DEPTH3-NEXT: successors: %bb.10(0x30000000), %bb.1(0x50000000)
+ ; DEPTH3-NEXT: liveins: $x0, $x1, $w2, $w3, $w4
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; DEPTH3-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1
+ ; DEPTH3-NEXT: [[COPY2:%[0-9]+]]:gpr32 = COPY $w2
+ ; DEPTH3-NEXT: [[COPY3:%[0-9]+]]:gpr32 = COPY $w3
+ ; DEPTH3-NEXT: [[COPY4:%[0-9]+]]:gpr32 = COPY $w4
+ ; DEPTH3-NEXT: [[COPY5:%[0-9]+]]:gpr64all = COPY $xzr
+ ; DEPTH3-NEXT: CBZW [[COPY2]], %bb.10
+ ; DEPTH3-NEXT: B %bb.1
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: bb.1:
+ ; DEPTH3-NEXT: successors: %bb.2(0x80000000)
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 0 :: (load (s32))
+ ; DEPTH3-NEXT: [[LDRWui1:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 1 :: (load (s32))
+ ; DEPTH3-NEXT: [[LDRWui2:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 2 :: (load (s32))
+ ; DEPTH3-NEXT: [[LDRWui3:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 3 :: (load (s32))
+ ; DEPTH3-NEXT: [[LDRWui4:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 4 :: (load (s32))
+ ; DEPTH3-NEXT: [[LDRWui5:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 5 :: (load (s32))
+ ; DEPTH3-NEXT: [[LDRWui6:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 6 :: (load (s32))
+ ; DEPTH3-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 1
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: bb.2:
+ ; DEPTH3-NEXT: successors: %bb.3(0x40000000), %bb.9(0x40000000)
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: [[PHI:%[0-9]+]]:gpr32 = PHI [[COPY2]], %bb.1, %15, %bb.9
+ ; DEPTH3-NEXT: [[LDRWui7:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load (s32))
+ ; DEPTH3-NEXT: [[MADDWrrr:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[LDRWui7]], $wzr
+ ; DEPTH3-NEXT: [[MADDWrrr1:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY4]], [[LDRWui7]], [[MADDWrrr]]
+ ; DEPTH3-NEXT: [[MADDWrrr2:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[LDRWui7]], [[MADDWrrr1]]
+ ; DEPTH3-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui]], implicit-def $nzcv
+ ; DEPTH3-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 7
+ ; DEPTH3-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; DEPTH3-NEXT: B %bb.3
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: bb.3:
+ ; DEPTH3-NEXT: successors: %bb.4(0x40000000), %bb.9(0x40000000)
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui1]], implicit-def $nzcv
+ ; DEPTH3-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 6
+ ; DEPTH3-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; DEPTH3-NEXT: B %bb.4
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: bb.4:
+ ; DEPTH3-NEXT: successors: %bb.5(0x40000000), %bb.9(0x40000000)
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: [[SUBSWrr2:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui2]], implicit-def $nzcv
+ ; DEPTH3-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 5
+ ; DEPTH3-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; DEPTH3-NEXT: B %bb.5
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: bb.5:
+ ; DEPTH3-NEXT: successors: %bb.6(0x40000000), %bb.9(0x40000000)
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: [[SUBSWrr3:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui3]], implicit-def $nzcv
+ ; DEPTH3-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 4
+ ; DEPTH3-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; DEPTH3-NEXT: B %bb.6
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: bb.6:
+ ; DEPTH3-NEXT: successors: %bb.7(0x40000000), %bb.9(0x40000000)
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: [[SUBSWrr4:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui4]], implicit-def $nzcv
+ ; DEPTH3-NEXT: [[MOVi64imm4:%[0-9]+]]:gpr64 = MOVi64imm 3
+ ; DEPTH3-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; DEPTH3-NEXT: B %bb.7
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: bb.7:
+ ; DEPTH3-NEXT: successors: %bb.8(0x40000000), %bb.9(0x40000000)
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: [[SUBSWrr5:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui5]], implicit-def $nzcv
+ ; DEPTH3-NEXT: [[MOVi64imm5:%[0-9]+]]:gpr64 = MOVi64imm 2
+ ; DEPTH3-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; DEPTH3-NEXT: B %bb.8
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: bb.8:
+ ; DEPTH3-NEXT: successors: %bb.9(0x80000000)
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: [[MOVi64imm6:%[0-9]+]]:gpr64 = MOVi64imm 1
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: bb.9:
+ ; DEPTH3-NEXT: successors: %bb.10(0x04000000), %bb.2(0x7c000000)
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: [[PHI1:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.2, [[MOVi64imm1]], %bb.3, [[MOVi64imm2]], %bb.4, [[MOVi64imm3]], %bb.5, [[MOVi64imm4]], %bb.6, [[MOVi64imm5]], %bb.7, [[MOVi64imm6]], %bb.8
+ ; DEPTH3-NEXT: [[SUBSWrr6:%[0-9]+]]:gpr32 = SUBSWrr [[PHI]], [[MOVi32imm]], implicit-def $nzcv
+ ; DEPTH3-NEXT: Bcc 1, %bb.2, implicit $nzcv
+ ; DEPTH3-NEXT: B %bb.10
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: bb.10:
+ ; DEPTH3-NEXT: [[PHI2:%[0-9]+]]:gpr64 = PHI [[COPY5]], %bb.0, [[PHI1]], %bb.9
+ ; DEPTH3-NEXT: $x0 = COPY [[PHI2]]
+ ; DEPTH3-NEXT: RET_ReallyLR implicit $x0
+ ;
+ ; LIMIT3-LABEL: name: cascade_selector_loop
+ ; LIMIT3: bb.0:
+ ; LIMIT3-NEXT: successors: %bb.10(0x30000000), %bb.1(0x50000000)
+ ; LIMIT3-NEXT: liveins: $x0, $x1, $w2, $w3, $w4
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; LIMIT3-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1
+ ; LIMIT3-NEXT: [[COPY2:%[0-9]+]]:gpr32 = COPY $w2
+ ; LIMIT3-NEXT: [[COPY3:%[0-9]+]]:gpr32 = COPY $w3
+ ; LIMIT3-NEXT: [[COPY4:%[0-9]+]]:gpr32 = COPY $w4
+ ; LIMIT3-NEXT: [[COPY5:%[0-9]+]]:gpr64all = COPY $xzr
+ ; LIMIT3-NEXT: CBZW [[COPY2]], %bb.10
+ ; LIMIT3-NEXT: B %bb.1
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: bb.1:
+ ; LIMIT3-NEXT: successors: %bb.2(0x80000000)
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 0 :: (load (s32))
+ ; LIMIT3-NEXT: [[LDRWui1:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 1 :: (load (s32))
+ ; LIMIT3-NEXT: [[LDRWui2:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 2 :: (load (s32))
+ ; LIMIT3-NEXT: [[LDRWui3:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 3 :: (load (s32))
+ ; LIMIT3-NEXT: [[LDRWui4:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 4 :: (load (s32))
+ ; LIMIT3-NEXT: [[LDRWui5:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 5 :: (load (s32))
+ ; LIMIT3-NEXT: [[LDRWui6:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 6 :: (load (s32))
+ ; LIMIT3-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 1
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: bb.2:
+ ; LIMIT3-NEXT: successors: %bb.3(0x40000000), %bb.9(0x40000000)
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: [[PHI:%[0-9]+]]:gpr32 = PHI [[COPY2]], %bb.1, %15, %bb.9
+ ; LIMIT3-NEXT: [[LDRWui7:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load (s32))
+ ; LIMIT3-NEXT: [[MADDWrrr:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[LDRWui7]], $wzr
+ ; LIMIT3-NEXT: [[MADDWrrr1:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY4]], [[LDRWui7]], [[MADDWrrr]]
+ ; LIMIT3-NEXT: [[MADDWrrr2:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[LDRWui7]], [[MADDWrrr1]]
+ ; LIMIT3-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui]], implicit-def $nzcv
+ ; LIMIT3-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 7
+ ; LIMIT3-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; LIMIT3-NEXT: B %bb.3
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: bb.3:
+ ; LIMIT3-NEXT: successors: %bb.4(0x40000000), %bb.9(0x40000000)
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui1]], implicit-def $nzcv
+ ; LIMIT3-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 6
+ ; LIMIT3-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; LIMIT3-NEXT: B %bb.4
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: bb.4:
+ ; LIMIT3-NEXT: successors: %bb.5(0x40000000), %bb.9(0x40000000)
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: [[SUBSWrr2:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui2]], implicit-def $nzcv
+ ; LIMIT3-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 5
+ ; LIMIT3-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; LIMIT3-NEXT: B %bb.5
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: bb.5:
+ ; LIMIT3-NEXT: successors: %bb.6(0x40000000), %bb.9(0x40000000)
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: [[SUBSWrr3:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui3]], implicit-def $nzcv
+ ; LIMIT3-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 4
+ ; LIMIT3-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; LIMIT3-NEXT: B %bb.6
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: bb.6:
+ ; LIMIT3-NEXT: successors: %bb.7(0x40000000), %bb.9(0x40000000)
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: [[SUBSWrr4:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui4]], implicit-def $nzcv
+ ; LIMIT3-NEXT: [[MOVi64imm4:%[0-9]+]]:gpr64 = MOVi64imm 3
+ ; LIMIT3-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; LIMIT3-NEXT: B %bb.7
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: bb.7:
+ ; LIMIT3-NEXT: successors: %bb.8(0x40000000), %bb.9(0x40000000)
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: [[SUBSWrr5:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui5]], implicit-def $nzcv
+ ; LIMIT3-NEXT: [[MOVi64imm5:%[0-9]+]]:gpr64 = MOVi64imm 2
+ ; LIMIT3-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; LIMIT3-NEXT: B %bb.8
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: bb.8:
+ ; LIMIT3-NEXT: successors: %bb.9(0x80000000)
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: [[MOVi64imm6:%[0-9]+]]:gpr64 = MOVi64imm 1
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: bb.9:
+ ; LIMIT3-NEXT: successors: %bb.10(0x04000000), %bb.2(0x7c000000)
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: [[PHI1:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.2, [[MOVi64imm1]], %bb.3, [[MOVi64imm2]], %bb.4, [[MOVi64imm3]], %bb.5, [[MOVi64imm4]], %bb.6, [[MOVi64imm5]], %bb.7, [[MOVi64imm6]], %bb.8
+ ; LIMIT3-NEXT: [[SUBSWrr6:%[0-9]+]]:gpr32 = SUBSWrr [[PHI]], [[MOVi32imm]], implicit-def $nzcv
+ ; LIMIT3-NEXT: Bcc 1, %bb.2, implicit $nzcv
+ ; LIMIT3-NEXT: B %bb.10
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: bb.10:
+ ; LIMIT3-NEXT: [[PHI2:%[0-9]+]]:gpr64 = PHI [[COPY5]], %bb.0, [[PHI1]], %bb.9
+ ; LIMIT3-NEXT: $x0 = COPY [[PHI2]]
+ ; LIMIT3-NEXT: RET_ReallyLR implicit $x0
bb.0:
successors: %bb.10(0x30000000), %bb.1(0x50000000)
liveins: $x0, $x1, $w2, $w3, $w4
@@ -139,6 +306,7 @@ body: |
%16:gpr32 = LDRWui %1, 6 :: (load (s32))
%17:gpr32 = MOVi32imm 1
+ ; Cascade head: data-dependent condition (derived from load + compute chain)
bb.2:
successors: %bb.3(0x40000000), %bb.9(0x40000000)
@@ -220,104 +388,104 @@ liveins:
- { reg: '$w4', virtual-reg: '%4' }
- { reg: '$w5', virtual-reg: '%5' }
body: |
- ; CHECK-LABEL: name: cascade_branches_from_args_not_dd
- ; CHECK: bb.0:
- ; CHECK-NEXT: successors: %bb.10(0x30000000), %bb.1(0x50000000)
- ; CHECK-NEXT: liveins: $x1, $w2, $w3, $w4, $w5
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x1
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr32 = COPY $w2
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr32 = COPY $w3
- ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gpr32 = COPY $w4
- ; CHECK-NEXT: [[COPY4:%[0-9]+]]:gpr32 = COPY $w5
- ; CHECK-NEXT: [[COPY5:%[0-9]+]]:gpr64all = COPY $xzr
- ; CHECK-NEXT: CBZW [[COPY1]], %bb.10
- ; CHECK-NEXT: B %bb.1
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.1:
- ; CHECK-NEXT: successors: %bb.2(0x80000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[COPY6:%[0-9]+]]:gpr32 = COPY [[COPY2]]
- ; CHECK-NEXT: [[COPY7:%[0-9]+]]:gpr32 = COPY [[COPY3]]
- ; CHECK-NEXT: [[COPY8:%[0-9]+]]:gpr32 = COPY [[COPY4]]
- ; CHECK-NEXT: [[COPY9:%[0-9]+]]:gpr32 = COPY [[COPY2]]
- ; CHECK-NEXT: [[COPY10:%[0-9]+]]:gpr32 = COPY [[COPY3]]
- ; CHECK-NEXT: [[COPY11:%[0-9]+]]:gpr32 = COPY [[COPY4]]
- ; CHECK-NEXT: [[COPY12:%[0-9]+]]:gpr32 = COPY [[COPY2]]
- ; CHECK-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 1
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.2:
- ; CHECK-NEXT: successors: %bb.3(0x40000000), %bb.9(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[PHI:%[0-9]+]]:gpr32 = PHI [[COPY1]], %bb.1, %15, %bb.9
- ; CHECK-NEXT: [[MADDWrrr:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY2]], [[COPY4]], $wzr
- ; CHECK-NEXT: [[MADDWrrr1:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[COPY4]], [[MADDWrrr]]
- ; CHECK-NEXT: [[MADDWrrr2:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY2]], [[COPY3]], [[MADDWrrr1]]
- ; CHECK-NEXT: [[MADDWrrr3:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[COPY2]], [[MADDWrrr2]]
- ; CHECK-NEXT: [[MADDWrrr4:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY4]], [[COPY3]], [[MADDWrrr3]]
- ; CHECK-NEXT: [[MADDWrrr5:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY2]], [[COPY4]], [[MADDWrrr4]]
- ; CHECK-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr5]], [[COPY6]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 7
- ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
- ; CHECK-NEXT: B %bb.3
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.3:
- ; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.9(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr5]], [[COPY7]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 6
- ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
- ; CHECK-NEXT: B %bb.4
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.4:
- ; CHECK-NEXT: successors: %bb.5(0x40000000), %bb.9(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[SUBSWrr2:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr5]], [[COPY8]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 5
- ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
- ; CHECK-NEXT: B %bb.5
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.5:
- ; CHECK-NEXT: successors: %bb.6(0x40000000), %bb.9(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[SUBSWrr3:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr5]], [[COPY9]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 4
- ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
- ; CHECK-NEXT: B %bb.6
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.6:
- ; CHECK-NEXT: successors: %bb.7(0x40000000), %bb.9(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[SUBSWrr4:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr5]], [[COPY10]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm4:%[0-9]+]]:gpr64 = MOVi64imm 3
- ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
- ; CHECK-NEXT: B %bb.7
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.7:
- ; CHECK-NEXT: successors: %bb.8(0x40000000), %bb.9(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[SUBSWrr5:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr5]], [[COPY11]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm5:%[0-9]+]]:gpr64 = MOVi64imm 2
- ; CHECK-NEXT: Bcc 10, %bb.9, implicit $nzcv
- ; CHECK-NEXT: B %bb.8
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.8:
- ; CHECK-NEXT: successors: %bb.9(0x80000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[MOVi64imm6:%[0-9]+]]:gpr64 = MOVi64imm 1
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.9:
- ; CHECK-NEXT: successors: %bb.10(0x04000000), %bb.2(0x7c000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[PHI1:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.2, [[MOVi64imm1]], %bb.3, [[MOVi64imm2]], %bb.4, [[MOVi64imm3]], %bb.5, [[MOVi64imm4]], %bb.6, [[MOVi64imm5]], %bb.7, [[MOVi64imm6]], %bb.8
- ; CHECK-NEXT: [[SUBSWrr6:%[0-9]+]]:gpr32 = SUBSWrr [[PHI]], [[MOVi32imm]], implicit-def $nzcv
- ; CHECK-NEXT: Bcc 1, %bb.2, implicit $nzcv
- ; CHECK-NEXT: B %bb.10
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.10:
- ; CHECK-NEXT: [[PHI2:%[0-9]+]]:gpr64 = PHI [[COPY5]], %bb.0, [[PHI1]], %bb.9
- ; CHECK-NEXT: $x0 = COPY [[PHI2]]
- ; CHECK-NEXT: RET_ReallyLR implicit $x0
+ ; COMMON-LABEL: name: cascade_branches_from_args_not_dd
+ ; COMMON: bb.0:
+ ; COMMON-NEXT: successors: %bb.10(0x30000000), %bb.1(0x50000000)
+ ; COMMON-NEXT: liveins: $x1, $w2, $w3, $w4, $w5
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x1
+ ; COMMON-NEXT: [[COPY1:%[0-9]+]]:gpr32 = COPY $w2
+ ; COMMON-NEXT: [[COPY2:%[0-9]+]]:gpr32 = COPY $w3
+ ; COMMON-NEXT: [[COPY3:%[0-9]+]]:gpr32 = COPY $w4
+ ; COMMON-NEXT: [[COPY4:%[0-9]+]]:gpr32 = COPY $w5
+ ; COMMON-NEXT: [[COPY5:%[0-9]+]]:gpr64all = COPY $xzr
+ ; COMMON-NEXT: CBZW [[COPY1]], %bb.10
+ ; COMMON-NEXT: B %bb.1
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.1:
+ ; COMMON-NEXT: successors: %bb.2(0x80000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[COPY6:%[0-9]+]]:gpr32 = COPY [[COPY2]]
+ ; COMMON-NEXT: [[COPY7:%[0-9]+]]:gpr32 = COPY [[COPY3]]
+ ; COMMON-NEXT: [[COPY8:%[0-9]+]]:gpr32 = COPY [[COPY4]]
+ ; COMMON-NEXT: [[COPY9:%[0-9]+]]:gpr32 = COPY [[COPY2]]
+ ; COMMON-NEXT: [[COPY10:%[0-9]+]]:gpr32 = COPY [[COPY3]]
+ ; COMMON-NEXT: [[COPY11:%[0-9]+]]:gpr32 = COPY [[COPY4]]
+ ; COMMON-NEXT: [[COPY12:%[0-9]+]]:gpr32 = COPY [[COPY2]]
+ ; COMMON-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 1
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.2:
+ ; COMMON-NEXT: successors: %bb.3(0x40000000), %bb.9(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[PHI:%[0-9]+]]:gpr32 = PHI [[COPY1]], %bb.1, %15, %bb.9
+ ; COMMON-NEXT: [[MADDWrrr:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY2]], [[COPY4]], $wzr
+ ; COMMON-NEXT: [[MADDWrrr1:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[COPY4]], [[MADDWrrr]]
+ ; COMMON-NEXT: [[MADDWrrr2:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY2]], [[COPY3]], [[MADDWrrr1]]
+ ; COMMON-NEXT: [[MADDWrrr3:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[COPY2]], [[MADDWrrr2]]
+ ; COMMON-NEXT: [[MADDWrrr4:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY4]], [[COPY3]], [[MADDWrrr3]]
+ ; COMMON-NEXT: [[MADDWrrr5:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY2]], [[COPY4]], [[MADDWrrr4]]
+ ; COMMON-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr5]], [[COPY6]], implicit-def $nzcv
+ ; COMMON-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 7
+ ; COMMON-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; COMMON-NEXT: B %bb.3
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.3:
+ ; COMMON-NEXT: successors: %bb.4(0x40000000), %bb.9(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr5]], [[COPY7]], implicit-def $nzcv
+ ; COMMON-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 6
+ ; COMMON-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; COMMON-NEXT: B %bb.4
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.4:
+ ; COMMON-NEXT: successors: %bb.5(0x40000000), %bb.9(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[SUBSWrr2:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr5]], [[COPY8]], implicit-def $nzcv
+ ; COMMON-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 5
+ ; COMMON-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; COMMON-NEXT: B %bb.5
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.5:
+ ; COMMON-NEXT: successors: %bb.6(0x40000000), %bb.9(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[SUBSWrr3:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr5]], [[COPY9]], implicit-def $nzcv
+ ; COMMON-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 4
+ ; COMMON-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; COMMON-NEXT: B %bb.6
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.6:
+ ; COMMON-NEXT: successors: %bb.7(0x40000000), %bb.9(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[SUBSWrr4:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr5]], [[COPY10]], implicit-def $nzcv
+ ; COMMON-NEXT: [[MOVi64imm4:%[0-9]+]]:gpr64 = MOVi64imm 3
+ ; COMMON-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; COMMON-NEXT: B %bb.7
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.7:
+ ; COMMON-NEXT: successors: %bb.8(0x40000000), %bb.9(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[SUBSWrr5:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr5]], [[COPY11]], implicit-def $nzcv
+ ; COMMON-NEXT: [[MOVi64imm5:%[0-9]+]]:gpr64 = MOVi64imm 2
+ ; COMMON-NEXT: Bcc 10, %bb.9, implicit $nzcv
+ ; COMMON-NEXT: B %bb.8
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.8:
+ ; COMMON-NEXT: successors: %bb.9(0x80000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[MOVi64imm6:%[0-9]+]]:gpr64 = MOVi64imm 1
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.9:
+ ; COMMON-NEXT: successors: %bb.10(0x04000000), %bb.2(0x7c000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[PHI1:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.2, [[MOVi64imm1]], %bb.3, [[MOVi64imm2]], %bb.4, [[MOVi64imm3]], %bb.5, [[MOVi64imm4]], %bb.6, [[MOVi64imm5]], %bb.7, [[MOVi64imm6]], %bb.8
+ ; COMMON-NEXT: [[SUBSWrr6:%[0-9]+]]:gpr32 = SUBSWrr [[PHI]], [[MOVi32imm]], implicit-def $nzcv
+ ; COMMON-NEXT: Bcc 1, %bb.2, implicit $nzcv
+ ; COMMON-NEXT: B %bb.10
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.10:
+ ; COMMON-NEXT: [[PHI2:%[0-9]+]]:gpr64 = PHI [[COPY5]], %bb.0, [[PHI1]], %bb.9
+ ; COMMON-NEXT: $x0 = COPY [[PHI2]]
+ ; COMMON-NEXT: RET_ReallyLR implicit $x0
bb.0:
successors: %bb.10(0x30000000), %bb.1(0x50000000)
liveins: $x1, $w2, $w3, $w4, $w5
@@ -428,107 +596,107 @@ liveins:
- { reg: '$w3', virtual-reg: '%3' }
- { reg: '$w4', virtual-reg: '%4' }
body: |
- ; CHECK-LABEL: name: cascade_exceeds_crit_limit
- ; CHECK: bb.0:
- ; CHECK-NEXT: successors: %bb.11(0x30000000), %bb.1(0x50000000)
- ; CHECK-NEXT: liveins: $x0, $x1, $w2, $w3, $w4
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr32 = COPY $w2
- ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gpr32 = COPY $w3
- ; CHECK-NEXT: [[COPY4:%[0-9]+]]:gpr32 = COPY $w4
- ; CHECK-NEXT: [[COPY5:%[0-9]+]]:gpr64all = COPY $xzr
- ; CHECK-NEXT: CBZW [[COPY2]], %bb.11
- ; CHECK-NEXT: B %bb.1
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.1:
- ; CHECK-NEXT: successors: %bb.2(0x80000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 0 :: (load (s32))
- ; CHECK-NEXT: [[LDRWui1:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 1 :: (load (s32))
- ; CHECK-NEXT: [[LDRWui2:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 2 :: (load (s32))
- ; CHECK-NEXT: [[LDRWui3:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 3 :: (load (s32))
- ; CHECK-NEXT: [[LDRWui4:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 4 :: (load (s32))
- ; CHECK-NEXT: [[LDRWui5:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 5 :: (load (s32))
- ; CHECK-NEXT: [[LDRWui6:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 6 :: (load (s32))
- ; CHECK-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 1
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.2:
- ; CHECK-NEXT: successors: %bb.3(0x40000000), %bb.10(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[PHI:%[0-9]+]]:gpr32 = PHI [[COPY2]], %bb.1, %15, %bb.10
- ; CHECK-NEXT: [[LDRWui7:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load (s32))
- ; CHECK-NEXT: [[MADDWrrr:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[LDRWui7]], $wzr
- ; CHECK-NEXT: [[MADDWrrr1:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY4]], [[LDRWui7]], [[MADDWrrr]]
- ; CHECK-NEXT: [[MADDWrrr2:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[LDRWui7]], [[MADDWrrr1]]
- ; CHECK-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 7
- ; CHECK-NEXT: Bcc 10, %bb.10, implicit $nzcv
- ; CHECK-NEXT: B %bb.3
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.3:
- ; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.10(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui1]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 6
- ; CHECK-NEXT: Bcc 10, %bb.10, implicit $nzcv
- ; CHECK-NEXT: B %bb.4
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.4:
- ; CHECK-NEXT: successors: %bb.6(0x40000000), %bb.5(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[SUBSWrr2:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui2]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 5
- ; CHECK-NEXT: Bcc 10, %bb.5, implicit $nzcv
- ; CHECK-NEXT: B %bb.6
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.5:
- ; CHECK-NEXT: successors: %bb.10(0x80000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: B %bb.10
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.6:
- ; CHECK-NEXT: successors: %bb.7(0x40000000), %bb.10(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[SUBSWrr3:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui3]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 4
- ; CHECK-NEXT: Bcc 10, %bb.10, implicit $nzcv
- ; CHECK-NEXT: B %bb.7
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.7:
- ; CHECK-NEXT: successors: %bb.8(0x40000000), %bb.10(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[SUBSWrr4:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui4]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm4:%[0-9]+]]:gpr64 = MOVi64imm 3
- ; CHECK-NEXT: Bcc 10, %bb.10, implicit $nzcv
- ; CHECK-NEXT: B %bb.8
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.8:
- ; CHECK-NEXT: successors: %bb.9(0x40000000), %bb.10(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[SUBSWrr5:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui5]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm5:%[0-9]+]]:gpr64 = MOVi64imm 2
- ; CHECK-NEXT: Bcc 10, %bb.10, implicit $nzcv
- ; CHECK-NEXT: B %bb.9
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.9:
- ; CHECK-NEXT: successors: %bb.10(0x80000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[MOVi64imm6:%[0-9]+]]:gpr64 = MOVi64imm 1
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.10:
- ; CHECK-NEXT: successors: %bb.11(0x04000000), %bb.2(0x7c000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[PHI1:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.2, [[MOVi64imm1]], %bb.3, [[MOVi64imm2]], %bb.5, [[MOVi64imm3]], %bb.6, [[MOVi64imm4]], %bb.7, [[MOVi64imm5]], %bb.8, [[MOVi64imm6]], %bb.9
- ; CHECK-NEXT: [[SUBSWrr6:%[0-9]+]]:gpr32 = SUBSWrr [[PHI]], [[MOVi32imm]], implicit-def $nzcv
- ; CHECK-NEXT: Bcc 1, %bb.2, implicit $nzcv
- ; CHECK-NEXT: B %bb.11
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.11:
- ; CHECK-NEXT: [[PHI2:%[0-9]+]]:gpr64 = PHI [[COPY5]], %bb.0, [[PHI1]], %bb.10
- ; CHECK-NEXT: $x0 = COPY [[PHI2]]
- ; CHECK-NEXT: RET_ReallyLR implicit $x0
+ ; COMMON-LABEL: name: cascade_exceeds_crit_limit
+ ; COMMON: bb.0:
+ ; COMMON-NEXT: successors: %bb.11(0x30000000), %bb.1(0x50000000)
+ ; COMMON-NEXT: liveins: $x0, $x1, $w2, $w3, $w4
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; COMMON-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1
+ ; COMMON-NEXT: [[COPY2:%[0-9]+]]:gpr32 = COPY $w2
+ ; COMMON-NEXT: [[COPY3:%[0-9]+]]:gpr32 = COPY $w3
+ ; COMMON-NEXT: [[COPY4:%[0-9]+]]:gpr32 = COPY $w4
+ ; COMMON-NEXT: [[COPY5:%[0-9]+]]:gpr64all = COPY $xzr
+ ; COMMON-NEXT: CBZW [[COPY2]], %bb.11
+ ; COMMON-NEXT: B %bb.1
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.1:
+ ; COMMON-NEXT: successors: %bb.2(0x80000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 0 :: (load (s32))
+ ; COMMON-NEXT: [[LDRWui1:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 1 :: (load (s32))
+ ; COMMON-NEXT: [[LDRWui2:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 2 :: (load (s32))
+ ; COMMON-NEXT: [[LDRWui3:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 3 :: (load (s32))
+ ; COMMON-NEXT: [[LDRWui4:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 4 :: (load (s32))
+ ; COMMON-NEXT: [[LDRWui5:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 5 :: (load (s32))
+ ; COMMON-NEXT: [[LDRWui6:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 6 :: (load (s32))
+ ; COMMON-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 1
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.2:
+ ; COMMON-NEXT: successors: %bb.3(0x40000000), %bb.10(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[PHI:%[0-9]+]]:gpr32 = PHI [[COPY2]], %bb.1, %15, %bb.10
+ ; COMMON-NEXT: [[LDRWui7:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load (s32))
+ ; COMMON-NEXT: [[MADDWrrr:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[LDRWui7]], $wzr
+ ; COMMON-NEXT: [[MADDWrrr1:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY4]], [[LDRWui7]], [[MADDWrrr]]
+ ; COMMON-NEXT: [[MADDWrrr2:%[0-9]+]]:gpr32 = nsw MADDWrrr [[COPY3]], [[LDRWui7]], [[MADDWrrr1]]
+ ; COMMON-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui]], implicit-def $nzcv
+ ; COMMON-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 7
+ ; COMMON-NEXT: Bcc 10, %bb.10, implicit $nzcv
+ ; COMMON-NEXT: B %bb.3
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.3:
+ ; COMMON-NEXT: successors: %bb.4(0x40000000), %bb.10(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui1]], implicit-def $nzcv
+ ; COMMON-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 6
+ ; COMMON-NEXT: Bcc 10, %bb.10, implicit $nzcv
+ ; COMMON-NEXT: B %bb.4
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.4:
+ ; COMMON-NEXT: successors: %bb.6(0x40000000), %bb.5(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[SUBSWrr2:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui2]], implicit-def $nzcv
+ ; COMMON-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 5
+ ; COMMON-NEXT: Bcc 10, %bb.5, implicit $nzcv
+ ; COMMON-NEXT: B %bb.6
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.5:
+ ; COMMON-NEXT: successors: %bb.10(0x80000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: B %bb.10
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.6:
+ ; COMMON-NEXT: successors: %bb.7(0x40000000), %bb.10(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[SUBSWrr3:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui3]], implicit-def $nzcv
+ ; COMMON-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 4
+ ; COMMON-NEXT: Bcc 10, %bb.10, implicit $nzcv
+ ; COMMON-NEXT: B %bb.7
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.7:
+ ; COMMON-NEXT: successors: %bb.8(0x40000000), %bb.10(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[SUBSWrr4:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui4]], implicit-def $nzcv
+ ; COMMON-NEXT: [[MOVi64imm4:%[0-9]+]]:gpr64 = MOVi64imm 3
+ ; COMMON-NEXT: Bcc 10, %bb.10, implicit $nzcv
+ ; COMMON-NEXT: B %bb.8
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.8:
+ ; COMMON-NEXT: successors: %bb.9(0x40000000), %bb.10(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[SUBSWrr5:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr2]], [[LDRWui5]], implicit-def $nzcv
+ ; COMMON-NEXT: [[MOVi64imm5:%[0-9]+]]:gpr64 = MOVi64imm 2
+ ; COMMON-NEXT: Bcc 10, %bb.10, implicit $nzcv
+ ; COMMON-NEXT: B %bb.9
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.9:
+ ; COMMON-NEXT: successors: %bb.10(0x80000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[MOVi64imm6:%[0-9]+]]:gpr64 = MOVi64imm 1
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.10:
+ ; COMMON-NEXT: successors: %bb.11(0x04000000), %bb.2(0x7c000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[PHI1:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.2, [[MOVi64imm1]], %bb.3, [[MOVi64imm2]], %bb.5, [[MOVi64imm3]], %bb.6, [[MOVi64imm4]], %bb.7, [[MOVi64imm5]], %bb.8, [[MOVi64imm6]], %bb.9
+ ; COMMON-NEXT: [[SUBSWrr6:%[0-9]+]]:gpr32 = SUBSWrr [[PHI]], [[MOVi32imm]], implicit-def $nzcv
+ ; COMMON-NEXT: Bcc 1, %bb.2, implicit $nzcv
+ ; COMMON-NEXT: B %bb.11
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.11:
+ ; COMMON-NEXT: [[PHI2:%[0-9]+]]:gpr64 = PHI [[COPY5]], %bb.0, [[PHI1]], %bb.10
+ ; COMMON-NEXT: $x0 = COPY [[PHI2]]
+ ; COMMON-NEXT: RET_ReallyLR implicit $x0
bb.0:
successors: %bb.11(0x30000000), %bb.1(0x50000000)
liveins: $x0, $x1, $w2, $w3, $w4
@@ -638,49 +806,49 @@ name: cascade_backedge_to_earlier_block
alignment: 4
tracksRegLiveness: true
body: |
- ; CHECK-LABEL: name: cascade_backedge_to_earlier_block
- ; CHECK: bb.0:
- ; CHECK-NEXT: successors: %bb.1(0x80000000)
- ; CHECK-NEXT: liveins: $x0, $x1
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.1:
- ; CHECK-NEXT: successors: %bb.2(0x40000000), %bb.5(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load (s32))
- ; CHECK-NEXT: [[LDRWui1:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 0 :: (load (s32))
- ; CHECK-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui1]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 100
- ; CHECK-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 200
- ; CHECK-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 400
- ; CHECK-NEXT: Bcc 10, %bb.5, implicit $nzcv
- ; CHECK-NEXT: B %bb.2
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.2:
- ; CHECK-NEXT: successors: %bb.3(0x40000000), %bb.5(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[LDRWui2:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 1 :: (load (s32))
- ; CHECK-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui2]], implicit-def $nzcv
- ; CHECK-NEXT: Bcc 10, %bb.5, implicit $nzcv
- ; CHECK-NEXT: B %bb.3
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.3:
- ; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[LDRWui3:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 2 :: (load (s32))
- ; CHECK-NEXT: [[SUBSWrr2:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui3]], implicit-def $nzcv
- ; CHECK-NEXT: Bcc 10, %bb.2, implicit $nzcv
- ; CHECK-NEXT: B %bb.4
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.4:
- ; CHECK-NEXT: successors: %bb.5(0x80000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.5:
- ; CHECK-NEXT: [[PHI:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.1, [[MOVi64imm1]], %bb.2, [[MOVi64imm2]], %bb.4
- ; CHECK-NEXT: $x0 = COPY [[PHI]]
- ; CHECK-NEXT: RET_ReallyLR implicit $x0
+ ; COMMON-LABEL: name: cascade_backedge_to_earlier_block
+ ; COMMON: bb.0:
+ ; COMMON-NEXT: successors: %bb.1(0x80000000)
+ ; COMMON-NEXT: liveins: $x0, $x1
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; COMMON-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.1:
+ ; COMMON-NEXT: successors: %bb.2(0x40000000), %bb.5(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load (s32))
+ ; COMMON-NEXT: [[LDRWui1:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 0 :: (load (s32))
+ ; COMMON-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui1]], implicit-def $nzcv
+ ; COMMON-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 100
+ ; COMMON-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 200
+ ; COMMON-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 400
+ ; COMMON-NEXT: Bcc 10, %bb.5, implicit $nzcv
+ ; COMMON-NEXT: B %bb.2
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.2:
+ ; COMMON-NEXT: successors: %bb.3(0x40000000), %bb.5(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[LDRWui2:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 1 :: (load (s32))
+ ; COMMON-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui2]], implicit-def $nzcv
+ ; COMMON-NEXT: Bcc 10, %bb.5, implicit $nzcv
+ ; COMMON-NEXT: B %bb.3
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.3:
+ ; COMMON-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[LDRWui3:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 2 :: (load (s32))
+ ; COMMON-NEXT: [[SUBSWrr2:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui3]], implicit-def $nzcv
+ ; COMMON-NEXT: Bcc 10, %bb.2, implicit $nzcv
+ ; COMMON-NEXT: B %bb.4
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.4:
+ ; COMMON-NEXT: successors: %bb.5(0x80000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.5:
+ ; COMMON-NEXT: [[PHI:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.1, [[MOVi64imm1]], %bb.2, [[MOVi64imm2]], %bb.4
+ ; COMMON-NEXT: $x0 = COPY [[PHI]]
+ ; COMMON-NEXT: RET_ReallyLR implicit $x0
bb.0:
successors: %bb.1(0x80000000)
liveins: $x0, $x1
@@ -731,60 +899,60 @@ liveins:
- { reg: '$x0', virtual-reg: '%0' }
- { reg: '$x1', virtual-reg: '%1' }
body: |
- ; CHECK-LABEL: name: cascade_forward_skip_to_later_block
- ; CHECK: bb.0:
- ; CHECK-NEXT: successors: %bb.1(0x80000000)
- ; CHECK-NEXT: liveins: $x0, $x1
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.1:
- ; CHECK-NEXT: successors: %bb.2(0x40000000), %bb.6(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load (s32))
- ; CHECK-NEXT: [[LDRWui1:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 0 :: (load (s32))
- ; CHECK-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui1]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 100
- ; CHECK-NEXT: Bcc 10, %bb.6, implicit $nzcv
- ; CHECK-NEXT: B %bb.2
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.2:
- ; CHECK-NEXT: successors: %bb.3(0x40000000), %bb.6(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[LDRWui2:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 1 :: (load (s32))
- ; CHECK-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui2]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 200
- ; CHECK-NEXT: Bcc 10, %bb.6, implicit $nzcv
- ; CHECK-NEXT: B %bb.3
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.3:
- ; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.5(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[LDRWui3:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 2 :: (load (s32))
- ; CHECK-NEXT: [[SUBSWrr2:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui3]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 300
- ; CHECK-NEXT: Bcc 10, %bb.5, implicit $nzcv
- ; CHECK-NEXT: B %bb.4
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.4:
- ; CHECK-NEXT: successors: %bb.5(0x40000000), %bb.6(0x40000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[LDRWui4:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 3 :: (load (s32))
- ; CHECK-NEXT: [[SUBSWrr3:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui4]], implicit-def $nzcv
- ; CHECK-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 400
- ; CHECK-NEXT: Bcc 10, %bb.6, implicit $nzcv
- ; CHECK-NEXT: B %bb.5
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.5:
- ; CHECK-NEXT: successors: %bb.6(0x80000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[MOVi64imm4:%[0-9]+]]:gpr64 = MOVi64imm 500
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.6:
- ; CHECK-NEXT: [[PHI:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.1, [[MOVi64imm1]], %bb.2, [[MOVi64imm3]], %bb.4, [[MOVi64imm4]], %bb.5
- ; CHECK-NEXT: $x0 = COPY [[PHI]]
- ; CHECK-NEXT: RET_ReallyLR implicit $x0
+ ; COMMON-LABEL: name: cascade_forward_skip_to_later_block
+ ; COMMON: bb.0:
+ ; COMMON-NEXT: successors: %bb.1(0x80000000)
+ ; COMMON-NEXT: liveins: $x0, $x1
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; COMMON-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.1:
+ ; COMMON-NEXT: successors: %bb.2(0x40000000), %bb.6(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load (s32))
+ ; COMMON-NEXT: [[LDRWui1:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 0 :: (load (s32))
+ ; COMMON-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui1]], implicit-def $nzcv
+ ; COMMON-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 100
+ ; COMMON-NEXT: Bcc 10, %bb.6, implicit $nzcv
+ ; COMMON-NEXT: B %bb.2
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.2:
+ ; COMMON-NEXT: successors: %bb.3(0x40000000), %bb.6(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[LDRWui2:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 1 :: (load (s32))
+ ; COMMON-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui2]], implicit-def $nzcv
+ ; COMMON-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 200
+ ; COMMON-NEXT: Bcc 10, %bb.6, implicit $nzcv
+ ; COMMON-NEXT: B %bb.3
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.3:
+ ; COMMON-NEXT: successors: %bb.4(0x40000000), %bb.5(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[LDRWui3:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 2 :: (load (s32))
+ ; COMMON-NEXT: [[SUBSWrr2:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui3]], implicit-def $nzcv
+ ; COMMON-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 300
+ ; COMMON-NEXT: Bcc 10, %bb.5, implicit $nzcv
+ ; COMMON-NEXT: B %bb.4
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.4:
+ ; COMMON-NEXT: successors: %bb.5(0x40000000), %bb.6(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[LDRWui4:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 3 :: (load (s32))
+ ; COMMON-NEXT: [[SUBSWrr3:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui4]], implicit-def $nzcv
+ ; COMMON-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 400
+ ; COMMON-NEXT: Bcc 10, %bb.6, implicit $nzcv
+ ; COMMON-NEXT: B %bb.5
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.5:
+ ; COMMON-NEXT: successors: %bb.6(0x80000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[MOVi64imm4:%[0-9]+]]:gpr64 = MOVi64imm 500
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.6:
+ ; COMMON-NEXT: [[PHI:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.1, [[MOVi64imm1]], %bb.2, [[MOVi64imm3]], %bb.4, [[MOVi64imm4]], %bb.5
+ ; COMMON-NEXT: $x0 = COPY [[PHI]]
+ ; COMMON-NEXT: RET_ReallyLR implicit $x0
bb.0:
successors: %bb.1(0x80000000)
liveins: $x0, $x1
@@ -840,4 +1008,890 @@ body: |
$x0 = COPY %90
RET_ReallyLR implicit $x0
...
+---
+name: cascade_per_phi_slack
+alignment: 4
+tracksRegLiveness: true
+body: |
+ ; COMMON-LABEL: name: cascade_per_phi_slack
+ ; COMMON: bb.0:
+ ; COMMON-NEXT: successors: %bb.1(0x80000000)
+ ; COMMON-NEXT: liveins: $x0
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.1:
+ ; COMMON-NEXT: successors: %bb.2(0x40000000), %bb.7(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load (s32))
+ ; COMMON-NEXT: [[MADDWrrr:%[0-9]+]]:gpr32 = MADDWrrr [[LDRWui]], [[LDRWui]], $wzr
+ ; COMMON-NEXT: [[MADDWrrr1:%[0-9]+]]:gpr32 = MADDWrrr [[MADDWrrr]], [[MADDWrrr]], $wzr
+ ; COMMON-NEXT: [[ADDWrr:%[0-9]+]]:gpr32 = ADDWrr [[MADDWrrr1]], [[MADDWrrr1]]
+ ; COMMON-NEXT: [[ADDWrr1:%[0-9]+]]:gpr32 = ADDWrr [[ADDWrr]], [[ADDWrr]]
+ ; COMMON-NEXT: [[ADDWrr2:%[0-9]+]]:gpr32 = ADDWrr [[ADDWrr1]], [[ADDWrr1]]
+ ; COMMON-NEXT: [[ADDWrr3:%[0-9]+]]:gpr32 = ADDWrr [[ADDWrr2]], [[ADDWrr2]]
+ ; COMMON-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 100
+ ; COMMON-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 1100
+ ; COMMON-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr1]], [[MADDWrrr1]], implicit-def $nzcv
+ ; COMMON-NEXT: Bcc 10, %bb.7, implicit $nzcv
+ ; COMMON-NEXT: B %bb.2
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.2:
+ ; COMMON-NEXT: successors: %bb.3(0x40000000), %bb.7(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 200
+ ; COMMON-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 1200
+ ; COMMON-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[ADDWrr]], [[ADDWrr]], implicit-def $nzcv
+ ; COMMON-NEXT: Bcc 10, %bb.7, implicit $nzcv
+ ; COMMON-NEXT: B %bb.3
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.3:
+ ; COMMON-NEXT: successors: %bb.4(0x40000000), %bb.7(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[MOVi64imm4:%[0-9]+]]:gpr64 = MOVi64imm 300
+ ; COMMON-NEXT: [[MOVi64imm5:%[0-9]+]]:gpr64 = MOVi64imm 1300
+ ; COMMON-NEXT: [[SUBSWrr2:%[0-9]+]]:gpr32 = SUBSWrr [[ADDWrr1]], [[ADDWrr1]], implicit-def $nzcv
+ ; COMMON-NEXT: Bcc 10, %bb.7, implicit $nzcv
+ ; COMMON-NEXT: B %bb.4
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.4:
+ ; COMMON-NEXT: successors: %bb.5(0x40000000), %bb.7(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[MOVi64imm6:%[0-9]+]]:gpr64 = MOVi64imm 400
+ ; COMMON-NEXT: [[MOVi64imm7:%[0-9]+]]:gpr64 = MOVi64imm 1400
+ ; COMMON-NEXT: [[SUBSWrr3:%[0-9]+]]:gpr32 = SUBSWrr [[ADDWrr2]], [[ADDWrr2]], implicit-def $nzcv
+ ; COMMON-NEXT: Bcc 10, %bb.7, implicit $nzcv
+ ; COMMON-NEXT: B %bb.5
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.5:
+ ; COMMON-NEXT: successors: %bb.6(0x40000000), %bb.7(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[MOVi64imm8:%[0-9]+]]:gpr64 = MOVi64imm 500
+ ; COMMON-NEXT: [[MOVi64imm9:%[0-9]+]]:gpr64 = MOVi64imm 1500
+ ; COMMON-NEXT: [[SUBSWrr4:%[0-9]+]]:gpr32 = SUBSWrr [[ADDWrr3]], [[ADDWrr3]], implicit-def $nzcv
+ ; COMMON-NEXT: Bcc 10, %bb.7, implicit $nzcv
+ ; COMMON-NEXT: B %bb.6
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.6:
+ ; COMMON-NEXT: successors: %bb.7(0x80000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[MOVi64imm10:%[0-9]+]]:gpr64 = MOVi64imm 600
+ ; COMMON-NEXT: [[MOVi64imm11:%[0-9]+]]:gpr64 = MOVi64imm 1600
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.7:
+ ; COMMON-NEXT: [[PHI:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.1, [[MOVi64imm2]], %bb.2, [[MOVi64imm4]], %bb.3, [[MOVi64imm6]], %bb.4, [[MOVi64imm8]], %bb.5, [[MOVi64imm10]], %bb.6
+ ; COMMON-NEXT: [[PHI1:%[0-9]+]]:gpr64 = PHI [[MOVi64imm1]], %bb.1, [[MOVi64imm3]], %bb.2, [[MOVi64imm5]], %bb.3, [[MOVi64imm7]], %bb.4, [[MOVi64imm9]], %bb.5, [[MOVi64imm11]], %bb.6
+ ; COMMON-NEXT: [[MADDXrrr:%[0-9]+]]:gpr64 = MADDXrrr [[PHI]], [[PHI]], $xzr
+ ; COMMON-NEXT: [[MADDXrrr1:%[0-9]+]]:gpr64 = MADDXrrr [[MADDXrrr]], [[MADDXrrr]], $xzr
+ ; COMMON-NEXT: [[MADDXrrr2:%[0-9]+]]:gpr64 = MADDXrrr [[MADDXrrr1]], [[MADDXrrr1]], $xzr
+ ; COMMON-NEXT: [[MADDXrrr3:%[0-9]+]]:gpr64 = MADDXrrr [[MADDXrrr2]], [[MADDXrrr2]], $xzr
+ ; COMMON-NEXT: [[ADDXrr:%[0-9]+]]:gpr64 = ADDXrr [[MADDXrrr3]], [[PHI1]]
+ ; COMMON-NEXT: $x0 = COPY [[ADDXrr]]
+ ; COMMON-NEXT: RET_ReallyLR implicit $x0
+ bb.0:
+ successors: %bb.1(0x80000000)
+ liveins: $x0
+
+ %0:gpr64common = COPY $x0
+
+ bb.1:
+ successors: %bb.2(0x40000000), %bb.7(0x40000000)
+
+ %10:gpr32 = LDRWui %0, 0 :: (load (s32))
+ %11:gpr32 = MADDWrrr %10, %10, $wzr
+ %12:gpr32 = MADDWrrr %11, %11, $wzr
+ %20:gpr32 = ADDWrr %12, %12
+ %21:gpr32 = ADDWrr %20, %20
+ %22:gpr32 = ADDWrr %21, %21
+ %23:gpr32 = ADDWrr %22, %22
+ %40:gpr64 = MOVi64imm 100
+ %50:gpr64 = MOVi64imm 1100
+ %30:gpr32 = SUBSWrr %12, %12, implicit-def $nzcv
+ Bcc 10, %bb.7, implicit $nzcv
+ B %bb.2
+
+ bb.2:
+ successors: %bb.3(0x40000000), %bb.7(0x40000000)
+
+ %41:gpr64 = MOVi64imm 200
+ %51:gpr64 = MOVi64imm 1200
+ %31:gpr32 = SUBSWrr %20, %20, implicit-def $nzcv
+ Bcc 10, %bb.7, implicit $nzcv
+ B %bb.3
+
+ bb.3:
+ successors: %bb.4(0x40000000), %bb.7(0x40000000)
+
+ %42:gpr64 = MOVi64imm 300
+ %52:gpr64 = MOVi64imm 1300
+ %32:gpr32 = SUBSWrr %21, %21, implicit-def $nzcv
+ Bcc 10, %bb.7, implicit $nzcv
+ B %bb.4
+
+ bb.4:
+ successors: %bb.5(0x40000000), %bb.7(0x40000000)
+
+ %43:gpr64 = MOVi64imm 400
+ %53:gpr64 = MOVi64imm 1400
+ %33:gpr32 = SUBSWrr %22, %22, implicit-def $nzcv
+ Bcc 10, %bb.7, implicit $nzcv
+ B %bb.5
+
+ bb.5:
+ successors: %bb.6(0x40000000), %bb.7(0x40000000)
+
+ %44:gpr64 = MOVi64imm 500
+ %54:gpr64 = MOVi64imm 1500
+ %34:gpr32 = SUBSWrr %23, %23, implicit-def $nzcv
+ Bcc 10, %bb.7, implicit $nzcv
+ B %bb.6
+
+ bb.6:
+ successors: %bb.7(0x80000000)
+
+ %45:gpr64 = MOVi64imm 600
+ %55:gpr64 = MOVi64imm 1600
+
+ bb.7:
+ %90:gpr64 = PHI %40, %bb.1, %41, %bb.2, %42, %bb.3, %43, %bb.4, %44, %bb.5, %45, %bb.6
+ %91:gpr64 = PHI %50, %bb.1, %51, %bb.2, %52, %bb.3, %53, %bb.4, %54, %bb.5, %55, %bb.6
+ %60:gpr64 = MADDXrrr %90, %90, $xzr
+ %61:gpr64 = MADDXrrr %60, %60, $xzr
+ %62:gpr64 = MADDXrrr %61, %61, $xzr
+ %63:gpr64 = MADDXrrr %62, %62, $xzr
+ %64:gpr64 = ADDXrr %63, %91
+ $x0 = COPY %64
+ RET_ReallyLR implicit $x0
+...
+---
+name: cascade_phi_subset
+alignment: 4
+tracksRegLiveness: true
+body: |
+ ; CASCADE-WITH-DD-LABEL: name: cascade_phi_subset
+ ; CASCADE-WITH-DD: bb.0:
+ ; CASCADE-WITH-DD-NEXT: successors: %bb.1(0x80000000)
+ ; CASCADE-WITH-DD-NEXT: liveins: $x0
+ ; CASCADE-WITH-DD-NEXT: {{ $}}
+ ; CASCADE-WITH-DD-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; CASCADE-WITH-DD-NEXT: {{ $}}
+ ; CASCADE-WITH-DD-NEXT: bb.1:
+ ; CASCADE-WITH-DD-NEXT: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load (s32))
+ ; CASCADE-WITH-DD-NEXT: [[MADDWrrr:%[0-9]+]]:gpr32 = MADDWrrr [[LDRWui]], [[LDRWui]], $wzr
+ ; CASCADE-WITH-DD-NEXT: [[MADDWrrr1:%[0-9]+]]:gpr32 = MADDWrrr [[MADDWrrr]], [[MADDWrrr]], $wzr
+ ; CASCADE-WITH-DD-NEXT: [[ADDWrr:%[0-9]+]]:gpr32 = ADDWrr [[MADDWrrr1]], [[MADDWrrr1]]
+ ; CASCADE-WITH-DD-NEXT: [[ADDWrr1:%[0-9]+]]:gpr32 = ADDWrr [[ADDWrr]], [[ADDWrr]]
+ ; CASCADE-WITH-DD-NEXT: [[ADDWrr2:%[0-9]+]]:gpr32 = ADDWrr [[ADDWrr1]], [[ADDWrr1]]
+ ; CASCADE-WITH-DD-NEXT: [[ADDWrr3:%[0-9]+]]:gpr32 = ADDWrr [[ADDWrr2]], [[ADDWrr2]]
+ ; CASCADE-WITH-DD-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 100
+ ; CASCADE-WITH-DD-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 200
+ ; CASCADE-WITH-DD-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 300
+ ; CASCADE-WITH-DD-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 400
+ ; CASCADE-WITH-DD-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[ADDWrr3]], [[ADDWrr3]], implicit-def $nzcv
+ ; CASCADE-WITH-DD-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[ADDWrr2]], [[ADDWrr2]], implicit-def $nzcv
+ ; CASCADE-WITH-DD-NEXT: [[SUBSWrr2:%[0-9]+]]:gpr32 = SUBSWrr [[ADDWrr1]], [[ADDWrr1]], implicit-def $nzcv
+ ; CASCADE-WITH-DD-NEXT: [[CSELXr:%[0-9]+]]:gpr64 = CSELXr [[MOVi64imm2]], [[MOVi64imm3]], 10, implicit $nzcv
+ ; CASCADE-WITH-DD-NEXT: [[SUBSWrr3:%[0-9]+]]:gpr32 = SUBSWrr [[ADDWrr]], [[ADDWrr]], implicit-def $nzcv
+ ; CASCADE-WITH-DD-NEXT: [[CSELXr1:%[0-9]+]]:gpr64 = CSELXr [[MOVi64imm1]], [[CSELXr]], 10, implicit $nzcv
+ ; CASCADE-WITH-DD-NEXT: [[SUBSWrr4:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr1]], [[MADDWrrr1]], implicit-def $nzcv
+ ; CASCADE-WITH-DD-NEXT: [[CSELXr2:%[0-9]+]]:gpr64 = CSELXr [[MOVi64imm]], [[CSELXr1]], 10, implicit $nzcv
+ ; CASCADE-WITH-DD-NEXT: $x0 = COPY [[CSELXr2]]
+ ; CASCADE-WITH-DD-NEXT: RET_ReallyLR implicit $x0
+ ;
+ ; DEPTH3-LABEL: name: cascade_phi_subset
+ ; DEPTH3: bb.0:
+ ; DEPTH3-NEXT: successors: %bb.1(0x80000000)
+ ; DEPTH3-NEXT: liveins: $x0
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: bb.1:
+ ; DEPTH3-NEXT: successors: %bb.2(0x40000000), %bb.7(0x40000000)
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load (s32))
+ ; DEPTH3-NEXT: [[MADDWrrr:%[0-9]+]]:gpr32 = MADDWrrr [[LDRWui]], [[LDRWui]], $wzr
+ ; DEPTH3-NEXT: [[MADDWrrr1:%[0-9]+]]:gpr32 = MADDWrrr [[MADDWrrr]], [[MADDWrrr]], $wzr
+ ; DEPTH3-NEXT: [[ADDWrr:%[0-9]+]]:gpr32 = ADDWrr [[MADDWrrr1]], [[MADDWrrr1]]
+ ; DEPTH3-NEXT: [[ADDWrr1:%[0-9]+]]:gpr32 = ADDWrr [[ADDWrr]], [[ADDWrr]]
+ ; DEPTH3-NEXT: [[ADDWrr2:%[0-9]+]]:gpr32 = ADDWrr [[ADDWrr1]], [[ADDWrr1]]
+ ; DEPTH3-NEXT: [[ADDWrr3:%[0-9]+]]:gpr32 = ADDWrr [[ADDWrr2]], [[ADDWrr2]]
+ ; DEPTH3-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 100
+ ; DEPTH3-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr1]], [[MADDWrrr1]], implicit-def $nzcv
+ ; DEPTH3-NEXT: Bcc 10, %bb.7, implicit $nzcv
+ ; DEPTH3-NEXT: B %bb.2
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: bb.2:
+ ; DEPTH3-NEXT: successors: %bb.3(0x40000000), %bb.7(0x40000000)
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 200
+ ; DEPTH3-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[ADDWrr]], [[ADDWrr]], implicit-def $nzcv
+ ; DEPTH3-NEXT: Bcc 10, %bb.7, implicit $nzcv
+ ; DEPTH3-NEXT: B %bb.3
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: bb.3:
+ ; DEPTH3-NEXT: successors: %bb.4(0x40000000), %bb.7(0x40000000)
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 300
+ ; DEPTH3-NEXT: [[SUBSWrr2:%[0-9]+]]:gpr32 = SUBSWrr [[ADDWrr1]], [[ADDWrr1]], implicit-def $nzcv
+ ; DEPTH3-NEXT: Bcc 10, %bb.7, implicit $nzcv
+ ; DEPTH3-NEXT: B %bb.4
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: bb.4:
+ ; DEPTH3-NEXT: successors: %bb.5(0x40000000), %bb.7(0x40000000)
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 400
+ ; DEPTH3-NEXT: [[SUBSWrr3:%[0-9]+]]:gpr32 = SUBSWrr [[ADDWrr2]], [[ADDWrr2]], implicit-def $nzcv
+ ; DEPTH3-NEXT: Bcc 10, %bb.7, implicit $nzcv
+ ; DEPTH3-NEXT: B %bb.5
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: bb.5:
+ ; DEPTH3-NEXT: successors: %bb.6(0x40000000), %bb.7(0x40000000)
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: [[SUBSWrr4:%[0-9]+]]:gpr32 = SUBSWrr [[ADDWrr3]], [[ADDWrr3]], implicit-def $nzcv
+ ; DEPTH3-NEXT: Bcc 10, %bb.7, implicit $nzcv
+ ; DEPTH3-NEXT: B %bb.6
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: bb.6:
+ ; DEPTH3-NEXT: successors: %bb.7(0x80000000)
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: bb.7:
+ ; DEPTH3-NEXT: [[PHI:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.1, [[MOVi64imm1]], %bb.2, [[MOVi64imm2]], %bb.3, [[MOVi64imm3]], %bb.4, [[MOVi64imm3]], %bb.5, [[MOVi64imm3]], %bb.6
+ ; DEPTH3-NEXT: $x0 = COPY [[PHI]]
+ ; DEPTH3-NEXT: RET_ReallyLR implicit $x0
+ ;
+ ; LIMIT3-LABEL: name: cascade_phi_subset
+ ; LIMIT3: bb.0:
+ ; LIMIT3-NEXT: successors: %bb.1(0x80000000)
+ ; LIMIT3-NEXT: liveins: $x0
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: bb.1:
+ ; LIMIT3-NEXT: successors: %bb.2(0x40000000), %bb.7(0x40000000)
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load (s32))
+ ; LIMIT3-NEXT: [[MADDWrrr:%[0-9]+]]:gpr32 = MADDWrrr [[LDRWui]], [[LDRWui]], $wzr
+ ; LIMIT3-NEXT: [[MADDWrrr1:%[0-9]+]]:gpr32 = MADDWrrr [[MADDWrrr]], [[MADDWrrr]], $wzr
+ ; LIMIT3-NEXT: [[ADDWrr:%[0-9]+]]:gpr32 = ADDWrr [[MADDWrrr1]], [[MADDWrrr1]]
+ ; LIMIT3-NEXT: [[ADDWrr1:%[0-9]+]]:gpr32 = ADDWrr [[ADDWrr]], [[ADDWrr]]
+ ; LIMIT3-NEXT: [[ADDWrr2:%[0-9]+]]:gpr32 = ADDWrr [[ADDWrr1]], [[ADDWrr1]]
+ ; LIMIT3-NEXT: [[ADDWrr3:%[0-9]+]]:gpr32 = ADDWrr [[ADDWrr2]], [[ADDWrr2]]
+ ; LIMIT3-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 100
+ ; LIMIT3-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[MADDWrrr1]], [[MADDWrrr1]], implicit-def $nzcv
+ ; LIMIT3-NEXT: Bcc 10, %bb.7, implicit $nzcv
+ ; LIMIT3-NEXT: B %bb.2
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: bb.2:
+ ; LIMIT3-NEXT: successors: %bb.3(0x40000000), %bb.7(0x40000000)
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 200
+ ; LIMIT3-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[ADDWrr]], [[ADDWrr]], implicit-def $nzcv
+ ; LIMIT3-NEXT: Bcc 10, %bb.7, implicit $nzcv
+ ; LIMIT3-NEXT: B %bb.3
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: bb.3:
+ ; LIMIT3-NEXT: successors: %bb.4(0x40000000), %bb.7(0x40000000)
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 300
+ ; LIMIT3-NEXT: [[SUBSWrr2:%[0-9]+]]:gpr32 = SUBSWrr [[ADDWrr1]], [[ADDWrr1]], implicit-def $nzcv
+ ; LIMIT3-NEXT: Bcc 10, %bb.7, implicit $nzcv
+ ; LIMIT3-NEXT: B %bb.4
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: bb.4:
+ ; LIMIT3-NEXT: successors: %bb.5(0x40000000), %bb.7(0x40000000)
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 400
+ ; LIMIT3-NEXT: [[SUBSWrr3:%[0-9]+]]:gpr32 = SUBSWrr [[ADDWrr2]], [[ADDWrr2]], implicit-def $nzcv
+ ; LIMIT3-NEXT: Bcc 10, %bb.7, implicit $nzcv
+ ; LIMIT3-NEXT: B %bb.5
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: bb.5:
+ ; LIMIT3-NEXT: successors: %bb.6(0x40000000), %bb.7(0x40000000)
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: [[SUBSWrr4:%[0-9]+]]:gpr32 = SUBSWrr [[ADDWrr3]], [[ADDWrr3]], implicit-def $nzcv
+ ; LIMIT3-NEXT: Bcc 10, %bb.7, implicit $nzcv
+ ; LIMIT3-NEXT: B %bb.6
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: bb.6:
+ ; LIMIT3-NEXT: successors: %bb.7(0x80000000)
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: bb.7:
+ ; LIMIT3-NEXT: [[PHI:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.1, [[MOVi64imm1]], %bb.2, [[MOVi64imm2]], %bb.3, [[MOVi64imm3]], %bb.4, [[MOVi64imm3]], %bb.5, [[MOVi64imm3]], %bb.6
+ ; LIMIT3-NEXT: $x0 = COPY [[PHI]]
+ ; LIMIT3-NEXT: RET_ReallyLR implicit $x0
+ bb.0:
+ successors: %bb.1(0x80000000)
+ liveins: $x0
+
+ %0:gpr64common = COPY $x0
+
+ bb.1:
+ successors: %bb.2(0x40000000), %bb.7(0x40000000)
+
+ %10:gpr32 = LDRWui %0, 0 :: (load (s32))
+ %11:gpr32 = MADDWrrr %10, %10, $wzr
+ %12:gpr32 = MADDWrrr %11, %11, $wzr
+ %20:gpr32 = ADDWrr %12, %12
+ %21:gpr32 = ADDWrr %20, %20
+ %22:gpr32 = ADDWrr %21, %21
+ %23:gpr32 = ADDWrr %22, %22
+ %40:gpr64 = MOVi64imm 100
+ %30:gpr32 = SUBSWrr %12, %12, implicit-def $nzcv
+ Bcc 10, %bb.7, implicit $nzcv
+ B %bb.2
+
+ bb.2:
+ successors: %bb.3(0x40000000), %bb.7(0x40000000)
+
+ %41:gpr64 = MOVi64imm 200
+ %31:gpr32 = SUBSWrr %20, %20, implicit-def $nzcv
+ Bcc 10, %bb.7, implicit $nzcv
+ B %bb.3
+
+ bb.3:
+ successors: %bb.4(0x40000000), %bb.7(0x40000000)
+
+ %42:gpr64 = MOVi64imm 300
+ %32:gpr32 = SUBSWrr %21, %21, implicit-def $nzcv
+ Bcc 10, %bb.7, implicit $nzcv
+ B %bb.4
+
+ bb.4:
+ successors: %bb.5(0x40000000), %bb.7(0x40000000)
+
+ %43:gpr64 = MOVi64imm 400
+ %33:gpr32 = SUBSWrr %22, %22, implicit-def $nzcv
+ Bcc 10, %bb.7, implicit $nzcv
+ B %bb.5
+
+ bb.5:
+ successors: %bb.6(0x40000000), %bb.7(0x40000000)
+
+ %34:gpr32 = SUBSWrr %23, %23, implicit-def $nzcv
+ Bcc 10, %bb.7, implicit $nzcv
+ B %bb.6
+
+ bb.6:
+ successors: %bb.7(0x80000000)
+
+ bb.7:
+ %90:gpr64 = PHI %40, %bb.1, %41, %bb.2, %42, %bb.3, %43, %bb.4, %43, %bb.5, %43, %bb.6
+ $x0 = COPY %90
+ RET_ReallyLR implicit $x0
+...
+---
+name: cascade_cbz_condition_not_converted
+tracksRegLiveness: true
+body: |
+ ; CASCADE-WITH-DD-LABEL: name: cascade_cbz_condition_not_converted
+ ; CASCADE-WITH-DD: bb.0:
+ ; CASCADE-WITH-DD-NEXT: successors: %bb.1(0x80000000)
+ ; CASCADE-WITH-DD-NEXT: liveins: $x0
+ ; CASCADE-WITH-DD-NEXT: {{ $}}
+ ; CASCADE-WITH-DD-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; CASCADE-WITH-DD-NEXT: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load (s32))
+ ; CASCADE-WITH-DD-NEXT: [[LDRWui1:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 1 :: (load (s32))
+ ; CASCADE-WITH-DD-NEXT: [[LDRWui2:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 2 :: (load (s32))
+ ; CASCADE-WITH-DD-NEXT: [[LDRWui3:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 3 :: (load (s32))
+ ; CASCADE-WITH-DD-NEXT: [[LDRWui4:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 4 :: (load (s32))
+ ; CASCADE-WITH-DD-NEXT: [[LDRWui5:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 5 :: (load (s32))
+ ; CASCADE-WITH-DD-NEXT: [[LDRWui6:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 6 :: (load (s32))
+ ; CASCADE-WITH-DD-NEXT: [[LDRWui7:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 7 :: (load (s32))
+ ; CASCADE-WITH-DD-NEXT: B %bb.1
+ ; CASCADE-WITH-DD-NEXT: {{ $}}
+ ; CASCADE-WITH-DD-NEXT: bb.1:
+ ; CASCADE-WITH-DD-NEXT: successors: %bb.9(0x40000000), %bb.2(0x40000000)
+ ; CASCADE-WITH-DD-NEXT: {{ $}}
+ ; CASCADE-WITH-DD-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 1
+ ; CASCADE-WITH-DD-NEXT: CBZW [[LDRWui]], %bb.9
+ ; CASCADE-WITH-DD-NEXT: B %bb.2
+ ; CASCADE-WITH-DD-NEXT: {{ $}}
+ ; CASCADE-WITH-DD-NEXT: bb.2:
+ ; CASCADE-WITH-DD-NEXT: successors: %bb.9(0x80000000)
+ ; CASCADE-WITH-DD-NEXT: {{ $}}
+ ; CASCADE-WITH-DD-NEXT: [[ADDWrr:%[0-9]+]]:gpr32 = ADDWrr [[LDRWui]], [[LDRWui1]]
+ ; CASCADE-WITH-DD-NEXT: [[ADDWrr1:%[0-9]+]]:gpr32 = ADDWrr [[ADDWrr]], [[LDRWui1]]
+ ; CASCADE-WITH-DD-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 2
+ ; CASCADE-WITH-DD-NEXT: [[ADDWrr2:%[0-9]+]]:gpr32 = ADDWrr [[LDRWui]], [[LDRWui2]]
+ ; CASCADE-WITH-DD-NEXT: [[ADDWrr3:%[0-9]+]]:gpr32 = ADDWrr [[ADDWrr2]], [[LDRWui2]]
+ ; CASCADE-WITH-DD-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 3
+ ; CASCADE-WITH-DD-NEXT: [[ADDWrr4:%[0-9]+]]:gpr32 = ADDWrr [[LDRWui]], [[LDRWui3]]
+ ; CASCADE-WITH-DD-NEXT: [[ADDWrr5:%[0-9]+]]:gpr32 = ADDWrr [[ADDWrr4]], [[LDRWui3]]
+ ; CASCADE-WITH-DD-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 4
+ ; CASCADE-WITH-DD-NEXT: [[ADDWrr6:%[0-9]+]]:gpr32 = ADDWrr [[LDRWui]], [[LDRWui4]]
+ ; CASCADE-WITH-DD-NEXT: [[ADDWrr7:%[0-9]+]]:gpr32 = ADDWrr [[ADDWrr6]], [[LDRWui4]]
+ ; CASCADE-WITH-DD-NEXT: [[MOVi64imm4:%[0-9]+]]:gpr64 = MOVi64imm 5
+ ; CASCADE-WITH-DD-NEXT: [[ADDWrr8:%[0-9]+]]:gpr32 = ADDWrr [[LDRWui]], [[LDRWui5]]
+ ; CASCADE-WITH-DD-NEXT: [[ADDWrr9:%[0-9]+]]:gpr32 = ADDWrr [[ADDWrr8]], [[LDRWui5]]
+ ; CASCADE-WITH-DD-NEXT: [[MOVi64imm5:%[0-9]+]]:gpr64 = MOVi64imm 6
+ ; CASCADE-WITH-DD-NEXT: [[ADDWrr10:%[0-9]+]]:gpr32 = ADDWrr [[LDRWui]], [[LDRWui6]]
+ ; CASCADE-WITH-DD-NEXT: [[ADDWrr11:%[0-9]+]]:gpr32 = ADDWrr [[ADDWrr10]], [[LDRWui6]]
+ ; CASCADE-WITH-DD-NEXT: [[MOVi64imm6:%[0-9]+]]:gpr64 = MOVi64imm 7
+ ; CASCADE-WITH-DD-NEXT: [[ADDWrr12:%[0-9]+]]:gpr32 = ADDWrr [[LDRWui]], [[LDRWui7]]
+ ; CASCADE-WITH-DD-NEXT: [[ADDWrr13:%[0-9]+]]:gpr32 = ADDWrr [[ADDWrr12]], [[LDRWui7]]
+ ; CASCADE-WITH-DD-NEXT: [[MOVi64imm7:%[0-9]+]]:gpr64 = MOVi64imm 8
+ ; CASCADE-WITH-DD-NEXT: $wzr = SUBSWri [[LDRWui6]], 0, 0, implicit-def $nzcv
+ ; CASCADE-WITH-DD-NEXT: [[CSELXr:%[0-9]+]]:gpr64 = CSELXr [[MOVi64imm6]], [[MOVi64imm7]], 0, implicit $nzcv
+ ; CASCADE-WITH-DD-NEXT: $wzr = SUBSWri [[LDRWui5]], 0, 0, implicit-def $nzcv
+ ; CASCADE-WITH-DD-NEXT: [[CSELXr1:%[0-9]+]]:gpr64 = CSELXr [[MOVi64imm5]], [[CSELXr]], 0, implicit $nzcv
+ ; CASCADE-WITH-DD-NEXT: $wzr = SUBSWri [[LDRWui4]], 0, 0, implicit-def $nzcv
+ ; CASCADE-WITH-DD-NEXT: [[CSELXr2:%[0-9]+]]:gpr64 = CSELXr [[MOVi64imm4]], [[CSELXr1]], 0, implicit $nzcv
+ ; CASCADE-WITH-DD-NEXT: $wzr = SUBSWri [[LDRWui3]], 0, 0, implicit-def $nzcv
+ ; CASCADE-WITH-DD-NEXT: [[CSELXr3:%[0-9]+]]:gpr64 = CSELXr [[MOVi64imm3]], [[CSELXr2]], 0, implicit $nzcv
+ ; CASCADE-WITH-DD-NEXT: $wzr = SUBSWri [[LDRWui2]], 0, 0, implicit-def $nzcv
+ ; CASCADE-WITH-DD-NEXT: [[CSELXr4:%[0-9]+]]:gpr64 = CSELXr [[MOVi64imm2]], [[CSELXr3]], 0, implicit $nzcv
+ ; CASCADE-WITH-DD-NEXT: $wzr = SUBSWri [[LDRWui1]], 0, 0, implicit-def $nzcv
+ ; CASCADE-WITH-DD-NEXT: [[CSELXr5:%[0-9]+]]:gpr64 = CSELXr [[MOVi64imm1]], [[CSELXr4]], 0, implicit $nzcv
+ ; CASCADE-WITH-DD-NEXT: B %bb.9
+ ; CASCADE-WITH-DD-NEXT: {{ $}}
+ ; CASCADE-WITH-DD-NEXT: bb.9:
+ ; CASCADE-WITH-DD-NEXT: [[PHI:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.1, [[CSELXr5]], %bb.2
+ ; CASCADE-WITH-DD-NEXT: $x0 = COPY [[PHI]]
+ ; CASCADE-WITH-DD-NEXT: RET_ReallyLR implicit $x0
+ ;
+ ; DEPTH3-LABEL: name: cascade_cbz_condition_not_converted
+ ; DEPTH3: bb.0:
+ ; DEPTH3-NEXT: successors: %bb.1(0x80000000)
+ ; DEPTH3-NEXT: liveins: $x0
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; DEPTH3-NEXT: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load (s32))
+ ; DEPTH3-NEXT: [[LDRWui1:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 1 :: (load (s32))
+ ; DEPTH3-NEXT: [[LDRWui2:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 2 :: (load (s32))
+ ; DEPTH3-NEXT: [[LDRWui3:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 3 :: (load (s32))
+ ; DEPTH3-NEXT: [[LDRWui4:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 4 :: (load (s32))
+ ; DEPTH3-NEXT: [[LDRWui5:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 5 :: (load (s32))
+ ; DEPTH3-NEXT: [[LDRWui6:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 6 :: (load (s32))
+ ; DEPTH3-NEXT: [[LDRWui7:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 7 :: (load (s32))
+ ; DEPTH3-NEXT: B %bb.1
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: bb.1:
+ ; DEPTH3-NEXT: successors: %bb.9(0x40000000), %bb.2(0x40000000)
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 1
+ ; DEPTH3-NEXT: CBZW [[LDRWui]], %bb.9
+ ; DEPTH3-NEXT: B %bb.2
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: bb.2:
+ ; DEPTH3-NEXT: successors: %bb.9(0x80000000)
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: [[ADDWrr:%[0-9]+]]:gpr32 = ADDWrr [[LDRWui]], [[LDRWui1]]
+ ; DEPTH3-NEXT: [[ADDWrr1:%[0-9]+]]:gpr32 = ADDWrr [[ADDWrr]], [[LDRWui1]]
+ ; DEPTH3-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 2
+ ; DEPTH3-NEXT: [[ADDWrr2:%[0-9]+]]:gpr32 = ADDWrr [[LDRWui]], [[LDRWui2]]
+ ; DEPTH3-NEXT: [[ADDWrr3:%[0-9]+]]:gpr32 = ADDWrr [[ADDWrr2]], [[LDRWui2]]
+ ; DEPTH3-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 3
+ ; DEPTH3-NEXT: [[ADDWrr4:%[0-9]+]]:gpr32 = ADDWrr [[LDRWui]], [[LDRWui3]]
+ ; DEPTH3-NEXT: [[ADDWrr5:%[0-9]+]]:gpr32 = ADDWrr [[ADDWrr4]], [[LDRWui3]]
+ ; DEPTH3-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 4
+ ; DEPTH3-NEXT: [[ADDWrr6:%[0-9]+]]:gpr32 = ADDWrr [[LDRWui]], [[LDRWui4]]
+ ; DEPTH3-NEXT: [[ADDWrr7:%[0-9]+]]:gpr32 = ADDWrr [[ADDWrr6]], [[LDRWui4]]
+ ; DEPTH3-NEXT: [[MOVi64imm4:%[0-9]+]]:gpr64 = MOVi64imm 5
+ ; DEPTH3-NEXT: [[ADDWrr8:%[0-9]+]]:gpr32 = ADDWrr [[LDRWui]], [[LDRWui5]]
+ ; DEPTH3-NEXT: [[ADDWrr9:%[0-9]+]]:gpr32 = ADDWrr [[ADDWrr8]], [[LDRWui5]]
+ ; DEPTH3-NEXT: [[MOVi64imm5:%[0-9]+]]:gpr64 = MOVi64imm 6
+ ; DEPTH3-NEXT: [[ADDWrr10:%[0-9]+]]:gpr32 = ADDWrr [[LDRWui]], [[LDRWui6]]
+ ; DEPTH3-NEXT: [[ADDWrr11:%[0-9]+]]:gpr32 = ADDWrr [[ADDWrr10]], [[LDRWui6]]
+ ; DEPTH3-NEXT: [[MOVi64imm6:%[0-9]+]]:gpr64 = MOVi64imm 7
+ ; DEPTH3-NEXT: [[ADDWrr12:%[0-9]+]]:gpr32 = ADDWrr [[LDRWui]], [[LDRWui7]]
+ ; DEPTH3-NEXT: [[ADDWrr13:%[0-9]+]]:gpr32 = ADDWrr [[ADDWrr12]], [[LDRWui7]]
+ ; DEPTH3-NEXT: [[MOVi64imm7:%[0-9]+]]:gpr64 = MOVi64imm 8
+ ; DEPTH3-NEXT: $wzr = SUBSWri [[LDRWui6]], 0, 0, implicit-def $nzcv
+ ; DEPTH3-NEXT: [[CSELXr:%[0-9]+]]:gpr64 = CSELXr [[MOVi64imm6]], [[MOVi64imm7]], 0, implicit $nzcv
+ ; DEPTH3-NEXT: $wzr = SUBSWri [[LDRWui5]], 0, 0, implicit-def $nzcv
+ ; DEPTH3-NEXT: [[CSELXr1:%[0-9]+]]:gpr64 = CSELXr [[MOVi64imm5]], [[CSELXr]], 0, implicit $nzcv
+ ; DEPTH3-NEXT: $wzr = SUBSWri [[LDRWui4]], 0, 0, implicit-def $nzcv
+ ; DEPTH3-NEXT: [[CSELXr2:%[0-9]+]]:gpr64 = CSELXr [[MOVi64imm4]], [[CSELXr1]], 0, implicit $nzcv
+ ; DEPTH3-NEXT: $wzr = SUBSWri [[LDRWui3]], 0, 0, implicit-def $nzcv
+ ; DEPTH3-NEXT: [[CSELXr3:%[0-9]+]]:gpr64 = CSELXr [[MOVi64imm3]], [[CSELXr2]], 0, implicit $nzcv
+ ; DEPTH3-NEXT: $wzr = SUBSWri [[LDRWui2]], 0, 0, implicit-def $nzcv
+ ; DEPTH3-NEXT: [[CSELXr4:%[0-9]+]]:gpr64 = CSELXr [[MOVi64imm2]], [[CSELXr3]], 0, implicit $nzcv
+ ; DEPTH3-NEXT: $wzr = SUBSWri [[LDRWui1]], 0, 0, implicit-def $nzcv
+ ; DEPTH3-NEXT: [[CSELXr5:%[0-9]+]]:gpr64 = CSELXr [[MOVi64imm1]], [[CSELXr4]], 0, implicit $nzcv
+ ; DEPTH3-NEXT: B %bb.9
+ ; DEPTH3-NEXT: {{ $}}
+ ; DEPTH3-NEXT: bb.9:
+ ; DEPTH3-NEXT: [[PHI:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.1, [[CSELXr5]], %bb.2
+ ; DEPTH3-NEXT: $x0 = COPY [[PHI]]
+ ; DEPTH3-NEXT: RET_ReallyLR implicit $x0
+ ;
+ ; LIMIT3-LABEL: name: cascade_cbz_condition_not_converted
+ ; LIMIT3: bb.0:
+ ; LIMIT3-NEXT: successors: %bb.1(0x80000000)
+ ; LIMIT3-NEXT: liveins: $x0
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; LIMIT3-NEXT: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load (s32))
+ ; LIMIT3-NEXT: [[LDRWui1:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 1 :: (load (s32))
+ ; LIMIT3-NEXT: [[LDRWui2:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 2 :: (load (s32))
+ ; LIMIT3-NEXT: [[LDRWui3:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 3 :: (load (s32))
+ ; LIMIT3-NEXT: [[LDRWui4:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 4 :: (load (s32))
+ ; LIMIT3-NEXT: [[LDRWui5:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 5 :: (load (s32))
+ ; LIMIT3-NEXT: [[LDRWui6:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 6 :: (load (s32))
+ ; LIMIT3-NEXT: [[LDRWui7:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 7 :: (load (s32))
+ ; LIMIT3-NEXT: B %bb.1
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: bb.1:
+ ; LIMIT3-NEXT: successors: %bb.9(0x40000000), %bb.2(0x40000000)
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 1
+ ; LIMIT3-NEXT: CBZW [[LDRWui]], %bb.9
+ ; LIMIT3-NEXT: B %bb.2
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: bb.2:
+ ; LIMIT3-NEXT: successors: %bb.9(0x40000000), %bb.3(0x40000000)
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: [[ADDWrr:%[0-9]+]]:gpr32 = ADDWrr [[LDRWui]], [[LDRWui1]]
+ ; LIMIT3-NEXT: [[ADDWrr1:%[0-9]+]]:gpr32 = ADDWrr [[ADDWrr]], [[LDRWui1]]
+ ; LIMIT3-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 2
+ ; LIMIT3-NEXT: CBZW [[LDRWui1]], %bb.9
+ ; LIMIT3-NEXT: B %bb.3
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: bb.3:
+ ; LIMIT3-NEXT: successors: %bb.9(0x40000000), %bb.4(0x40000000)
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: [[ADDWrr2:%[0-9]+]]:gpr32 = ADDWrr [[LDRWui]], [[LDRWui2]]
+ ; LIMIT3-NEXT: [[ADDWrr3:%[0-9]+]]:gpr32 = ADDWrr [[ADDWrr2]], [[LDRWui2]]
+ ; LIMIT3-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 3
+ ; LIMIT3-NEXT: CBZW [[LDRWui2]], %bb.9
+ ; LIMIT3-NEXT: B %bb.4
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: bb.4:
+ ; LIMIT3-NEXT: successors: %bb.9(0x40000000), %bb.5(0x40000000)
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: [[ADDWrr4:%[0-9]+]]:gpr32 = ADDWrr [[LDRWui]], [[LDRWui3]]
+ ; LIMIT3-NEXT: [[ADDWrr5:%[0-9]+]]:gpr32 = ADDWrr [[ADDWrr4]], [[LDRWui3]]
+ ; LIMIT3-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 4
+ ; LIMIT3-NEXT: CBZW [[LDRWui3]], %bb.9
+ ; LIMIT3-NEXT: B %bb.5
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: bb.5:
+ ; LIMIT3-NEXT: successors: %bb.9(0x40000000), %bb.6(0x40000000)
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: [[ADDWrr6:%[0-9]+]]:gpr32 = ADDWrr [[LDRWui]], [[LDRWui4]]
+ ; LIMIT3-NEXT: [[ADDWrr7:%[0-9]+]]:gpr32 = ADDWrr [[ADDWrr6]], [[LDRWui4]]
+ ; LIMIT3-NEXT: [[MOVi64imm4:%[0-9]+]]:gpr64 = MOVi64imm 5
+ ; LIMIT3-NEXT: CBZW [[LDRWui4]], %bb.9
+ ; LIMIT3-NEXT: B %bb.6
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: bb.6:
+ ; LIMIT3-NEXT: successors: %bb.9(0x40000000), %bb.7(0x40000000)
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: [[ADDWrr8:%[0-9]+]]:gpr32 = ADDWrr [[LDRWui]], [[LDRWui5]]
+ ; LIMIT3-NEXT: [[ADDWrr9:%[0-9]+]]:gpr32 = ADDWrr [[ADDWrr8]], [[LDRWui5]]
+ ; LIMIT3-NEXT: [[MOVi64imm5:%[0-9]+]]:gpr64 = MOVi64imm 6
+ ; LIMIT3-NEXT: CBZW [[LDRWui5]], %bb.9
+ ; LIMIT3-NEXT: B %bb.7
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: bb.7:
+ ; LIMIT3-NEXT: successors: %bb.9(0x80000000)
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: [[ADDWrr10:%[0-9]+]]:gpr32 = ADDWrr [[LDRWui]], [[LDRWui6]]
+ ; LIMIT3-NEXT: [[ADDWrr11:%[0-9]+]]:gpr32 = ADDWrr [[ADDWrr10]], [[LDRWui6]]
+ ; LIMIT3-NEXT: [[MOVi64imm6:%[0-9]+]]:gpr64 = MOVi64imm 7
+ ; LIMIT3-NEXT: [[ADDWrr12:%[0-9]+]]:gpr32 = ADDWrr [[LDRWui]], [[LDRWui7]]
+ ; LIMIT3-NEXT: [[ADDWrr13:%[0-9]+]]:gpr32 = ADDWrr [[ADDWrr12]], [[LDRWui7]]
+ ; LIMIT3-NEXT: [[MOVi64imm7:%[0-9]+]]:gpr64 = MOVi64imm 8
+ ; LIMIT3-NEXT: $wzr = SUBSWri [[LDRWui6]], 0, 0, implicit-def $nzcv
+ ; LIMIT3-NEXT: [[CSELXr:%[0-9]+]]:gpr64 = CSELXr [[MOVi64imm6]], [[MOVi64imm7]], 0, implicit $nzcv
+ ; LIMIT3-NEXT: B %bb.9
+ ; LIMIT3-NEXT: {{ $}}
+ ; LIMIT3-NEXT: bb.9:
+ ; LIMIT3-NEXT: [[PHI:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.1, [[MOVi64imm1]], %bb.2, [[MOVi64imm2]], %bb.3, [[MOVi64imm3]], %bb.4, [[MOVi64imm4]], %bb.5, [[MOVi64imm5]], %bb.6, [[CSELXr]], %bb.7
+ ; LIMIT3-NEXT: $x0 = COPY [[PHI]]
+ ; LIMIT3-NEXT: RET_ReallyLR implicit $x0
+ bb.0:
+ successors: %bb.1(0x80000000)
+ liveins: $x0
+ %0:gpr64common = COPY $x0
+ %1:gpr32 = LDRWui %0, 0 :: (load (s32))
+ %2:gpr32 = LDRWui %0, 1 :: (load (s32))
+ %3:gpr32 = LDRWui %0, 2 :: (load (s32))
+ %4:gpr32 = LDRWui %0, 3 :: (load (s32))
+ %5:gpr32 = LDRWui %0, 4 :: (load (s32))
+ %6:gpr32 = LDRWui %0, 5 :: (load (s32))
+ %7:gpr32 = LDRWui %0, 6 :: (load (s32))
+ %8:gpr32 = LDRWui %0, 7 :: (load (s32))
+ B %bb.1
+
+ bb.1:
+ successors: %bb.9(0x40000000), %bb.2(0x40000000)
+ %100:gpr64 = MOVi64imm 1
+ CBZW %1, %bb.9
+ B %bb.2
+
+ bb.2:
+ successors: %bb.9(0x40000000), %bb.3(0x40000000)
+ %201:gpr32 = ADDWrr %1, %2
+ %202:gpr32 = ADDWrr %201, %2
+ %200:gpr64 = MOVi64imm 2
+ CBZW %2, %bb.9
+ B %bb.3
+
+ bb.3:
+ successors: %bb.9(0x40000000), %bb.4(0x40000000)
+ %301:gpr32 = ADDWrr %1, %3
+ %302:gpr32 = ADDWrr %301, %3
+ %300:gpr64 = MOVi64imm 3
+ CBZW %3, %bb.9
+ B %bb.4
+
+ bb.4:
+ successors: %bb.9(0x40000000), %bb.5(0x40000000)
+ %401:gpr32 = ADDWrr %1, %4
+ %402:gpr32 = ADDWrr %401, %4
+ %400:gpr64 = MOVi64imm 4
+ CBZW %4, %bb.9
+ B %bb.5
+
+ bb.5:
+ successors: %bb.9(0x40000000), %bb.6(0x40000000)
+ %501:gpr32 = ADDWrr %1, %5
+ %502:gpr32 = ADDWrr %501, %5
+ %500:gpr64 = MOVi64imm 5
+ CBZW %5, %bb.9
+ B %bb.6
+
+ bb.6:
+ successors: %bb.9(0x40000000), %bb.7(0x40000000)
+ %601:gpr32 = ADDWrr %1, %6
+ %602:gpr32 = ADDWrr %601, %6
+ %600:gpr64 = MOVi64imm 6
+ CBZW %6, %bb.9
+ B %bb.7
+
+ bb.7:
+ successors: %bb.9(0x40000000), %bb.8(0x40000000)
+ %701:gpr32 = ADDWrr %1, %7
+ %702:gpr32 = ADDWrr %701, %7
+ %700:gpr64 = MOVi64imm 7
+ CBZW %7, %bb.9
+ B %bb.8
+
+ bb.8:
+ successors: %bb.9(0x80000000)
+ %801:gpr32 = ADDWrr %1, %8
+ %802:gpr32 = ADDWrr %801, %8
+ %800:gpr64 = MOVi64imm 8
+ B %bb.9
+
+ bb.9:
+ %900:gpr64 = PHI %100, %bb.1, %200, %bb.2, %300, %bb.3, %400, %bb.4, %500, %bb.5, %600, %bb.6, %700, %bb.7, %800, %bb.8
+ $x0 = COPY %900
+ RET_ReallyLR implicit $x0
+...
+---
+name: cascade_head_def_after_condition_not_converted
+tracksRegLiveness: true
+body: |
+ ; COMMON-LABEL: name: cascade_head_def_after_condition_not_converted
+ ; COMMON: bb.0:
+ ; COMMON-NEXT: successors: %bb.1(0x80000000)
+ ; COMMON-NEXT: liveins: $x0
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; COMMON-NEXT: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load (s32))
+ ; COMMON-NEXT: [[LDRWui1:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 1 :: (load (s32))
+ ; COMMON-NEXT: [[LDRWui2:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 2 :: (load (s32))
+ ; COMMON-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 100
+ ; COMMON-NEXT: B %bb.1
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.1:
+ ; COMMON-NEXT: successors: %bb.2(0x40000000), %bb.4(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui1]], implicit-def $nzcv
+ ; COMMON-NEXT: [[ADDXrr:%[0-9]+]]:gpr64 = ADDXrr [[MOVi64imm]], [[MOVi64imm]]
+ ; COMMON-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 1
+ ; COMMON-NEXT: Bcc 10, %bb.4, implicit $nzcv
+ ; COMMON-NEXT: B %bb.2
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.2:
+ ; COMMON-NEXT: successors: %bb.4(0x80000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui2]], implicit-def $nzcv
+ ; COMMON-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 2
+ ; COMMON-NEXT: [[ADDXrr1:%[0-9]+]]:gpr64 = ADDXrr [[ADDXrr]], [[ADDXrr]]
+ ; COMMON-NEXT: [[CSELXr:%[0-9]+]]:gpr64 = CSELXr [[MOVi64imm2]], [[ADDXrr1]], 10, implicit $nzcv
+ ; COMMON-NEXT: B %bb.4
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.4:
+ ; COMMON-NEXT: [[PHI:%[0-9]+]]:gpr64 = PHI [[MOVi64imm1]], %bb.1, [[CSELXr]], %bb.2
+ ; COMMON-NEXT: $x0 = COPY [[PHI]]
+ ; COMMON-NEXT: RET_ReallyLR implicit $x0
+ bb.0:
+ successors: %bb.1(0x80000000)
+ liveins: $x0
+ %0:gpr64common = COPY $x0
+ %1:gpr32 = LDRWui %0, 0 :: (load (s32))
+ %2:gpr32 = LDRWui %0, 1 :: (load (s32))
+ %3:gpr32 = LDRWui %0, 2 :: (load (s32))
+ %20:gpr64 = MOVi64imm 100
+ B %bb.1
+
+ bb.1:
+ successors: %bb.2(0x40000000), %bb.5(0x40000000)
+ %10:gpr32 = SUBSWrr %1, %2, implicit-def $nzcv
+ %11:gpr64 = ADDXrr %20, %20
+ %31:gpr64 = MOVi64imm 1
+ Bcc 10, %bb.5, implicit $nzcv
+ B %bb.2
+
+ bb.2:
+ successors: %bb.3(0x40000000), %bb.5(0x40000000)
+ %12:gpr32 = SUBSWrr %1, %3, implicit-def $nzcv
+ %32:gpr64 = MOVi64imm 2
+ Bcc 10, %bb.5, implicit $nzcv
+ B %bb.3
+
+ bb.3:
+ successors: %bb.5(0x80000000)
+ %33:gpr64 = ADDXrr %11, %11
+ B %bb.5
+ bb.5:
+ %40:gpr64 = PHI %31, %bb.1, %32, %bb.2, %33, %bb.3
+ $x0 = COPY %40
+ RET_ReallyLR implicit $x0
+...
+---
+name: cascade_head_def_reaching_select_not_converted
+tracksRegLiveness: true
+body: |
+ ; COMMON-LABEL: name: cascade_head_def_reaching_select_not_converted
+ ; COMMON: bb.0:
+ ; COMMON-NEXT: successors: %bb.1(0x80000000)
+ ; COMMON-NEXT: liveins: $x0
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; COMMON-NEXT: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load (s32))
+ ; COMMON-NEXT: [[LDRWui1:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 1 :: (load (s32))
+ ; COMMON-NEXT: [[LDRWui2:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 2 :: (load (s32))
+ ; COMMON-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 100
+ ; COMMON-NEXT: B %bb.1
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.1:
+ ; COMMON-NEXT: successors: %bb.2(0x40000000), %bb.4(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 1
+ ; COMMON-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui1]], implicit-def $nzcv
+ ; COMMON-NEXT: [[ADDXrr:%[0-9]+]]:gpr64 = ADDXrr [[MOVi64imm]], [[MOVi64imm]]
+ ; COMMON-NEXT: Bcc 10, %bb.4, implicit $nzcv
+ ; COMMON-NEXT: B %bb.2
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.2:
+ ; COMMON-NEXT: successors: %bb.4(0x80000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui2]], implicit-def $nzcv
+ ; COMMON-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 3
+ ; COMMON-NEXT: [[CSELXr:%[0-9]+]]:gpr64 = CSELXr [[ADDXrr]], [[MOVi64imm2]], 10, implicit $nzcv
+ ; COMMON-NEXT: B %bb.4
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.4:
+ ; COMMON-NEXT: [[PHI:%[0-9]+]]:gpr64 = PHI [[MOVi64imm1]], %bb.1, [[CSELXr]], %bb.2
+ ; COMMON-NEXT: $x0 = COPY [[PHI]]
+ ; COMMON-NEXT: RET_ReallyLR implicit $x0
+ bb.0:
+ successors: %bb.1(0x80000000)
+ liveins: $x0
+ %0:gpr64common = COPY $x0
+ %1:gpr32 = LDRWui %0, 0 :: (load (s32))
+ %2:gpr32 = LDRWui %0, 1 :: (load (s32))
+ %3:gpr32 = LDRWui %0, 2 :: (load (s32))
+ %20:gpr64 = MOVi64imm 100
+ B %bb.1
+
+ bb.1:
+ successors: %bb.2(0x40000000), %bb.5(0x40000000)
+ %30:gpr64 = MOVi64imm 1
+ %10:gpr32 = SUBSWrr %1, %2, implicit-def $nzcv
+ %11:gpr64 = ADDXrr %20, %20
+ Bcc 10, %bb.5, implicit $nzcv
+ B %bb.2
+
+ bb.2:
+ successors: %bb.3(0x40000000), %bb.5(0x40000000)
+ %12:gpr32 = SUBSWrr %1, %3, implicit-def $nzcv
+ Bcc 10, %bb.5, implicit $nzcv
+ B %bb.3
+
+ bb.3:
+ successors: %bb.5(0x80000000)
+ %33:gpr64 = MOVi64imm 3
+ B %bb.5
+
+ bb.5:
+ %40:gpr64 = PHI %30, %bb.1, %11, %bb.2, %33, %bb.3
+ $x0 = COPY %40
+ RET_ReallyLR implicit $x0
+...
+---
+name: cascade_head_def_reaching_select_inverted_branch
+tracksRegLiveness: true
+body: |
+ ; COMMON-LABEL: name: cascade_head_def_reaching_select_inverted_branch
+ ; COMMON: bb.0:
+ ; COMMON-NEXT: successors: %bb.1(0x80000000)
+ ; COMMON-NEXT: liveins: $x0
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; COMMON-NEXT: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load (s32))
+ ; COMMON-NEXT: [[LDRWui1:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 1 :: (load (s32))
+ ; COMMON-NEXT: [[LDRWui2:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 2 :: (load (s32))
+ ; COMMON-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 100
+ ; COMMON-NEXT: B %bb.1
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.1:
+ ; COMMON-NEXT: successors: %bb.2(0x40000000), %bb.4(0x40000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 1
+ ; COMMON-NEXT: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui1]], implicit-def $nzcv
+ ; COMMON-NEXT: [[ADDXrr:%[0-9]+]]:gpr64 = ADDXrr [[MOVi64imm]], [[MOVi64imm]]
+ ; COMMON-NEXT: Bcc 10, %bb.2, implicit $nzcv
+ ; COMMON-NEXT: B %bb.4
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.2:
+ ; COMMON-NEXT: successors: %bb.4(0x80000000)
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: [[SUBSWrr1:%[0-9]+]]:gpr32 = SUBSWrr [[LDRWui]], [[LDRWui2]], implicit-def $nzcv
+ ; COMMON-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 3
+ ; COMMON-NEXT: [[CSELXr:%[0-9]+]]:gpr64 = CSELXr [[MOVi64imm2]], [[ADDXrr]], 10, implicit $nzcv
+ ; COMMON-NEXT: B %bb.4
+ ; COMMON-NEXT: {{ $}}
+ ; COMMON-NEXT: bb.4:
+ ; COMMON-NEXT: [[PHI:%[0-9]+]]:gpr64 = PHI [[MOVi64imm1]], %bb.1, [[CSELXr]], %bb.2
+ ; COMMON-NEXT: $x0 = COPY [[PHI]]
+ ; COMMON-NEXT: RET_ReallyLR implicit $x0
+ bb.0:
+ successors: %bb.1(0x80000000)
+ liveins: $x0
+ %0:gpr64common = COPY $x0
+ %1:gpr32 = LDRWui %0, 0 :: (load (s32))
+ %2:gpr32 = LDRWui %0, 1 :: (load (s32))
+ %3:gpr32 = LDRWui %0, 2 :: (load (s32))
+ %20:gpr64 = MOVi64imm 100
+ B %bb.1
+
+ bb.1:
+ successors: %bb.2(0x40000000), %bb.5(0x40000000)
+ %30:gpr64 = MOVi64imm 1
+ %10:gpr32 = SUBSWrr %1, %2, implicit-def $nzcv
+ %11:gpr64 = ADDXrr %20, %20
+ Bcc 10, %bb.2, implicit $nzcv
+ B %bb.5
+
+ bb.2:
+ successors: %bb.3(0x40000000), %bb.5(0x40000000)
+ %12:gpr32 = SUBSWrr %1, %3, implicit-def $nzcv
+ Bcc 10, %bb.3, implicit $nzcv
+ B %bb.5
+
+ bb.3:
+ successors: %bb.5(0x80000000)
+ %33:gpr64 = MOVi64imm 3
+ B %bb.5
+
+ bb.5:
+ %40:gpr64 = PHI %30, %bb.1, %11, %bb.2, %33, %bb.3
+ $x0 = COPY %40
+ RET_ReallyLR implicit $x0
+...
diff --git a/llvm/test/CodeGen/AArch64/early-ifcvt-load-to-cond-br-limit.mir b/llvm/test/CodeGen/AArch64/early-ifcvt-load-to-cond-br-limit.mir
index 12170733e2332..c8ab9bb5da684 100644
--- a/llvm/test/CodeGen/AArch64/early-ifcvt-load-to-cond-br-limit.mir
+++ b/llvm/test/CodeGen/AArch64/early-ifcvt-load-to-cond-br-limit.mir
@@ -2,6 +2,7 @@
# RUN: llc -mtriple=aarch64 -mcpu=apple-m1 -run-pass=early-ifcvt -verify-machine-dom-info -o - %s | FileCheck %s --check-prefix=CHECK-DEFAULT
# RUN: llc -mtriple=aarch64 -mcpu=apple-m1 -run-pass=early-ifcvt -enable-early-ifcvt-data-dependent -early-ifcvt-max-region-instrs=3 -verify-machine-dom-info -o - %s | FileCheck %s --check-prefix=CHECK-SMALL-BUDGET
# RUN: llc -mtriple=aarch64 -mcpu=apple-m1 -run-pass=early-ifcvt -enable-early-ifcvt-data-dependent -verify-machine-dom-info -o - %s | FileCheck %s --check-prefix=CHECK-LARGE-BUDGET
+# RUN: llc -mtriple=aarch64 -mcpu=apple-m1 -run-pass=early-ifcvt -enable-early-ifcvt-cascade -enable-early-ifcvt-data-dependent -early-ifcvt-limit=50 -early-ifcvt-max-region-instrs=65 -verify-machine-dom-info -o - %s | FileCheck %s --check-prefix=CHECK-CASCADE-BUDGET
---
name: load_in_pred_region_over_instruction_budget
@@ -138,6 +139,43 @@ body: |
; CHECK-LARGE-BUDGET-NEXT: [[CSELWr:%[0-9]+]]:gpr32 = CSELWr [[COPY1]], [[MADDWrrr1]], 12, implicit $nzcv
; CHECK-LARGE-BUDGET-NEXT: $w0 = COPY [[CSELWr]]
; CHECK-LARGE-BUDGET-NEXT: RET_ReallyLR implicit $w0
+ ;
+ ; CHECK-CASCADE-BUDGET-LABEL: name: load_in_pred_region_over_instruction_budget
+ ; CHECK-CASCADE-BUDGET: bb.0:
+ ; CHECK-CASCADE-BUDGET-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-CASCADE-BUDGET-NEXT: liveins: $x0, $w1, $w2
+ ; CHECK-CASCADE-BUDGET-NEXT: {{ $}}
+ ; CHECK-CASCADE-BUDGET-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[COPY1:%[0-9]+]]:gpr32 = COPY $w1
+ ; CHECK-CASCADE-BUDGET-NEXT: [[COPY2:%[0-9]+]]:gpr32 = COPY $w2
+ ; CHECK-CASCADE-BUDGET-NEXT: {{ $}}
+ ; CHECK-CASCADE-BUDGET-NEXT: bb.1:
+ ; CHECK-CASCADE-BUDGET-NEXT: successors: %bb.2(0x40000000), %bb.3(0x40000000)
+ ; CHECK-CASCADE-BUDGET-NEXT: {{ $}}
+ ; CHECK-CASCADE-BUDGET-NEXT: [[LDRWui:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 0 :: (load (s32))
+ ; CHECK-CASCADE-BUDGET-NEXT: [[SUBSWri:%[0-9]+]]:gpr32 = SUBSWri [[LDRWui]], 0, 0, implicit-def $nzcv
+ ; CHECK-CASCADE-BUDGET-NEXT: Bcc 0, %bb.2, implicit $nzcv
+ ; CHECK-CASCADE-BUDGET-NEXT: B %bb.3
+ ; CHECK-CASCADE-BUDGET-NEXT: {{ $}}
+ ; CHECK-CASCADE-BUDGET-NEXT: bb.2:
+ ; CHECK-CASCADE-BUDGET-NEXT: successors: %bb.3(0x80000000)
+ ; CHECK-CASCADE-BUDGET-NEXT: {{ $}}
+ ; CHECK-CASCADE-BUDGET-NEXT: STRWui $wzr, [[COPY]], 0 :: (store (s32))
+ ; CHECK-CASCADE-BUDGET-NEXT: STRWui $wzr, [[COPY]], 1 :: (store (s32))
+ ; CHECK-CASCADE-BUDGET-NEXT: STRWui $wzr, [[COPY]], 2 :: (store (s32))
+ ; CHECK-CASCADE-BUDGET-NEXT: STRWui $wzr, [[COPY]], 3 :: (store (s32))
+ ; CHECK-CASCADE-BUDGET-NEXT: STRWui $wzr, [[COPY]], 4 :: (store (s32))
+ ; CHECK-CASCADE-BUDGET-NEXT: STRWui $wzr, [[COPY]], 5 :: (store (s32))
+ ; CHECK-CASCADE-BUDGET-NEXT: B %bb.3
+ ; CHECK-CASCADE-BUDGET-NEXT: {{ $}}
+ ; CHECK-CASCADE-BUDGET-NEXT: bb.3:
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri:%[0-9]+]]:gpr32common = ADDWri [[LDRWui]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[SUBSWri1:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri]], 100, 0, implicit-def $nzcv
+ ; CHECK-CASCADE-BUDGET-NEXT: [[MADDWrrr:%[0-9]+]]:gpr32 = MADDWrrr [[COPY1]], [[COPY2]], $wzr
+ ; CHECK-CASCADE-BUDGET-NEXT: [[MADDWrrr1:%[0-9]+]]:gpr32 = MADDWrrr [[MADDWrrr]], [[COPY1]], $wzr
+ ; CHECK-CASCADE-BUDGET-NEXT: [[CSELWr:%[0-9]+]]:gpr32 = CSELWr [[COPY1]], [[MADDWrrr1]], 12, implicit $nzcv
+ ; CHECK-CASCADE-BUDGET-NEXT: $w0 = COPY [[CSELWr]]
+ ; CHECK-CASCADE-BUDGET-NEXT: RET_ReallyLR implicit $w0
bb.0:
successors: %bb.1(0x80000000)
liveins: $x0, $w1, $w2
@@ -184,3 +222,542 @@ body: |
$w0 = COPY %8
RET_ReallyLR implicit $w0
...
+---
+name: cascade_load_to_cond_br_scan_budget
+alignment: 4
+tracksRegLiveness: true
+body: |
+ ; CHECK-DEFAULT-LABEL: name: cascade_load_to_cond_br_scan_budget
+ ; CHECK-DEFAULT: bb.0:
+ ; CHECK-DEFAULT-NEXT: successors: %bb.8(0x40000000), %bb.1(0x40000000)
+ ; CHECK-DEFAULT-NEXT: liveins: $x0
+ ; CHECK-DEFAULT-NEXT: {{ $}}
+ ; CHECK-DEFAULT-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; CHECK-DEFAULT-NEXT: [[LDRWui:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 0 :: (load (s32))
+ ; CHECK-DEFAULT-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 1
+ ; CHECK-DEFAULT-NEXT: [[MOVi32imm1:%[0-9]+]]:gpr32 = MOVi32imm 2
+ ; CHECK-DEFAULT-NEXT: [[MOVi32imm2:%[0-9]+]]:gpr32 = MOVi32imm 3
+ ; CHECK-DEFAULT-NEXT: [[MOVi32imm3:%[0-9]+]]:gpr32 = MOVi32imm 4
+ ; CHECK-DEFAULT-NEXT: [[LDRWui1:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 1 :: (load (s32))
+ ; CHECK-DEFAULT-NEXT: [[LDRWui2:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 2 :: (load (s32))
+ ; CHECK-DEFAULT-NEXT: [[LDRWui3:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 3 :: (load (s32))
+ ; CHECK-DEFAULT-NEXT: [[LDRWui4:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 4 :: (load (s32))
+ ; CHECK-DEFAULT-NEXT: [[LDRWui5:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 5 :: (load (s32))
+ ; CHECK-DEFAULT-NEXT: [[LDRWui6:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 6 :: (load (s32))
+ ; CHECK-DEFAULT-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 100
+ ; CHECK-DEFAULT-NEXT: [[ADDWri:%[0-9]+]]:gpr32common = ADDWri [[LDRWui]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri1:%[0-9]+]]:gpr32common = ADDWri [[ADDWri]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri2:%[0-9]+]]:gpr32common = ADDWri [[ADDWri1]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri3:%[0-9]+]]:gpr32common = ADDWri [[ADDWri2]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri4:%[0-9]+]]:gpr32common = ADDWri [[ADDWri3]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[SUBSWri:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri4]], 1, 0, implicit-def $nzcv
+ ; CHECK-DEFAULT-NEXT: Bcc 10, %bb.8, implicit $nzcv
+ ; CHECK-DEFAULT-NEXT: B %bb.1
+ ; CHECK-DEFAULT-NEXT: {{ $}}
+ ; CHECK-DEFAULT-NEXT: bb.1:
+ ; CHECK-DEFAULT-NEXT: successors: %bb.8(0x40000000), %bb.2(0x40000000)
+ ; CHECK-DEFAULT-NEXT: {{ $}}
+ ; CHECK-DEFAULT-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 101
+ ; CHECK-DEFAULT-NEXT: [[ADDWri5:%[0-9]+]]:gpr32common = ADDWri [[LDRWui1]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri6:%[0-9]+]]:gpr32common = ADDWri [[ADDWri5]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri7:%[0-9]+]]:gpr32common = ADDWri [[ADDWri6]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri8:%[0-9]+]]:gpr32common = ADDWri [[ADDWri7]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri9:%[0-9]+]]:gpr32common = ADDWri [[ADDWri8]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[SUBSWri1:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri9]], 2, 0, implicit-def $nzcv
+ ; CHECK-DEFAULT-NEXT: Bcc 11, %bb.8, implicit $nzcv
+ ; CHECK-DEFAULT-NEXT: B %bb.2
+ ; CHECK-DEFAULT-NEXT: {{ $}}
+ ; CHECK-DEFAULT-NEXT: bb.2:
+ ; CHECK-DEFAULT-NEXT: successors: %bb.8(0x40000000), %bb.3(0x40000000)
+ ; CHECK-DEFAULT-NEXT: {{ $}}
+ ; CHECK-DEFAULT-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 102
+ ; CHECK-DEFAULT-NEXT: [[ADDWri10:%[0-9]+]]:gpr32common = ADDWri [[LDRWui2]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri11:%[0-9]+]]:gpr32common = ADDWri [[ADDWri10]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri12:%[0-9]+]]:gpr32common = ADDWri [[ADDWri11]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri13:%[0-9]+]]:gpr32common = ADDWri [[ADDWri12]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri14:%[0-9]+]]:gpr32common = ADDWri [[ADDWri13]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[SUBSWri2:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri14]], 3, 0, implicit-def $nzcv
+ ; CHECK-DEFAULT-NEXT: Bcc 12, %bb.8, implicit $nzcv
+ ; CHECK-DEFAULT-NEXT: B %bb.3
+ ; CHECK-DEFAULT-NEXT: {{ $}}
+ ; CHECK-DEFAULT-NEXT: bb.3:
+ ; CHECK-DEFAULT-NEXT: successors: %bb.8(0x40000000), %bb.4(0x40000000)
+ ; CHECK-DEFAULT-NEXT: {{ $}}
+ ; CHECK-DEFAULT-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 103
+ ; CHECK-DEFAULT-NEXT: [[ADDWri15:%[0-9]+]]:gpr32common = ADDWri [[LDRWui3]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri16:%[0-9]+]]:gpr32common = ADDWri [[ADDWri15]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri17:%[0-9]+]]:gpr32common = ADDWri [[ADDWri16]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri18:%[0-9]+]]:gpr32common = ADDWri [[ADDWri17]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri19:%[0-9]+]]:gpr32common = ADDWri [[ADDWri18]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[SUBSWri3:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri19]], 4, 0, implicit-def $nzcv
+ ; CHECK-DEFAULT-NEXT: Bcc 13, %bb.8, implicit $nzcv
+ ; CHECK-DEFAULT-NEXT: B %bb.4
+ ; CHECK-DEFAULT-NEXT: {{ $}}
+ ; CHECK-DEFAULT-NEXT: bb.4:
+ ; CHECK-DEFAULT-NEXT: successors: %bb.8(0x40000000), %bb.5(0x40000000)
+ ; CHECK-DEFAULT-NEXT: {{ $}}
+ ; CHECK-DEFAULT-NEXT: [[MOVi64imm4:%[0-9]+]]:gpr64 = MOVi64imm 104
+ ; CHECK-DEFAULT-NEXT: [[ADDWri20:%[0-9]+]]:gpr32common = ADDWri [[LDRWui4]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri21:%[0-9]+]]:gpr32common = ADDWri [[ADDWri20]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri22:%[0-9]+]]:gpr32common = ADDWri [[ADDWri21]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri23:%[0-9]+]]:gpr32common = ADDWri [[ADDWri22]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri24:%[0-9]+]]:gpr32common = ADDWri [[ADDWri23]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[SUBSWri4:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri24]], 5, 0, implicit-def $nzcv
+ ; CHECK-DEFAULT-NEXT: Bcc 14, %bb.8, implicit $nzcv
+ ; CHECK-DEFAULT-NEXT: B %bb.5
+ ; CHECK-DEFAULT-NEXT: {{ $}}
+ ; CHECK-DEFAULT-NEXT: bb.5:
+ ; CHECK-DEFAULT-NEXT: successors: %bb.8(0x40000000), %bb.6(0x40000000)
+ ; CHECK-DEFAULT-NEXT: {{ $}}
+ ; CHECK-DEFAULT-NEXT: [[MOVi64imm5:%[0-9]+]]:gpr64 = MOVi64imm 105
+ ; CHECK-DEFAULT-NEXT: [[ADDWri25:%[0-9]+]]:gpr32common = ADDWri [[LDRWui5]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri26:%[0-9]+]]:gpr32common = ADDWri [[ADDWri25]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri27:%[0-9]+]]:gpr32common = ADDWri [[ADDWri26]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri28:%[0-9]+]]:gpr32common = ADDWri [[ADDWri27]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri29:%[0-9]+]]:gpr32common = ADDWri [[ADDWri28]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[SUBSWri5:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri29]], 6, 0, implicit-def $nzcv
+ ; CHECK-DEFAULT-NEXT: Bcc 15, %bb.8, implicit $nzcv
+ ; CHECK-DEFAULT-NEXT: B %bb.6
+ ; CHECK-DEFAULT-NEXT: {{ $}}
+ ; CHECK-DEFAULT-NEXT: bb.6:
+ ; CHECK-DEFAULT-NEXT: successors: %bb.8(0x40000000), %bb.7(0x40000000)
+ ; CHECK-DEFAULT-NEXT: {{ $}}
+ ; CHECK-DEFAULT-NEXT: [[MOVi64imm6:%[0-9]+]]:gpr64 = MOVi64imm 106
+ ; CHECK-DEFAULT-NEXT: [[ADDWri30:%[0-9]+]]:gpr32common = ADDWri [[LDRWui6]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri31:%[0-9]+]]:gpr32common = ADDWri [[ADDWri30]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri32:%[0-9]+]]:gpr32common = ADDWri [[ADDWri31]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri33:%[0-9]+]]:gpr32common = ADDWri [[ADDWri32]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[ADDWri34:%[0-9]+]]:gpr32common = ADDWri [[ADDWri33]], 1, 0
+ ; CHECK-DEFAULT-NEXT: [[SUBSWri6:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri34]], 7, 0, implicit-def $nzcv
+ ; CHECK-DEFAULT-NEXT: Bcc 16, %bb.8, implicit $nzcv
+ ; CHECK-DEFAULT-NEXT: B %bb.7
+ ; CHECK-DEFAULT-NEXT: {{ $}}
+ ; CHECK-DEFAULT-NEXT: bb.7:
+ ; CHECK-DEFAULT-NEXT: successors: %bb.8(0x80000000)
+ ; CHECK-DEFAULT-NEXT: {{ $}}
+ ; CHECK-DEFAULT-NEXT: [[MOVi64imm7:%[0-9]+]]:gpr64 = MOVi64imm 107
+ ; CHECK-DEFAULT-NEXT: {{ $}}
+ ; CHECK-DEFAULT-NEXT: bb.8:
+ ; CHECK-DEFAULT-NEXT: [[PHI:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.0, [[MOVi64imm1]], %bb.1, [[MOVi64imm2]], %bb.2, [[MOVi64imm3]], %bb.3, [[MOVi64imm4]], %bb.4, [[MOVi64imm5]], %bb.5, [[MOVi64imm6]], %bb.6, [[MOVi64imm7]], %bb.7
+ ; CHECK-DEFAULT-NEXT: $x0 = COPY [[PHI]]
+ ; CHECK-DEFAULT-NEXT: RET_ReallyLR implicit $x0
+ ;
+ ; CHECK-SMALL-BUDGET-LABEL: name: cascade_load_to_cond_br_scan_budget
+ ; CHECK-SMALL-BUDGET: bb.0:
+ ; CHECK-SMALL-BUDGET-NEXT: successors: %bb.8(0x40000000), %bb.1(0x40000000)
+ ; CHECK-SMALL-BUDGET-NEXT: liveins: $x0
+ ; CHECK-SMALL-BUDGET-NEXT: {{ $}}
+ ; CHECK-SMALL-BUDGET-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; CHECK-SMALL-BUDGET-NEXT: [[LDRWui:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 0 :: (load (s32))
+ ; CHECK-SMALL-BUDGET-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 1
+ ; CHECK-SMALL-BUDGET-NEXT: [[MOVi32imm1:%[0-9]+]]:gpr32 = MOVi32imm 2
+ ; CHECK-SMALL-BUDGET-NEXT: [[MOVi32imm2:%[0-9]+]]:gpr32 = MOVi32imm 3
+ ; CHECK-SMALL-BUDGET-NEXT: [[MOVi32imm3:%[0-9]+]]:gpr32 = MOVi32imm 4
+ ; CHECK-SMALL-BUDGET-NEXT: [[LDRWui1:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 1 :: (load (s32))
+ ; CHECK-SMALL-BUDGET-NEXT: [[LDRWui2:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 2 :: (load (s32))
+ ; CHECK-SMALL-BUDGET-NEXT: [[LDRWui3:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 3 :: (load (s32))
+ ; CHECK-SMALL-BUDGET-NEXT: [[LDRWui4:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 4 :: (load (s32))
+ ; CHECK-SMALL-BUDGET-NEXT: [[LDRWui5:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 5 :: (load (s32))
+ ; CHECK-SMALL-BUDGET-NEXT: [[LDRWui6:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 6 :: (load (s32))
+ ; CHECK-SMALL-BUDGET-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 100
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri:%[0-9]+]]:gpr32common = ADDWri [[LDRWui]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri1:%[0-9]+]]:gpr32common = ADDWri [[ADDWri]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri2:%[0-9]+]]:gpr32common = ADDWri [[ADDWri1]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri3:%[0-9]+]]:gpr32common = ADDWri [[ADDWri2]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri4:%[0-9]+]]:gpr32common = ADDWri [[ADDWri3]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[SUBSWri:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri4]], 1, 0, implicit-def $nzcv
+ ; CHECK-SMALL-BUDGET-NEXT: Bcc 10, %bb.8, implicit $nzcv
+ ; CHECK-SMALL-BUDGET-NEXT: B %bb.1
+ ; CHECK-SMALL-BUDGET-NEXT: {{ $}}
+ ; CHECK-SMALL-BUDGET-NEXT: bb.1:
+ ; CHECK-SMALL-BUDGET-NEXT: successors: %bb.8(0x40000000), %bb.2(0x40000000)
+ ; CHECK-SMALL-BUDGET-NEXT: {{ $}}
+ ; CHECK-SMALL-BUDGET-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 101
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri5:%[0-9]+]]:gpr32common = ADDWri [[LDRWui1]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri6:%[0-9]+]]:gpr32common = ADDWri [[ADDWri5]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri7:%[0-9]+]]:gpr32common = ADDWri [[ADDWri6]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri8:%[0-9]+]]:gpr32common = ADDWri [[ADDWri7]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri9:%[0-9]+]]:gpr32common = ADDWri [[ADDWri8]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[SUBSWri1:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri9]], 2, 0, implicit-def $nzcv
+ ; CHECK-SMALL-BUDGET-NEXT: Bcc 11, %bb.8, implicit $nzcv
+ ; CHECK-SMALL-BUDGET-NEXT: B %bb.2
+ ; CHECK-SMALL-BUDGET-NEXT: {{ $}}
+ ; CHECK-SMALL-BUDGET-NEXT: bb.2:
+ ; CHECK-SMALL-BUDGET-NEXT: successors: %bb.8(0x40000000), %bb.3(0x40000000)
+ ; CHECK-SMALL-BUDGET-NEXT: {{ $}}
+ ; CHECK-SMALL-BUDGET-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 102
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri10:%[0-9]+]]:gpr32common = ADDWri [[LDRWui2]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri11:%[0-9]+]]:gpr32common = ADDWri [[ADDWri10]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri12:%[0-9]+]]:gpr32common = ADDWri [[ADDWri11]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri13:%[0-9]+]]:gpr32common = ADDWri [[ADDWri12]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri14:%[0-9]+]]:gpr32common = ADDWri [[ADDWri13]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[SUBSWri2:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri14]], 3, 0, implicit-def $nzcv
+ ; CHECK-SMALL-BUDGET-NEXT: Bcc 12, %bb.8, implicit $nzcv
+ ; CHECK-SMALL-BUDGET-NEXT: B %bb.3
+ ; CHECK-SMALL-BUDGET-NEXT: {{ $}}
+ ; CHECK-SMALL-BUDGET-NEXT: bb.3:
+ ; CHECK-SMALL-BUDGET-NEXT: successors: %bb.8(0x40000000), %bb.4(0x40000000)
+ ; CHECK-SMALL-BUDGET-NEXT: {{ $}}
+ ; CHECK-SMALL-BUDGET-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 103
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri15:%[0-9]+]]:gpr32common = ADDWri [[LDRWui3]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri16:%[0-9]+]]:gpr32common = ADDWri [[ADDWri15]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri17:%[0-9]+]]:gpr32common = ADDWri [[ADDWri16]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri18:%[0-9]+]]:gpr32common = ADDWri [[ADDWri17]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri19:%[0-9]+]]:gpr32common = ADDWri [[ADDWri18]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[SUBSWri3:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri19]], 4, 0, implicit-def $nzcv
+ ; CHECK-SMALL-BUDGET-NEXT: Bcc 13, %bb.8, implicit $nzcv
+ ; CHECK-SMALL-BUDGET-NEXT: B %bb.4
+ ; CHECK-SMALL-BUDGET-NEXT: {{ $}}
+ ; CHECK-SMALL-BUDGET-NEXT: bb.4:
+ ; CHECK-SMALL-BUDGET-NEXT: successors: %bb.8(0x40000000), %bb.5(0x40000000)
+ ; CHECK-SMALL-BUDGET-NEXT: {{ $}}
+ ; CHECK-SMALL-BUDGET-NEXT: [[MOVi64imm4:%[0-9]+]]:gpr64 = MOVi64imm 104
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri20:%[0-9]+]]:gpr32common = ADDWri [[LDRWui4]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri21:%[0-9]+]]:gpr32common = ADDWri [[ADDWri20]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri22:%[0-9]+]]:gpr32common = ADDWri [[ADDWri21]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri23:%[0-9]+]]:gpr32common = ADDWri [[ADDWri22]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri24:%[0-9]+]]:gpr32common = ADDWri [[ADDWri23]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[SUBSWri4:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri24]], 5, 0, implicit-def $nzcv
+ ; CHECK-SMALL-BUDGET-NEXT: Bcc 14, %bb.8, implicit $nzcv
+ ; CHECK-SMALL-BUDGET-NEXT: B %bb.5
+ ; CHECK-SMALL-BUDGET-NEXT: {{ $}}
+ ; CHECK-SMALL-BUDGET-NEXT: bb.5:
+ ; CHECK-SMALL-BUDGET-NEXT: successors: %bb.8(0x40000000), %bb.6(0x40000000)
+ ; CHECK-SMALL-BUDGET-NEXT: {{ $}}
+ ; CHECK-SMALL-BUDGET-NEXT: [[MOVi64imm5:%[0-9]+]]:gpr64 = MOVi64imm 105
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri25:%[0-9]+]]:gpr32common = ADDWri [[LDRWui5]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri26:%[0-9]+]]:gpr32common = ADDWri [[ADDWri25]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri27:%[0-9]+]]:gpr32common = ADDWri [[ADDWri26]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri28:%[0-9]+]]:gpr32common = ADDWri [[ADDWri27]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri29:%[0-9]+]]:gpr32common = ADDWri [[ADDWri28]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[SUBSWri5:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri29]], 6, 0, implicit-def $nzcv
+ ; CHECK-SMALL-BUDGET-NEXT: Bcc 15, %bb.8, implicit $nzcv
+ ; CHECK-SMALL-BUDGET-NEXT: B %bb.6
+ ; CHECK-SMALL-BUDGET-NEXT: {{ $}}
+ ; CHECK-SMALL-BUDGET-NEXT: bb.6:
+ ; CHECK-SMALL-BUDGET-NEXT: successors: %bb.8(0x40000000), %bb.7(0x40000000)
+ ; CHECK-SMALL-BUDGET-NEXT: {{ $}}
+ ; CHECK-SMALL-BUDGET-NEXT: [[MOVi64imm6:%[0-9]+]]:gpr64 = MOVi64imm 106
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri30:%[0-9]+]]:gpr32common = ADDWri [[LDRWui6]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri31:%[0-9]+]]:gpr32common = ADDWri [[ADDWri30]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri32:%[0-9]+]]:gpr32common = ADDWri [[ADDWri31]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri33:%[0-9]+]]:gpr32common = ADDWri [[ADDWri32]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[ADDWri34:%[0-9]+]]:gpr32common = ADDWri [[ADDWri33]], 1, 0
+ ; CHECK-SMALL-BUDGET-NEXT: [[SUBSWri6:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri34]], 7, 0, implicit-def $nzcv
+ ; CHECK-SMALL-BUDGET-NEXT: Bcc 16, %bb.8, implicit $nzcv
+ ; CHECK-SMALL-BUDGET-NEXT: B %bb.7
+ ; CHECK-SMALL-BUDGET-NEXT: {{ $}}
+ ; CHECK-SMALL-BUDGET-NEXT: bb.7:
+ ; CHECK-SMALL-BUDGET-NEXT: successors: %bb.8(0x80000000)
+ ; CHECK-SMALL-BUDGET-NEXT: {{ $}}
+ ; CHECK-SMALL-BUDGET-NEXT: [[MOVi64imm7:%[0-9]+]]:gpr64 = MOVi64imm 107
+ ; CHECK-SMALL-BUDGET-NEXT: {{ $}}
+ ; CHECK-SMALL-BUDGET-NEXT: bb.8:
+ ; CHECK-SMALL-BUDGET-NEXT: [[PHI:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.0, [[MOVi64imm1]], %bb.1, [[MOVi64imm2]], %bb.2, [[MOVi64imm3]], %bb.3, [[MOVi64imm4]], %bb.4, [[MOVi64imm5]], %bb.5, [[MOVi64imm6]], %bb.6, [[MOVi64imm7]], %bb.7
+ ; CHECK-SMALL-BUDGET-NEXT: $x0 = COPY [[PHI]]
+ ; CHECK-SMALL-BUDGET-NEXT: RET_ReallyLR implicit $x0
+ ;
+ ; CHECK-LARGE-BUDGET-LABEL: name: cascade_load_to_cond_br_scan_budget
+ ; CHECK-LARGE-BUDGET: bb.0:
+ ; CHECK-LARGE-BUDGET-NEXT: successors: %bb.8(0x40000000), %bb.1(0x40000000)
+ ; CHECK-LARGE-BUDGET-NEXT: liveins: $x0
+ ; CHECK-LARGE-BUDGET-NEXT: {{ $}}
+ ; CHECK-LARGE-BUDGET-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; CHECK-LARGE-BUDGET-NEXT: [[LDRWui:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 0 :: (load (s32))
+ ; CHECK-LARGE-BUDGET-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 1
+ ; CHECK-LARGE-BUDGET-NEXT: [[MOVi32imm1:%[0-9]+]]:gpr32 = MOVi32imm 2
+ ; CHECK-LARGE-BUDGET-NEXT: [[MOVi32imm2:%[0-9]+]]:gpr32 = MOVi32imm 3
+ ; CHECK-LARGE-BUDGET-NEXT: [[MOVi32imm3:%[0-9]+]]:gpr32 = MOVi32imm 4
+ ; CHECK-LARGE-BUDGET-NEXT: [[LDRWui1:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 1 :: (load (s32))
+ ; CHECK-LARGE-BUDGET-NEXT: [[LDRWui2:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 2 :: (load (s32))
+ ; CHECK-LARGE-BUDGET-NEXT: [[LDRWui3:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 3 :: (load (s32))
+ ; CHECK-LARGE-BUDGET-NEXT: [[LDRWui4:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 4 :: (load (s32))
+ ; CHECK-LARGE-BUDGET-NEXT: [[LDRWui5:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 5 :: (load (s32))
+ ; CHECK-LARGE-BUDGET-NEXT: [[LDRWui6:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 6 :: (load (s32))
+ ; CHECK-LARGE-BUDGET-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 100
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri:%[0-9]+]]:gpr32common = ADDWri [[LDRWui]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri1:%[0-9]+]]:gpr32common = ADDWri [[ADDWri]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri2:%[0-9]+]]:gpr32common = ADDWri [[ADDWri1]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri3:%[0-9]+]]:gpr32common = ADDWri [[ADDWri2]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri4:%[0-9]+]]:gpr32common = ADDWri [[ADDWri3]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[SUBSWri:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri4]], 1, 0, implicit-def $nzcv
+ ; CHECK-LARGE-BUDGET-NEXT: Bcc 10, %bb.8, implicit $nzcv
+ ; CHECK-LARGE-BUDGET-NEXT: B %bb.1
+ ; CHECK-LARGE-BUDGET-NEXT: {{ $}}
+ ; CHECK-LARGE-BUDGET-NEXT: bb.1:
+ ; CHECK-LARGE-BUDGET-NEXT: successors: %bb.8(0x40000000), %bb.2(0x40000000)
+ ; CHECK-LARGE-BUDGET-NEXT: {{ $}}
+ ; CHECK-LARGE-BUDGET-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 101
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri5:%[0-9]+]]:gpr32common = ADDWri [[LDRWui1]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri6:%[0-9]+]]:gpr32common = ADDWri [[ADDWri5]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri7:%[0-9]+]]:gpr32common = ADDWri [[ADDWri6]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri8:%[0-9]+]]:gpr32common = ADDWri [[ADDWri7]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri9:%[0-9]+]]:gpr32common = ADDWri [[ADDWri8]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[SUBSWri1:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri9]], 2, 0, implicit-def $nzcv
+ ; CHECK-LARGE-BUDGET-NEXT: Bcc 11, %bb.8, implicit $nzcv
+ ; CHECK-LARGE-BUDGET-NEXT: B %bb.2
+ ; CHECK-LARGE-BUDGET-NEXT: {{ $}}
+ ; CHECK-LARGE-BUDGET-NEXT: bb.2:
+ ; CHECK-LARGE-BUDGET-NEXT: successors: %bb.8(0x40000000), %bb.3(0x40000000)
+ ; CHECK-LARGE-BUDGET-NEXT: {{ $}}
+ ; CHECK-LARGE-BUDGET-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 102
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri10:%[0-9]+]]:gpr32common = ADDWri [[LDRWui2]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri11:%[0-9]+]]:gpr32common = ADDWri [[ADDWri10]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri12:%[0-9]+]]:gpr32common = ADDWri [[ADDWri11]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri13:%[0-9]+]]:gpr32common = ADDWri [[ADDWri12]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri14:%[0-9]+]]:gpr32common = ADDWri [[ADDWri13]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[SUBSWri2:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri14]], 3, 0, implicit-def $nzcv
+ ; CHECK-LARGE-BUDGET-NEXT: Bcc 12, %bb.8, implicit $nzcv
+ ; CHECK-LARGE-BUDGET-NEXT: B %bb.3
+ ; CHECK-LARGE-BUDGET-NEXT: {{ $}}
+ ; CHECK-LARGE-BUDGET-NEXT: bb.3:
+ ; CHECK-LARGE-BUDGET-NEXT: successors: %bb.8(0x40000000), %bb.4(0x40000000)
+ ; CHECK-LARGE-BUDGET-NEXT: {{ $}}
+ ; CHECK-LARGE-BUDGET-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 103
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri15:%[0-9]+]]:gpr32common = ADDWri [[LDRWui3]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri16:%[0-9]+]]:gpr32common = ADDWri [[ADDWri15]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri17:%[0-9]+]]:gpr32common = ADDWri [[ADDWri16]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri18:%[0-9]+]]:gpr32common = ADDWri [[ADDWri17]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri19:%[0-9]+]]:gpr32common = ADDWri [[ADDWri18]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[SUBSWri3:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri19]], 4, 0, implicit-def $nzcv
+ ; CHECK-LARGE-BUDGET-NEXT: Bcc 13, %bb.8, implicit $nzcv
+ ; CHECK-LARGE-BUDGET-NEXT: B %bb.4
+ ; CHECK-LARGE-BUDGET-NEXT: {{ $}}
+ ; CHECK-LARGE-BUDGET-NEXT: bb.4:
+ ; CHECK-LARGE-BUDGET-NEXT: successors: %bb.8(0x40000000), %bb.5(0x40000000)
+ ; CHECK-LARGE-BUDGET-NEXT: {{ $}}
+ ; CHECK-LARGE-BUDGET-NEXT: [[MOVi64imm4:%[0-9]+]]:gpr64 = MOVi64imm 104
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri20:%[0-9]+]]:gpr32common = ADDWri [[LDRWui4]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri21:%[0-9]+]]:gpr32common = ADDWri [[ADDWri20]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri22:%[0-9]+]]:gpr32common = ADDWri [[ADDWri21]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri23:%[0-9]+]]:gpr32common = ADDWri [[ADDWri22]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri24:%[0-9]+]]:gpr32common = ADDWri [[ADDWri23]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[SUBSWri4:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri24]], 5, 0, implicit-def $nzcv
+ ; CHECK-LARGE-BUDGET-NEXT: Bcc 14, %bb.8, implicit $nzcv
+ ; CHECK-LARGE-BUDGET-NEXT: B %bb.5
+ ; CHECK-LARGE-BUDGET-NEXT: {{ $}}
+ ; CHECK-LARGE-BUDGET-NEXT: bb.5:
+ ; CHECK-LARGE-BUDGET-NEXT: successors: %bb.8(0x40000000), %bb.6(0x40000000)
+ ; CHECK-LARGE-BUDGET-NEXT: {{ $}}
+ ; CHECK-LARGE-BUDGET-NEXT: [[MOVi64imm5:%[0-9]+]]:gpr64 = MOVi64imm 105
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri25:%[0-9]+]]:gpr32common = ADDWri [[LDRWui5]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri26:%[0-9]+]]:gpr32common = ADDWri [[ADDWri25]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri27:%[0-9]+]]:gpr32common = ADDWri [[ADDWri26]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri28:%[0-9]+]]:gpr32common = ADDWri [[ADDWri27]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri29:%[0-9]+]]:gpr32common = ADDWri [[ADDWri28]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[SUBSWri5:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri29]], 6, 0, implicit-def $nzcv
+ ; CHECK-LARGE-BUDGET-NEXT: Bcc 15, %bb.8, implicit $nzcv
+ ; CHECK-LARGE-BUDGET-NEXT: B %bb.6
+ ; CHECK-LARGE-BUDGET-NEXT: {{ $}}
+ ; CHECK-LARGE-BUDGET-NEXT: bb.6:
+ ; CHECK-LARGE-BUDGET-NEXT: successors: %bb.8(0x40000000), %bb.7(0x40000000)
+ ; CHECK-LARGE-BUDGET-NEXT: {{ $}}
+ ; CHECK-LARGE-BUDGET-NEXT: [[MOVi64imm6:%[0-9]+]]:gpr64 = MOVi64imm 106
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri30:%[0-9]+]]:gpr32common = ADDWri [[LDRWui6]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri31:%[0-9]+]]:gpr32common = ADDWri [[ADDWri30]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri32:%[0-9]+]]:gpr32common = ADDWri [[ADDWri31]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri33:%[0-9]+]]:gpr32common = ADDWri [[ADDWri32]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[ADDWri34:%[0-9]+]]:gpr32common = ADDWri [[ADDWri33]], 1, 0
+ ; CHECK-LARGE-BUDGET-NEXT: [[SUBSWri6:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri34]], 7, 0, implicit-def $nzcv
+ ; CHECK-LARGE-BUDGET-NEXT: Bcc 16, %bb.8, implicit $nzcv
+ ; CHECK-LARGE-BUDGET-NEXT: B %bb.7
+ ; CHECK-LARGE-BUDGET-NEXT: {{ $}}
+ ; CHECK-LARGE-BUDGET-NEXT: bb.7:
+ ; CHECK-LARGE-BUDGET-NEXT: successors: %bb.8(0x80000000)
+ ; CHECK-LARGE-BUDGET-NEXT: {{ $}}
+ ; CHECK-LARGE-BUDGET-NEXT: [[MOVi64imm7:%[0-9]+]]:gpr64 = MOVi64imm 107
+ ; CHECK-LARGE-BUDGET-NEXT: {{ $}}
+ ; CHECK-LARGE-BUDGET-NEXT: bb.8:
+ ; CHECK-LARGE-BUDGET-NEXT: [[PHI:%[0-9]+]]:gpr64 = PHI [[MOVi64imm]], %bb.0, [[MOVi64imm1]], %bb.1, [[MOVi64imm2]], %bb.2, [[MOVi64imm3]], %bb.3, [[MOVi64imm4]], %bb.4, [[MOVi64imm5]], %bb.5, [[MOVi64imm6]], %bb.6, [[MOVi64imm7]], %bb.7
+ ; CHECK-LARGE-BUDGET-NEXT: $x0 = COPY [[PHI]]
+ ; CHECK-LARGE-BUDGET-NEXT: RET_ReallyLR implicit $x0
+ ;
+ ; CHECK-CASCADE-BUDGET-LABEL: name: cascade_load_to_cond_br_scan_budget
+ ; CHECK-CASCADE-BUDGET: bb.0:
+ ; CHECK-CASCADE-BUDGET-NEXT: liveins: $x0
+ ; CHECK-CASCADE-BUDGET-NEXT: {{ $}}
+ ; CHECK-CASCADE-BUDGET-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[LDRWui:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 0 :: (load (s32))
+ ; CHECK-CASCADE-BUDGET-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 1
+ ; CHECK-CASCADE-BUDGET-NEXT: [[MOVi32imm1:%[0-9]+]]:gpr32 = MOVi32imm 2
+ ; CHECK-CASCADE-BUDGET-NEXT: [[MOVi32imm2:%[0-9]+]]:gpr32 = MOVi32imm 3
+ ; CHECK-CASCADE-BUDGET-NEXT: [[MOVi32imm3:%[0-9]+]]:gpr32 = MOVi32imm 4
+ ; CHECK-CASCADE-BUDGET-NEXT: [[LDRWui1:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 1 :: (load (s32))
+ ; CHECK-CASCADE-BUDGET-NEXT: [[LDRWui2:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 2 :: (load (s32))
+ ; CHECK-CASCADE-BUDGET-NEXT: [[LDRWui3:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 3 :: (load (s32))
+ ; CHECK-CASCADE-BUDGET-NEXT: [[LDRWui4:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 4 :: (load (s32))
+ ; CHECK-CASCADE-BUDGET-NEXT: [[LDRWui5:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 5 :: (load (s32))
+ ; CHECK-CASCADE-BUDGET-NEXT: [[LDRWui6:%[0-9]+]]:gpr32common = LDRWui [[COPY]], 6 :: (load (s32))
+ ; CHECK-CASCADE-BUDGET-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 100
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri:%[0-9]+]]:gpr32common = ADDWri [[LDRWui]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri1:%[0-9]+]]:gpr32common = ADDWri [[ADDWri]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri2:%[0-9]+]]:gpr32common = ADDWri [[ADDWri1]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri3:%[0-9]+]]:gpr32common = ADDWri [[ADDWri2]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri4:%[0-9]+]]:gpr32common = ADDWri [[ADDWri3]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 101
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri5:%[0-9]+]]:gpr32common = ADDWri [[LDRWui1]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri6:%[0-9]+]]:gpr32common = ADDWri [[ADDWri5]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri7:%[0-9]+]]:gpr32common = ADDWri [[ADDWri6]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri8:%[0-9]+]]:gpr32common = ADDWri [[ADDWri7]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri9:%[0-9]+]]:gpr32common = ADDWri [[ADDWri8]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[MOVi64imm2:%[0-9]+]]:gpr64 = MOVi64imm 102
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri10:%[0-9]+]]:gpr32common = ADDWri [[LDRWui2]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri11:%[0-9]+]]:gpr32common = ADDWri [[ADDWri10]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri12:%[0-9]+]]:gpr32common = ADDWri [[ADDWri11]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri13:%[0-9]+]]:gpr32common = ADDWri [[ADDWri12]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri14:%[0-9]+]]:gpr32common = ADDWri [[ADDWri13]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[MOVi64imm3:%[0-9]+]]:gpr64 = MOVi64imm 103
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri15:%[0-9]+]]:gpr32common = ADDWri [[LDRWui3]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri16:%[0-9]+]]:gpr32common = ADDWri [[ADDWri15]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri17:%[0-9]+]]:gpr32common = ADDWri [[ADDWri16]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri18:%[0-9]+]]:gpr32common = ADDWri [[ADDWri17]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri19:%[0-9]+]]:gpr32common = ADDWri [[ADDWri18]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[MOVi64imm4:%[0-9]+]]:gpr64 = MOVi64imm 104
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri20:%[0-9]+]]:gpr32common = ADDWri [[LDRWui4]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri21:%[0-9]+]]:gpr32common = ADDWri [[ADDWri20]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri22:%[0-9]+]]:gpr32common = ADDWri [[ADDWri21]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri23:%[0-9]+]]:gpr32common = ADDWri [[ADDWri22]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri24:%[0-9]+]]:gpr32common = ADDWri [[ADDWri23]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[MOVi64imm5:%[0-9]+]]:gpr64 = MOVi64imm 105
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri25:%[0-9]+]]:gpr32common = ADDWri [[LDRWui5]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri26:%[0-9]+]]:gpr32common = ADDWri [[ADDWri25]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri27:%[0-9]+]]:gpr32common = ADDWri [[ADDWri26]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri28:%[0-9]+]]:gpr32common = ADDWri [[ADDWri27]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri29:%[0-9]+]]:gpr32common = ADDWri [[ADDWri28]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[MOVi64imm6:%[0-9]+]]:gpr64 = MOVi64imm 106
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri30:%[0-9]+]]:gpr32common = ADDWri [[LDRWui6]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri31:%[0-9]+]]:gpr32common = ADDWri [[ADDWri30]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri32:%[0-9]+]]:gpr32common = ADDWri [[ADDWri31]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri33:%[0-9]+]]:gpr32common = ADDWri [[ADDWri32]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[ADDWri34:%[0-9]+]]:gpr32common = ADDWri [[ADDWri33]], 1, 0
+ ; CHECK-CASCADE-BUDGET-NEXT: [[SUBSWri:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri34]], 7, 0, implicit-def $nzcv
+ ; CHECK-CASCADE-BUDGET-NEXT: [[MOVi64imm7:%[0-9]+]]:gpr64 = MOVi64imm 107
+ ; CHECK-CASCADE-BUDGET-NEXT: [[CSELXr:%[0-9]+]]:gpr64 = CSELXr [[MOVi64imm6]], [[MOVi64imm7]], 16, implicit $nzcv
+ ; CHECK-CASCADE-BUDGET-NEXT: [[SUBSWri1:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri29]], 6, 0, implicit-def $nzcv
+ ; CHECK-CASCADE-BUDGET-NEXT: [[CSELXr1:%[0-9]+]]:gpr64 = CSELXr [[MOVi64imm5]], [[CSELXr]], 15, implicit $nzcv
+ ; CHECK-CASCADE-BUDGET-NEXT: [[SUBSWri2:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri24]], 5, 0, implicit-def $nzcv
+ ; CHECK-CASCADE-BUDGET-NEXT: [[CSELXr2:%[0-9]+]]:gpr64 = CSELXr [[MOVi64imm4]], [[CSELXr1]], 14, implicit $nzcv
+ ; CHECK-CASCADE-BUDGET-NEXT: [[SUBSWri3:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri19]], 4, 0, implicit-def $nzcv
+ ; CHECK-CASCADE-BUDGET-NEXT: [[CSELXr3:%[0-9]+]]:gpr64 = CSELXr [[MOVi64imm3]], [[CSELXr2]], 13, implicit $nzcv
+ ; CHECK-CASCADE-BUDGET-NEXT: [[SUBSWri4:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri14]], 3, 0, implicit-def $nzcv
+ ; CHECK-CASCADE-BUDGET-NEXT: [[CSELXr4:%[0-9]+]]:gpr64 = CSELXr [[MOVi64imm2]], [[CSELXr3]], 12, implicit $nzcv
+ ; CHECK-CASCADE-BUDGET-NEXT: [[SUBSWri5:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri9]], 2, 0, implicit-def $nzcv
+ ; CHECK-CASCADE-BUDGET-NEXT: [[CSELXr5:%[0-9]+]]:gpr64 = CSELXr [[MOVi64imm1]], [[CSELXr4]], 11, implicit $nzcv
+ ; CHECK-CASCADE-BUDGET-NEXT: [[SUBSWri6:%[0-9]+]]:gpr32 = SUBSWri [[ADDWri4]], 1, 0, implicit-def $nzcv
+ ; CHECK-CASCADE-BUDGET-NEXT: [[CSELXr6:%[0-9]+]]:gpr64 = CSELXr [[MOVi64imm]], [[CSELXr5]], 10, implicit $nzcv
+ ; CHECK-CASCADE-BUDGET-NEXT: $x0 = COPY [[CSELXr6]]
+ ; CHECK-CASCADE-BUDGET-NEXT: RET_ReallyLR implicit $x0
+ bb.0:
+ successors: %bb.8(0x40000000), %bb.1(0x40000000)
+ liveins: $x0
+
+ %0:gpr64common = COPY $x0
+ %1:gpr32common = LDRWui %0, 0 :: (load (s32))
+ %200:gpr32 = MOVi32imm 1
+ %201:gpr32 = MOVi32imm 2
+ %202:gpr32 = MOVi32imm 3
+ %203:gpr32 = MOVi32imm 4
+ %2:gpr32common = LDRWui %0, 1 :: (load (s32))
+ %3:gpr32common = LDRWui %0, 2 :: (load (s32))
+ %4:gpr32common = LDRWui %0, 3 :: (load (s32))
+ %5:gpr32common = LDRWui %0, 4 :: (load (s32))
+ %6:gpr32common = LDRWui %0, 5 :: (load (s32))
+ %7:gpr32common = LDRWui %0, 6 :: (load (s32))
+ %50:gpr64 = MOVi64imm 100
+ %204:gpr32common = ADDWri %1, 1, 0
+ %205:gpr32common = ADDWri %204, 1, 0
+ %206:gpr32common = ADDWri %205, 1, 0
+ %207:gpr32common = ADDWri %206, 1, 0
+ %208:gpr32common = ADDWri %207, 1, 0
+ %209:gpr32 = SUBSWri %208, 1, 0, implicit-def $nzcv
+ Bcc 10, %bb.8, implicit $nzcv
+ B %bb.1
+
+ bb.1:
+ successors: %bb.8(0x40000000), %bb.2(0x40000000)
+
+ %51:gpr64 = MOVi64imm 101
+ %210:gpr32common = ADDWri %2, 1, 0
+ %211:gpr32common = ADDWri %210, 1, 0
+ %212:gpr32common = ADDWri %211, 1, 0
+ %213:gpr32common = ADDWri %212, 1, 0
+ %214:gpr32common = ADDWri %213, 1, 0
+ %215:gpr32 = SUBSWri %214, 2, 0, implicit-def $nzcv
+ Bcc 11, %bb.8, implicit $nzcv
+ B %bb.2
+
+ bb.2:
+ successors: %bb.8(0x40000000), %bb.3(0x40000000)
+
+ %52:gpr64 = MOVi64imm 102
+ %216:gpr32common = ADDWri %3, 1, 0
+ %217:gpr32common = ADDWri %216, 1, 0
+ %218:gpr32common = ADDWri %217, 1, 0
+ %219:gpr32common = ADDWri %218, 1, 0
+ %220:gpr32common = ADDWri %219, 1, 0
+ %221:gpr32 = SUBSWri %220, 3, 0, implicit-def $nzcv
+ Bcc 12, %bb.8, implicit $nzcv
+ B %bb.3
+
+ bb.3:
+ successors: %bb.8(0x40000000), %bb.4(0x40000000)
+
+ %53:gpr64 = MOVi64imm 103
+ %222:gpr32common = ADDWri %4, 1, 0
+ %223:gpr32common = ADDWri %222, 1, 0
+ %224:gpr32common = ADDWri %223, 1, 0
+ %225:gpr32common = ADDWri %224, 1, 0
+ %226:gpr32common = ADDWri %225, 1, 0
+ %227:gpr32 = SUBSWri %226, 4, 0, implicit-def $nzcv
+ Bcc 13, %bb.8, implicit $nzcv
+ B %bb.4
+
+ bb.4:
+ successors: %bb.8(0x40000000), %bb.5(0x40000000)
+
+ %54:gpr64 = MOVi64imm 104
+ %228:gpr32common = ADDWri %5, 1, 0
+ %229:gpr32common = ADDWri %228, 1, 0
+ %230:gpr32common = ADDWri %229, 1, 0
+ %231:gpr32common = ADDWri %230, 1, 0
+ %232:gpr32common = ADDWri %231, 1, 0
+ %233:gpr32 = SUBSWri %232, 5, 0, implicit-def $nzcv
+ Bcc 14, %bb.8, implicit $nzcv
+ B %bb.5
+
+ bb.5:
+ successors: %bb.8(0x40000000), %bb.6(0x40000000)
+
+ %55:gpr64 = MOVi64imm 105
+ %234:gpr32common = ADDWri %6, 1, 0
+ %235:gpr32common = ADDWri %234, 1, 0
+ %236:gpr32common = ADDWri %235, 1, 0
+ %237:gpr32common = ADDWri %236, 1, 0
+ %238:gpr32common = ADDWri %237, 1, 0
+ %239:gpr32 = SUBSWri %238, 6, 0, implicit-def $nzcv
+ Bcc 15, %bb.8, implicit $nzcv
+ B %bb.6
+
+ bb.6:
+ successors: %bb.8(0x40000000), %bb.7(0x40000000)
+
+ %56:gpr64 = MOVi64imm 106
+ %240:gpr32common = ADDWri %7, 1, 0
+ %241:gpr32common = ADDWri %240, 1, 0
+ %242:gpr32common = ADDWri %241, 1, 0
+ %243:gpr32common = ADDWri %242, 1, 0
+ %244:gpr32common = ADDWri %243, 1, 0
+ %245:gpr32 = SUBSWri %244, 7, 0, implicit-def $nzcv
+ Bcc 16, %bb.8, implicit $nzcv
+ B %bb.7
+
+ bb.7:
+ successors: %bb.8(0x80000000)
+
+ %57:gpr64 = MOVi64imm 107
+
+ bb.8:
+ %90:gpr64 = PHI %50, %bb.0, %51, %bb.1, %52, %bb.2, %53, %bb.3, %54, %bb.4, %55, %bb.5, %56, %bb.6, %57, %bb.7
+ $x0 = COPY %90
+ RET_ReallyLR implicit $x0
+...
diff --git a/llvm/test/CodeGen/AArch64/early-ifcvt-remarks.mir b/llvm/test/CodeGen/AArch64/early-ifcvt-remarks.mir
index 6f01d48601fa2..0b3ad1efe79a1 100644
--- a/llvm/test/CodeGen/AArch64/early-ifcvt-remarks.mir
+++ b/llvm/test/CodeGen/AArch64/early-ifcvt-remarks.mir
@@ -2,11 +2,44 @@
# RUN: -enable-early-ifcvt-data-dependent \
# RUN: -pass-remarks-analysis='early-ifcvt' -o /dev/null %s 2>&1 | FileCheck %s
+# RUN: llc -mtriple=aarch64 -mcpu=apple-m1 -run-pass=early-ifcvt \
+# RUN: -enable-early-ifcvt-cascade -enable-early-ifcvt-data-dependent \
+# RUN: -pass-remarks='early-ifcvt' -o /dev/null %s 2>&1 | FileCheck %s --check-prefix=CASCADE-SUCCESS \
+# RUN: --implicit-check-not="performing cascade if-conversion"
+
+# RUN: llc -mtriple=aarch64 -mcpu=apple-m1 -run-pass=early-ifcvt \
+# RUN: -enable-early-ifcvt-cascade -enable-early-ifcvt-data-dependent \
+# RUN: -pass-remarks-missed='early-ifcvt' -o /dev/null %s 2>&1 | FileCheck %s --check-prefix=CASCADE-MISSED
+
+# RUN: llc -mtriple=aarch64 -mcpu=apple-m1 -run-pass=early-ifcvt \
+# RUN: -enable-early-ifcvt-cascade \
+# RUN: -pass-remarks='early-ifcvt' -o /dev/null %s 2>&1 \
+# RUN: | FileCheck %s --check-prefix=CASCADE-NO-DD --implicit-check-not="cascade if-conversion"
+
+# RUN: llc -mtriple=aarch64 -mcpu=apple-m1 -run-pass=early-ifcvt \
+# RUN: -enable-early-ifcvt-cascade -enable-early-ifcvt-data-dependent \
+# RUN: -early-ifcvt-limit=3 -pass-remarks='early-ifcvt' \
+# RUN: -pass-remarks-missed='early-ifcvt' -o /dev/null %s 2>&1 \
+# RUN: | FileCheck %s --check-prefix=CASCADE-OVER-LIMIT \
+# RUN: --implicit-check-not="performing cascade if-conversion"
+
# Test that we emit an analysis remark when the branch condition is data-dependent
# (derived from a memory load).
# CHECK: remark: <unknown>:0:0: branch condition is data-dependent (from memory load), using higher CritLimit of {{[0-9]+}} cycles
+# Test cascade conversion success remark.
+# CASCADE-SUCCESS: remark: <unknown>:0:0: performing cascade if-conversion on 3 blocks: critical path extension is {{[0-9]+}} cycle
+
+# CASCADE-MISSED: remark: <unknown>:0:0: did not if-convert cascade with 5 blocks: critical path extension of 21 cycles exceeds the threshold of 20 cycles.
+# CASCADE-MISSED: remark: <unknown>:0:0: did not if-convert cascade with 3 blocks: critical path extension of 30 cycles exceeds the threshold of 12 cycles.
+
+# CASCADE-NO-DD: remark: <unknown>:0:0: performing if-conversion on branch: {{.*}} staying under the threshold of 8 cycles.
+
+# CASCADE-OVER-LIMIT: remark: <unknown>:0:0: did not if-convert cascade with 3 blocks: the speculated blocks would hold up to 10 instructions, exceeding the limit of 3.
+# CASCADE-OVER-LIMIT: remark: <unknown>:0:0: did not if-convert cascade with 5 blocks: the speculated blocks would hold up to 14 instructions, exceeding the limit of 3.
+# CASCADE-OVER-LIMIT: remark: <unknown>:0:0: did not if-convert cascade with 3 blocks: the speculated blocks would hold up to 11 instructions, exceeding the limit of 3.
+
--- |
define i32 @data_dependent_branch(ptr %p, i32 %x, i32 %y) {
ret i32 0
@@ -17,6 +50,9 @@
define i64 @cascade_increasing_cond_depth(ptr %p) {
ret i64 0
}
+ define i64 @cascade_mixed_reg_classes(ptr %p) {
+ ret i64 0
+ }
...
---
name: data_dependent_branch
@@ -202,3 +238,64 @@ body: |
$x0 = COPY %90
RET_ReallyLR implicit $x0
...
+---
+name: cascade_mixed_reg_classes
+alignment: 4
+tracksRegLiveness: true
+body: |
+ bb.0:
+ successors: %bb.1(0x80000000)
+ liveins: $x0, $d0, $d1, $d2, $d3
+
+ %0:gpr64common = COPY $x0
+ %5:fpr64 = COPY $d0
+ %6:fpr64 = COPY $d1
+ %7:fpr64 = COPY $d2
+ %8:fpr64 = COPY $d3
+
+ bb.1:
+ successors: %bb.2(0x40000000), %bb.5(0x40000000)
+
+ %10:gpr32 = LDRWui %0, 0 :: (load (s32))
+ %11:gpr32 = MADDWrrr %10, %10, $wzr
+ %12:gpr32 = MADDWrrr %11, %11, $wzr
+ %20:gpr32 = ADDWrr %12, %12
+ %21:gpr32 = ADDWrr %20, %20
+ %40:gpr64 = MOVi64imm 100
+ %30:gpr32 = SUBSWrr %12, %12, implicit-def $nzcv
+ Bcc 10, %bb.5, implicit $nzcv
+ B %bb.2
+
+ bb.2:
+ successors: %bb.3(0x40000000), %bb.5(0x40000000)
+
+ %41:gpr64 = MOVi64imm 200
+ %31:gpr32 = SUBSWrr %20, %20, implicit-def $nzcv
+ Bcc 10, %bb.5, implicit $nzcv
+ B %bb.3
+
+ bb.3:
+ successors: %bb.4(0x40000000), %bb.5(0x40000000)
+
+ %42:gpr64 = MOVi64imm 300
+ %32:gpr32 = SUBSWrr %21, %21, implicit-def $nzcv
+ Bcc 10, %bb.5, implicit $nzcv
+ B %bb.4
+
+ bb.4:
+ successors: %bb.5(0x80000000)
+
+ %43:gpr64 = MOVi64imm 400
+
+ bb.5:
+ %90:gpr64 = PHI %40, %bb.1, %41, %bb.2, %42, %bb.3, %43, %bb.4
+ %91:fpr64 = PHI %5, %bb.1, %6, %bb.2, %7, %bb.3, %8, %bb.4
+ %60:fpr64 = FMULDrr %91, %91, implicit $fpcr
+ %61:fpr64 = FMULDrr %60, %60, implicit $fpcr
+ %62:fpr64 = FMULDrr %61, %61, implicit $fpcr
+ %63:fpr64 = FMULDrr %62, %62, implicit $fpcr
+ %64:fpr64 = FMULDrr %63, %63, implicit $fpcr
+ $x0 = COPY %90
+ $d0 = COPY %64
+ RET_ReallyLR implicit $x0, implicit $d0
+...
More information about the llvm-commits
mailing list