[llvm] [EarlyIfConversion] Limit critical-path growth for strongly biased branches (PR #228709)
Dávid Bolvanský via llvm-commits
llvm-commits at lists.llvm.org
Sat Oct 3 06:30:16 PDT 2026
https://github.com/davidbolvansky updated https://github.com/llvm/llvm-project/pull/228709
>From 3631223099ecca31845cc04f7b53b66d1ca599ad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?D=C3=A1vid=20Bolvansk=C3=BD?= <david.bolvansky at gmail.com>
Date: Sat, 3 Oct 2026 15:15:20 +0200
Subject: [PATCH 1/4] [EarlyIfConversion] Limit critical-path growth for
strongly biased branches
Scale the critical-path extension budget by the cold-edge probability for
branches with more than 99% bias. Provide branch probability information in
both pass managers even when data-dependent branch analysis is disabled.
This keeps biased bitstream reads off the loop-carried dependency chain in
zstd's sequence decoder. Add a biased/balanced regression test and update
i256 conversion checks for the resulting branchy rounding path.
Assisted-by: OpenAI Codex
---
llvm/lib/CodeGen/EarlyIfConversion.cpp | 22 +++-
.../AArch64/early-ifcvt-biased-bitstream.ll | 84 ++++++++++++++
llvm/test/CodeGen/AArch64/fcvt-i256.ll | 103 ++++++++++--------
3 files changed, 157 insertions(+), 52 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/early-ifcvt-biased-bitstream.ll
diff --git a/llvm/lib/CodeGen/EarlyIfConversion.cpp b/llvm/lib/CodeGen/EarlyIfConversion.cpp
index bf2664fd4225c1..f9cbb0dffdc403 100644
--- a/llvm/lib/CodeGen/EarlyIfConversion.cpp
+++ b/llvm/lib/CodeGen/EarlyIfConversion.cpp
@@ -1230,8 +1230,20 @@ bool EarlyIfConverter::shouldConvertIf() {
if (EnableDataDependentBranchAnalysis)
DataDependent = isConditionDataDependent();
+ // A strongly biased branch has little misprediction cost to hide the
+ // critical-path extension introduced by speculative instructions and selects.
+ // Keep the existing budget for branches without a strongly preferred edge.
unsigned CritLimit = DataDependent ? STI->getMispredictionPenalty()
: STI->getMispredictionPenalty() / 2;
+ if (!DataDependent) {
+ BranchProbability TBBProb =
+ MBPI->getEdgeProbability(IfConv.Head, IfConv.TBB);
+ BranchProbability FBBProb =
+ MBPI->getEdgeProbability(IfConv.Head, IfConv.FBB);
+ BranchProbability ColdProb = std::min(TBBProb, FBBProb);
+ if (ColdProb < BranchProbability(1, 100))
+ CritLimit = ColdProb.scale(STI->getMispredictionPenalty());
+ }
MachineBasicBlock &MBB = *IfConv.Head;
MachineOptimizationRemarkEmitter MORE(*MBB.getParent(), nullptr);
@@ -1444,9 +1456,8 @@ EarlyIfConverterPass::run(MachineFunction &MF,
MachineDominatorTree &MDT = MFAM.getResult<MachineDominatorTreeAnalysis>(MF);
MachineLoopInfo &LI = MFAM.getResult<MachineLoopAnalysis>(MF);
MachineTraceMetrics &MTM = MFAM.getResult<MachineTraceMetricsAnalysis>(MF);
- MachineBranchProbabilityInfo *MBPI = nullptr;
- if (EnableDataDependentBranchAnalysis)
- MBPI = &MFAM.getResult<MachineBranchProbabilityAnalysis>(MF);
+ MachineBranchProbabilityInfo *MBPI =
+ &MFAM.getResult<MachineBranchProbabilityAnalysis>(MF);
EarlyIfConverter Impl(MDT, LI, MTM, MBPI);
bool Changed = Impl.run(MF);
@@ -1469,9 +1480,8 @@ bool EarlyIfConverterLegacy::runOnMachineFunction(MachineFunction &MF) {
MachineLoopInfo &LI = getAnalysis<MachineLoopInfoWrapperPass>().getLI();
MachineTraceMetrics &MTM =
getAnalysis<MachineTraceMetricsWrapperPass>().getMTM();
- MachineBranchProbabilityInfo *MBPI = nullptr;
- if (EnableDataDependentBranchAnalysis)
- MBPI = &getAnalysis<MachineBranchProbabilityInfoWrapperPass>().getMBPI();
+ MachineBranchProbabilityInfo *MBPI =
+ &getAnalysis<MachineBranchProbabilityInfoWrapperPass>().getMBPI();
return EarlyIfConverter(MDT, LI, MTM, MBPI).run(MF);
}
diff --git a/llvm/test/CodeGen/AArch64/early-ifcvt-biased-bitstream.ll b/llvm/test/CodeGen/AArch64/early-ifcvt-biased-bitstream.ll
new file mode 100644
index 00000000000000..46f57811a822a1
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/early-ifcvt-biased-bitstream.ll
@@ -0,0 +1,84 @@
+; RUN: llc -mcpu=apple-m1 -verify-machineinstrs < %s | FileCheck %s
+;
+; Reading zero bits must not add speculative shifts and selects to the
+; loop-carried consumed-bits dependency when the branch is strongly biased.
+
+target triple = "aarch64-unknown-linux-gnu"
+
+; CHECK-LABEL: decode_bits:
+; CHECK-NOT: csel
+; CHECK: cbnz
+; CHECK-NOT: csel
+; CHECK: ret
+define i32 @decode_bits(ptr %counts, i64 %count, i64 %bits, i32 %initial, ptr %out) {
+entry:
+ br label %loop
+loop:
+ %i = phi i64 [0, %entry], [%nexti, %merge]
+ %consumed = phi i32 [%initial, %entry], [%nextconsumed, %merge]
+ %p = getelementptr i8, ptr %counts, i64 %i
+ %n8 = load i8, ptr %p
+ %zero = icmp eq i8 %n8, 0
+ br i1 %zero, label %merge, label %read, !prof !0
+read:
+ %n = zext i8 %n8 to i32
+ %start = and i32 %consumed, 63
+ %start64 = zext i32 %start to i64
+ %shifted = shl i64 %bits, %start64
+ %neg = sub i32 0, %n
+ %shift = and i32 %neg, 63
+ %shift64 = zext i32 %shift to i64
+ %value = lshr i64 %shifted, %shift64
+ %added = add i32 %consumed, %n
+ br label %merge
+merge:
+ %result = phi i64 [0, %loop], [%value, %read]
+ %nextconsumed = phi i32 [%consumed, %loop], [%added, %read]
+ %op = getelementptr i64, ptr %out, i64 %i
+ store i64 %result, ptr %op
+ %nexti = add i64 %i, 1
+ %done = icmp eq i64 %nexti, %count
+ br i1 %done, label %exit, label %loop
+exit:
+ ret i32 %nextconsumed
+}
+
+; CHECK-LABEL: decode_bits_balanced:
+; CHECK: csel
+; CHECK: csel
+; CHECK: ret
+define i32 @decode_bits_balanced(ptr %counts, i64 %count, i64 %bits, i32 %initial, ptr %out) {
+entry:
+ br label %loop
+loop:
+ %i = phi i64 [0, %entry], [%nexti, %merge]
+ %consumed = phi i32 [%initial, %entry], [%nextconsumed, %merge]
+ %p = getelementptr i8, ptr %counts, i64 %i
+ %n8 = load i8, ptr %p
+ %zero = icmp eq i8 %n8, 0
+ br i1 %zero, label %merge, label %read, !prof !1
+read:
+ %n = zext i8 %n8 to i32
+ %start = and i32 %consumed, 63
+ %start64 = zext i32 %start to i64
+ %shifted = shl i64 %bits, %start64
+ %neg = sub i32 0, %n
+ %shift = and i32 %neg, 63
+ %shift64 = zext i32 %shift to i64
+ %value = lshr i64 %shifted, %shift64
+ %added = add i32 %consumed, %n
+ br label %merge
+merge:
+ %result = phi i64 [0, %loop], [%value, %read]
+ %nextconsumed = phi i32 [%consumed, %loop], [%added, %read]
+ %op = getelementptr i64, ptr %out, i64 %i
+ store i64 %result, ptr %op
+ %nexti = add i64 %i, 1
+ %done = icmp eq i64 %nexti, %count
+ br i1 %done, label %exit, label %loop
+exit:
+ ret i32 %nextconsumed
+}
+
+!0 = !{!"branch_weights", i32 2000, i32 1}
+!1 = !{!"branch_weights", i32 1, i32 1}
diff --git a/llvm/test/CodeGen/AArch64/fcvt-i256.ll b/llvm/test/CodeGen/AArch64/fcvt-i256.ll
index 10b16c0b205d5f..b0f1511834f0a8 100644
--- a/llvm/test/CodeGen/AArch64/fcvt-i256.ll
+++ b/llvm/test/CodeGen/AArch64/fcvt-i256.ll
@@ -13,7 +13,7 @@ define float @s256_to_f32(i256 %val) {
; CHECK-NEXT: orr x8, x1, x3
; CHECK-NEXT: orr x9, x0, x2
; CHECK-NEXT: orr x8, x9, x8
-; CHECK-NEXT: cbz x8, .LBB0_7
+; CHECK-NEXT: cbz x8, .LBB0_9
; CHECK-NEXT: // %bb.1: // %itofp-if-end
; CHECK-NEXT: sub sp, sp, #192
; CHECK-NEXT: .cfi_def_cfa_offset 192
@@ -44,13 +44,13 @@ define float @s256_to_f32(i256 %val) {
; CHECK-NEXT: mov w8, #255 // =0xff
; CHECK-NEXT: cmp w9, #25
; CHECK-NEXT: sub w8, w8, w15
-; CHECK-NEXT: b.lt .LBB0_8
+; CHECK-NEXT: b.lt .LBB0_10
; CHECK-NEXT: // %bb.2: // %itofp-if-then4
; CHECK-NEXT: cmp w9, #26
; CHECK-NEXT: b.eq .LBB0_5
; CHECK-NEXT: // %bb.3: // %itofp-if-then4
; CHECK-NEXT: cmp w9, #25
-; CHECK-NEXT: b.eq .LBB0_9
+; CHECK-NEXT: b.eq .LBB0_12
; CHECK-NEXT: // %bb.4: // %itofp-sw-default
; CHECK-NEXT: mov w14, #230 // =0xe6
; CHECK-NEXT: movi v0.2d, #0000000000000000
@@ -100,12 +100,12 @@ define float @s256_to_f32(i256 %val) {
; CHECK-NEXT: ubfx w11, w10, #2, #1
; CHECK-NEXT: orr x10, x10, x11
; CHECK-NEXT: adds x10, x10, #1
-; CHECK-NEXT: lsr x11, x10, #2
-; CHECK-NEXT: lsr x12, x10, #3
-; CHECK-NEXT: tst w10, #0x4000000
-; CHECK-NEXT: csel w9, w8, w9, eq
-; CHECK-NEXT: csel w10, w11, w12, eq
-; CHECK-NEXT: .LBB0_6: // %itofp-if-end26
+; CHECK-NEXT: tbnz w10, #26, .LBB0_11
+; CHECK-NEXT: // %bb.6:
+; CHECK-NEXT: lsr x10, x10, #2
+; CHECK-NEXT: .LBB0_7: // %itofp-if-end26
+; CHECK-NEXT: mov w9, w8
+; CHECK-NEXT: .LBB0_8: // %itofp-if-end26
; CHECK-NEXT: lsr x11, x3, #32
; CHECK-NEXT: mov w12, #1065353216 // =0x3f800000
; CHECK-NEXT: mov w13, #2139095040 // =0x7f800000
@@ -123,10 +123,10 @@ define float @s256_to_f32(i256 %val) {
; CHECK-NEXT: fcsel s0, s0, s1, hi
; CHECK-NEXT: add sp, sp, #192
; CHECK-NEXT: ret
-; CHECK-NEXT: .LBB0_7:
+; CHECK-NEXT: .LBB0_9:
; CHECK-NEXT: movi d0, #0000000000000000
; CHECK-NEXT: ret
-; CHECK-NEXT: .LBB0_8: // %itofp-if-else
+; CHECK-NEXT: .LBB0_10: // %itofp-if-else
; CHECK-NEXT: movi v0.2d, #0000000000000000
; CHECK-NEXT: sub w9, w15, #232
; CHECK-NEXT: str x10, [sp, #160]
@@ -138,9 +138,13 @@ define float @s256_to_f32(i256 %val) {
; CHECK-NEXT: sub x10, x10, x11
; CHECK-NEXT: ldr x10, [x10]
; CHECK-NEXT: lsl x10, x10, x9
-; CHECK-NEXT: mov w9, w8
-; CHECK-NEXT: b .LBB0_6
-; CHECK-NEXT: .LBB0_9: // %itofp-sw-bb
+; CHECK-NEXT: // kill: def $w10 killed $w10 killed $x10 def $x10
+; CHECK-NEXT: b .LBB0_7
+; CHECK-NEXT: .LBB0_11: // %itofp-if-then20
+; CHECK-NEXT: lsr x10, x10, #3
+; CHECK-NEXT: // kill: def $w10 killed $w10 killed $x10 def $x10
+; CHECK-NEXT: b .LBB0_8
+; CHECK-NEXT: .LBB0_12: // %itofp-sw-bb
; CHECK-NEXT: lsl x10, x10, #1
; CHECK-NEXT: b .LBB0_5
%result = sitofp i256 %val to float
@@ -153,7 +157,7 @@ define float @u256_to_f32(i256 %val) {
; CHECK-NEXT: orr x8, x1, x3
; CHECK-NEXT: orr x9, x0, x2
; CHECK-NEXT: orr x8, x9, x8
-; CHECK-NEXT: cbz x8, .LBB1_7
+; CHECK-NEXT: cbz x8, .LBB1_9
; CHECK-NEXT: // %bb.1: // %itofp-if-end
; CHECK-NEXT: sub sp, sp, #192
; CHECK-NEXT: .cfi_def_cfa_offset 192
@@ -176,13 +180,13 @@ define float @u256_to_f32(i256 %val) {
; CHECK-NEXT: mov w8, #255 // =0xff
; CHECK-NEXT: cmp w9, #25
; CHECK-NEXT: sub w8, w8, w11
-; CHECK-NEXT: b.lt .LBB1_8
+; CHECK-NEXT: b.lt .LBB1_10
; CHECK-NEXT: // %bb.2: // %itofp-if-then4
; CHECK-NEXT: cmp w9, #26
; CHECK-NEXT: b.eq .LBB1_5
; CHECK-NEXT: // %bb.3: // %itofp-if-then4
; CHECK-NEXT: cmp w9, #25
-; CHECK-NEXT: b.eq .LBB1_9
+; CHECK-NEXT: b.eq .LBB1_12
; CHECK-NEXT: // %bb.4: // %itofp-sw-default
; CHECK-NEXT: mov w10, #230 // =0xe6
; CHECK-NEXT: movi v0.2d, #0000000000000000
@@ -232,12 +236,12 @@ define float @u256_to_f32(i256 %val) {
; CHECK-NEXT: ubfx w10, w0, #2, #1
; CHECK-NEXT: orr x10, x0, x10
; CHECK-NEXT: adds x10, x10, #1
-; CHECK-NEXT: lsr x11, x10, #2
-; CHECK-NEXT: lsr x12, x10, #3
-; CHECK-NEXT: tst w10, #0x4000000
-; CHECK-NEXT: csel w9, w8, w9, eq
-; CHECK-NEXT: csel w10, w11, w12, eq
-; CHECK-NEXT: .LBB1_6: // %itofp-if-end26
+; CHECK-NEXT: tbnz w10, #26, .LBB1_11
+; CHECK-NEXT: // %bb.6:
+; CHECK-NEXT: lsr x10, x10, #2
+; CHECK-NEXT: .LBB1_7: // %itofp-if-end26
+; CHECK-NEXT: mov w9, w8
+; CHECK-NEXT: .LBB1_8: // %itofp-if-end26
; CHECK-NEXT: bfi w10, w9, #23, #9
; CHECK-NEXT: mov w9, #1065353216 // =0x3f800000
; CHECK-NEXT: cmp w8, #127
@@ -248,10 +252,10 @@ define float @u256_to_f32(i256 %val) {
; CHECK-NEXT: fcsel s0, s1, s0, hi
; CHECK-NEXT: add sp, sp, #192
; CHECK-NEXT: ret
-; CHECK-NEXT: .LBB1_7:
+; CHECK-NEXT: .LBB1_9:
; CHECK-NEXT: movi d0, #0000000000000000
; CHECK-NEXT: ret
-; CHECK-NEXT: .LBB1_8: // %itofp-if-else
+; CHECK-NEXT: .LBB1_10: // %itofp-if-else
; CHECK-NEXT: movi v0.2d, #0000000000000000
; CHECK-NEXT: sub w9, w11, #232
; CHECK-NEXT: add x11, sp, #128
@@ -263,9 +267,13 @@ define float @u256_to_f32(i256 %val) {
; CHECK-NEXT: sub x10, x11, x10
; CHECK-NEXT: ldr x10, [x10]
; CHECK-NEXT: lsl x10, x10, x9
-; CHECK-NEXT: mov w9, w8
-; CHECK-NEXT: b .LBB1_6
-; CHECK-NEXT: .LBB1_9: // %itofp-sw-bb
+; CHECK-NEXT: // kill: def $w10 killed $w10 killed $x10 def $x10
+; CHECK-NEXT: b .LBB1_7
+; CHECK-NEXT: .LBB1_11: // %itofp-if-then20
+; CHECK-NEXT: lsr x10, x10, #3
+; CHECK-NEXT: // kill: def $w10 killed $w10 killed $x10 def $x10
+; CHECK-NEXT: b .LBB1_8
+; CHECK-NEXT: .LBB1_12: // %itofp-sw-bb
; CHECK-NEXT: lsl x0, x0, #1
; CHECK-NEXT: b .LBB1_5
%result = uitofp i256 %val to float
@@ -423,7 +431,7 @@ define double @u256_to_f64(i256 %val) {
; CHECK-NEXT: orr x8, x1, x3
; CHECK-NEXT: orr x9, x0, x2
; CHECK-NEXT: orr x8, x9, x8
-; CHECK-NEXT: cbz x8, .LBB3_7
+; CHECK-NEXT: cbz x8, .LBB3_8
; CHECK-NEXT: // %bb.1: // %itofp-if-end
; CHECK-NEXT: sub sp, sp, #192
; CHECK-NEXT: .cfi_def_cfa_offset 192
@@ -446,13 +454,13 @@ define double @u256_to_f64(i256 %val) {
; CHECK-NEXT: mov w8, #255 // =0xff
; CHECK-NEXT: cmp w9, #54
; CHECK-NEXT: sub w8, w8, w10
-; CHECK-NEXT: b.lt .LBB3_8
+; CHECK-NEXT: b.lt .LBB3_9
; CHECK-NEXT: // %bb.2: // %itofp-if-then4
; CHECK-NEXT: cmp w9, #55
; CHECK-NEXT: b.eq .LBB3_5
; CHECK-NEXT: // %bb.3: // %itofp-if-then4
; CHECK-NEXT: cmp w9, #54
-; CHECK-NEXT: b.eq .LBB3_9
+; CHECK-NEXT: b.eq .LBB3_11
; CHECK-NEXT: // %bb.4: // %itofp-sw-default
; CHECK-NEXT: mov w11, #201 // =0xc9
; CHECK-NEXT: movi v0.2d, #0000000000000000
@@ -507,17 +515,13 @@ define double @u256_to_f64(i256 %val) {
; CHECK-NEXT: .LBB3_5: // %itofp-sw-epilog
; CHECK-NEXT: ubfx w10, w0, #2, #1
; CHECK-NEXT: orr x10, x0, x10
-; CHECK-NEXT: adds x10, x10, #1
-; CHECK-NEXT: adcs x11, x1, xzr
-; CHECK-NEXT: extr x12, x11, x10, #2
-; CHECK-NEXT: extr x13, x11, x10, #34
-; CHECK-NEXT: extr x14, x11, x10, #3
-; CHECK-NEXT: extr x11, x11, x10, #35
-; CHECK-NEXT: tst x10, #0x80000000000000
-; CHECK-NEXT: csel x10, x12, x14, eq
-; CHECK-NEXT: csel w8, w8, w9, eq
-; CHECK-NEXT: csel w11, w13, w11, eq
-; CHECK-NEXT: .LBB3_6: // %itofp-if-end26
+; CHECK-NEXT: adds x11, x10, #1
+; CHECK-NEXT: adcs x12, x1, xzr
+; CHECK-NEXT: tbnz x11, #55, .LBB3_10
+; CHECK-NEXT: // %bb.6:
+; CHECK-NEXT: extr x10, x12, x11, #2
+; CHECK-NEXT: extr x11, x12, x11, #34
+; CHECK-NEXT: .LBB3_7: // %itofp-if-end26
; CHECK-NEXT: bfi w11, w8, #20, #12
; CHECK-NEXT: mov w8, #1072693248 // =0x3ff00000
; CHECK-NEXT: add w8, w11, w8
@@ -525,10 +529,10 @@ define double @u256_to_f64(i256 %val) {
; CHECK-NEXT: fmov d0, x10
; CHECK-NEXT: add sp, sp, #192
; CHECK-NEXT: ret
-; CHECK-NEXT: .LBB3_7:
+; CHECK-NEXT: .LBB3_8:
; CHECK-NEXT: movi d0, #0000000000000000
; CHECK-NEXT: ret
-; CHECK-NEXT: .LBB3_8: // %itofp-if-else
+; CHECK-NEXT: .LBB3_9: // %itofp-if-else
; CHECK-NEXT: movi v0.2d, #0000000000000000
; CHECK-NEXT: sub w9, w10, #203
; CHECK-NEXT: add x11, sp, #128
@@ -542,8 +546,15 @@ define double @u256_to_f64(i256 %val) {
; CHECK-NEXT: ldr x10, [x10]
; CHECK-NEXT: lsl x10, x10, x9
; CHECK-NEXT: lsr x11, x10, #32
-; CHECK-NEXT: b .LBB3_6
-; CHECK-NEXT: .LBB3_9: // %itofp-sw-bb
+; CHECK-NEXT: // kill: def $w11 killed $w11 killed $x11 def $x11
+; CHECK-NEXT: b .LBB3_7
+; CHECK-NEXT: .LBB3_10: // %itofp-if-then20
+; CHECK-NEXT: extr x8, x12, x11, #35
+; CHECK-NEXT: extr x10, x12, x11, #3
+; CHECK-NEXT: mov w11, w8
+; CHECK-NEXT: mov w8, w9
+; CHECK-NEXT: b .LBB3_7
+; CHECK-NEXT: .LBB3_11: // %itofp-sw-bb
; CHECK-NEXT: extr x1, x1, x0, #63
; CHECK-NEXT: lsl x0, x0, #1
; CHECK-NEXT: b .LBB3_5
>From 5af26d622aeac120cd8534295a53f2302f7a5647 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?D=C3=A1vid=20Bolvansk=C3=BD?= <david.bolvansky at gmail.com>
Date: Sat, 3 Oct 2026 15:22:32 +0200
Subject: [PATCH 2/4] Generate bitstream checks with update_llc_test_checks.py
---
.../AArch64/early-ifcvt-biased-bitstream.ll | 48 +++++++++++++++++++
1 file changed, 48 insertions(+)
diff --git a/llvm/test/CodeGen/AArch64/early-ifcvt-biased-bitstream.ll b/llvm/test/CodeGen/AArch64/early-ifcvt-biased-bitstream.ll
index 46f57811a822a1..34bd415063a311 100644
--- a/llvm/test/CodeGen/AArch64/early-ifcvt-biased-bitstream.ll
+++ b/llvm/test/CodeGen/AArch64/early-ifcvt-biased-bitstream.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -mcpu=apple-m1 -verify-machineinstrs < %s | FileCheck %s
;
; Reading zero bits must not add speculative shifts and selects to the
@@ -11,6 +12,31 @@ target triple = "aarch64-unknown-linux-gnu"
; CHECK-NOT: csel
; CHECK: ret
define i32 @decode_bits(ptr %counts, i64 %count, i64 %bits, i32 %initial, ptr %out) {
+; CHECK-LABEL: decode_bits:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: // kill: def $w3 killed $w3 def $x3
+; CHECK-NEXT: .LBB0_1: // %loop
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ldrb w9, [x0], #1
+; CHECK-NEXT: cbnz w9, .LBB0_4
+; CHECK-NEXT: // %bb.2: // in Loop: Header=BB0_1 Depth=1
+; CHECK-NEXT: mov x8, #0 // =0x0
+; CHECK-NEXT: .LBB0_3: // %merge
+; CHECK-NEXT: // in Loop: Header=BB0_1 Depth=1
+; CHECK-NEXT: str x8, [x4], #8
+; CHECK-NEXT: subs x1, x1, #1
+; CHECK-NEXT: b.ne .LBB0_1
+; CHECK-NEXT: b .LBB0_5
+; CHECK-NEXT: .LBB0_4: // %read
+; CHECK-NEXT: // in Loop: Header=BB0_1 Depth=1
+; CHECK-NEXT: lsl x8, x2, x3
+; CHECK-NEXT: neg w10, w9
+; CHECK-NEXT: lsr x8, x8, x10
+; CHECK-NEXT: add w3, w3, w9
+; CHECK-NEXT: b .LBB0_3
+; CHECK-NEXT: .LBB0_5: // %exit
+; CHECK-NEXT: mov x0, x3
+; CHECK-NEXT: ret
entry:
br label %loop
loop:
@@ -48,6 +74,28 @@ exit:
; CHECK: csel
; CHECK: ret
define i32 @decode_bits_balanced(ptr %counts, i64 %count, i64 %bits, i32 %initial, ptr %out) {
+; CHECK-LABEL: decode_bits_balanced:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: // kill: def $w3 killed $w3 def $x3
+; CHECK-NEXT: .LBB1_1: // %loop
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ldrb w8, [x0], #1
+; CHECK-NEXT: lsl x9, x2, x3
+; CHECK-NEXT: neg w10, w8
+; CHECK-NEXT: lsr x9, x9, x10
+; CHECK-NEXT: add w10, w3, w8
+; CHECK-NEXT: .p2align 6, , 4
+; CHECK-NEXT: // %bb.2: // %loop
+; CHECK-NEXT: // in Loop: Header=BB1_1 Depth=1
+; CHECK-NEXT: cmp w8, #0
+; CHECK-NEXT: csel w3, w3, w10, eq
+; CHECK-NEXT: csel x8, xzr, x9, eq
+; CHECK-NEXT: str x8, [x4], #8
+; CHECK-NEXT: subs x1, x1, #1
+; CHECK-NEXT: b.ne .LBB1_1
+; CHECK-NEXT: // %bb.3: // %exit
+; CHECK-NEXT: mov x0, x3
+; CHECK-NEXT: ret
entry:
br label %loop
loop:
>From 84c3215feb6ffe506f8412d659fedbd6215040d0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?D=C3=A1vid=20Bolvansk=C3=BD?= <david.bolvansky at gmail.com>
Date: Sat, 3 Oct 2026 15:28:16 +0200
Subject: [PATCH 3/4] Simplify the biased branch budget calculation and clean
up checks
Assisted-by: OpenAI Codex
---
llvm/lib/CodeGen/EarlyIfConversion.cpp | 16 +++++++---------
.../AArch64/early-ifcvt-biased-bitstream.ll | 9 ---------
2 files changed, 7 insertions(+), 18 deletions(-)
diff --git a/llvm/lib/CodeGen/EarlyIfConversion.cpp b/llvm/lib/CodeGen/EarlyIfConversion.cpp
index f9cbb0dffdc403..91ce47ab3a54e9 100644
--- a/llvm/lib/CodeGen/EarlyIfConversion.cpp
+++ b/llvm/lib/CodeGen/EarlyIfConversion.cpp
@@ -1225,7 +1225,7 @@ bool EarlyIfConverter::shouldConvertIf() {
// Set a somewhat arbitrary limit on the critical path extension we accept.
// When hard-to-predict analysis is enabled, use full MispredictPenalty for
- // hard-to-predict branches, half for others. Otherwise use half for all.
+ // hard-to-predict branches, half for others unless strongly biased.
bool DataDependent = false;
if (EnableDataDependentBranchAnalysis)
DataDependent = isConditionDataDependent();
@@ -1233,16 +1233,14 @@ bool EarlyIfConverter::shouldConvertIf() {
// A strongly biased branch has little misprediction cost to hide the
// critical-path extension introduced by speculative instructions and selects.
// Keep the existing budget for branches without a strongly preferred edge.
- unsigned CritLimit = DataDependent ? STI->getMispredictionPenalty()
- : STI->getMispredictionPenalty() / 2;
+ const unsigned MispredictPenalty = STI->getMispredictionPenalty();
+ unsigned CritLimit = DataDependent ? MispredictPenalty : MispredictPenalty / 2;
if (!DataDependent) {
- BranchProbability TBBProb =
- MBPI->getEdgeProbability(IfConv.Head, IfConv.TBB);
- BranchProbability FBBProb =
- MBPI->getEdgeProbability(IfConv.Head, IfConv.FBB);
- BranchProbability ColdProb = std::min(TBBProb, FBBProb);
+ const BranchProbability ColdProb =
+ std::min(MBPI->getEdgeProbability(IfConv.Head, IfConv.TBB),
+ MBPI->getEdgeProbability(IfConv.Head, IfConv.FBB));
if (ColdProb < BranchProbability(1, 100))
- CritLimit = ColdProb.scale(STI->getMispredictionPenalty());
+ CritLimit = ColdProb.scale(MispredictPenalty);
}
MachineBasicBlock &MBB = *IfConv.Head;
diff --git a/llvm/test/CodeGen/AArch64/early-ifcvt-biased-bitstream.ll b/llvm/test/CodeGen/AArch64/early-ifcvt-biased-bitstream.ll
index 34bd415063a311..d06a9dc5989530 100644
--- a/llvm/test/CodeGen/AArch64/early-ifcvt-biased-bitstream.ll
+++ b/llvm/test/CodeGen/AArch64/early-ifcvt-biased-bitstream.ll
@@ -6,11 +6,6 @@
target triple = "aarch64-unknown-linux-gnu"
-; CHECK-LABEL: decode_bits:
-; CHECK-NOT: csel
-; CHECK: cbnz
-; CHECK-NOT: csel
-; CHECK: ret
define i32 @decode_bits(ptr %counts, i64 %count, i64 %bits, i32 %initial, ptr %out) {
; CHECK-LABEL: decode_bits:
; CHECK: // %bb.0: // %entry
@@ -69,10 +64,6 @@ exit:
ret i32 %nextconsumed
}
-; CHECK-LABEL: decode_bits_balanced:
-; CHECK: csel
-; CHECK: csel
-; CHECK: ret
define i32 @decode_bits_balanced(ptr %counts, i64 %count, i64 %bits, i32 %initial, ptr %out) {
; CHECK-LABEL: decode_bits_balanced:
; CHECK: // %bb.0: // %entry
>From c5f056d82e43e31ed859a8b1afd5d3d15b86b3db Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?D=C3=A1vid=20Bolvansk=C3=BD?= <david.bolvansky at gmail.com>
Date: Sat, 3 Oct 2026 15:30:01 +0200
Subject: [PATCH 4/4] Compute each critical-path budget once
Assisted-by: OpenAI Codex
---
llvm/lib/CodeGen/EarlyIfConversion.cpp | 11 +++++++----
1 file changed, 7 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/CodeGen/EarlyIfConversion.cpp b/llvm/lib/CodeGen/EarlyIfConversion.cpp
index 91ce47ab3a54e9..cff8fa6ac74b37 100644
--- a/llvm/lib/CodeGen/EarlyIfConversion.cpp
+++ b/llvm/lib/CodeGen/EarlyIfConversion.cpp
@@ -1234,13 +1234,16 @@ bool EarlyIfConverter::shouldConvertIf() {
// critical-path extension introduced by speculative instructions and selects.
// Keep the existing budget for branches without a strongly preferred edge.
const unsigned MispredictPenalty = STI->getMispredictionPenalty();
- unsigned CritLimit = DataDependent ? MispredictPenalty : MispredictPenalty / 2;
- if (!DataDependent) {
+ unsigned CritLimit;
+ if (DataDependent) {
+ CritLimit = MispredictPenalty;
+ } else {
const BranchProbability ColdProb =
std::min(MBPI->getEdgeProbability(IfConv.Head, IfConv.TBB),
MBPI->getEdgeProbability(IfConv.Head, IfConv.FBB));
- if (ColdProb < BranchProbability(1, 100))
- CritLimit = ColdProb.scale(MispredictPenalty);
+ CritLimit = ColdProb < BranchProbability(1, 100)
+ ? ColdProb.scale(MispredictPenalty)
+ : MispredictPenalty / 2;
}
MachineBasicBlock &MBB = *IfConv.Head;
More information about the llvm-commits
mailing list