[llvm] [CodeGen] [EarlyIfConversion] Prioritize conversion of hard to predict branches (PR #174457)
Jonathan Cohen via llvm-commits
llvm-commits at lists.llvm.org
Thu Jan 8 12:06:16 PST 2026
https://github.com/jcohen-apple updated https://github.com/llvm/llvm-project/pull/174457
>From 60880f97aca77a92aa63a4d40f67d708dfc55e47 Mon Sep 17 00:00:00 2001
From: Jonathan Cohen <joncoh at apple.com>
Date: Wed, 7 Jan 2026 12:58:45 +0200
Subject: [PATCH 01/11] [CodeGen][EarlyIfConvert] Test case documenting current
behavior (before typo fix)
In this case the branch is converted even though it's operands are loop invariant - we should let the branch predictor handle this
---
.../early-ifcvt-likely-predictable.mir | 62 +++++++++++++++++++
1 file changed, 62 insertions(+)
diff --git a/llvm/test/CodeGen/AArch64/early-ifcvt-likely-predictable.mir b/llvm/test/CodeGen/AArch64/early-ifcvt-likely-predictable.mir
index ab5e320725d5b..dc697fd72940c 100644
--- a/llvm/test/CodeGen/AArch64/early-ifcvt-likely-predictable.mir
+++ b/llvm/test/CodeGen/AArch64/early-ifcvt-likely-predictable.mir
@@ -17,6 +17,11 @@
entry:
ret void
}
+
+ define void @test_cond_def_inside_loop_with_invariant_operands() {
+ entry:
+ ret void
+ }
...
---
name: test_cond_is_load_with_invariant_ops
@@ -243,3 +248,60 @@ body: |
B %bb.1
...
+---
+name: test_cond_def_inside_loop_with_invariant_operands
+alignment: 4
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: test_cond_def_inside_loop_with_invariant_operands
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: liveins: $x0, $x1, $x2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr64common = COPY $x2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load (s32))
+ ; CHECK-NEXT: [[LDRWui1:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 0 :: (load (s32))
+ ; CHECK-NEXT: [[SUBSWrs:%[0-9]+]]:gpr32common = SUBSWrs [[LDRWui]], [[LDRWui1]], 0, implicit-def $nzcv
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gpr32all = COPY $wzr
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:gpr32 = COPY [[COPY3]]
+ ; CHECK-NEXT: [[COPY5:%[0-9]+]]:gpr32common = COPY [[LDRWui]]
+ ; CHECK-NEXT: [[ADDWri:%[0-9]+]]:gpr32common = ADDWri [[COPY5]], 1, 0
+ ; CHECK-NEXT: [[COPY6:%[0-9]+]]:gpr32all = COPY [[ADDWri]]
+ ; CHECK-NEXT: $wzr = SUBSWri [[SUBSWrs]], 0, 0, implicit-def $nzcv
+ ; CHECK-NEXT: [[CSINCWr:%[0-9]+]]:gpr32 = CSINCWr [[COPY4]], [[COPY5]], 0, implicit $nzcv
+ ; CHECK-NEXT: STRWui [[CSINCWr]], [[COPY2]], 0 :: (store (s32))
+ ; CHECK-NEXT: B %bb.1
+ bb.0:
+ liveins: $x0, $x1, $x2
+ %3:gpr64common = COPY $x0
+ %4:gpr64common = COPY $x1
+ %12:gpr64common = COPY $x2
+
+ bb.1:
+ successors: %bb.3(0x30000000), %bb.2(0x50000000)
+
+ %5:gpr32 = LDRWui %3, 0 :: (load (s32))
+ %6:gpr32 = LDRWui %4, 0 :: (load (s32))
+ %7:gpr32 = SUBSWrs %5, %6, 0, implicit-def $nzcv
+ %8:gpr32all = COPY $wzr
+ %9:gpr32all = COPY %8
+ CBZW killed %7, %bb.3
+ B %bb.2
+
+ bb.2:
+ %10:gpr32sp = COPY %5
+ %11:gpr32common = ADDWri %10, 1, 0
+ %0:gpr32all = COPY %11
+
+ bb.3:
+ %1:gpr32 = PHI %9, %bb.1, %0, %bb.2
+ STRWui %1, %12, 0 :: (store (s32))
+ B %bb.1
+
+...
>From 1d5c6d3802b0455745ef1001f46f0da34e5862d7 Mon Sep 17 00:00:00 2001
From: Jonathan Cohen <joncoh at apple.com>
Date: Wed, 7 Jan 2026 12:00:52 +0200
Subject: [PATCH 02/11] [CodeGen][EarlyIfConversion] Fix typo in loop invariant
check lambda
---
llvm/lib/CodeGen/EarlyIfConversion.cpp | 6 ++---
.../early-ifcvt-likely-predictable.mir | 23 +++++++++++++------
2 files changed, 19 insertions(+), 10 deletions(-)
diff --git a/llvm/lib/CodeGen/EarlyIfConversion.cpp b/llvm/lib/CodeGen/EarlyIfConversion.cpp
index 28993c47c094d..af5f366f92c4a 100644
--- a/llvm/lib/CodeGen/EarlyIfConversion.cpp
+++ b/llvm/lib/CodeGen/EarlyIfConversion.cpp
@@ -942,9 +942,9 @@ bool EarlyIfConverter::shouldConvertIf() {
all_of(Def->operands(), [&](MachineOperand &Op) {
if (Op.isImm())
return true;
- if (!MO.isReg() || !MO.isUse())
- return false;
- Register Reg = MO.getReg();
+ if (!Op.isReg() || !Op.isUse())
+ return true;
+ Register Reg = Op.getReg();
if (Reg.isPhysical())
return false;
diff --git a/llvm/test/CodeGen/AArch64/early-ifcvt-likely-predictable.mir b/llvm/test/CodeGen/AArch64/early-ifcvt-likely-predictable.mir
index dc697fd72940c..7f1d99bec6f48 100644
--- a/llvm/test/CodeGen/AArch64/early-ifcvt-likely-predictable.mir
+++ b/llvm/test/CodeGen/AArch64/early-ifcvt-likely-predictable.mir
@@ -263,19 +263,28 @@ body: |
; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr64common = COPY $x2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.1:
- ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: successors: %bb.3(0x30000000), %bb.2(0x50000000)
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load (s32))
; CHECK-NEXT: [[LDRWui1:%[0-9]+]]:gpr32 = LDRWui [[COPY1]], 0 :: (load (s32))
- ; CHECK-NEXT: [[SUBSWrs:%[0-9]+]]:gpr32common = SUBSWrs [[LDRWui]], [[LDRWui1]], 0, implicit-def $nzcv
+ ; CHECK-NEXT: [[SUBSWrs:%[0-9]+]]:gpr32 = SUBSWrs [[LDRWui]], [[LDRWui1]], 0, implicit-def $nzcv
; CHECK-NEXT: [[COPY3:%[0-9]+]]:gpr32all = COPY $wzr
- ; CHECK-NEXT: [[COPY4:%[0-9]+]]:gpr32 = COPY [[COPY3]]
- ; CHECK-NEXT: [[COPY5:%[0-9]+]]:gpr32common = COPY [[LDRWui]]
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:gpr32all = COPY [[COPY3]]
+ ; CHECK-NEXT: CBZW killed [[SUBSWrs]], %bb.3
+ ; CHECK-NEXT: B %bb.2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: successors: %bb.3(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY5:%[0-9]+]]:gpr32sp = COPY [[LDRWui]]
; CHECK-NEXT: [[ADDWri:%[0-9]+]]:gpr32common = ADDWri [[COPY5]], 1, 0
; CHECK-NEXT: [[COPY6:%[0-9]+]]:gpr32all = COPY [[ADDWri]]
- ; CHECK-NEXT: $wzr = SUBSWri [[SUBSWrs]], 0, 0, implicit-def $nzcv
- ; CHECK-NEXT: [[CSINCWr:%[0-9]+]]:gpr32 = CSINCWr [[COPY4]], [[COPY5]], 0, implicit $nzcv
- ; CHECK-NEXT: STRWui [[CSINCWr]], [[COPY2]], 0 :: (store (s32))
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.3:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[PHI:%[0-9]+]]:gpr32 = PHI [[COPY4]], %bb.1, [[COPY6]], %bb.2
+ ; CHECK-NEXT: STRWui [[PHI]], [[COPY2]], 0 :: (store (s32))
; CHECK-NEXT: B %bb.1
bb.0:
liveins: $x0, $x1, $x2
>From 7e70d413f8d74eadf54bcde3cc28c9edd587aed2 Mon Sep 17 00:00:00 2001
From: Jonathan Cohen <joncoh at apple.com>
Date: Mon, 5 Jan 2026 15:55:28 +0200
Subject: [PATCH 03/11] [CodeGen] add unit tests for early if-cvt
Documents the current if conversion behavior before updating the
heuristics.
---
.../early-ifcvt-hard-to-predict.mir | 57 +++++++++++++++++++
1 file changed, 57 insertions(+)
create mode 100644 llvm/test/CodeGen/AArch64/early-ifcvt-cascade-tests/early-ifcvt-hard-to-predict.mir
diff --git a/llvm/test/CodeGen/AArch64/early-ifcvt-cascade-tests/early-ifcvt-hard-to-predict.mir b/llvm/test/CodeGen/AArch64/early-ifcvt-cascade-tests/early-ifcvt-hard-to-predict.mir
new file mode 100644
index 0000000000000..d7f1d4ea83065
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/early-ifcvt-cascade-tests/early-ifcvt-hard-to-predict.mir
@@ -0,0 +1,57 @@
+# RUN: llc -mtriple=aarch64 -mcpu=apple-m1 -run-pass=early-ifcvt -o - %s | FileCheck %s
+
+# Test early if-conversion with a hard-to-predict branch (load-dependent condition).
+
+# The new heuristic in If Conversion added by this PR assumes that conditional
+# branches that originate from memory (and not function arguments) are data
+# dependent, and therefore more difficult to predict based on branch history alone.
+# In that case, we can be more aggressive with if-conversion and apply the full
+# mispredict penalty.
+
+# CHECK-LABEL: name: load_to_cond_branch
+
+# Current behavior: NOT converted
+# CHECK: bb.0:
+# CHECK: LDRWui
+# CHECK: Bcc
+# CHECK: bb.1:
+# CHECK: bb.2:
+# CHECK: PHI
+
+--- |
+ define i32 @load_to_cond_branch(ptr %p, i32 %x, i32 %y) {
+ ret i32 0
+ }
+...
+---
+name: load_to_cond_branch
+alignment: 4
+tracksRegLiveness: true
+liveins:
+ - { reg: '$x0', virtual-reg: '%0' }
+ - { reg: '$w1', virtual-reg: '%1' }
+ - { reg: '$w2', virtual-reg: '%2' }
+body: |
+ bb.1:
+ successors: %bb.2(0x40000000), %bb.3(0x40000000)
+ liveins: $x0, $w1, $w2
+
+ %0:gpr64common = COPY $x0
+ %1:gpr32 = COPY $w1
+ %2:gpr32 = COPY $w2
+ %10:gpr32common = LDRWui %0, 0 :: (load (s32))
+ %11:gpr32 = SUBSWri %10, 100, 0, implicit-def $nzcv
+ Bcc 12, %bb.3, implicit $nzcv
+ B %bb.2
+
+ bb.2:
+ successors: %bb.3(0x80000000)
+
+ %20:gpr32 = MADDWrrr %1, %2, $wzr
+ %21:gpr32 = MADDWrrr %20, %1, $wzr
+
+ bb.3:
+ %100:gpr32 = PHI %1, %bb.1, %21, %bb.2
+ $w0 = COPY %100
+ RET_ReallyLR implicit $w0
+...
>From 0dbd864642e396694050d5a7e657d072b6ca8ea9 Mon Sep 17 00:00:00 2001
From: Jonathan Cohen <joncoh at apple.com>
Date: Mon, 5 Jan 2026 17:02:40 +0200
Subject: [PATCH 04/11] [CodeGen][AArch64] Add additional heuristic to check
for data dependent branches
Branches which depend on values which are read from memory within the
scope of the machine function will be considered hard to predict. In
that case we can be more aggressive in early if-conversion by setting a
critical path limit.
---
.../include/llvm/CodeGen/TargetRegisterInfo.h | 4 +
llvm/lib/CodeGen/EarlyIfConversion.cpp | 171 +++++++++++++++++-
.../Target/AArch64/AArch64RegisterInfo.cpp | 4 +
llvm/lib/Target/AArch64/AArch64RegisterInfo.h | 2 +
.../early-ifcvt-easy-to-predict.mir | 45 +++++
.../early-ifcvt-hard-to-predict.mir | 20 +-
6 files changed, 230 insertions(+), 16 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/early-ifcvt-cascade-tests/early-ifcvt-easy-to-predict.mir
diff --git a/llvm/include/llvm/CodeGen/TargetRegisterInfo.h b/llvm/include/llvm/CodeGen/TargetRegisterInfo.h
index 35b14e8b8fd30..921a1e64b2ccb 100644
--- a/llvm/include/llvm/CodeGen/TargetRegisterInfo.h
+++ b/llvm/include/llvm/CodeGen/TargetRegisterInfo.h
@@ -662,6 +662,10 @@ class LLVM_ABI TargetRegisterInfo : public MCRegisterInfo {
return false;
}
+ /// Returns the physical register that holds condition flags (status flags)
+ /// for conditional branches on this target.
+ virtual MCRegister getStatusFlagsRegister() const { return MCRegister(); }
+
/// Prior to adding the live-out mask to a stackmap or patchpoint
/// instruction, provide the target the opportunity to adjust it (mainly to
/// remove pseudo-registers that should be ignored).
diff --git a/llvm/lib/CodeGen/EarlyIfConversion.cpp b/llvm/lib/CodeGen/EarlyIfConversion.cpp
index af5f366f92c4a..07f2871a0347c 100644
--- a/llvm/lib/CodeGen/EarlyIfConversion.cpp
+++ b/llvm/lib/CodeGen/EarlyIfConversion.cpp
@@ -55,10 +55,17 @@ BlockInstrLimit("early-ifcvt-limit", cl::init(30), cl::Hidden,
static cl::opt<bool> Stress("stress-early-ifcvt", cl::Hidden,
cl::desc("Turn all knobs to 11"));
+// Enable analysis of hard-to-predict branches (conditions derived from loads).
+static cl::opt<bool> EnableHardToPredictAnalysis(
+ "enable-early-ifcvt-hard-to-predict", cl::Hidden, cl::init(true),
+ cl::desc("Enable hard-to-predict branch analysis for if-conversion"));
+
STATISTIC(NumDiamondsSeen, "Number of diamonds");
STATISTIC(NumDiamondsConv, "Number of diamonds converted");
STATISTIC(NumTrianglesSeen, "Number of triangles");
STATISTIC(NumTrianglesConv, "Number of triangles converted");
+STATISTIC(NumDataDependant,
+ "Number of data dependent conditional branches encountered");
//===----------------------------------------------------------------------===//
// SSAIfConv
@@ -811,6 +818,9 @@ class EarlyIfConverter {
MachineTraceMetrics::Ensemble *MinInstr = nullptr;
SSAIfConv IfConv;
+ // Cache for operand memory analysis.
+ DenseMap<Register, bool> OperandMemoryCache;
+
public:
EarlyIfConverter(MachineDominatorTree &DT, MachineLoopInfo &LI,
MachineTraceMetrics &MTM)
@@ -823,6 +833,8 @@ class EarlyIfConverter {
bool tryConvertIf(MachineBasicBlock *);
void invalidateTraces();
bool shouldConvertIf();
+ bool isConditionDataDependent();
+ bool doOperandsComeFromMemory(Register Reg);
};
class EarlyIfConverterLegacy : public MachineFunctionPass {
@@ -896,6 +908,142 @@ void EarlyIfConverter::invalidateTraces() {
Traces->verifyAnalysis();
}
+/// Check if a register's value comes from a memory load by walking the
+/// def-use chain. Results are cached for virtual registers only.
+bool EarlyIfConverter::doOperandsComeFromMemory(Register Reg) {
+ if (!Reg.isVirtual())
+ return false;
+
+ // Check cache first.
+ auto It = OperandMemoryCache.find(Reg);
+ if (It != OperandMemoryCache.end())
+ return It->second;
+
+ // Walk the def-use chain.
+ SmallPtrSet<const MachineInstr *, 16> Visited;
+ SmallVector<const MachineInstr *, 16> Worklist;
+ SmallVector<Register, 16> VisitedRegs;
+
+ MachineInstr *DefMI = MRI->getVRegDef(Reg);
+ // The operand is defined outside of the function - it does not
+ // come from memory access.
+ if (!DefMI) {
+ OperandMemoryCache[Reg] = false;
+ return false;
+ }
+
+ Worklist.push_back(DefMI);
+ VisitedRegs.push_back(Reg);
+
+ while (!Worklist.empty()) {
+ const MachineInstr *MI = Worklist.pop_back_val();
+ if (!Visited.insert(MI).second)
+ continue;
+
+ // Check if this instruction is a load.
+ if (MI->mayLoad()) {
+ LLVM_DEBUG(dbgs() << "Operand comes from load: " << *MI);
+ // Cache all visited virtual registers as coming from memory.
+ for (Register Reg : VisitedRegs)
+ OperandMemoryCache[Reg] = true;
+ return true;
+ }
+
+ // Walk through all register use operands and find their definitions.
+ for (const MachineOperand &MO : MI->operands()) {
+ if (!MO.isReg() || !MO.isUse())
+ continue;
+ Register UseReg = MO.getReg();
+ if (!UseReg.isVirtual())
+ continue;
+
+ // Check cache for this operand.
+ auto CacheIt = OperandMemoryCache.find(UseReg);
+ if (CacheIt != OperandMemoryCache.end()) {
+ if (CacheIt->second) {
+ for (Register Reg : VisitedRegs)
+ OperandMemoryCache[Reg] = true;
+ return true;
+ }
+ continue;
+ }
+
+ MachineInstr *UseDef = MRI->getVRegDef(UseReg);
+ if (UseDef && !Visited.count(UseDef)) {
+ Worklist.push_back(UseDef);
+ VisitedRegs.push_back(UseReg);
+ }
+ }
+ }
+
+ // No load found, cache all visited virtual registers as not from memory.
+ for (Register Reg : VisitedRegs)
+ OperandMemoryCache[Reg] = false;
+ return false;
+}
+
+/// Check if the branch condition is data-dependent (comes from memory loads).
+bool EarlyIfConverter::isConditionDataDependent() {
+ MachineBasicBlock::iterator BranchIt = IfConv.Head->getFirstTerminator();
+ if (BranchIt == IfConv.Head->end())
+ return false;
+
+ // Get the status flags register from the target. If the target hasn't
+ // implemented this hook, we skip physical register analysis entirely
+ // and fallback to the previous behavior.
+ MCRegister StatusFlagsReg = TRI->getStatusFlagsRegister();
+
+ if (!StatusFlagsReg.isValid())
+ return false;
+
+ for (const MachineOperand &MO : BranchIt->operands()) {
+ if (!MO.isReg() || !MO.isUse())
+ continue;
+ Register Reg = MO.getReg();
+
+ if (Reg.isVirtual()) {
+ if (doOperandsComeFromMemory(Reg)) {
+ LLVM_DEBUG(dbgs() << "Condition is data-dependent (from memory)\n");
+ ++NumDataDependant;
+ return true;
+ }
+ } else if (Reg.isPhysical() && TRI->regsOverlap(Reg, StatusFlagsReg)) {
+ // For the status flags register (e.g., NZCV on AArch64),
+ // search backwards to find the defining instruction.
+ for (auto I = std::prev(BranchIt);; --I) {
+ // Check if this instruction defines the status flags register.
+ bool FoundDef = false;
+ for (const MachineOperand &DefMO : I->operands()) {
+ if (DefMO.isReg() && DefMO.isDef() &&
+ TRI->regsOverlap(DefMO.getReg(), StatusFlagsReg)) {
+ FoundDef = true;
+ break;
+ }
+ }
+
+ if (FoundDef) {
+ for (const MachineOperand &DefMO : I->operands()) {
+ if (!DefMO.isReg() || !DefMO.isUse())
+ continue;
+ Register DefReg = DefMO.getReg();
+ if (DefReg.isVirtual() && doOperandsComeFromMemory(DefReg)) {
+ LLVM_DEBUG(dbgs()
+ << "Condition is data-dependent (from memory)\n");
+ ++NumDataDependant;
+ return true;
+ }
+ }
+ break;
+ }
+
+ if (I == IfConv.Head->begin())
+ break;
+ }
+ }
+ }
+ return false;
+}
+
// Adjust cycles with downward saturation.
static unsigned adjCycles(unsigned Cyc, int Delta) {
if (Delta < 0 && Cyc + Delta > Cyc)
@@ -963,12 +1111,31 @@ bool EarlyIfConverter::shouldConvertIf() {
unsigned MinCrit = std::min(TBBTrace.getCriticalPath(),
FBBTrace.getCriticalPath());
- // Set a somewhat arbitrary limit on the critical path extension we accept.
- unsigned CritLimit = SchedModel.MispredictPenalty/2;
+ // Set a limit on the critical path extension we accept.
+ // When hard-to-predict analysis is enabled, use full MispredictPenalty for
+ // hard-to-predict branches, half for others. Otherwise use half for all.
+ bool DataDependent = false;
+ if (EnableHardToPredictAnalysis) {
+ DataDependent = isConditionDataDependent();
+ }
+ unsigned CritLimit = DataDependent ? SchedModel.MispredictPenalty
+ : SchedModel.MispredictPenalty / 2;
MachineBasicBlock &MBB = *IfConv.Head;
MachineOptimizationRemarkEmitter MORE(*MBB.getParent(), nullptr);
+ // Emit analysis remark about data-dependent condition.
+ if (DataDependent) {
+ MORE.emit([&]() {
+ return MachineOptimizationRemarkAnalysis(DEBUG_TYPE,
+ "DataDependentCondition",
+ MBB.back().getDebugLoc(), &MBB)
+ << "branch condition is data-dependent (from memory load), "
+ << "using higher CritLimit of " << ore::NV("CritLimit", CritLimit)
+ << " cycles";
+ });
+ }
+
// If-conversion only makes sense when there is unexploited ILP. Compute the
// maximum-ILP resource length of the trace after if-conversion. Compare it
// to the shortest critical path.
diff --git a/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp b/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
index 5ebec9c4d11d9..a10a39bcf2487 100644
--- a/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
@@ -1437,3 +1437,7 @@ bool AArch64RegisterInfo::isIgnoredCVReg(MCRegister LLVMReg) const {
return (LLVMReg >= AArch64::Z0 && LLVMReg <= AArch64::Z31) ||
(LLVMReg >= AArch64::P0 && LLVMReg <= AArch64::P15);
}
+
+MCRegister AArch64RegisterInfo::getStatusFlagsRegister() const {
+ return AArch64::NZCV;
+}
diff --git a/llvm/lib/Target/AArch64/AArch64RegisterInfo.h b/llvm/lib/Target/AArch64/AArch64RegisterInfo.h
index 89d1802ab98d5..b40b34b6dd0ee 100644
--- a/llvm/lib/Target/AArch64/AArch64RegisterInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64RegisterInfo.h
@@ -154,6 +154,8 @@ class AArch64RegisterInfo final : public AArch64GenRegisterInfo {
bool shouldAnalyzePhysregInMachineLoopInfo(MCRegister R) const override;
bool isIgnoredCVReg(MCRegister LLVMReg) const override;
+
+ MCRegister getStatusFlagsRegister() const override;
};
} // end namespace llvm
diff --git a/llvm/test/CodeGen/AArch64/early-ifcvt-cascade-tests/early-ifcvt-easy-to-predict.mir b/llvm/test/CodeGen/AArch64/early-ifcvt-cascade-tests/early-ifcvt-easy-to-predict.mir
new file mode 100644
index 0000000000000..2560c4ade5c73
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/early-ifcvt-cascade-tests/early-ifcvt-easy-to-predict.mir
@@ -0,0 +1,45 @@
+# RUN: llc -mtriple=aarch64 -mcpu=apple-m1 -run-pass=early-ifcvt -o - %s | FileCheck %s
+
+# CHECK: bb.0:
+# CHECK: SUBSWri
+# CHECK: Bcc
+# CHECK: bb.1:
+# CHECK: bb.2:
+# CHECK: PHI
+
+--- |
+ define i32 @arg_to_cond_branch(i32 %val, i32 %x, i32 %y) {
+ ret i32 0
+ }
+...
+---
+name: arg_to_cond_branch
+alignment: 4
+tracksRegLiveness: true
+liveins:
+ - { reg: '$w0', virtual-reg: '%0' }
+ - { reg: '$w1', virtual-reg: '%1' }
+ - { reg: '$w2', virtual-reg: '%2' }
+body: |
+ bb.1:
+ successors: %bb.2(0x40000000), %bb.3(0x40000000)
+ liveins: $w0, $w1, $w2
+
+ %0:gpr32common = COPY $w0
+ %1:gpr32 = COPY $w1
+ %2:gpr32 = COPY $w2
+ %11:gpr32 = SUBSWri %0, 100, 0, implicit-def $nzcv
+ Bcc 12, %bb.3, implicit $nzcv
+ B %bb.2
+
+ bb.2:
+ successors: %bb.3(0x80000000)
+
+ %20:gpr32 = MADDWrrr %1, %2, $wzr
+ %21:gpr32 = MADDWrrr %20, %1, $wzr
+
+ bb.3:
+ %100:gpr32 = PHI %1, %bb.1, %21, %bb.2
+ $w0 = COPY %100
+ RET_ReallyLR implicit $w0
+...
diff --git a/llvm/test/CodeGen/AArch64/early-ifcvt-cascade-tests/early-ifcvt-hard-to-predict.mir b/llvm/test/CodeGen/AArch64/early-ifcvt-cascade-tests/early-ifcvt-hard-to-predict.mir
index d7f1d4ea83065..0cffedf4533fc 100644
--- a/llvm/test/CodeGen/AArch64/early-ifcvt-cascade-tests/early-ifcvt-hard-to-predict.mir
+++ b/llvm/test/CodeGen/AArch64/early-ifcvt-cascade-tests/early-ifcvt-hard-to-predict.mir
@@ -1,22 +1,14 @@
# RUN: llc -mtriple=aarch64 -mcpu=apple-m1 -run-pass=early-ifcvt -o - %s | FileCheck %s
-# Test early if-conversion with a hard-to-predict branch (load-dependent condition).
-
-# The new heuristic in If Conversion added by this PR assumes that conditional
-# branches that originate from memory (and not function arguments) are data
-# dependent, and therefore more difficult to predict based on branch history alone.
-# In that case, we can be more aggressive with if-conversion and apply the full
-# mispredict penalty.
-
# CHECK-LABEL: name: load_to_cond_branch
-
-# Current behavior: NOT converted
# CHECK: bb.0:
# CHECK: LDRWui
-# CHECK: Bcc
-# CHECK: bb.1:
-# CHECK: bb.2:
-# CHECK: PHI
+# CHECK: SUBSWri
+# CHECK: MADDWrrr
+# CHECK: MADDWrrr
+# CHECK: CSELWr
+# CHECK-NOT: Bcc
+# CHECK-NOT: PHI
--- |
define i32 @load_to_cond_branch(ptr %p, i32 %x, i32 %y) {
>From ca71d7b4f9e17bedcdcc1f3a1bcf59c0ee21ca75 Mon Sep 17 00:00:00 2001
From: Jonathan Cohen <joncoh at apple.com>
Date: Tue, 6 Jan 2026 16:16:43 +0200
Subject: [PATCH 05/11] review comments - consider loads from constant pool as
predictable values
---
llvm/lib/CodeGen/EarlyIfConversion.cpp | 18 ++++--
.../early-ifcvt-easy-to-predict.mir | 59 ++++++++++++++++---
2 files changed, 66 insertions(+), 11 deletions(-)
diff --git a/llvm/lib/CodeGen/EarlyIfConversion.cpp b/llvm/lib/CodeGen/EarlyIfConversion.cpp
index 07f2871a0347c..5d54ea412f110 100644
--- a/llvm/lib/CodeGen/EarlyIfConversion.cpp
+++ b/llvm/lib/CodeGen/EarlyIfConversion.cpp
@@ -32,6 +32,7 @@
#include "llvm/CodeGen/MachineOptimizationRemarkEmitter.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/MachineTraceMetrics.h"
+#include "llvm/CodeGen/PseudoSourceValue.h"
#include "llvm/CodeGen/Register.h"
#include "llvm/CodeGen/TargetInstrInfo.h"
#include "llvm/CodeGen/TargetRegisterInfo.h"
@@ -943,10 +944,19 @@ bool EarlyIfConverter::doOperandsComeFromMemory(Register Reg) {
// Check if this instruction is a load.
if (MI->mayLoad()) {
LLVM_DEBUG(dbgs() << "Operand comes from load: " << *MI);
- // Cache all visited virtual registers as coming from memory.
- for (Register Reg : VisitedRegs)
- OperandMemoryCache[Reg] = true;
- return true;
+ // Check if the load is from a constant pool
+ bool IsConstantPoolLoad = false;
+ for (const auto &MOp : MI->memoperands()) {
+ if (const PseudoSourceValue *PSV = MOp->getPseudoValue())
+ if (PSV->isConstantPool()) {
+ LLVM_DEBUG(dbgs() << "Found constant pool load!\n");
+ IsConstantPoolLoad = true;
+ }
+ // Cache all visited virtual registers as coming from memory.
+ for (Register Reg : VisitedRegs)
+ OperandMemoryCache[Reg] = !IsConstantPoolLoad;
+ return !IsConstantPoolLoad;
+ }
}
// Walk through all register use operands and find their definitions.
diff --git a/llvm/test/CodeGen/AArch64/early-ifcvt-cascade-tests/early-ifcvt-easy-to-predict.mir b/llvm/test/CodeGen/AArch64/early-ifcvt-cascade-tests/early-ifcvt-easy-to-predict.mir
index 2560c4ade5c73..63f3af6336b2d 100644
--- a/llvm/test/CodeGen/AArch64/early-ifcvt-cascade-tests/early-ifcvt-easy-to-predict.mir
+++ b/llvm/test/CodeGen/AArch64/early-ifcvt-cascade-tests/early-ifcvt-easy-to-predict.mir
@@ -1,16 +1,13 @@
# RUN: llc -mtriple=aarch64 -mcpu=apple-m1 -run-pass=early-ifcvt -o - %s | FileCheck %s
-# CHECK: bb.0:
-# CHECK: SUBSWri
-# CHECK: Bcc
-# CHECK: bb.1:
-# CHECK: bb.2:
-# CHECK: PHI
-
--- |
define i32 @arg_to_cond_branch(i32 %val, i32 %x, i32 %y) {
ret i32 0
}
+
+ define i32 @load_from_constant_pool_to_cond_branch(ptr %p, i32 %x, i32 %y) {
+ ret i32 0
+ }
...
---
name: arg_to_cond_branch
@@ -21,6 +18,14 @@ liveins:
- { reg: '$w1', virtual-reg: '%1' }
- { reg: '$w2', virtual-reg: '%2' }
body: |
+ ; CHECK-LABEL: name: arg_to_cond_branch
+ ; CHECK: bb.0:
+ ; CHECK: SUBSWri
+ ; CHECK: Bcc
+ ; CHECK: bb.1:
+ ; CHECK: bb.2:
+ ; CHECK: PHI
+
bb.1:
successors: %bb.2(0x40000000), %bb.3(0x40000000)
liveins: $w0, $w1, $w2
@@ -43,3 +48,43 @@ body: |
$w0 = COPY %100
RET_ReallyLR implicit $w0
...
+---
+name: load_from_constant_pool_to_cond_branch
+alignment: 4
+tracksRegLiveness: true
+constants:
+ - id: 0
+ value: i32 100
+ alignment: 4
+body: |
+ ; CHECK-LABEL: name: load_from_constant_pool_to_cond_branch
+ ; CHECK: bb.0:
+ ; CHECK: SUBSWrr
+ ; CHECK: Bcc
+ ; CHECK: bb.1:
+ ; CHECK: bb.2:
+ ; CHECK: PHI
+ bb.0:
+ successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ liveins: $w0, $w1, $w2
+
+ %0:gpr32 = COPY $w0
+ %1:gpr32 = COPY $w1
+ %2:gpr32 = COPY $w2
+ renamable $x8 = ADRP target-flags(aarch64-page) %const.0
+ %10:gpr32common = LDRWui killed renamable $x8, target-flags(aarch64-pageoff, aarch64-nc) %const.0 :: (load (s32) from constant-pool)
+ %11:gpr32 = SUBSWrr %0, %10, implicit-def $nzcv
+ Bcc 12, %bb.2, implicit $nzcv
+ B %bb.1
+
+ bb.1:
+ successors: %bb.2(0x80000000)
+
+ %20:gpr32 = MADDWrrr %1, %2, $wzr
+ %21:gpr32 = MADDWrrr %20, %1, $wzr
+
+ bb.2:
+ %100:gpr32 = PHI %1, %bb.0, %21, %bb.1
+ $w0 = COPY %100
+ RET_ReallyLR implicit $w0
+...
>From 39618803fbee1300cf573a7ad7e0000ee3a6b04d Mon Sep 17 00:00:00 2001
From: Jonathan Cohen <joncoh at apple.com>
Date: Tue, 6 Jan 2026 23:35:12 +0200
Subject: [PATCH 06/11] Add test for new remark
---
.../CodeGen/AArch64/early-ifcvt-remarks.mir | 44 +++++++++++++++++++
1 file changed, 44 insertions(+)
create mode 100644 llvm/test/CodeGen/AArch64/early-ifcvt-remarks.mir
diff --git a/llvm/test/CodeGen/AArch64/early-ifcvt-remarks.mir b/llvm/test/CodeGen/AArch64/early-ifcvt-remarks.mir
new file mode 100644
index 0000000000000..63d61ce9bd47e
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/early-ifcvt-remarks.mir
@@ -0,0 +1,44 @@
+# RUN: llc -mtriple=aarch64 -mcpu=apple-m1 -run-pass=early-ifcvt \
+# RUN: -pass-remarks-analysis='early-ifcvt' -o /dev/null %s 2>&1 | FileCheck %s
+
+# Test that we emit an analysis remark when the branch condition is data-dependent
+# (derived from a memory load).
+
+# CHECK: remark: <unknown>:0:0: branch condition is data-dependent (from memory load), using higher CritLimit of {{[0-9]+}} cycles
+
+--- |
+ define i32 @data_dependent_branch(ptr %p, i32 %x, i32 %y) {
+ ret i32 0
+ }
+...
+---
+name: data_dependent_branch
+alignment: 4
+tracksRegLiveness: true
+liveins:
+ - { reg: '$x0', virtual-reg: '%0' }
+ - { reg: '$w1', virtual-reg: '%1' }
+ - { reg: '$w2', virtual-reg: '%2' }
+body: |
+ bb.0:
+ successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ liveins: $x0, $w1, $w2
+
+ %0:gpr64common = COPY $x0
+ %1:gpr32 = COPY $w1
+ %2:gpr32 = COPY $w2
+ %10:gpr32common = LDRWui %0, 0 :: (load (s32))
+ %11:gpr32 = SUBSWri %10, 100, 0, implicit-def $nzcv
+ Bcc 12, %bb.2, implicit $nzcv
+ B %bb.1
+
+ bb.1:
+ successors: %bb.2(0x80000000)
+
+ %20:gpr32 = MADDWrrr %1, %2, $wzr
+
+ bb.2:
+ %100:gpr32 = PHI %1, %bb.0, %20, %bb.1
+ $w0 = COPY %100
+ RET_ReallyLR implicit $w0
+...
>From cde8376ab919621dca738a835742cfdcf226bc94 Mon Sep 17 00:00:00 2001
From: Jonathan Cohen <joncoh at apple.com>
Date: Wed, 7 Jan 2026 00:11:16 +0200
Subject: [PATCH 07/11] Formatting, comments and moved test files
---
llvm/lib/CodeGen/EarlyIfConversion.cpp | 27 +++++-----
.../early-ifcvt-hard-to-predict.mir | 49 -------------------
...ct.mir => early-ifcvt-load-to-cond-br.mir} | 43 ++++++++++++++--
3 files changed, 51 insertions(+), 68 deletions(-)
delete mode 100644 llvm/test/CodeGen/AArch64/early-ifcvt-cascade-tests/early-ifcvt-hard-to-predict.mir
rename llvm/test/CodeGen/AArch64/{early-ifcvt-cascade-tests/early-ifcvt-easy-to-predict.mir => early-ifcvt-load-to-cond-br.mir} (70%)
diff --git a/llvm/lib/CodeGen/EarlyIfConversion.cpp b/llvm/lib/CodeGen/EarlyIfConversion.cpp
index 5d54ea412f110..cc8269b9c01f0 100644
--- a/llvm/lib/CodeGen/EarlyIfConversion.cpp
+++ b/llvm/lib/CodeGen/EarlyIfConversion.cpp
@@ -819,7 +819,7 @@ class EarlyIfConverter {
MachineTraceMetrics::Ensemble *MinInstr = nullptr;
SSAIfConv IfConv;
- // Cache for operand memory analysis.
+ /// Cache for operand memory analysis.
DenseMap<Register, bool> OperandMemoryCache;
public:
@@ -910,7 +910,10 @@ void EarlyIfConverter::invalidateTraces() {
}
/// Check if a register's value comes from a memory load by walking the
-/// def-use chain. Results are cached for virtual registers only.
+/// def-use chain. We want to prioritize converting branches which
+/// depend on values loaded from memory (unless they are loop invariant,
+/// or come from a constant pool).
+/// Results are cached for virtual registers only.
bool EarlyIfConverter::doOperandsComeFromMemory(Register Reg) {
if (!Reg.isVirtual())
return false;
@@ -978,10 +981,11 @@ bool EarlyIfConverter::doOperandsComeFromMemory(Register Reg) {
continue;
}
- MachineInstr *UseDef = MRI->getVRegDef(UseReg);
- if (UseDef && !Visited.count(UseDef)) {
- Worklist.push_back(UseDef);
- VisitedRegs.push_back(UseReg);
+ if (MachineInstr *UseDef = MRI->getVRegDef(UseReg)) {
+ if (!Visited.count(UseDef)) {
+ Worklist.push_back(UseDef);
+ VisitedRegs.push_back(UseReg);
+ }
}
}
}
@@ -1020,7 +1024,7 @@ bool EarlyIfConverter::isConditionDataDependent() {
} else if (Reg.isPhysical() && TRI->regsOverlap(Reg, StatusFlagsReg)) {
// For the status flags register (e.g., NZCV on AArch64),
// search backwards to find the defining instruction.
- for (auto I = std::prev(BranchIt);; --I) {
+ for (auto I = std::prev(BranchIt); I != IfConv.Head->begin(); --I) {
// Check if this instruction defines the status flags register.
bool FoundDef = false;
for (const MachineOperand &DefMO : I->operands()) {
@@ -1037,17 +1041,12 @@ bool EarlyIfConverter::isConditionDataDependent() {
continue;
Register DefReg = DefMO.getReg();
if (DefReg.isVirtual() && doOperandsComeFromMemory(DefReg)) {
- LLVM_DEBUG(dbgs()
- << "Condition is data-dependent (from memory)\n");
++NumDataDependant;
return true;
}
}
break;
}
-
- if (I == IfConv.Head->begin())
- break;
}
}
}
@@ -1125,9 +1124,9 @@ bool EarlyIfConverter::shouldConvertIf() {
// When hard-to-predict analysis is enabled, use full MispredictPenalty for
// hard-to-predict branches, half for others. Otherwise use half for all.
bool DataDependent = false;
- if (EnableHardToPredictAnalysis) {
+ if (EnableHardToPredictAnalysis)
DataDependent = isConditionDataDependent();
- }
+
unsigned CritLimit = DataDependent ? SchedModel.MispredictPenalty
: SchedModel.MispredictPenalty / 2;
diff --git a/llvm/test/CodeGen/AArch64/early-ifcvt-cascade-tests/early-ifcvt-hard-to-predict.mir b/llvm/test/CodeGen/AArch64/early-ifcvt-cascade-tests/early-ifcvt-hard-to-predict.mir
deleted file mode 100644
index 0cffedf4533fc..0000000000000
--- a/llvm/test/CodeGen/AArch64/early-ifcvt-cascade-tests/early-ifcvt-hard-to-predict.mir
+++ /dev/null
@@ -1,49 +0,0 @@
-# RUN: llc -mtriple=aarch64 -mcpu=apple-m1 -run-pass=early-ifcvt -o - %s | FileCheck %s
-
-# CHECK-LABEL: name: load_to_cond_branch
-# CHECK: bb.0:
-# CHECK: LDRWui
-# CHECK: SUBSWri
-# CHECK: MADDWrrr
-# CHECK: MADDWrrr
-# CHECK: CSELWr
-# CHECK-NOT: Bcc
-# CHECK-NOT: PHI
-
---- |
- define i32 @load_to_cond_branch(ptr %p, i32 %x, i32 %y) {
- ret i32 0
- }
-...
----
-name: load_to_cond_branch
-alignment: 4
-tracksRegLiveness: true
-liveins:
- - { reg: '$x0', virtual-reg: '%0' }
- - { reg: '$w1', virtual-reg: '%1' }
- - { reg: '$w2', virtual-reg: '%2' }
-body: |
- bb.1:
- successors: %bb.2(0x40000000), %bb.3(0x40000000)
- liveins: $x0, $w1, $w2
-
- %0:gpr64common = COPY $x0
- %1:gpr32 = COPY $w1
- %2:gpr32 = COPY $w2
- %10:gpr32common = LDRWui %0, 0 :: (load (s32))
- %11:gpr32 = SUBSWri %10, 100, 0, implicit-def $nzcv
- Bcc 12, %bb.3, implicit $nzcv
- B %bb.2
-
- bb.2:
- successors: %bb.3(0x80000000)
-
- %20:gpr32 = MADDWrrr %1, %2, $wzr
- %21:gpr32 = MADDWrrr %20, %1, $wzr
-
- bb.3:
- %100:gpr32 = PHI %1, %bb.1, %21, %bb.2
- $w0 = COPY %100
- RET_ReallyLR implicit $w0
-...
diff --git a/llvm/test/CodeGen/AArch64/early-ifcvt-cascade-tests/early-ifcvt-easy-to-predict.mir b/llvm/test/CodeGen/AArch64/early-ifcvt-load-to-cond-br.mir
similarity index 70%
rename from llvm/test/CodeGen/AArch64/early-ifcvt-cascade-tests/early-ifcvt-easy-to-predict.mir
rename to llvm/test/CodeGen/AArch64/early-ifcvt-load-to-cond-br.mir
index 63f3af6336b2d..67f332e948a20 100644
--- a/llvm/test/CodeGen/AArch64/early-ifcvt-cascade-tests/early-ifcvt-easy-to-predict.mir
+++ b/llvm/test/CodeGen/AArch64/early-ifcvt-load-to-cond-br.mir
@@ -4,19 +4,17 @@
define i32 @arg_to_cond_branch(i32 %val, i32 %x, i32 %y) {
ret i32 0
}
-
define i32 @load_from_constant_pool_to_cond_branch(ptr %p, i32 %x, i32 %y) {
ret i32 0
}
+ define i32 @load_to_cond_branch(ptr %p, i32 %x, i32 %y) {
+ ret i32 0
+ }
...
---
name: arg_to_cond_branch
alignment: 4
tracksRegLiveness: true
-liveins:
- - { reg: '$w0', virtual-reg: '%0' }
- - { reg: '$w1', virtual-reg: '%1' }
- - { reg: '$w2', virtual-reg: '%2' }
body: |
; CHECK-LABEL: name: arg_to_cond_branch
; CHECK: bb.0:
@@ -88,3 +86,38 @@ body: |
$w0 = COPY %100
RET_ReallyLR implicit $w0
...
+---
+name: load_to_cond_branch
+alignment: 4
+tracksRegLiveness: true
+body: |
+ ; CHECK: bb.0:
+ ; CHECK: LDRWui
+ ; CHECK: SUBSWri
+ ; CHECK: MADDWrrr
+ ; CHECK: MADDWrrr
+ ; CHECK: CSELWr
+ ; CHECK-NOT: Bcc
+ ; CHECK-NOT: PHI
+ bb.1:
+ successors: %bb.2(0x40000000), %bb.3(0x40000000)
+ liveins: $x0, $w1, $w2
+
+ %0:gpr64common = COPY $x0
+ %1:gpr32 = COPY $w1
+ %2:gpr32 = COPY $w2
+ %10:gpr32common = LDRWui %0, 0 :: (load (s32))
+ %11:gpr32 = SUBSWri %10, 100, 0, implicit-def $nzcv
+ Bcc 12, %bb.3, implicit $nzcv
+ B %bb.2
+
+ bb.2:
+ successors: %bb.3(0x80000000)
+
+ %20:gpr32 = MADDWrrr %1, %2, $wzr
+ %21:gpr32 = MADDWrrr %20, %1, $wzr
+
+ bb.3:
+ %100:gpr32 = PHI %1, %bb.1, %21, %bb.2
+ $w0 = COPY %100
+ RET_ReallyLR implicit $w0
>From e2ccfa0873be02f7fd58f4c65e6af5c3df9f3640 Mon Sep 17 00:00:00 2001
From: Jonathan Cohen <joncoh at apple.com>
Date: Wed, 7 Jan 2026 20:19:47 +0200
Subject: [PATCH 08/11] [AArch64InstrInfo] Support returning ConditionDef from
analyzeBranchPredicate
Also add support for the `Bcc` instruction
---
llvm/lib/Target/AArch64/AArch64InstrInfo.cpp | 82 ++++++++++++--------
1 file changed, 50 insertions(+), 32 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index 7e1dd8f16b337..c00905eee3cb2 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -523,52 +523,70 @@ bool AArch64InstrInfo::analyzeBranch(MachineBasicBlock &MBB,
bool AArch64InstrInfo::analyzeBranchPredicate(MachineBasicBlock &MBB,
MachineBranchPredicate &MBP,
bool AllowModify) const {
- // For the moment, handle only a block which ends with a cb(n)zx followed by
- // a fallthrough. Why this? Because it is a common form.
- // TODO: Should we handle b.cc?
+ // Use analyzeBranch to validate the branch pattern.
+ MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
+ SmallVector<MachineOperand, 4> Cond;
+ if (analyzeBranch(MBB, TBB, FBB, Cond, AllowModify))
+ return true;
+ // analyzeBranch returns success with empty Cond for unconditional branches.
+ if (Cond.empty())
+ return true;
+
+ MBP.TrueDest = TBB;
+ assert(MBP.TrueDest && "expected!");
+ MBP.FalseDest = FBB ? FBB : MBB.getNextNode();
+
+ MBP.ConditionDef = nullptr;
+ MBP.SingleUseCondition = false;
+
+ // Find the conditional branch. After analyzeBranch succeeds with non-empty
+ // Cond, there's exactly one conditional branch - either last (fallthrough)
+ // or second-to-last (followed by unconditional B).
MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
if (I == MBB.end())
return true;
- // Skip over SpeculationBarrierEndBB terminators
- if (I->getOpcode() == AArch64::SpeculationBarrierISBDSBEndBB ||
- I->getOpcode() == AArch64::SpeculationBarrierSBEndBB) {
+ if (isUncondBranchOpcode(I->getOpcode())) {
+ if (I == MBB.begin())
+ return true;
--I;
}
- if (!isUnpredicatedTerminator(*I))
- return true;
-
- // Get the last instruction in the block.
- MachineInstr *LastInst = &*I;
- unsigned LastOpc = LastInst->getOpcode();
- if (!isCondBranchOpcode(LastOpc))
- return true;
+ MachineInstr *CondBranch = &*I;
+ MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
- switch (LastOpc) {
+ switch (CondBranch->getOpcode()) {
default:
return true;
+
+ case AArch64::Bcc:
+ // Bcc takes the NZCV flag as the operand to branch on, walk up the
+ // instruction stream to find the last instruction to define NZCV.
+ for (MachineInstr &MI : llvm::drop_begin(llvm::reverse(MBB))) {
+ if (MI.modifiesRegister(AArch64::NZCV, /*TRI=*/nullptr)) {
+ MBP.ConditionDef = &MI;
+ break;
+ }
+ }
+ return false;
+
case AArch64::CBZW:
case AArch64::CBZX:
case AArch64::CBNZW:
- case AArch64::CBNZX:
- break;
- };
-
- MBP.TrueDest = LastInst->getOperand(1).getMBB();
- assert(MBP.TrueDest && "expected!");
- MBP.FalseDest = MBB.getNextNode();
-
- MBP.ConditionDef = nullptr;
- MBP.SingleUseCondition = false;
-
- MBP.LHS = LastInst->getOperand(0);
- MBP.RHS = MachineOperand::CreateImm(0);
- MBP.Predicate = (LastOpc == AArch64::CBNZX || LastOpc == AArch64::CBNZW)
- ? MachineBranchPredicate::PRED_NE
- : MachineBranchPredicate::PRED_EQ;
- return false;
+ case AArch64::CBNZX: {
+ MBP.LHS = CondBranch->getOperand(0);
+ MBP.RHS = MachineOperand::CreateImm(0);
+ unsigned Opc = CondBranch->getOpcode();
+ MBP.Predicate = (Opc == AArch64::CBNZX || Opc == AArch64::CBNZW)
+ ? MachineBranchPredicate::PRED_NE
+ : MachineBranchPredicate::PRED_EQ;
+ Register CondReg = MBP.LHS.getReg();
+ if (CondReg.isVirtual())
+ MBP.ConditionDef = MRI.getVRegDef(CondReg);
+ return false;
+ }
+ }
}
bool AArch64InstrInfo::reverseBranchCondition(
>From 38677f0b8139f42d20aea3c8138c644c7a7cc3fc Mon Sep 17 00:00:00 2001
From: Jonathan Cohen <joncoh at apple.com>
Date: Wed, 7 Jan 2026 20:22:22 +0200
Subject: [PATCH 09/11] [CodeGen][EarlyIfConvert] Use the
analyzeBranchPredicate function to get the branch condition operands
---
llvm/lib/CodeGen/EarlyIfConversion.cpp | 69 +++++++----------
.../AArch64/early-ifcvt-load-to-cond-br.mir | 76 +++++++++++++++++++
2 files changed, 102 insertions(+), 43 deletions(-)
diff --git a/llvm/lib/CodeGen/EarlyIfConversion.cpp b/llvm/lib/CodeGen/EarlyIfConversion.cpp
index cc8269b9c01f0..c294754413e28 100644
--- a/llvm/lib/CodeGen/EarlyIfConversion.cpp
+++ b/llvm/lib/CodeGen/EarlyIfConversion.cpp
@@ -998,58 +998,41 @@ bool EarlyIfConverter::doOperandsComeFromMemory(Register Reg) {
/// Check if the branch condition is data-dependent (comes from memory loads).
bool EarlyIfConverter::isConditionDataDependent() {
- MachineBasicBlock::iterator BranchIt = IfConv.Head->getFirstTerminator();
- if (BranchIt == IfConv.Head->end())
+ TargetInstrInfo::MachineBranchPredicate MBP;
+ if (TII->analyzeBranchPredicate(*IfConv.Head, MBP, /*AllowModify=*/false))
return false;
- // Get the status flags register from the target. If the target hasn't
- // implemented this hook, we skip physical register analysis entirely
- // and fallback to the previous behavior.
- MCRegister StatusFlagsReg = TRI->getStatusFlagsRegister();
-
- if (!StatusFlagsReg.isValid())
+ if (!MBP.ConditionDef)
return false;
- for (const MachineOperand &MO : BranchIt->operands()) {
+ // Check if ConditionDef itself is a load (e.g., CBZ with loaded value).
+ if (MBP.ConditionDef->mayLoad()) {
+ // Check if it's a constant pool load. In that case treat it as predictable
+ // since the branch does not depend on potentially random data.
+ bool IsConstantPoolLoad = false;
+ for (const auto &MOp : MBP.ConditionDef->memoperands()) {
+ if (const PseudoSourceValue *PSV = MOp->getPseudoValue())
+ if (PSV->isConstantPool())
+ IsConstantPoolLoad = true;
+ }
+ if (!IsConstantPoolLoad) {
+ ++NumDataDependant;
+ return true;
+ }
+ }
+
+ // Check if any of the operands of the condition defining instruction
+ // come from memory.
+ for (const MachineOperand &MO : MBP.ConditionDef->operands()) {
if (!MO.isReg() || !MO.isUse())
continue;
Register Reg = MO.getReg();
-
- if (Reg.isVirtual()) {
- if (doOperandsComeFromMemory(Reg)) {
- LLVM_DEBUG(dbgs() << "Condition is data-dependent (from memory)\n");
- ++NumDataDependant;
- return true;
- }
- } else if (Reg.isPhysical() && TRI->regsOverlap(Reg, StatusFlagsReg)) {
- // For the status flags register (e.g., NZCV on AArch64),
- // search backwards to find the defining instruction.
- for (auto I = std::prev(BranchIt); I != IfConv.Head->begin(); --I) {
- // Check if this instruction defines the status flags register.
- bool FoundDef = false;
- for (const MachineOperand &DefMO : I->operands()) {
- if (DefMO.isReg() && DefMO.isDef() &&
- TRI->regsOverlap(DefMO.getReg(), StatusFlagsReg)) {
- FoundDef = true;
- break;
- }
- }
-
- if (FoundDef) {
- for (const MachineOperand &DefMO : I->operands()) {
- if (!DefMO.isReg() || !DefMO.isUse())
- continue;
- Register DefReg = DefMO.getReg();
- if (DefReg.isVirtual() && doOperandsComeFromMemory(DefReg)) {
- ++NumDataDependant;
- return true;
- }
- }
- break;
- }
- }
+ if (Reg.isVirtual() && doOperandsComeFromMemory(Reg)) {
+ ++NumDataDependant;
+ return true;
}
}
+
return false;
}
diff --git a/llvm/test/CodeGen/AArch64/early-ifcvt-load-to-cond-br.mir b/llvm/test/CodeGen/AArch64/early-ifcvt-load-to-cond-br.mir
index 67f332e948a20..82b1b1aa3f4d4 100644
--- a/llvm/test/CodeGen/AArch64/early-ifcvt-load-to-cond-br.mir
+++ b/llvm/test/CodeGen/AArch64/early-ifcvt-load-to-cond-br.mir
@@ -10,6 +10,12 @@
define i32 @load_to_cond_branch(ptr %p, i32 %x, i32 %y) {
ret i32 0
}
+ define i32 @cbz_arg_to_cond_branch(i32 %val, i32 %x, i32 %y) {
+ ret i32 0
+ }
+ define i32 @cbz_load_to_cond_branch(ptr %p, i32 %x, i32 %y) {
+ ret i32 0
+ }
...
---
name: arg_to_cond_branch
@@ -121,3 +127,73 @@ body: |
%100:gpr32 = PHI %1, %bb.1, %21, %bb.2
$w0 = COPY %100
RET_ReallyLR implicit $w0
+...
+---
+name: cbz_arg_to_cond_branch
+alignment: 4
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: cbz_arg_to_cond_branch
+ ; CHECK: bb.0:
+ ; CHECK: CBZW
+ ; CHECK: bb.1:
+ ; CHECK: bb.2:
+ ; CHECK: PHI
+
+ bb.0:
+ successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ liveins: $w0, $w1, $w2
+
+ %0:gpr32 = COPY $w0
+ %1:gpr32 = COPY $w1
+ %2:gpr32 = COPY $w2
+ CBZW %0, %bb.2
+ B %bb.1
+
+ bb.1:
+ successors: %bb.2(0x80000000)
+
+ %20:gpr32 = MADDWrrr %1, %2, $wzr
+ %21:gpr32 = MADDWrrr %20, %1, $wzr
+
+ bb.2:
+ %100:gpr32 = PHI %1, %bb.0, %21, %bb.1
+ $w0 = COPY %100
+ RET_ReallyLR implicit $w0
+...
+---
+name: cbz_load_to_cond_branch
+alignment: 4
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: cbz_load_to_cond_branch
+ ; CHECK: bb.0:
+ ; CHECK: LDRWui
+ ; CHECK: MADDWrrr
+ ; CHECK: MADDWrrr
+ ; CHECK: CSELWr
+ ; CHECK-NOT: CBZW
+ ; CHECK-NOT: PHI
+
+ bb.0:
+ successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ liveins: $x0, $w1, $w2
+
+ %0:gpr64common = COPY $x0
+ %1:gpr32 = COPY $w1
+ %2:gpr32 = COPY $w2
+ %10:gpr32 = LDRWui %0, 0 :: (load (s32))
+ CBZW %10, %bb.2
+ B %bb.1
+
+ bb.1:
+ successors: %bb.2(0x80000000)
+
+ %20:gpr32 = MADDWrrr %1, %2, $wzr
+ %21:gpr32 = MADDWrrr %20, %1, $wzr
+
+ bb.2:
+ %100:gpr32 = PHI %1, %bb.0, %21, %bb.1
+ $w0 = COPY %100
+ RET_ReallyLR implicit $w0
+
>From 7f3506a01e74e9795e44a216eea50734280aa93d Mon Sep 17 00:00:00 2001
From: Jonathan Cohen <joncoh at apple.com>
Date: Wed, 7 Jan 2026 21:17:24 +0200
Subject: [PATCH 10/11] [AArch64InstrInfo] Add TB(N)Z support to
`analyzeBranchPredicate`
---
llvm/lib/Target/AArch64/AArch64InstrInfo.cpp | 10 +++
.../AArch64/early-ifcvt-load-to-cond-br.mir | 75 +++++++++++++++++++
2 files changed, 85 insertions(+)
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index c00905eee3cb2..dff43ca57a7c0 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -586,6 +586,16 @@ bool AArch64InstrInfo::analyzeBranchPredicate(MachineBasicBlock &MBB,
MBP.ConditionDef = MRI.getVRegDef(CondReg);
return false;
}
+
+ case AArch64::TBZW:
+ case AArch64::TBZX:
+ case AArch64::TBNZW:
+ case AArch64::TBNZX: {
+ Register CondReg = CondBranch->getOperand(0).getReg();
+ if (CondReg.isVirtual())
+ MBP.ConditionDef = MRI.getVRegDef(CondReg);
+ return false;
+ }
}
}
diff --git a/llvm/test/CodeGen/AArch64/early-ifcvt-load-to-cond-br.mir b/llvm/test/CodeGen/AArch64/early-ifcvt-load-to-cond-br.mir
index 82b1b1aa3f4d4..048f796169758 100644
--- a/llvm/test/CodeGen/AArch64/early-ifcvt-load-to-cond-br.mir
+++ b/llvm/test/CodeGen/AArch64/early-ifcvt-load-to-cond-br.mir
@@ -16,6 +16,12 @@
define i32 @cbz_load_to_cond_branch(ptr %p, i32 %x, i32 %y) {
ret i32 0
}
+ define i32 @tbz_arg_to_cond_branch(i32 %val, i32 %x, i32 %y) {
+ ret i32 0
+ }
+ define i32 @tbz_load_to_cond_branch(ptr %p, i32 %x, i32 %y) {
+ ret i32 0
+ }
...
---
name: arg_to_cond_branch
@@ -197,3 +203,72 @@ body: |
$w0 = COPY %100
RET_ReallyLR implicit $w0
+...
+---
+name: tbz_arg_to_cond_branch
+alignment: 4
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: tbz_arg_to_cond_branch
+ ; CHECK: bb.0:
+ ; CHECK: TBZW
+ ; CHECK: bb.1:
+ ; CHECK: bb.2:
+ ; CHECK: PHI
+
+ bb.0:
+ successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ liveins: $w0, $w1, $w2
+
+ %0:gpr32 = COPY $w0
+ %1:gpr32 = COPY $w1
+ %2:gpr32 = COPY $w2
+ TBZW %0, 0, %bb.2
+ B %bb.1
+
+ bb.1:
+ successors: %bb.2(0x80000000)
+
+ %20:gpr32 = MADDWrrr %1, %2, $wzr
+ %21:gpr32 = MADDWrrr %20, %1, $wzr
+
+ bb.2:
+ %100:gpr32 = PHI %1, %bb.0, %21, %bb.1
+ $w0 = COPY %100
+ RET_ReallyLR implicit $w0
+...
+---
+name: tbz_load_to_cond_branch
+alignment: 4
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: tbz_load_to_cond_branch
+ ; CHECK: bb.0:
+ ; CHECK: LDRWui
+ ; CHECK: MADDWrrr
+ ; CHECK: MADDWrrr
+ ; CHECK: CSELWr
+ ; CHECK-NOT: TBZW
+ ; CHECK-NOT: PHI
+
+ bb.0:
+ successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ liveins: $x0, $w1, $w2
+
+ %0:gpr64common = COPY $x0
+ %1:gpr32 = COPY $w1
+ %2:gpr32 = COPY $w2
+ %10:gpr32 = LDRWui %0, 0 :: (load (s32))
+ TBZW %10, 0, %bb.2
+ B %bb.1
+
+ bb.1:
+ successors: %bb.2(0x80000000)
+
+ %20:gpr32 = MADDWrrr %1, %2, $wzr
+ %21:gpr32 = MADDWrrr %20, %1, $wzr
+
+ bb.2:
+ %100:gpr32 = PHI %1, %bb.0, %21, %bb.1
+ $w0 = COPY %100
+ RET_ReallyLR implicit $w0
>From b547049c9be6664402e39d20238dad190fc17d15 Mon Sep 17 00:00:00 2001
From: Jonathan Cohen <joncoh at apple.com>
Date: Thu, 8 Jan 2026 13:10:53 +0200
Subject: [PATCH 11/11] [CodeGen][EarlyIfConversion] Clear OperandMemoryCache
after successful if-conversion.
---
llvm/lib/CodeGen/EarlyIfConversion.cpp | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/CodeGen/EarlyIfConversion.cpp b/llvm/lib/CodeGen/EarlyIfConversion.cpp
index c294754413e28..15ff16dc0bc8b 100644
--- a/llvm/lib/CodeGen/EarlyIfConversion.cpp
+++ b/llvm/lib/CodeGen/EarlyIfConversion.cpp
@@ -924,8 +924,8 @@ bool EarlyIfConverter::doOperandsComeFromMemory(Register Reg) {
return It->second;
// Walk the def-use chain.
- SmallPtrSet<const MachineInstr *, 16> Visited;
- SmallVector<const MachineInstr *, 16> Worklist;
+ SmallPtrSet<const MachineInstr *, 8> Visited;
+ SmallVector<const MachineInstr *> Worklist;
SmallVector<Register, 16> VisitedRegs;
MachineInstr *DefMI = MRI->getVRegDef(Reg);
@@ -1281,6 +1281,7 @@ bool EarlyIfConverter::tryConvertIf(MachineBasicBlock *MBB) {
for (MachineBasicBlock *MBB : RemoveBlocks)
MBB->eraseFromParent();
updateLoops(Loops, RemoveBlocks);
+ OperandMemoryCache.clear();
}
return Changed;
}
More information about the llvm-commits
mailing list