[llvm] [InstCombine] Support well-defined recurrences in isGuaranteedNotToBeUndefOrPoison (PR #150420)
Cullen Rhodes via llvm-commits
llvm-commits at lists.llvm.org
Tue Jul 29 02:13:03 PDT 2025
https://github.com/c-rhodes updated https://github.com/llvm/llvm-project/pull/150420
>From 2e21b63a93fe4a3c7de4160f336846853184aece Mon Sep 17 00:00:00 2001
From: Cullen Rhodes <cullen.rhodes at arm.com>
Date: Thu, 17 Jul 2025 09:08:46 +0000
Subject: [PATCH 1/3] [InstCombine] Support well-defined recurrences in
isGuaranteedNotToBeUndefOrPoison
This function currently doesn't detect well-defined recurrences that
can't produce undef or poison. This prevents instcombine from pushing
freezes through GEPs to the ptr which may be undef or poison, when the
offset is a well-defined recurrence, as the tests added in #145541
demonstrate.
This patch fixes this by pulling existing code from
foldFreezeIntoRecurrence in instcombine, which can detect whether a PHI
produces undef/poison, so it can be reused by
isGuaranteedNotToBeUndefOrPoison in ValueTracking.
---
llvm/include/llvm/Analysis/ValueTracking.h | 8 +++
llvm/lib/Analysis/ValueTracking.cpp | 67 +++++++++++++++++++
.../InstCombine/InstructionCombining.cpp | 62 +++--------------
.../Attributor/dereferenceable-1.ll | 4 +-
.../Transforms/Attributor/value-simplify.ll | 8 ++-
.../InstCombine/freeze-landingpad.ll | 9 ++-
llvm/test/Transforms/InstCombine/freeze.ll | 22 +++---
.../runtime-exit-phi-scev-invalidation.ll | 8 +--
.../nontrivial-unswitch-select.ll | 3 +-
9 files changed, 111 insertions(+), 80 deletions(-)
diff --git a/llvm/include/llvm/Analysis/ValueTracking.h b/llvm/include/llvm/Analysis/ValueTracking.h
index 02990a3cb44f7..c37d00380e0e0 100644
--- a/llvm/include/llvm/Analysis/ValueTracking.h
+++ b/llvm/include/llvm/Analysis/ValueTracking.h
@@ -773,6 +773,14 @@ LLVM_ABI bool canCreatePoison(const Operator *Op,
/// impliesPoison returns true.
LLVM_ABI bool impliesPoison(const Value *ValAssumedPoison, const Value *V);
+/// Detect if PN is a recurrence with a start value and some number of backedge
+/// values. We'll check whether we can push the freeze through the backedge
+/// values (possibly dropping poison flags along the way) until we reach the
+/// phi again. In that case, we can move the freeze to the start value.
+LLVM_ABI Use *canFoldFreezeIntoRecurrence(
+ PHINode *PN, DominatorTree *DT, bool &StartNeedsFreeze,
+ SmallVectorImpl<Instruction *> *DropFlags = nullptr);
+
/// Return true if this function can prove that V does not have undef bits
/// and is never poison. If V is an aggregate value or vector, check whether
/// all elements (except padding) are not undef or poison.
diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index af85ce4077ec8..0373e5cd4c301 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -7569,6 +7569,66 @@ bool llvm::impliesPoison(const Value *ValAssumedPoison, const Value *V) {
static bool programUndefinedIfUndefOrPoison(const Value *V, bool PoisonOnly);
+Use *llvm::canFoldFreezeIntoRecurrence(
+ PHINode *PN, DominatorTree *DT, bool &StartNeedsFreeze,
+ SmallVectorImpl<Instruction *> *DropFlags) {
+ // Detect whether this is a recurrence with a start value and some number of
+ // backedge values. We'll check whether we can push the freeze through the
+ // backedge values (possibly dropping poison flags along the way) until we
+ // reach the phi again. In that case, we can move the freeze to the start
+ // value.
+ Use *StartU = nullptr;
+ SmallVector<Value *> Worklist;
+ for (Use &U : PN->incoming_values()) {
+ if (DT && DT->dominates(PN->getParent(), PN->getIncomingBlock(U))) {
+ // Add backedge value to worklist.
+ Worklist.push_back(U.get());
+ continue;
+ }
+
+ // Don't bother handling multiple start values.
+ if (StartU)
+ return nullptr;
+ StartU = &U;
+ }
+
+ if (!StartU || Worklist.empty())
+ return nullptr; // Not a recurrence.
+
+ Value *StartV = StartU->get();
+ BasicBlock *StartBB = PN->getIncomingBlock(*StartU);
+ StartNeedsFreeze = !isGuaranteedNotToBeUndefOrPoison(StartV);
+ // We can't insert freeze if the start value is the result of the
+ // terminator (e.g. an invoke).
+ if (StartNeedsFreeze && StartBB->getTerminator() == StartV)
+ return nullptr;
+
+ SmallPtrSet<Value *, 32> Visited;
+ while (!Worklist.empty()) {
+ Value *V = Worklist.pop_back_val();
+ if (!Visited.insert(V).second)
+ continue;
+
+ if (Visited.size() > 32)
+ return nullptr; // Limit the total number of values we inspect.
+
+ // Assume that PN is non-poison, because it will be after the transform.
+ if (V == PN || isGuaranteedNotToBeUndefOrPoison(V))
+ continue;
+
+ Instruction *I = dyn_cast<Instruction>(V);
+ if (!I || canCreateUndefOrPoison(cast<Operator>(I),
+ /*ConsiderFlagsAndMetadata*/ false))
+ return nullptr;
+
+ if (DropFlags)
+ DropFlags->push_back(I);
+ append_range(Worklist, I->operands());
+ }
+
+ return StartU;
+}
+
static bool isGuaranteedNotToBeUndefOrPoison(
const Value *V, AssumptionCache *AC, const Instruction *CtxI,
const DominatorTree *DT, unsigned Depth, UndefPoisonKind Kind) {
@@ -7657,6 +7717,13 @@ static bool isGuaranteedNotToBeUndefOrPoison(
}
if (IsWellDefined)
return true;
+
+ bool StartNeedsFreeze;
+ if (canFoldFreezeIntoRecurrence(const_cast<PHINode *>(PN),
+ const_cast<DominatorTree *>(DT),
+ StartNeedsFreeze) &&
+ !StartNeedsFreeze)
+ return true;
} else if (!::canCreateUndefOrPoison(Opr, Kind,
/*ConsiderFlagsAndMetadata*/ true) &&
all_of(Opr->operands(), OpCheck))
diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
index e2a9255ca9c6e..e28f2abdede7f 100644
--- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
@@ -4933,7 +4933,8 @@ InstCombinerImpl::pushFreezeToPreventPoisonFromPropagating(FreezeInst &OrigFI) {
// poison.
Value *MaybePoisonOperand = nullptr;
for (Value *V : OrigOpInst->operands()) {
- if (isa<MetadataAsValue>(V) || isGuaranteedNotToBeUndefOrPoison(V) ||
+ if (isa<MetadataAsValue>(V) ||
+ isGuaranteedNotToBeUndefOrPoison(V, &AC, &OrigFI, &DT) ||
// Treat identical operands as a single operand.
(MaybePoisonOperand && MaybePoisonOperand == V))
continue;
@@ -4959,64 +4960,19 @@ InstCombinerImpl::pushFreezeToPreventPoisonFromPropagating(FreezeInst &OrigFI) {
Instruction *InstCombinerImpl::foldFreezeIntoRecurrence(FreezeInst &FI,
PHINode *PN) {
- // Detect whether this is a recurrence with a start value and some number of
- // backedge values. We'll check whether we can push the freeze through the
- // backedge values (possibly dropping poison flags along the way) until we
- // reach the phi again. In that case, we can move the freeze to the start
- // value.
- Use *StartU = nullptr;
- SmallVector<Value *> Worklist;
- for (Use &U : PN->incoming_values()) {
- if (DT.dominates(PN->getParent(), PN->getIncomingBlock(U))) {
- // Add backedge value to worklist.
- Worklist.push_back(U.get());
- continue;
- }
-
- // Don't bother handling multiple start values.
- if (StartU)
- return nullptr;
- StartU = &U;
- }
-
- if (!StartU || Worklist.empty())
- return nullptr; // Not a recurrence.
-
- Value *StartV = StartU->get();
- BasicBlock *StartBB = PN->getIncomingBlock(*StartU);
- bool StartNeedsFreeze = !isGuaranteedNotToBeUndefOrPoison(StartV);
- // We can't insert freeze if the start value is the result of the
- // terminator (e.g. an invoke).
- if (StartNeedsFreeze && StartBB->getTerminator() == StartV)
- return nullptr;
-
- SmallPtrSet<Value *, 32> Visited;
+ bool StartNeedsFreeze;
SmallVector<Instruction *> DropFlags;
- while (!Worklist.empty()) {
- Value *V = Worklist.pop_back_val();
- if (!Visited.insert(V).second)
- continue;
-
- if (Visited.size() > 32)
- return nullptr; // Limit the total number of values we inspect.
-
- // Assume that PN is non-poison, because it will be after the transform.
- if (V == PN || isGuaranteedNotToBeUndefOrPoison(V))
- continue;
-
- Instruction *I = dyn_cast<Instruction>(V);
- if (!I || canCreateUndefOrPoison(cast<Operator>(I),
- /*ConsiderFlagsAndMetadata*/ false))
- return nullptr;
-
- DropFlags.push_back(I);
- append_range(Worklist, I->operands());
- }
+ Use *StartU =
+ canFoldFreezeIntoRecurrence(PN, &DT, StartNeedsFreeze, &DropFlags);
+ if (!StartU)
+ return nullptr;
for (Instruction *I : DropFlags)
I->dropPoisonGeneratingAnnotations();
if (StartNeedsFreeze) {
+ Value *StartV = StartU->get();
+ BasicBlock *StartBB = PN->getIncomingBlock(*StartU);
Builder.SetInsertPoint(StartBB->getTerminator());
Value *FrozenStartV = Builder.CreateFreeze(StartV,
StartV->getName() + ".fr");
diff --git a/llvm/test/Transforms/Attributor/dereferenceable-1.ll b/llvm/test/Transforms/Attributor/dereferenceable-1.ll
index 5bff2a2e6b208..32102e64351d0 100644
--- a/llvm/test/Transforms/Attributor/dereferenceable-1.ll
+++ b/llvm/test/Transforms/Attributor/dereferenceable-1.ll
@@ -95,7 +95,7 @@ define void @deref_phi_growing(ptr dereferenceable(4000) %a) {
; CHECK: for.cond:
; CHECK-NEXT: [[I_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_INC:%.*]] ]
; CHECK-NEXT: [[A_ADDR_0:%.*]] = phi ptr [ [[A]], [[ENTRY]] ], [ [[INCDEC_PTR:%.*]], [[FOR_INC]] ]
-; CHECK-NEXT: call void @deref_phi_user(ptr nonnull [[A_ADDR_0]])
+; CHECK-NEXT: call void @deref_phi_user(ptr noundef nonnull [[A_ADDR_0]])
; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[A_ADDR_0]], align 4
; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[I_0]], [[VAL]]
; CHECK-NEXT: br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]]
@@ -146,7 +146,7 @@ define void @deref_phi_shrinking(ptr dereferenceable(4000) %a) {
; CHECK: for.cond:
; CHECK-NEXT: [[I_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_INC:%.*]] ]
; CHECK-NEXT: [[A_ADDR_0:%.*]] = phi ptr [ [[A]], [[ENTRY]] ], [ [[INCDEC_PTR:%.*]], [[FOR_INC]] ]
-; CHECK-NEXT: call void @deref_phi_user(ptr nonnull [[A_ADDR_0]])
+; CHECK-NEXT: call void @deref_phi_user(ptr noundef nonnull [[A_ADDR_0]])
; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[A_ADDR_0]], align 4
; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[I_0]], [[VAL]]
; CHECK-NEXT: br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]]
diff --git a/llvm/test/Transforms/Attributor/value-simplify.ll b/llvm/test/Transforms/Attributor/value-simplify.ll
index beceab7ce9ed7..981964ec1efed 100644
--- a/llvm/test/Transforms/Attributor/value-simplify.ll
+++ b/llvm/test/Transforms/Attributor/value-simplify.ll
@@ -626,7 +626,7 @@ define internal ptr @test_byval2(ptr byval(%struct.X) %a) {
; CHECK-NEXT: [[A_PRIV:%.*]] = alloca [[STRUCT_X:%.*]], align 8
; CHECK-NEXT: store ptr [[TMP0]], ptr [[A_PRIV]], align 8
; CHECK-NEXT: call void @sync()
-; CHECK-NEXT: [[L:%.*]] = load ptr, ptr [[A_PRIV]], align 8
+; CHECK-NEXT: [[L:%.*]] = load ptr, ptr [[A_PRIV]], align 8, !invariant.load [[META0:![0-9]+]]
; CHECK-NEXT: ret ptr [[L]]
;
call void @sync()
@@ -1359,7 +1359,7 @@ define internal i32 @ret_speculatable_expr(ptr %mem, i32 %a2) {
; CGSCC-SAME: (i32 [[TMP0:%.*]]) #[[ATTR1]] {
; CGSCC-NEXT: [[MEM_PRIV:%.*]] = alloca i32, align 4
; CGSCC-NEXT: store i32 [[TMP0]], ptr [[MEM_PRIV]], align 4
-; CGSCC-NEXT: [[L:%.*]] = load i32, ptr [[MEM_PRIV]], align 4
+; CGSCC-NEXT: [[L:%.*]] = load i32, ptr [[MEM_PRIV]], align 4, !invariant.load [[META0]]
; CGSCC-NEXT: [[MUL:%.*]] = mul i32 [[L]], 13
; CGSCC-NEXT: [[ADD:%.*]] = add i32 [[MUL]], 7
; CGSCC-NEXT: ret i32 [[ADD]]
@@ -1709,3 +1709,7 @@ define i32 @readExtInitZeroInit() {
; CGSCC: attributes #[[ATTR17]] = { nosync }
; CGSCC: attributes #[[ATTR18]] = { nounwind }
;.
+; TUNIT: [[META0]] = !{}
+;.
+; CGSCC: [[META0]] = !{}
+;.
diff --git a/llvm/test/Transforms/InstCombine/freeze-landingpad.ll b/llvm/test/Transforms/InstCombine/freeze-landingpad.ll
index 7221dc14dfaf4..126fd43bbdebf 100644
--- a/llvm/test/Transforms/InstCombine/freeze-landingpad.ll
+++ b/llvm/test/Transforms/InstCombine/freeze-landingpad.ll
@@ -9,19 +9,18 @@ define i32 @propagate_freeze_in_landingpad() personality ptr null {
; CHECK: invoke.bb1:
; CHECK-NEXT: [[X:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[NORMAL_RETURN:%.*]] ]
; CHECK-NEXT: [[RES0:%.*]] = invoke i32 @foo()
-; CHECK-NEXT: to label [[INVOKE_BB2:%.*]] unwind label [[EXCEPTIONAL_RETURN:%.*]]
+; CHECK-NEXT: to label [[INVOKE_BB2:%.*]] unwind label [[EXCEPTIONAL_RETURN:%.*]]
; CHECK: invoke.bb2:
; CHECK-NEXT: [[RES1:%.*]] = invoke i32 @foo()
-; CHECK-NEXT: to label [[NORMAL_RETURN]] unwind label [[EXCEPTIONAL_RETURN]]
+; CHECK-NEXT: to label [[NORMAL_RETURN]] unwind label [[EXCEPTIONAL_RETURN]]
; CHECK: normal_return:
; CHECK-NEXT: [[INC]] = add nuw nsw i32 [[X]], 1
; CHECK-NEXT: br label [[INVOKE_BB1]]
; CHECK: exceptional_return:
; CHECK-NEXT: [[PHI:%.*]] = phi i32 [ [[X]], [[INVOKE_BB1]] ], [ 0, [[INVOKE_BB2]] ]
; CHECK-NEXT: [[LANDING_PAD:%.*]] = landingpad { ptr, i32 }
-; CHECK-NEXT: cleanup
-; CHECK-NEXT: [[FR:%.*]] = freeze i32 [[PHI]]
-; CHECK-NEXT: [[RES:%.*]] = shl i32 [[FR]], 1
+; CHECK-NEXT: cleanup
+; CHECK-NEXT: [[RES:%.*]] = shl nuw i32 [[PHI]], 1
; CHECK-NEXT: ret i32 [[RES]]
;
entry:
diff --git a/llvm/test/Transforms/InstCombine/freeze.ll b/llvm/test/Transforms/InstCombine/freeze.ll
index 3fedead2feab8..671ad32322c0f 100644
--- a/llvm/test/Transforms/InstCombine/freeze.ll
+++ b/llvm/test/Transforms/InstCombine/freeze.ll
@@ -889,17 +889,17 @@ exit: ; preds = %loop
}
; The recurrence for the GEP offset can't produce poison so the freeze should
-; be pushed through to the ptr, but this is not currently supported.
+; be pushed through to the ptr.
define void @fold_phi_gep_phi_offset(ptr %init, ptr %end, i64 noundef %n) {
; CHECK-LABEL: @fold_phi_gep_phi_offset(
; CHECK-NEXT: entry:
+; CHECK-NEXT: [[INIT:%.*]] = freeze ptr [[INIT1:%.*]]
; CHECK-NEXT: br label [[LOOP:%.*]]
; CHECK: loop:
-; CHECK-NEXT: [[I:%.*]] = phi ptr [ [[INIT:%.*]], [[ENTRY:%.*]] ], [ [[I_NEXT_FR:%.*]], [[LOOP]] ]
+; CHECK-NEXT: [[I:%.*]] = phi ptr [ [[INIT]], [[ENTRY:%.*]] ], [ [[I_NEXT_FR:%.*]], [[LOOP]] ]
; CHECK-NEXT: [[OFF:%.*]] = phi i64 [ [[N:%.*]], [[ENTRY]] ], [ [[OFF_NEXT:%.*]], [[LOOP]] ]
; CHECK-NEXT: [[OFF_NEXT]] = shl i64 [[OFF]], 3
-; CHECK-NEXT: [[I_NEXT:%.*]] = getelementptr i8, ptr [[I]], i64 [[OFF_NEXT]]
-; CHECK-NEXT: [[I_NEXT_FR]] = freeze ptr [[I_NEXT]]
+; CHECK-NEXT: [[I_NEXT_FR]] = getelementptr i8, ptr [[I]], i64 [[OFF_NEXT]]
; CHECK-NEXT: [[COND:%.*]] = icmp eq ptr [[I_NEXT_FR]], [[END:%.*]]
; CHECK-NEXT: br i1 [[COND]], label [[LOOP]], label [[EXIT:%.*]]
; CHECK: exit:
@@ -921,18 +921,18 @@ exit: ; preds = %loop
ret void
}
-; Offset is still guaranteed not to be poison, so the freeze could be moved
-; here if we strip inbounds from the GEP, but this is not currently supported.
+; Offset is still guaranteed not to be poison, so the freeze can be moved
+; here if we strip inbounds from the GEP.
define void @fold_phi_gep_inbounds_phi_offset(ptr %init, ptr %end, i64 noundef %n) {
; CHECK-LABEL: @fold_phi_gep_inbounds_phi_offset(
; CHECK-NEXT: entry:
+; CHECK-NEXT: [[INIT:%.*]] = freeze ptr [[INIT1:%.*]]
; CHECK-NEXT: br label [[LOOP:%.*]]
; CHECK: loop:
-; CHECK-NEXT: [[I:%.*]] = phi ptr [ [[INIT:%.*]], [[ENTRY:%.*]] ], [ [[I_NEXT_FR:%.*]], [[LOOP]] ]
+; CHECK-NEXT: [[I:%.*]] = phi ptr [ [[INIT]], [[ENTRY:%.*]] ], [ [[I_NEXT_FR:%.*]], [[LOOP]] ]
; CHECK-NEXT: [[OFF:%.*]] = phi i64 [ [[N:%.*]], [[ENTRY]] ], [ [[OFF_NEXT:%.*]], [[LOOP]] ]
; CHECK-NEXT: [[OFF_NEXT]] = shl i64 [[OFF]], 3
-; CHECK-NEXT: [[I_NEXT:%.*]] = getelementptr inbounds i8, ptr [[I]], i64 [[OFF_NEXT]]
-; CHECK-NEXT: [[I_NEXT_FR]] = freeze ptr [[I_NEXT]]
+; CHECK-NEXT: [[I_NEXT_FR]] = getelementptr i8, ptr [[I]], i64 [[OFF_NEXT]]
; CHECK-NEXT: [[COND:%.*]] = icmp eq ptr [[I_NEXT_FR]], [[END:%.*]]
; CHECK-NEXT: br i1 [[COND]], label [[LOOP]], label [[EXIT:%.*]]
; CHECK: exit:
@@ -994,7 +994,7 @@ define void @fold_phi_multiple_insts(i32 %init, i32 %n) {
; CHECK: loop:
; CHECK-NEXT: [[I:%.*]] = phi i32 [ [[INIT_FR]], [[ENTRY:%.*]] ], [ [[I_NEXT:%.*]], [[LOOP]] ]
; CHECK-NEXT: [[I_SQ:%.*]] = mul i32 [[I]], [[I]]
-; CHECK-NEXT: [[I_NEXT]] = add i32 [[I_SQ]], 1
+; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i32 [[I_SQ]], 1
; CHECK-NEXT: [[COND:%.*]] = icmp eq i32 [[I_NEXT]], [[N:%.*]]
; CHECK-NEXT: br i1 [[COND]], label [[LOOP]], label [[EXIT:%.*]]
; CHECK: exit:
@@ -1127,7 +1127,7 @@ define void @fold_phi_invoke_noundef_start_value(i32 %n) personality ptr undef {
; CHECK-NEXT: to label [[LOOP:%.*]] unwind label [[UNWIND:%.*]]
; CHECK: loop:
; CHECK-NEXT: [[I:%.*]] = phi i32 [ [[INIT]], [[ENTRY:%.*]] ], [ [[I_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i32 [[I]], 1
; CHECK-NEXT: [[COND:%.*]] = icmp eq i32 [[I_NEXT]], [[N:%.*]]
; CHECK-NEXT: br i1 [[COND]], label [[LOOP]], label [[EXIT:%.*]]
; CHECK: unwind:
diff --git a/llvm/test/Transforms/LoopUnroll/runtime-exit-phi-scev-invalidation.ll b/llvm/test/Transforms/LoopUnroll/runtime-exit-phi-scev-invalidation.ll
index a97b39494b2ef..22de6d5a0d6b6 100644
--- a/llvm/test/Transforms/LoopUnroll/runtime-exit-phi-scev-invalidation.ll
+++ b/llvm/test/Transforms/LoopUnroll/runtime-exit-phi-scev-invalidation.ll
@@ -16,13 +16,11 @@ define void @pr56282() {
; CHECK: outer.header:
; CHECK-NEXT: [[OUTER_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[OUTER_IV_NEXT:%.*]], [[INNER_2:%.*]] ]
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[OUTER_IV]], 1
-; CHECK-NEXT: [[TMP1:%.*]] = freeze i64 [[TMP0]]
-; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[TMP1]], -1
-; CHECK-NEXT: [[XTRAITER:%.*]] = and i64 [[TMP1]], 7
-; CHECK-NEXT: [[TMP3:%.*]] = icmp ult i64 [[TMP2]], 7
+; CHECK-NEXT: [[XTRAITER:%.*]] = and i64 [[TMP0]], 7
+; CHECK-NEXT: [[TMP3:%.*]] = icmp ult i64 [[OUTER_IV]], 7
; CHECK-NEXT: br i1 [[TMP3]], label [[OUTER_MIDDLE_UNR_LCSSA:%.*]], label [[OUTER_HEADER_NEW:%.*]]
; CHECK: outer.header.new:
-; CHECK-NEXT: [[UNROLL_ITER:%.*]] = sub i64 [[TMP1]], [[XTRAITER]]
+; CHECK-NEXT: [[UNROLL_ITER:%.*]] = sub i64 [[TMP0]], [[XTRAITER]]
; CHECK-NEXT: br label [[INNER_1_HEADER:%.*]]
; CHECK: inner.1.header:
; CHECK-NEXT: [[INNER_1_IV:%.*]] = phi i64 [ 0, [[OUTER_HEADER_NEW]] ], [ [[INNER_1_IV_NEXT_7:%.*]], [[INNER_1_LATCH_7:%.*]] ]
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch-select.ll b/llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch-select.ll
index c86fa349200c5..7a9c33637521a 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch-select.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch-select.ll
@@ -693,8 +693,7 @@ define dso_local void @select_invariant_outer_loop(i1 noundef zeroext %cond, i32
; CHECK-NEXT: [[I_021_US:%.*]] = phi i32 [ [[INC9_US:%.*]], [[FOR_COND1_FOR_COND_CLEANUP3_CRIT_EDGE_US:%.*]] ], [ 0, [[FOR_COND1_PREHEADER_US_PREHEADER]] ]
; CHECK-NEXT: [[REM_US:%.*]] = and i32 [[I_021_US]], 1
; CHECK-NEXT: [[CMP5_US:%.*]] = icmp eq i32 [[REM_US]], 0
-; CHECK-NEXT: [[CMP5_US_FR:%.*]] = freeze i1 [[CMP5_US]]
-; CHECK-NEXT: br i1 [[CMP5_US_FR]], label [[FOR_COND1_PREHEADER_US_SPLIT_US:%.*]], label [[FOR_COND1_PREHEADER_US_SPLIT:%.*]]
+; CHECK-NEXT: br i1 [[CMP5_US]], label [[FOR_COND1_PREHEADER_US_SPLIT_US:%.*]], label [[FOR_COND1_PREHEADER_US_SPLIT:%.*]]
; CHECK: for.cond1.preheader.us.split.us:
; CHECK-NEXT: br label [[FOR_BODY4_US_US:%.*]]
; CHECK: for.body4.us.us:
>From d7eae333dded35c29b53fba6a414e32736aabffc Mon Sep 17 00:00:00 2001
From: Cullen Rhodes <cullen.rhodes at arm.com>
Date: Fri, 25 Jul 2025 09:29:59 +0000
Subject: [PATCH 2/3] address comments
---
llvm/include/llvm/Analysis/ValueTracking.h | 5 ++++-
llvm/lib/Analysis/ValueTracking.cpp | 17 ++++++++++-------
2 files changed, 14 insertions(+), 8 deletions(-)
diff --git a/llvm/include/llvm/Analysis/ValueTracking.h b/llvm/include/llvm/Analysis/ValueTracking.h
index c37d00380e0e0..a9dcc65d02a07 100644
--- a/llvm/include/llvm/Analysis/ValueTracking.h
+++ b/llvm/include/llvm/Analysis/ValueTracking.h
@@ -50,6 +50,9 @@ constexpr unsigned MaxAnalysisRecursionDepth = 6;
/// getUnderlyingObject().
constexpr unsigned MaxLookupSearchDepth = 10;
+/// The max limit of the search depth in canFoldFreezeIntoRecurrence().
+constexpr unsigned MaxRecurrenceSearchDepth = 32;
+
/// Determine which bits of V are known to be either zero or one and return
/// them in the KnownZero/KnownOne bit sets.
///
@@ -779,7 +782,7 @@ LLVM_ABI bool impliesPoison(const Value *ValAssumedPoison, const Value *V);
/// phi again. In that case, we can move the freeze to the start value.
LLVM_ABI Use *canFoldFreezeIntoRecurrence(
PHINode *PN, DominatorTree *DT, bool &StartNeedsFreeze,
- SmallVectorImpl<Instruction *> *DropFlags = nullptr);
+ SmallVectorImpl<Instruction *> *DropFlags = nullptr, unsigned Depth = 0);
/// Return true if this function can prove that V does not have undef bits
/// and is never poison. If V is an aggregate value or vector, check whether
diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index 0373e5cd4c301..948562c907093 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -7571,7 +7571,7 @@ static bool programUndefinedIfUndefOrPoison(const Value *V, bool PoisonOnly);
Use *llvm::canFoldFreezeIntoRecurrence(
PHINode *PN, DominatorTree *DT, bool &StartNeedsFreeze,
- SmallVectorImpl<Instruction *> *DropFlags) {
+ SmallVectorImpl<Instruction *> *DropFlags, unsigned Depth) {
// Detect whether this is a recurrence with a start value and some number of
// backedge values. We'll check whether we can push the freeze through the
// backedge values (possibly dropping poison flags along the way) until we
@@ -7597,7 +7597,8 @@ Use *llvm::canFoldFreezeIntoRecurrence(
Value *StartV = StartU->get();
BasicBlock *StartBB = PN->getIncomingBlock(*StartU);
- StartNeedsFreeze = !isGuaranteedNotToBeUndefOrPoison(StartV);
+ StartNeedsFreeze = !isGuaranteedNotToBeUndefOrPoison(
+ StartV, /*AC=*/nullptr, /*CtxI=*/nullptr, /*DT=*/nullptr, Depth);
// We can't insert freeze if the start value is the result of the
// terminator (e.g. an invoke).
if (StartNeedsFreeze && StartBB->getTerminator() == StartV)
@@ -7609,11 +7610,13 @@ Use *llvm::canFoldFreezeIntoRecurrence(
if (!Visited.insert(V).second)
continue;
- if (Visited.size() > 32)
+ if (Visited.size() > MaxRecurrenceSearchDepth)
return nullptr; // Limit the total number of values we inspect.
// Assume that PN is non-poison, because it will be after the transform.
- if (V == PN || isGuaranteedNotToBeUndefOrPoison(V))
+ if (V == PN ||
+ isGuaranteedNotToBeUndefOrPoison(V, /*AC=*/nullptr, /*CtxI=*/nullptr,
+ /*DT=*/nullptr, Depth))
continue;
Instruction *I = dyn_cast<Instruction>(V);
@@ -7719,9 +7722,9 @@ static bool isGuaranteedNotToBeUndefOrPoison(
return true;
bool StartNeedsFreeze;
- if (canFoldFreezeIntoRecurrence(const_cast<PHINode *>(PN),
- const_cast<DominatorTree *>(DT),
- StartNeedsFreeze) &&
+ if (canFoldFreezeIntoRecurrence(
+ const_cast<PHINode *>(PN), const_cast<DominatorTree *>(DT),
+ StartNeedsFreeze, /*DropFlags=*/nullptr, Depth) &&
!StartNeedsFreeze)
return true;
} else if (!::canCreateUndefOrPoison(Opr, Kind,
>From f68aa927bc0fc002149a335a144db8c4addf6e84 Mon Sep 17 00:00:00 2001
From: Cullen Rhodes <cullen.rhodes at arm.com>
Date: Tue, 29 Jul 2025 09:11:54 +0000
Subject: [PATCH 3/3] address comments
---
llvm/test/Transforms/InstCombine/freeze.ll | 39 ++++++++++++++++++++--
1 file changed, 37 insertions(+), 2 deletions(-)
diff --git a/llvm/test/Transforms/InstCombine/freeze.ll b/llvm/test/Transforms/InstCombine/freeze.ll
index 671ad32322c0f..bc02e63689207 100644
--- a/llvm/test/Transforms/InstCombine/freeze.ll
+++ b/llvm/test/Transforms/InstCombine/freeze.ll
@@ -460,7 +460,7 @@ invoke.unwind:
define i32 @freeze_callbr_use_after_phi(i1 %c) {
; CHECK-LABEL: @freeze_callbr_use_after_phi(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[X:%.*]] = callbr i32 asm sideeffect "", "=r"() #[[ATTR1:[0-9]+]]
+; CHECK-NEXT: [[X:%.*]] = callbr i32 asm sideeffect "", "=r"() #[[ATTR2:[0-9]+]]
; CHECK-NEXT: to label [[CALLBR_CONT:%.*]] []
; CHECK: callbr.cont:
; CHECK-NEXT: [[PHI:%.*]] = phi i32 [ [[X]], [[ENTRY:%.*]] ], [ 0, [[CALLBR_CONT]] ]
@@ -986,6 +986,40 @@ exit: ; preds = %loop
ret void
}
+; 'assume' says GEP ptr can't produce poison, check freeze is pushed to offset.
+define void @fold_phi_gep_phi_offset_assume(ptr %init, ptr %end, i64 %n) {
+; CHECK-LABEL: @fold_phi_gep_phi_offset_assume(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[N:%.*]] = freeze i64 [[N1:%.*]]
+; CHECK-NEXT: br label [[LOOP:%.*]]
+; CHECK: loop:
+; CHECK-NEXT: [[I:%.*]] = phi ptr [ [[INIT:%.*]], [[ENTRY:%.*]] ], [ [[I_NEXT_FR:%.*]], [[LOOP]] ]
+; CHECK-NEXT: [[OFF:%.*]] = phi i64 [ [[N]], [[ENTRY]] ], [ [[OFF_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT: [[OFF_NEXT]] = shl i64 [[OFF]], 3
+; CHECK-NEXT: call void @llvm.assume(i1 true) [ "noundef"(ptr [[I]]) ]
+; CHECK-NEXT: [[I_NEXT_FR]] = getelementptr i8, ptr [[I]], i64 [[OFF_NEXT]]
+; CHECK-NEXT: [[COND:%.*]] = icmp eq ptr [[I_NEXT_FR]], [[END:%.*]]
+; CHECK-NEXT: br i1 [[COND]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK: exit:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop: ; preds = %loop, %entry
+ %i = phi ptr [ %init, %entry ], [ %i.next.fr, %loop ]
+ %off = phi i64 [ %n, %entry ], [ %off.next, %loop ]
+ %off.next = shl i64 %off, 3
+ call void @llvm.assume(i1 true) [ "noundef"(ptr %i) ]
+ %i.next = getelementptr inbounds i8, ptr %i, i64 %off.next
+ %i.next.fr = freeze ptr %i.next
+ %cond = icmp eq ptr %i.next.fr, %end
+ br i1 %cond, label %loop, label %exit
+
+exit: ; preds = %loop
+ ret void
+}
+
define void @fold_phi_multiple_insts(i32 %init, i32 %n) {
; CHECK-LABEL: @fold_phi_multiple_insts(
; CHECK-NEXT: entry:
@@ -1346,7 +1380,8 @@ define ptr @freeze_ptrmask_nonnull(ptr %p, i64 noundef %m) {
!2 = !{i32 0, i32 100}
;.
; CHECK: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
-; CHECK: attributes #[[ATTR1]] = { nounwind }
+; CHECK: attributes #[[ATTR1:[0-9]+]] = { nocallback nofree nosync nounwind willreturn memory(inaccessiblemem: write) }
+; CHECK: attributes #[[ATTR2]] = { nounwind }
;.
; CHECK: [[META0]] = !{}
; CHECK: [[META1]] = !{i64 4}
More information about the llvm-commits
mailing list