[llvm] [IndVarSimplify] Widen select-controlled recurrences (PR #209825)
via llvm-commits
llvm-commits at lists.llvm.org
Sun Jul 19 23:37:18 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-transforms
Author: Madhur Amilkanthwar (madhur13490)
<details>
<summary>Changes</summary>
Add widenSelectRecurrence to run a select-controlled recurrence in the
wider type used by its sole wide use, sinking a truncation onto the
narrow uses. SCEV does not classify select-controlled phis as induction
variables, so createWideIV never fires for them; check the structural
preconditions directly and run the widener as a fallback in
simplifyAndExtend.
---
Full diff: https://github.com/llvm/llvm-project/pull/209825.diff
2 Files Affected:
- (modified) llvm/lib/Transforms/Scalar/IndVarSimplify.cpp (+116)
- (added) llvm/test/Transforms/IndVarSimplify/widen-select-recurrence.ll (+152)
``````````diff
diff --git a/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp b/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp
index c92efadded635..0201a6ce680ae 100644
--- a/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp
+++ b/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp
@@ -663,6 +663,115 @@ static void visitIVCast(CastInst *Cast, WideIVInfo &WI,
WI.IsSigned |= IsSigned;
}
+/// Widen a select-controlled recurrence so it runs in the wider type used by
+/// its sole wide use, sinking a truncation onto the narrow uses.
+///
+/// %wide.next is any loop-defined iM value (typically the next-value of a
+/// wider companion IV); %c is the i1 select condition. The matcher only
+/// requires the trunc to be one-use so dropping it is safe.
+///
+/// Before:
+/// loop:
+/// %iv = phi iN [ %init, ph ], [ %sel, latch ]
+/// %ext = sext iN %iv to iM ; %iv's only wide-cast use
+/// %narrow = trunc nsw iM %wide.next to iN ; one-use
+/// %sel = select i1 %c, iN %iv, iN %narrow ; %iv's other use;
+/// ; arms in any order
+///
+/// After:
+/// preheader:
+/// %init.wide = sext iN %init to iM ; one-time widen of init
+/// loop:
+/// %iv.wide = phi iM [ %init.wide, ph ], [ %sel.wide, latch ]
+/// %sel.wide = select i1 %c, iM %iv.wide, iM %wide.next ; now in iM
+/// %sel.trunc = trunc nsw iM %sel.wide to iN ; for narrow consumers
+///
+/// What changed and why:
+/// - The recurrence runs in iM instead of iN. %ext's uses are rewired to
+/// %iv.wide, eliminating the per-iteration sign-extend on the wide use.
+/// - %narrow (trunc feeding the select) is gone; the select consumes
+/// %wide.next directly. %sel.trunc replaces it on the output side, so
+/// any narrow consumer of the recurrence still gets an iN value. Net
+/// trunc count per iteration is unchanged, but the recurrence itself
+/// is now wide.
+/// - %init is sign-extended once in the preheader, never inside the loop.
+///
+/// SCEV does not classify select-controlled phis as induction variables, so
+/// simplifyUsersOfIV's IV walker never visits them and createWideIV never
+/// runs. The structural preconditions are therefore checked here directly.
+static PHINode *
+widenSelectRecurrence(PHINode *PN, LoopInfo *LI,
+ SmallVectorImpl<WeakTrackingVH> &DeadInsts,
+ unsigned &NumWidened) {
+ Type *SrcTy = PN->getType();
+ if (!SrcTy->isIntegerTy() || PN->getNumIncomingValues() != 2 ||
+ !PN->hasNUses(2))
+ return nullptr;
+
+ Loop *L = LI->getLoopFor(PN->getParent());
+ if (!L || PN->getParent() != L->getHeader())
+ return nullptr;
+ BasicBlock *PH = L->getLoopPreheader();
+ BasicBlock *Latch = L->getLoopLatch();
+ if (!PH || !Latch)
+ return nullptr;
+
+ auto *Sel = dyn_cast<SelectInst>(PN->getIncomingValueForBlock(Latch));
+ if (!Sel)
+ return nullptr;
+
+ Value *V;
+ if (!match(Sel, m_c_Select(m_Specific(PN), m_OneUse(m_NSWTrunc(m_Value(V))))))
+ return nullptr;
+
+ // Widen to the truncated arm's source type, which must be a strictly wider
+ // legal integer.
+ Type *DestTy = V->getType();
+ if (!DestTy->isIntegerTy() ||
+ DestTy->getIntegerBitWidth() <= SrcTy->getIntegerBitWidth() ||
+ !PN->getDataLayout().isLegalInteger(DestTy->getIntegerBitWidth()))
+ return nullptr;
+
+ // The non-select use must be a sign-extend to DestTy.
+ SExtInst *Ext = nullptr;
+ for (User *U : PN->users()) {
+ if (U == Sel)
+ continue;
+ Ext = dyn_cast<SExtInst>(U);
+ if (!Ext || Ext->getType() != DestTy)
+ return nullptr;
+ }
+ if (!Ext)
+ return nullptr;
+
+ auto *WidePN =
+ PHINode::Create(DestTy, 2, PN->getName() + ".wide", PN->getIterator());
+ auto *WideInit =
+ CastInst::CreateSExtOrBitCast(PN->getIncomingValueForBlock(PH), DestTy,
+ "", PH->getTerminator()->getIterator());
+ WidePN->addIncoming(WideInit, PH);
+
+ Value *TVal = WidePN, *FVal = V;
+ if (Sel->getTrueValue() != PN)
+ std::swap(TVal, FVal);
+ auto *WideSel =
+ SelectInst::Create(Sel->getCondition(), TVal, FVal,
+ Sel->getName() + ".wide", Sel->getIterator());
+ WidePN->addIncoming(WideSel, Latch);
+
+ auto *Trunc =
+ CastInst::CreateTruncOrBitCast(WideSel, SrcTy, "", Sel->getIterator());
+ Trunc->setHasNoSignedWrap(true);
+
+ Ext->replaceAllUsesWith(WidePN);
+ Sel->replaceAllUsesWith(Trunc);
+ DeadInsts.emplace_back(Ext);
+ DeadInsts.emplace_back(Sel);
+ DeadInsts.emplace_back(PN);
+ ++NumWidened;
+ return WidePN;
+}
+
//===----------------------------------------------------------------------===//
// Live IV Reduction - Minimize IVs live across the loop.
//===----------------------------------------------------------------------===//
@@ -754,6 +863,13 @@ bool IndVarSimplify::simplifyAndExtend(Loop *L,
NumWidened += Widened;
Changed = true;
LoopPhis.push_back(WidePhi);
+ } else if (PHINode *WidePhi = widenSelectRecurrence(
+ WideIVs.back().NarrowIV, LI, DeadInsts, Widened)) {
+ // createWideIV did not fire (the phi is not an AddRec). Try the
+ // structural select-recurrence widener.
+ NumWidened += Widened;
+ Changed = true;
+ LoopPhis.push_back(WidePhi);
}
}
}
diff --git a/llvm/test/Transforms/IndVarSimplify/widen-select-recurrence.ll b/llvm/test/Transforms/IndVarSimplify/widen-select-recurrence.ll
new file mode 100644
index 0000000000000..785a3a439cb10
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/widen-select-recurrence.ll
@@ -0,0 +1,152 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -S -passes='indvars' -verify-loop-info -verify-dom-info -verify-scev | FileCheck %s
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32"
+target triple = "aarch64-unknown-linux-gnu"
+
+; Down-counting floating-point argmin, reduced from the search loop that reaches
+; IndVarSimplify in the real pipeline. The narrow i32 min-index recurrence has a
+; sext feeding the address of the a[minidx] reload and a latch select whose
+; new-value arm is the decremented scan index. SCEV does not classify
+; select-controlled recurrences as induction variables, so the widening relies
+; on a structural match. Once the scan induction is widened, the select's arm
+; becomes a one-use `trunc nsw` of the wide counter; the min-index recurrence
+; should then be widened to i64 and the truncation sunk onto the narrow exit use.
+define i32 @fp_argmin_decreasing(ptr %a, i32 %start, i64 %tc0) {
+; CHECK-LABEL: @fp_argmin_decreasing(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = sext i32 [[START:%.*]] to i64
+; CHECK-NEXT: [[TMP1:%.*]] = sext i32 [[START]] to i64
+; CHECK-NEXT: br label [[LOOP:%.*]]
+; CHECK: loop:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[LOOP]] ], [ [[TMP0]], [[ENTRY:%.*]] ]
+; CHECK-NEXT: [[CNT:%.*]] = phi i64 [ [[TC0:%.*]], [[ENTRY]] ], [ [[CNT_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT: [[MIN_WIDE:%.*]] = phi i64 [ [[TMP1]], [[ENTRY]] ], [ [[MIN_NEXT_WIDE:%.*]], [[LOOP]] ]
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nsw i64 [[INDVARS_IV]], -1
+; CHECK-NEXT: [[SCAN_P0:%.*]] = getelementptr [4 x i8], ptr [[A:%.*]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[SCAN_P:%.*]] = getelementptr i8, ptr [[SCAN_P0]], i64 -8
+; CHECK-NEXT: [[SCAN_V:%.*]] = load float, ptr [[SCAN_P]], align 4
+; CHECK-NEXT: [[MIN_P0:%.*]] = getelementptr [4 x i8], ptr [[A]], i64 [[MIN_WIDE]]
+; CHECK-NEXT: [[MIN_P:%.*]] = getelementptr i8, ptr [[MIN_P0]], i64 -4
+; CHECK-NEXT: [[MIN_V:%.*]] = load float, ptr [[MIN_P]], align 4
+; CHECK-NEXT: [[C:%.*]] = fcmp fast olt float [[SCAN_V]], [[MIN_V]]
+; CHECK-NEXT: [[MIN_NEXT_WIDE]] = select i1 [[C]], i64 [[INDVARS_IV_NEXT]], i64 [[MIN_WIDE]]
+; CHECK-NEXT: [[TMP2:%.*]] = trunc nsw i64 [[MIN_NEXT_WIDE]] to i32
+; CHECK-NEXT: [[CNT_NEXT]] = add nsw i64 [[CNT]], -1
+; CHECK-NEXT: [[AGAIN:%.*]] = icmp sgt i64 [[CNT]], 1
+; CHECK-NEXT: br i1 [[AGAIN]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK: exit:
+; CHECK-NEXT: [[MIN_LCSSA:%.*]] = phi i32 [ [[TMP2]], [[LOOP]] ]
+; CHECK-NEXT: ret i32 [[MIN_LCSSA]]
+;
+entry:
+ br label %loop
+
+loop:
+ %cnt = phi i64 [ %tc0, %entry ], [ %cnt.next, %loop ]
+ %scan = phi i32 [ %start, %entry ], [ %scan.next, %loop ]
+ %min = phi i32 [ %start, %entry ], [ %min.next, %loop ]
+ %scan.next = add nsw i32 %scan, -1
+ %scan.ext = sext i32 %scan to i64
+ %scan.p0 = getelementptr [4 x i8], ptr %a, i64 %scan.ext
+ %scan.p = getelementptr i8, ptr %scan.p0, i64 -8
+ %scan.v = load float, ptr %scan.p, align 4
+ %min.ext = sext i32 %min to i64
+ %min.p0 = getelementptr [4 x i8], ptr %a, i64 %min.ext
+ %min.p = getelementptr i8, ptr %min.p0, i64 -4
+ %min.v = load float, ptr %min.p, align 4
+ %c = fcmp fast olt float %scan.v, %min.v
+ %min.next = select i1 %c, i32 %scan.next, i32 %min
+ %cnt.next = add nsw i64 %cnt, -1
+ %again = icmp sgt i64 %cnt, 1
+ br i1 %again, label %loop, label %exit
+
+exit:
+ %min.lcssa = phi i32 [ %min.next, %loop ]
+ ret i32 %min.lcssa
+}
+
+; Negative: the latch select's new-value arm has multiple uses, so m_OneUse on
+; the trunc fails; the recurrence is not widened.
+define i32 @widen_select_recurrence_multi_use_trunc(ptr %arr) {
+; CHECK-LABEL: @widen_select_recurrence_multi_use_trunc(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br label [[LOOP:%.*]]
+; CHECK: loop:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT: [[MIN_IDX_EXT:%.*]] = sext i32 [[MIN_IDX]] to i64
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr float, ptr [[ARR:%.*]], i64 [[MIN_IDX_EXT]]
+; CHECK-NEXT: [[VAL:%.*]] = load float, ptr [[GEP]], align 4
+; CHECK-NEXT: [[CMP:%.*]] = fcmp olt float [[VAL]], 0.000000e+00
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[NEW_IDX:%.*]] = trunc nsw i64 [[IV_NEXT]] to i32
+; CHECK-NEXT: store i32 [[NEW_IDX]], ptr [[ARR]], align 4
+; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i32 [[NEW_IDX]], i32 [[MIN_IDX]]
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp ne i64 [[IV_NEXT]], 100
+; CHECK-NEXT: br i1 [[EXITCOND]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK: exit:
+; CHECK-NEXT: [[MIN_IDX_NEXT_LCSSA:%.*]] = phi i32 [ [[MIN_IDX_NEXT]], [[LOOP]] ]
+; CHECK-NEXT: ret i32 [[MIN_IDX_NEXT_LCSSA]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %min.idx = phi i32 [ 0, %entry ], [ %min.idx.next, %loop ]
+ %min.idx.ext = sext i32 %min.idx to i64
+ %gep = getelementptr float, ptr %arr, i64 %min.idx.ext
+ %val = load float, ptr %gep, align 4
+ %cmp = fcmp olt float %val, 0.0
+ %iv.next = add nsw i64 %iv, 1
+ %new.idx = trunc nsw i64 %iv.next to i32
+ store i32 %new.idx, ptr %arr, align 4
+ %min.idx.next = select i1 %cmp, i32 %new.idx, i32 %min.idx
+ %trip = icmp slt i64 %iv.next, 100
+ br i1 %trip, label %loop, label %exit
+
+exit:
+ ret i32 %min.idx.next
+}
+
+; Negative: the wide cast user is a zext, not a sext; the widener requires sext.
+define i32 @widen_select_recurrence_zext_user(ptr %arr) {
+; CHECK-LABEL: @widen_select_recurrence_zext_user(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br label [[LOOP:%.*]]
+; CHECK: loop:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT: [[MIN_IDX_EXT:%.*]] = zext i32 [[MIN_IDX]] to i64
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr float, ptr [[ARR:%.*]], i64 [[MIN_IDX_EXT]]
+; CHECK-NEXT: [[VAL:%.*]] = load float, ptr [[GEP]], align 4
+; CHECK-NEXT: [[CMP:%.*]] = fcmp olt float [[VAL]], 0.000000e+00
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[NEW_IDX:%.*]] = trunc nsw i64 [[IV_NEXT]] to i32
+; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i32 [[NEW_IDX]], i32 [[MIN_IDX]]
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp ne i64 [[IV_NEXT]], 100
+; CHECK-NEXT: br i1 [[EXITCOND]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK: exit:
+; CHECK-NEXT: [[MIN_IDX_NEXT_LCSSA:%.*]] = phi i32 [ [[MIN_IDX_NEXT]], [[LOOP]] ]
+; CHECK-NEXT: ret i32 [[MIN_IDX_NEXT_LCSSA]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %min.idx = phi i32 [ 0, %entry ], [ %min.idx.next, %loop ]
+ %min.idx.ext = zext i32 %min.idx to i64
+ %gep = getelementptr float, ptr %arr, i64 %min.idx.ext
+ %val = load float, ptr %gep, align 4
+ %cmp = fcmp olt float %val, 0.0
+ %iv.next = add nsw i64 %iv, 1
+ %new.idx = trunc nsw i64 %iv.next to i32
+ %min.idx.next = select i1 %cmp, i32 %new.idx, i32 %min.idx
+ %trip = icmp slt i64 %iv.next, 100
+ br i1 %trip, label %loop, label %exit
+
+exit:
+ ret i32 %min.idx.next
+}
``````````
</details>
https://github.com/llvm/llvm-project/pull/209825
More information about the llvm-commits
mailing list