[llvm-branch-commits] [llvm] [LV] Vectorize down-counting floating-point argmin/argmax reductions (PR #209827)
Madhur Amilkanthwar via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Mon Aug 10 03:01:47 PDT 2026
https://github.com/madhur13490 updated https://github.com/llvm/llvm-project/pull/209827
>From f0829eece7f53e22fee5ca358a535d99e6b13d2e Mon Sep 17 00:00:00 2001
From: Madhur Amilkanthwar <madhura at nvidia.com>
Date: Mon, 10 Aug 2026 02:00:14 -0700
Subject: [PATCH 1/2] [GVN] Replace SCEV address recovery with GEP peeling
Avoid introducing ScalarEvolution into GVN by teaching PHITransAddr to
recover affine-equivalent select-arm addresses from nested GEPs.
---
.../llvm/Analysis/MemoryDependenceAnalysis.h | 12 --
llvm/include/llvm/Analysis/PHITransAddr.h | 6 +
llvm/include/llvm/Transforms/Scalar/GVN.h | 3 -
.../lib/Analysis/MemoryDependenceAnalysis.cpp | 81 -------------
llvm/lib/Analysis/PHITransAddr.cpp | 106 +++++++++++++++---
llvm/lib/Transforms/Scalar/GVN.cpp | 6 -
...ll => phitrans-gep-select-load-address.ll} | 16 +--
7 files changed, 100 insertions(+), 130 deletions(-)
rename llvm/test/Transforms/GVN/{scev-select-load-address.ll => phitrans-gep-select-load-address.ll} (79%)
diff --git a/llvm/include/llvm/Analysis/MemoryDependenceAnalysis.h b/llvm/include/llvm/Analysis/MemoryDependenceAnalysis.h
index 98e7f84855b4a..459c8aeb5ab5b 100644
--- a/llvm/include/llvm/Analysis/MemoryDependenceAnalysis.h
+++ b/llvm/include/llvm/Analysis/MemoryDependenceAnalysis.h
@@ -32,7 +32,6 @@ namespace llvm {
class AssumptionCache;
class DominatorTree;
class PHITransAddr;
-class ScalarEvolution;
/// A memory dependence query can return one of three different answers.
class MemDepResult {
@@ -383,12 +382,6 @@ class MemoryDependenceResults {
PredIteratorCache PredCache;
EarliestEscapeAnalysis EEA;
- /// Optional, opt-in ScalarEvolution used only to recover select-dependent
- /// load addresses that are affine-equal (but not syntactically identical) to
- /// an existing pointer. Null unless a client (currently GVN) sets it, so all
- /// other MemDep users are unaffected.
- ScalarEvolution *SE = nullptr;
-
unsigned DefaultBlockScanLimit;
/// Offsets to dependant clobber loads.
@@ -402,11 +395,6 @@ class MemoryDependenceResults {
: AA(AA), AC(AC), TLI(TLI), DT(DT), EEA(DT),
DefaultBlockScanLimit(DefaultBlockScanLimit) {}
- /// Opt in to SCEV-based recovery of affine-equal select-dependent
- /// addresses. Passing null (the default) preserves the syntactic-only
- /// behavior.
- void setScalarEvolution(ScalarEvolution *S) { SE = S; }
-
/// Handle invalidation in the new PM.
LLVM_ABI bool invalidate(Function &F, const PreservedAnalyses &PA,
FunctionAnalysisManager::Invalidator &Inv);
diff --git a/llvm/include/llvm/Analysis/PHITransAddr.h b/llvm/include/llvm/Analysis/PHITransAddr.h
index 90df01269057f..2a45539a70156 100644
--- a/llvm/include/llvm/Analysis/PHITransAddr.h
+++ b/llvm/include/llvm/Analysis/PHITransAddr.h
@@ -146,6 +146,12 @@ class PHITransAddr {
LLVM_ABI bool verify() const;
private:
+ /// Recover an available address when select-arm translation only fails
+ /// because a constant index delta is folded into a nested i8 GEP offset.
+ Value *findAvailableSelectArmAddr(Value *Cond, BasicBlock *CurBB,
+ BasicBlock *PredBB, const DominatorTree *DT,
+ bool CondVal) const;
+
Value *translateSubExpr(Value *V, BasicBlock *CurBB, BasicBlock *PredBB,
const DominatorTree *DT, Value *Cond = nullptr,
bool CondVal = false);
diff --git a/llvm/include/llvm/Transforms/Scalar/GVN.h b/llvm/include/llvm/Transforms/Scalar/GVN.h
index 554d66deecf1c..0275c01b28020 100644
--- a/llvm/include/llvm/Transforms/Scalar/GVN.h
+++ b/llvm/include/llvm/Transforms/Scalar/GVN.h
@@ -49,7 +49,6 @@ class GetElementPtrInst;
class ImplicitControlFlowTracking;
class LoadInst;
class LoopInfo;
-class ScalarEvolution;
class MemDepResult;
class MemoryAccess;
class MemoryDependenceResults;
@@ -261,8 +260,6 @@ class GVNPass : public OptionalPassInfoMixin<GVNPass> {
LoopInfo *LI = nullptr;
AAResults *AA = nullptr;
MemorySSAUpdater *MSSAU = nullptr;
- // Prototype: SCEV handed to MemDep for affine select-address recovery.
- ScalarEvolution *SE = nullptr;
ValueTable VN;
diff --git a/llvm/lib/Analysis/MemoryDependenceAnalysis.cpp b/llvm/lib/Analysis/MemoryDependenceAnalysis.cpp
index 7cf1c7f35879b..6fda89af4867c 100644
--- a/llvm/lib/Analysis/MemoryDependenceAnalysis.cpp
+++ b/llvm/lib/Analysis/MemoryDependenceAnalysis.cpp
@@ -25,8 +25,6 @@
#include "llvm/Analysis/MemoryBuiltins.h"
#include "llvm/Analysis/MemoryLocation.h"
#include "llvm/Analysis/PHITransAddr.h"
-#include "llvm/Analysis/ScalarEvolution.h"
-#include "llvm/Analysis/ScalarEvolutionExpressions.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/IR/BasicBlock.h"
@@ -1058,73 +1056,6 @@ MemoryDependenceResults::lookupNonLocalPointerDepVisited(BasicBlock *BB) const {
return NonLocalPointerDepVisited[BB->getNumber()].first;
}
-// When the syntactic PHI translation of a select-dependent load address fails
-// for one arm, try to recover an equivalent, already-existing address with
-// SCEV. The arm's address is the original address \p A with the recurrence
-// phi (whose \p PredBB incoming is a select on \p Cond) replaced by the value
-// chosen by \p CondVal. If SCEV proves that address equals the SCEV of an
-// existing pointer that is loaded from and dominates \p PredBB's terminator,
-// that pointer is returned so the load-PRE-through-select path can reuse it.
-// This closes the affine-address gap (different index with compensating
-// offsets) that the syntactic operand match cannot see. Returns null on
-// failure.
-static Value *recoverSelectArmAddr(Value *A, Value *Cond, BasicBlock *PredBB,
- bool CondVal, ScalarEvolution &SE,
- const DominatorTree &DT) {
- if (!A || !SE.isSCEVable(A->getType()))
- return nullptr;
-
- const SCEV *S = SE.getSCEV(A);
-
- // Locate the recurrence phi in the address whose PredBB incoming is a select
- // on Cond, and the arm chosen by CondVal.
- PHINode *RecPhi = nullptr;
- Value *Arm = nullptr;
- SCEVExprContains(S, [&](const SCEV *Sub) {
- if (RecPhi)
- return true;
- auto *U = dyn_cast<SCEVUnknown>(Sub);
- if (!U)
- return false;
- auto *PN = dyn_cast<PHINode>(U->getValue());
- if (!PN || PN->getBasicBlockIndex(PredBB) < 0)
- return false;
- auto *SI = dyn_cast<SelectInst>(PN->getIncomingValueForBlock(PredBB));
- if (!SI || SI->getCondition() != Cond)
- return false;
- RecPhi = PN;
- Arm = CondVal ? SI->getTrueValue() : SI->getFalseValue();
- return true;
- });
- if (!RecPhi || !Arm || !SE.isSCEVable(Arm->getType()))
- return nullptr;
-
- // The "keep" arm reproduces the original address exactly.
- if (Arm == RecPhi)
- return A;
-
- ValueToSCEVMapTy Map;
- Map[RecPhi] = SE.getSCEV(Arm);
- const SCEV *Target = SCEVParameterRewriter::rewrite(S, SE, Map);
- if (Target == S)
- return nullptr;
-
- // Return an existing loaded-from pointer with a matching address SCEV.
- for (BasicBlock *BB = PredBB; BB; BB = BB->getSinglePredecessor())
- for (Instruction &I : *BB) {
- auto *LD = dyn_cast<LoadInst>(&I);
- if (!LD)
- continue;
- Value *Ptr = LD->getPointerOperand();
- if (!SE.isSCEVable(Ptr->getType()) ||
- !DT.dominates(LD, PredBB->getTerminator()))
- continue;
- if (SE.getSCEV(Ptr) == Target)
- return Ptr;
- }
- return nullptr;
-}
-
/// Perform a dependency query based on pointer/pointeesize starting at the end
/// of StartBB.
///
@@ -1447,18 +1378,6 @@ bool MemoryDependenceResults::getNonLocalPointerDepFromBB(
if (Value *Cond = PredPointer.getSelectCondition()) {
SelectAddr::SelectAddrs SelAddrs =
PHITransAddr(Pointer).translateValue(BB, Pred, &DT, Cond);
- // If a side failed the syntactic match, try to recover an existing
- // affine-equal address with SCEV (opt-in via setScalarEvolution).
- if (SE && (!SelAddrs.first || !SelAddrs.second)) {
- Value *A = Pointer.getAddr();
- if (!SelAddrs.first)
- SelAddrs.first = recoverSelectArmAddr(A, Cond, Pred,
- /*CondVal=*/true, *SE, DT);
- if (!SelAddrs.second)
- SelAddrs.second =
- recoverSelectArmAddr(A, Cond, Pred,
- /*CondVal=*/false, *SE, DT);
- }
if (SelAddrs.first && SelAddrs.second) {
Result.push_back(NonLocalDepResult(Pred, MemDepResult::getSelect(),
SelectAddr(Cond, SelAddrs)));
diff --git a/llvm/lib/Analysis/PHITransAddr.cpp b/llvm/lib/Analysis/PHITransAddr.cpp
index 03b931a2f0587..203a2d4050d00 100644
--- a/llvm/lib/Analysis/PHITransAddr.cpp
+++ b/llvm/lib/Analysis/PHITransAddr.cpp
@@ -15,12 +15,15 @@
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/Config/llvm-config.h"
#include "llvm/IR/Constants.h"
+#include "llvm/IR/DataLayout.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/Instructions.h"
+#include "llvm/IR/PatternMatch.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/raw_ostream.h"
using namespace llvm;
+using namespace llvm::PatternMatch;
static cl::opt<bool> EnableAddPhiTranslation(
"gvn-add-phi-translation", cl::init(false), cl::Hidden,
@@ -37,6 +40,31 @@ static bool canPHITrans(Instruction *Inst) {
return false;
}
+/// Return an existing GEP of \p SrcTy over \p Ops that dominates \p PredBB.
+static GetElementPtrInst *findAvailableGEP(Type *ResultTy, Type *SrcTy,
+ ArrayRef<Value *> Ops,
+ BasicBlock *CurBB,
+ BasicBlock *PredBB,
+ const DominatorTree *DT) {
+ assert(!Ops.empty() && "GEP needs a pointer operand");
+ Value *Ptr = Ops[0];
+ if (isa<ConstantData>(Ptr))
+ return nullptr;
+
+ for (User *U : Ptr->users()) {
+ auto *GEPI = dyn_cast<GetElementPtrInst>(U);
+ if (!GEPI || GEPI->getType() != ResultTy ||
+ GEPI->getSourceElementType() != SrcTy ||
+ GEPI->getNumOperands() != Ops.size() ||
+ GEPI->getParent()->getParent() != CurBB->getParent() ||
+ (DT && !DT->dominates(GEPI->getParent(), PredBB)))
+ continue;
+ if (std::equal(Ops.begin(), Ops.end(), GEPI->op_begin()))
+ return GEPI;
+ }
+ return nullptr;
+}
+
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
LLVM_DUMP_METHOD void PHITransAddr::dump() const {
if (!Addr) {
@@ -236,22 +264,8 @@ Value *PHITransAddr::translateSubExpr(Value *V, BasicBlock *CurBB,
}
// Scan to see if we have this GEP available.
- Value *APHIOp = GEPOps[0];
- if (isa<ConstantData>(APHIOp))
- return nullptr;
-
- for (User *U : APHIOp->users()) {
- if (GetElementPtrInst *GEPI = dyn_cast<GetElementPtrInst>(U))
- if (GEPI->getType() == GEP->getType() &&
- GEPI->getSourceElementType() == GEP->getSourceElementType() &&
- GEPI->getNumOperands() == GEPOps.size() &&
- GEPI->getParent()->getParent() == CurBB->getParent() &&
- (!DT || DT->dominates(GEPI->getParent(), PredBB))) {
- if (std::equal(GEPOps.begin(), GEPOps.end(), GEPI->op_begin()))
- return GEPI;
- }
- }
- return nullptr;
+ return findAvailableGEP(GEP->getType(), GEP->getSourceElementType(), GEPOps,
+ CurBB, PredBB, DT);
}
// Handle add with a constant RHS.
@@ -347,12 +361,70 @@ SelectAddr::SelectAddrs PHITransAddr::translateValue(BasicBlock *CurBB,
// Work on a copy so that the original address state is preserved and the
// other side can be translated independently.
PHITransAddr Tmp(*this);
- return Tmp.translateSubExpr(Tmp.Addr, CurBB, PredBB, DT, Cond, CondVal);
+ if (Value *V =
+ Tmp.translateSubExpr(Tmp.Addr, CurBB, PredBB, DT, Cond, CondVal))
+ return V;
+ // Syntactic match misses when a constant index delta is folded into an i8
+ // offset. Recover the affine-equal available pointer instead.
+ return findAvailableSelectArmAddr(Cond, CurBB, PredBB, DT, CondVal);
};
return {TranslateSide(/*CondVal=*/true), TranslateSide(/*CondVal=*/false)};
}
+Value *PHITransAddr::findAvailableSelectArmAddr(Value *Cond, BasicBlock *CurBB,
+ BasicBlock *PredBB,
+ const DominatorTree *DT,
+ bool CondVal) const {
+ // Addr must be: gep i8, (gep Ty, Base, Index), Off.
+ auto *Outer = dyn_cast<GetElementPtrInst>(Addr);
+ if (!Outer || Outer->getNumIndices() != 1 ||
+ !Outer->getSourceElementType()->isIntegerTy(8))
+ return nullptr;
+ auto *OffCI = dyn_cast<ConstantInt>(Outer->getOperand(1));
+ auto *Inner = dyn_cast<GetElementPtrInst>(Outer->getPointerOperand());
+ if (!OffCI || !Inner || Inner->getNumIndices() != 1)
+ return nullptr;
+
+ auto *RecPhi = dyn_cast<PHINode>(Inner->getOperand(1));
+ if (!RecPhi || RecPhi->getParent() != CurBB)
+ return nullptr;
+ auto *SI = dyn_cast<SelectInst>(RecPhi->getIncomingValueForBlock(PredBB));
+ if (!SI || SI->getCondition() != Cond)
+ return nullptr;
+
+ Value *Arm = CondVal ? SI->getTrueValue() : SI->getFalseValue();
+ if (Arm == RecPhi)
+ return Addr;
+
+ // Fold add(IV, C) into the outer byte offset and look for that GEP.
+ Value *IV = nullptr;
+ const APInt *C = nullptr;
+ if (!match(Arm, m_c_Add(m_Value(IV), m_APInt(C))))
+ return nullptr;
+
+ Type *Ty = Inner->getSourceElementType();
+ TypeSize ElemSize = DL.getTypeAllocSize(Ty);
+ if (ElemSize.isScalable())
+ return nullptr;
+
+ unsigned BitWidth = OffCI->getBitWidth();
+ APInt TargetOff =
+ OffCI->getValue() +
+ C->sextOrTrunc(BitWidth) * APInt(BitWidth, ElemSize.getFixedValue());
+
+ Value *InnerOps[] = {Inner->getPointerOperand(), IV};
+ auto *InnerAvail =
+ findAvailableGEP(Inner->getType(), Ty, InnerOps, CurBB, PredBB, DT);
+ if (!InnerAvail)
+ return nullptr;
+
+ Value *OuterOps[] = {InnerAvail,
+ ConstantInt::get(Outer->getContext(), TargetOff)};
+ return findAvailableGEP(Outer->getType(), Outer->getSourceElementType(),
+ OuterOps, CurBB, PredBB, DT);
+}
+
Value *PHITransAddr::getSelectCondition() const {
for (Instruction *I : InstInputs)
if (auto *SI = dyn_cast<SelectInst>(I))
diff --git a/llvm/lib/Transforms/Scalar/GVN.cpp b/llvm/lib/Transforms/Scalar/GVN.cpp
index 2c5725a7817dd..517d33bfba103 100644
--- a/llvm/lib/Transforms/Scalar/GVN.cpp
+++ b/llvm/lib/Transforms/Scalar/GVN.cpp
@@ -41,7 +41,6 @@
#include "llvm/Analysis/MemorySSAUpdater.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/Analysis/PHITransAddr.h"
-#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/IR/Attributes.h"
@@ -886,11 +885,6 @@ PreservedAnalyses GVNPass::run(Function &F, FunctionAnalysisManager &AM) {
MSSA = &AM.getResult<MemorySSAAnalysis>(F);
}
auto &ORE = AM.getResult<OptimizationRemarkEmitterAnalysis>(F);
- // Hand SCEV to MemDep so its select-address translation can recover
- // affine-equal addresses (reuses the load-PRE-through-select path).
- SE = &AM.getResult<ScalarEvolutionAnalysis>(F);
- if (MemDep)
- MemDep->setScalarEvolution(SE);
bool Changed = runImpl(F, AC, DT, TLI, AA, MemDep, LI, &ORE,
MSSA ? &MSSA->getMSSA() : nullptr);
if (!Changed)
diff --git a/llvm/test/Transforms/GVN/scev-select-load-address.ll b/llvm/test/Transforms/GVN/phitrans-gep-select-load-address.ll
similarity index 79%
rename from llvm/test/Transforms/GVN/scev-select-load-address.ll
rename to llvm/test/Transforms/GVN/phitrans-gep-select-load-address.ll
index 2786b6759fc10..854a232cbcbf5 100644
--- a/llvm/test/Transforms/GVN/scev-select-load-address.ll
+++ b/llvm/test/Transforms/GVN/phitrans-gep-select-load-address.ll
@@ -4,17 +4,11 @@
target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32"
target triple = "aarch64-unknown-linux-gnu"
-; Down-counting floating-point argmin after IndVarSimplify has widened the index
-; recurrence to i64, as it reaches GVN in the real pipeline. Each iteration
-; reloads a[minidx] to compare against the scanned element. minidx is the latch
-; select of the previous iteration, so the reload address is select-dependent.
-; The address is a typed [4 x i8] GEP composed with a byte-offset i8 GEP, so the
-; select arm is affine-equal to the scanned-element pointer but not syntactically
-; identical: upstream's syntactic PHITransAddr match fails on that arm. MemDep
-; rewrites the recurrence phi with the selected arm's SCEV, proves the reloaded
-; address equals a previously loaded pointer, and forwards the value. The
-; a[minidx] reload should be eliminated and replaced by a running-minimum phi
-; fed by the value select.
+; Down-counting FP argmin after IndVarSimplify widening. The reload of
+; a[minidx] uses gep([4 x i8], ...) + gep(i8, Off), so the select update arm
+; is affine-equal to the scanned pointer but not syntactically identical.
+; PHITransAddr folds the constant index delta into the byte offset and GVN
+; eliminates the reload.
define i32 @fp_argmin_decreasing(ptr %a, i32 %start, i64 %tc0) {
; CHECK-LABEL: @fp_argmin_decreasing(
; CHECK-NEXT: entry:
>From 20b9e608e4f98dead14310f2e76ff623eb966a56 Mon Sep 17 00:00:00 2001
From: Madhur Amilkanthwar <madhura at nvidia.com>
Date: Fri, 3 Jul 2026 06:08:51 -0700
Subject: [PATCH 2/2] [LV] Vectorize down-counting floating-point argmin/argmax
reductions
Extend the multi-use min/max reduction coupling so a floating-point
min/max value reduction can be paired with a down-counting (FindFirst)
index reduction, enabling argmin/argmax vectorization for loops whose
induction counts down.
Analysis (IVDescriptors):
- Recognize select-based FP min/max recurrences, not just the
intrinsic form.
- For the shared-compare argmin/argmax shape (the reduction compare
also feeds an index select), take the required NaN-free and
signed-zero-free facts from the compare when the select itself does
not carry them. Plain min/max reductions keep the strict
flags-on-the-select rule.
VPlan (handleMultiUseReductions):
- Accept a select as the value reduction's min/max operation and read
its value operands accordingly.
- Tolerate the shared compare feeding both the value select and the
index select, skipping the value select when locating the index
select.
- Support the down-counting FindFirst index reduction (UMin/SMin) for
floating-point, reconstructing the stored index from the base
induction plus an additive offset (e.g. iv-1).
- Fall back to scalar safely for shapes that are not handled.
Scope is intentionally limited to keep the change small:
- TODO: floating-point up-counting (FindLast) argmin/argmax.
- TODO: integer down-counting (FindFirst) argmin/argmax.
Add lit tests covering the down-counting floating-point argmin,
including the VF=1 scalar-plan path.
---
llvm/lib/Analysis/IVDescriptors.cpp | 34 ++-
.../Vectorize/VPlanConstruction.cpp | 264 +++++++++++++++---
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 1 +
.../Transforms/Vectorize/VPlanTransforms.cpp | 28 +-
.../AArch64/select-index-decreasing.ll | 127 +++++++++
5 files changed, 404 insertions(+), 50 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/select-index-decreasing.ll
diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp
index 9d30928d1751c..c9cb3fe6fddd0 100644
--- a/llvm/lib/Analysis/IVDescriptors.cpp
+++ b/llvm/lib/Analysis/IVDescriptors.cpp
@@ -319,15 +319,33 @@ static RecurrenceDescriptor getMinMaxRecurrence(PHINode *Phi, Loop *TheLoop,
return {};
RK = CurRK;
- // Check required fast-math flags for FP recurrences.
+ // A select-based min/max whose compare also drives another select is the
+ // argmin/argmax shape: one select carries the value, the other the index.
+ // Detect it here so the fast-math check and chain handling treat it as a
+ // multi-use reduction rather than a plain one.
+ auto *SI = dyn_cast<SelectInst>(Cur);
+ bool FeedsOtherSelect =
+ SI && any_of(SI->getCondition()->users(),
+ [&](User *U) { return U != SI && isa<SelectInst>(U); });
+
+ // Check required fast-math flags for FP recurrences. A plain min/max
+ // reduction must carry the flags on the select itself. Only for the
+ // shared-compare argmin/argmax shape do the NaN-free/signed-zero-free facts
+ // legitimately live on the compare, so fall back to the condition's flags
+ // there.
if (RecurrenceDescriptor::isFPMinMaxRecurrenceKind(CurRK)) {
auto CurFMF = hasRequiredFastMathFlags(cast<FPMathOperator>(Cur), RK);
+ if (!CurFMF && FeedsOtherSelect)
+ if (auto *FC = dyn_cast<FCmpInst>(SI->getCondition()))
+ CurFMF = hasRequiredFastMathFlags(cast<FPMathOperator>(FC), RK);
if (!CurFMF)
return {};
FMF &= *CurFMF;
}
- if (auto *SI = dyn_cast<SelectInst>(I))
+ // Keep the compare external only for the argmin/argmax shape, so the
+ // multi-use reduction path below can recognize it.
+ if (SI && !FeedsOtherSelect)
Chain.insert(SI->getCondition());
if (A == Phi || B == Phi)
@@ -355,12 +373,18 @@ static RecurrenceDescriptor getMinMaxRecurrence(PHINode *Phi, Loop *TheLoop,
GetMinMaxRK(U, A, B) == RecurKind::None;
});
if (PhiHasInvalidUses) {
- if (!RecurrenceDescriptor::isIntMinMaxRecurrenceKind(RK) ||
- !BackedgeValue->hasOneUse())
+ // Accept integer (llvm.smin/smax intrinsic) and floating-point
+ // (select-based) min/max value reductions with a single-use backedge value.
+ // NaN-free semantics are required to reorder the parallel FP reduction.
+ bool IsIntArgmin = RecurrenceDescriptor::isIntMinMaxRecurrenceKind(RK);
+ bool IsFPArgmin =
+ RecurrenceDescriptor::isFPMinMaxRecurrenceKind(RK) && FMF.noNaNs();
+ if ((!IsIntArgmin && !IsFPArgmin) || !BackedgeValue->hasOneUse())
return {};
return RecurrenceDescriptor(
Phi->getIncomingValueForBlock(TheLoop->getLoopPreheader()),
- /*Exit=*/nullptr, /*Store=*/nullptr, RK, FastMathFlags(),
+ /*Exit=*/nullptr, /*Store=*/nullptr, RK,
+ IsIntArgmin ? FastMathFlags() : FMF,
/*ExactFP=*/nullptr, Phi->getType(), /*IsMultiUse=*/true);
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 1a135b8549514..b14cf058eac23 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -1626,6 +1626,36 @@ static VPInstruction *findFindIVSelect(VPValue *BackedgeVal) {
}));
}
+/// Recover the wide induction underlying the value \p IVOp that a FindIV select
+/// stores. \p IVOp is either the induction itself (then \p Offset is left null)
+/// or an affine increment add(induction, loop-invariant), such as the iv-1
+/// stored by a down-counting argmin loop (then \p Offset is set to the value to
+/// add to the reconstructed index). Returns the base induction, or nullptr if
+/// \p IVOp is not a (possibly offset) wide induction.
+static VPWidenIntOrFpInductionRecipe *getFindIVBaseInduction(VPValue *IVOp,
+ VPValue *&Offset) {
+ Offset = nullptr;
+ // A narrowed store keeps the wide IV; any offset is reapplied after the
+ // index is reconstructed.
+ match(IVOp, m_TruncOrSelf(m_VPValue(IVOp)));
+ if (auto *WideIV = dyn_cast<VPWidenIntOrFpInductionRecipe>(IVOp))
+ return WideIV;
+
+ // Otherwise accept add(IV, loop-invariant), e.g. iv-1 as add(iv, -1).
+ VPValue *LHS, *RHS;
+ if (!match(IVOp, m_Add(m_VPValue(LHS), m_VPValue(RHS))))
+ return nullptr;
+ auto *WideIV = dyn_cast<VPWidenIntOrFpInductionRecipe>(LHS);
+ if (!WideIV) {
+ std::swap(LHS, RHS);
+ WideIV = dyn_cast<VPWidenIntOrFpInductionRecipe>(LHS);
+ }
+ if (!WideIV || !RHS->isDefinedOutsideLoopRegions())
+ return nullptr;
+ Offset = RHS;
+ return WideIV;
+}
+
bool VPlanTransforms::handleMaxMinNumReductions(VPlan &Plan) {
auto GetMinOrMaxCompareValue =
[](VPReductionPHIRecipe *RedPhiR) -> VPValue * {
@@ -1926,11 +1956,13 @@ bool VPlanTransforms::handleFindLastReductions(VPlan &Plan) {
/// \p FindIVSelect, \p FindIVCmp, and \p FindIVRdxResult, which are replaced
/// and removed.
/// Returns true if the pattern was handled successfully, false otherwise.
-static bool handleFirstArgMinOrMax(
- VPlan &Plan, VPReductionPHIRecipe *MinOrMaxPhiR,
- VPReductionPHIRecipe *FindLastIVPhiR, VPWidenIntOrFpInductionRecipe *WideIV,
- VPInstruction *MinOrMaxResult, VPInstruction *FindIVSelect,
- VPRecipeBase *FindIVCmp, VPInstruction *FindIVRdxResult) {
+static bool
+handleFirstArgMinOrMax(VPlan &Plan, VPReductionPHIRecipe *MinOrMaxPhiR,
+ VPReductionPHIRecipe *FindLastIVPhiR,
+ VPWidenIntOrFpInductionRecipe *WideIV,
+ VPValue *IndexOffset, VPInstruction *MinOrMaxResult,
+ VPInstruction *FindIVSelect, VPRecipeBase *FindIVCmp,
+ VPInstruction *FindIVRdxResult) {
assert(!FindLastIVPhiR->isInLoop() && !FindLastIVPhiR->isOrdered() &&
"inloop and ordered reductions not supported");
assert(FindLastIVPhiR->getVFScaleFactor() == 1 &&
@@ -1947,9 +1979,24 @@ static bool handleFirstArgMinOrMax(
assert(
match(FindIVSelectR, m_Select(m_VPValue(), m_VPValue(), m_VPValue())) &&
"backedge value must be a select");
- if (FindIVSelectR->getOperand(1) != WideIV &&
- FindIVSelectR->getOperand(2) != WideIV)
- return false;
+ // Identify the select arm that stores the index; the other arm is the
+ // reduction phi. Without an offset the stored arm is WideIV itself (the
+ // exact-match requirement bails cleanly on any other expression); with an
+ // offset it is add(WideIV, offset), so match against the phi instead.
+ unsigned StoredIdx;
+ if (IndexOffset) {
+ if (FindIVSelectR->getOperand(1) == FindLastIVPhiR)
+ StoredIdx = 2;
+ else if (FindIVSelectR->getOperand(2) == FindLastIVPhiR)
+ StoredIdx = 1;
+ else
+ return false;
+ } else {
+ if (FindIVSelectR->getOperand(1) != WideIV &&
+ FindIVSelectR->getOperand(2) != WideIV)
+ return false;
+ StoredIdx = FindIVSelectR->getOperand(1) == WideIV ? 1 : 2;
+ }
// If the original wide IV is not canonical, create a new one. The canonical
// wide IV is guaranteed to not wrap for all lanes that are active in the
@@ -1965,8 +2012,7 @@ static bool handleFirstArgMinOrMax(
WidenCanIV->insertBefore(WideIV);
// Update the select to use the wide canonical IV.
- FindIVSelectR->setOperand(FindIVSelectR->getOperand(1) == WideIV ? 1 : 2,
- WidenCanIV);
+ FindIVSelectR->setOperand(StoredIdx, WidenCanIV);
}
FindLastIVPhiR->setOperand(0, Plan.getPoison(Ty));
@@ -2020,10 +2066,20 @@ static bool handleFirstArgMinOrMax(
// vp<%final.idx> = select vp<%always.false>, ir<10>,
// vp<%scaled.idx>
+ // The min/max value comparisons below must use fcmp for FP recurrences; the
+ // index/IV comparisons stay integer. NaN-free semantics (required to reach
+ // here) make the ordered equality exact.
+ bool IsFPMinMax = RecurrenceDescriptor::isFPMinMaxRecurrenceKind(
+ MinOrMaxPhiR->getRecurrenceKind());
+ auto CreateValueEqCmp = [&](VPBuilder &B, VPValue *A, VPValue *C) {
+ return IsFPMinMax ? B.createFCmp(CmpInst::FCMP_OEQ, A, C)
+ : B.createICmp(CmpInst::ICMP_EQ, A, C);
+ };
+
VPBuilder Builder(FindIVRdxResult);
VPValue *MinOrMaxExiting = MinOrMaxResult->getOperand(0);
auto *FinalMinOrMaxCmp =
- Builder.createICmp(CmpInst::ICMP_EQ, MinOrMaxExiting, MinOrMaxResult);
+ CreateValueEqCmp(Builder, MinOrMaxExiting, MinOrMaxResult);
VPValue *LastIVExiting = FindIVRdxResult->getOperand(0);
VPValue *MaxIV =
Plan.getConstantInt(APInt::getMaxValue(Ty->getIntegerBitWidth()));
@@ -2045,11 +2101,19 @@ static bool handleFirstArgMinOrMax(
FinalCanIV = DerivedIVRecipe;
}
+ // Apply the additive offset of a stored IV increment (e.g. iv-1) so the
+ // reconstructed index matches the value the scalar loop stored.
+ if (IndexOffset)
+ FinalCanIV = Builder.createNaryOp(
+ Instruction::Add, {FinalCanIV, IndexOffset},
+ VPIRFlags(VPIRFlags::WrapFlagsTy(/*HasNUW=*/false, /*HasNSW=*/false)),
+ FindIVRdxResult->getDebugLoc());
+
// If the final min/max value matches its start value, the condition in the
// loop was always false, i.e. no induction value has been selected. If that's
// the case, set the result of the IV reduction to its start value.
- VPValue *AlwaysFalse = Builder.createICmp(CmpInst::ICMP_EQ, MinOrMaxResult,
- MinOrMaxPhiR->getStartValue());
+ VPValue *AlwaysFalse =
+ CreateValueEqCmp(Builder, MinOrMaxResult, MinOrMaxPhiR->getStartValue());
VPValue *FinalIV = Builder.createSelect(
AlwaysFalse, FindIVSelect->getOperand(2), FinalCanIV);
FindIVSelect->replaceAllUsesWith(FinalIV);
@@ -2080,8 +2144,10 @@ bool VPlanTransforms::handleMultiUseReductions(VPlan &Plan,
// min/max operation, and be used only by the select of the FindLastIV
// reduction cycle.
RecurKind RdxKind = MinOrMaxPhiR->getRecurrenceKind();
+ bool IsFPMinMax = RecurrenceDescriptor::isFPMinMaxRecurrenceKind(RdxKind);
assert(
- RecurrenceDescriptor::isIntMinMaxRecurrenceKind(RdxKind) &&
+ (RecurrenceDescriptor::isIntMinMaxRecurrenceKind(RdxKind) ||
+ IsFPMinMax) &&
"only min/max recurrences support users outside the reduction chain");
auto *MinOrMaxOp =
@@ -2089,26 +2155,37 @@ bool VPlanTransforms::handleMultiUseReductions(VPlan &Plan,
if (!MinOrMaxOp)
return false;
- // Check that MinOrMaxOp is a VPWidenIntrinsicRecipe or VPReplicateRecipe
- // with an intrinsic that matches the reduction kind.
+ // The value reduction's backedge op is either a min/max intrinsic (the
+ // canonical form) or, for a select-based FP min/max, a select whose
+ // condition is the reduction compare and whose arms are the value operands.
Intrinsic::ID ExpectedIntrinsicID = getMinMaxReductionIntrinsicOp(RdxKind);
- if (!match(MinOrMaxOp, m_Intrinsic(ExpectedIntrinsicID)))
+ bool IsSelectOp =
+ match(MinOrMaxOp, m_Select(m_VPValue(), m_VPValue(), m_VPValue()));
+ if (!match(MinOrMaxOp, m_Intrinsic(ExpectedIntrinsicID)) && !IsSelectOp)
return false;
// MinOrMaxOp must have 2 users: 1) MinOrMaxPhiR and 2)
// ComputeReductionResult.
assert(MinOrMaxOp->getNumUsers() == 2 &&
"MinOrMaxOp must have exactly 2 users");
- VPValue *MinOrMaxOpValue = MinOrMaxOp->getOperand(0);
+ // A select carries its condition in operand 0, so its value operands are 1
+ // and 2; an intrinsic's are 0 and 1.
+ unsigned FirstValOp = IsSelectOp ? 1 : 0;
+ VPValue *MinOrMaxOpValue = MinOrMaxOp->getOperand(FirstValOp);
if (MinOrMaxOpValue == MinOrMaxPhiR)
- MinOrMaxOpValue = MinOrMaxOp->getOperand(1);
+ MinOrMaxOpValue = MinOrMaxOp->getOperand(FirstValOp + 1);
VPValue *CmpOpA;
VPValue *CmpOpB;
CmpPredicate Pred;
auto *Cmp = dyn_cast_or_null<VPRecipeWithIRFlags>(findUserOf(
MinOrMaxPhiR, m_Cmp(Pred, m_VPValue(CmpOpA), m_VPValue(CmpOpB))));
- if (!Cmp || Cmp->getNumUsers() != 1 ||
+ // The shared compare normally feeds only the FindIV select. A down-counting
+ // (FindFirst) FP argmin lowered via the AnyOf path adds an extra Or user,
+ // which is tolerated: the FindIV select is located explicitly below and the
+ // AnyOf scaffolding is torn down once the reductions are combined. Integer
+ // argmin keeps the strict single-user requirement.
+ if (!Cmp || (!IsFPMinMax && Cmp->getNumUsers() != 1) ||
(CmpOpA != MinOrMaxOpValue && CmpOpB != MinOrMaxOpValue))
return false;
@@ -2127,12 +2204,25 @@ bool VPlanTransforms::handleMultiUseReductions(VPlan &Plan,
"one user must be MinOrMaxOp");
assert(MinOrMaxResult && "MinOrMaxResult must be a user of MinOrMaxOp");
- // Cmp must be used by the select of a FindLastIV chain.
- VPValue *Sel = dyn_cast<VPSingleDefRecipe>(Cmp->getSingleUser());
- VPValue *IVOp, *FindIV;
- if (!Sel || Sel->getNumUsers() != 2 ||
- !match(Sel,
- m_Select(m_Specific(Cmp), m_VPValue(IVOp), m_VPValue(FindIV))))
+ // Locate the FindIV select among the compare's users. There is exactly one
+ // select; a down-counting FP AnyOf reduction additionally uses the compare
+ // in an Or, which is tolerated (and cleaned up) below.
+ VPSingleDefRecipe *Sel = nullptr;
+ VPValue *IVOp = nullptr, *FindIV = nullptr;
+ for (VPUser *U : Cmp->users()) {
+ auto *R = dyn_cast<VPSingleDefRecipe>(U);
+ // Skip the value reduction's own select (select-based min/max); only the
+ // index select should be captured here.
+ if (R == MinOrMaxOp)
+ continue;
+ if (R && match(R, m_Select(m_Specific(Cmp), m_VPValue(IVOp),
+ m_VPValue(FindIV)))) {
+ if (Sel)
+ return false;
+ Sel = R;
+ }
+ }
+ if (!Sel || Sel->getNumUsers() != 2)
return false;
if (!isa<VPReductionPHIRecipe>(FindIV)) {
@@ -2148,21 +2238,38 @@ bool VPlanTransforms::handleMultiUseReductions(VPlan &Plan,
assert(!FindIVPhiR->isInLoop() && !FindIVPhiR->isOrdered() &&
"cannot handle inloop/ordered reductions yet");
- // Check if FindIVPhiR is a FindLast pattern by checking the MinMaxKind
- // on its ComputeReductionResult. SMax/UMax indicates FindLast.
+ // A scalar-only VPlan (VF=1) lowers the stored index as a scalar recipe
+ // rather than a widened induction. Such a plan needs no cross-lane
+ // combining, so skip this reduction; bailing would drop the scalar VPlan
+ // and trip a planner assertion once vector plans are built.
+ if (auto *R = IVOp->getDefiningRecipe())
+ if (isa<VPReplicateRecipe, VPScalarIVStepsRecipe>(R))
+ continue;
+
+ // Classify the index reduction: SMax/UMax is FindLast (up-counting),
+ // SMin/UMin is FindFirst (down-counting). Support is scoped to integer
+ // FindLast (upstream) and FP FindFirst.
+ // TODO: FP up-counting (FindLast) and integer down-counting (FindFirst).
VPInstruction *FindIVResult =
findUserOf<VPInstruction::ComputeReductionResult>(
FindIVPhiR->getBackedgeValue());
assert(FindIVResult &&
"must be able to retrieve the FindIVResult VPInstruction");
RecurKind FindIVMinMaxKind = FindIVResult->getRecurKind();
- if (FindIVMinMaxKind != RecurKind::SMax &&
- FindIVMinMaxKind != RecurKind::UMax)
+ bool IsFindLast = !IsFPMinMax && (FindIVMinMaxKind == RecurKind::SMax ||
+ FindIVMinMaxKind == RecurKind::UMax);
+ bool IsFindFirst = IsFPMinMax && (FindIVMinMaxKind == RecurKind::SMin ||
+ FindIVMinMaxKind == RecurKind::UMin);
+ if (!IsFindLast && !IsFindFirst)
return false;
- // TODO: Support cases where IVOp is the IV increment.
- if (!match(IVOp, m_TruncOrSelf(m_VPValue(IVOp))) ||
- !isa<VPWidenIntOrFpInductionRecipe>(IVOp))
+ // The stored index is the induction WideIV, or (for a down-counting loop
+ // storing the IV increment, e.g. iv-1) add(WideIV, invariant). Recover the
+ // base induction and any additive offset to reapply when the final index is
+ // reconstructed. Offsets are only accepted for FindFirst.
+ VPValue *IndexOffset = nullptr;
+ auto *WideIV = getFindIVBaseInduction(IVOp, IndexOffset);
+ if (!WideIV || (IndexOffset && !IsFindFirst))
return false;
// Check if the predicate is compatible with the reduction kind.
@@ -2176,8 +2283,18 @@ bool VPlanTransforms::handleMultiUseReductions(VPlan &Plan,
return Pred == CmpInst::ICMP_SLE || Pred == CmpInst::ICMP_SLT;
case RecurKind::SMin:
return Pred == CmpInst::ICMP_SGE || Pred == CmpInst::ICMP_SGT;
+ case RecurKind::FMin:
+ case RecurKind::FMinNum:
+ return Pred == CmpInst::FCMP_OGE || Pred == CmpInst::FCMP_OGT ||
+ Pred == CmpInst::FCMP_UGE || Pred == CmpInst::FCMP_UGT;
+ case RecurKind::FMax:
+ case RecurKind::FMaxNum:
+ return Pred == CmpInst::FCMP_OLE || Pred == CmpInst::FCMP_OLT ||
+ Pred == CmpInst::FCMP_ULE || Pred == CmpInst::FCMP_ULT;
default:
- llvm_unreachable("unhandled recurrence kind");
+ // FMinimum/FMaximum and their *Num variants have different NaN and
+ // signed-zero semantics; do not combine them with an index reduction.
+ return false;
}
}();
if (!IsValidKindPred) {
@@ -2192,27 +2309,86 @@ bool VPlanTransforms::handleMultiUseReductions(VPlan &Plan,
return false;
}
- auto *FindIVSelect = findFindIVSelect(FindIVPhiR->getBackedgeValue());
- auto *FindIVCmp = FindIVSelect->getOperand(0)->getDefiningRecipe();
- auto *FindIVRdxResult = cast<VPInstruction>(FindIVCmp->getOperand(0));
+ VPInstruction *FindIVSelect = nullptr;
+ VPRecipeBase *FindIVCmp = nullptr;
+ VPInstruction *FindIVRdxResult = nullptr;
+ if (IsFindFirst) {
+ // For down-counting loops sinking is disabled for the multi-use argmin,
+ // so the min/max index reduction result feeds the middle-block select
+ // directly, for either lowering:
+ // sentinel: select(icmp ne <rdx>, Sentinel), <rdx>, Start
+ // AnyOf: select(freeze(<or-reduce>), <rdx>, Start
+ FindIVRdxResult = FindIVResult;
+ for (VPUser *U : FindIVRdxResult->users()) {
+ auto *R = dyn_cast<VPInstruction>(U);
+ if (R && R->getOpcode() == Instruction::Select &&
+ R->getOperand(1) == FindIVRdxResult) {
+ FindIVSelect = R;
+ break;
+ }
+ }
+ if (!FindIVSelect)
+ return false;
+ FindIVCmp = FindIVSelect->getOperand(0)->getDefiningRecipe();
+ if (!FindIVCmp)
+ return false;
+ } else {
+ FindIVSelect = findFindIVSelect(FindIVPhiR->getBackedgeValue());
+ FindIVCmp = FindIVSelect->getOperand(0)->getDefiningRecipe();
+ FindIVRdxResult = cast<VPInstruction>(FindIVCmp->getOperand(0));
+ }
assert(FindIVSelect->getParent() == MinOrMaxResult->getParent() &&
"both results must be computed in the same block");
+
+ // For the AnyOf lowering FindIVCmp is a freeze of an Or reduction. Capture
+ // that Or reduction result now so its scaffolding can be erased after the
+ // final index is rebuilt below.
+ auto *FindIVCmpI = dyn_cast<VPInstruction>(FindIVCmp);
+ bool IsAnyOf = FindIVCmpI && FindIVCmpI->getOpcode() == Instruction::Freeze;
+ VPValue *OrReduceVal = IsAnyOf ? FindIVCmpI->getOperand(0) : nullptr;
// Reducing to a scalar min or max value is placed right before reducing to
// its scalar iteration, in order to generate instructions that use both
// their operands.
MinOrMaxResult->moveBefore(*FindIVRdxResult->getParent(),
FindIVRdxResult->getIterator());
- bool IsStrictPredicate = ICmpInst::isLT(Pred) || ICmpInst::isGT(Pred);
+ bool IsStrictPredicate =
+ CmpInst::isFPPredicate(Pred)
+ ? (Pred == CmpInst::FCMP_OLT || Pred == CmpInst::FCMP_OGT ||
+ Pred == CmpInst::FCMP_ULT || Pred == CmpInst::FCMP_UGT)
+ : (ICmpInst::isLT(Pred) || ICmpInst::isGT(Pred));
if (IsStrictPredicate) {
- if (!handleFirstArgMinOrMax(Plan, MinOrMaxPhiR, FindIVPhiR,
- cast<VPWidenIntOrFpInductionRecipe>(IVOp),
- MinOrMaxResult, FindIVSelect, FindIVCmp,
- FindIVRdxResult))
+ if (!handleFirstArgMinOrMax(Plan, MinOrMaxPhiR, FindIVPhiR, WideIV,
+ IndexOffset, MinOrMaxResult, FindIVSelect,
+ FindIVCmp, FindIVRdxResult))
return false;
+ // handleFirstArgMinOrMax erased the freeze; for the AnyOf lowering the Or
+ // reduction (result, in-loop Or, and reduction phi) is now dead. Erase
+ // it, breaking the phi<->Or cycle first.
+ if (IsAnyOf) {
+ auto *OrReduce = cast<VPInstruction>(OrReduceVal);
+ auto *OrR = cast<VPSingleDefRecipe>(
+ OrReduce->getOperand(0)->getDefiningRecipe());
+ VPReductionPHIRecipe *AnyOfPhi = nullptr;
+ for (VPValue *Op : OrR->operands())
+ if (auto *P = dyn_cast_or_null<VPReductionPHIRecipe>(
+ Op->getDefiningRecipe()))
+ AnyOfPhi = P;
+ assert(AnyOfPhi && "AnyOf Or must have a reduction phi operand");
+ OrReduce->eraseFromParent();
+ AnyOfPhi->setOperand(1, AnyOfPhi->getOperand(0));
+ OrR->eraseFromParent();
+ AnyOfPhi->eraseFromParent();
+ }
continue;
}
+ // The non-strict lowering below expects the sentinel form; the AnyOf
+ // lowering has no sentinel operand, so bail (only the strict-predicate
+ // argmin/argmax is handled above).
+ if (IsAnyOf)
+ return false;
+
// The reduction using MinOrMaxPhiR needs adjusting to compute the correct
// result:
// 1. We need to find the last IV for which the condition based on the
@@ -2239,7 +2415,9 @@ bool VPlanTransforms::handleMultiUseReductions(VPlan &Plan,
VPBuilder B(FindIVRdxResult);
VPValue *MinOrMaxExiting = MinOrMaxResult->getOperand(0);
auto *FinalMinOrMaxCmp =
- B.createICmp(CmpInst::ICMP_EQ, MinOrMaxExiting, MinOrMaxResult);
+ IsFPMinMax
+ ? B.createFCmp(CmpInst::FCMP_OEQ, MinOrMaxExiting, MinOrMaxResult)
+ : B.createICmp(CmpInst::ICMP_EQ, MinOrMaxExiting, MinOrMaxResult);
VPValue *Sentinel = FindIVCmp->getOperand(1);
VPValue *LastIVExiting = FindIVRdxResult->getOperand(0);
auto *FinalIVSelect =
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 3fbeb7e772a2e..7424493a6976b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -701,6 +701,7 @@ bool VPInstruction::canGenerateScalarForFirstLane() const {
switch (Opcode) {
case Instruction::Freeze:
case Instruction::ICmp:
+ case Instruction::FCmp:
case Instruction::PHI:
case Instruction::Select:
case VPInstruction::BranchOnCond:
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index c526f95311d5f..b89ab577afa6d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6423,6 +6423,18 @@ void VPlanTransforms::optimizeFindIVReductions(VPlan &Plan,
return std::nullopt;
};
+ // For a multi-use FP argmin/argmax loop handleMultiUseReductions rebuilds the
+ // final index from the stored index's induction, so a stored increment's
+ // offset (e.g. iv-1) must stay in the loop rather than being sunk to the
+ // exit. Disable sinking when such a reduction is present.
+ bool LoopHasMultiUseFPMinMax = any_of(
+ VectorLoopRegion->getEntryBasicBlock()->phis(), [](VPRecipeBase &R) {
+ auto *P = dyn_cast<VPReductionPHIRecipe>(&R);
+ return P && P->hasUsesOutsideReductionChain() &&
+ RecurrenceDescriptor::isFPMinMaxRecurrenceKind(
+ P->getRecurrenceKind());
+ });
+
VPValue *HeaderMask = VectorLoopRegion->getHeaderMask();
for (VPRecipeBase &Phi :
make_early_inc_range(VectorLoopRegion->getEntryBasicBlock()->phis())) {
@@ -6457,8 +6469,11 @@ void VPlanTransforms::optimizeFindIVReductions(VPlan &Plan,
continue;
// Check if FindLastExpression is a simple expression of a widened IV. If
- // so, we can track the underlying IV instead and sink the expression.
- auto *IVOfExpressionToSink = getExpressionIV(FindLastExpression);
+ // so, we can track the underlying IV instead and sink the expression. For a
+ // multi-use argmin/argmax the offset must stay in the loop (see above), so
+ // skip sinking in that case.
+ auto *IVOfExpressionToSink =
+ LoopHasMultiUseFPMinMax ? nullptr : getExpressionIV(FindLastExpression);
const SCEV *IVSCEV = vputils::getSCEVExprForVPValue(
IVOfExpressionToSink ? IVOfExpressionToSink : FindLastExpression, PSE,
&L);
@@ -6514,6 +6529,15 @@ void VPlanTransforms::optimizeFindIVReductions(VPlan &Plan,
UseSigned = true;
else if (AR->hasNoUnsignedWrap())
UseSigned = false;
+ else if (LoopHasMultiUseFPMinMax &&
+ AR->getNoWrapFlags(SCEV::FlagNW) != SCEV::FlagAnyWrap)
+ // Only no-self-wrap (FlagNW), not nsw/nuw: the recurrence never wraps
+ // back onto its start, which is weaker than signed/unsigned
+ // monotonicity. The chosen signedness is provisional and safe: the
+ // multi-use coupler reduces a canonical non-negative index instead, so
+ // the final result is independent of it. FlagNW is read off the
+ // plan-independent SCEV so scalar and vector VPlans accept alike.
+ UseSigned = true;
else
continue;
}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/select-index-decreasing.ll b/llvm/test/Transforms/LoopVectorize/AArch64/select-index-decreasing.ll
new file mode 100644
index 0000000000000..9b0bc5aedcfb2
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/select-index-decreasing.ll
@@ -0,0 +1,127 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes=loop-vectorize \
+; RUN: -force-vector-width=4 -force-vector-interleave=1 \
+; RUN: -mtriple=aarch64-unknown-linux-gnu -S %s | FileCheck %s
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32"
+target triple = "aarch64-unknown-linux-gnu"
+
+; Down-counting floating-point argmin as it reaches the loop vectorizer in the
+; real pipeline: IndVarSimplify widened the index recurrence to i64, GVN forwarded
+; the a[minidx] reload into a running-minimum float phi, and InstCombine sank the
+; index truncation onto the exit. The scanned-element address is a typed
+; [4 x i8] GEP composed with a byte-offset i8 GEP. The value reduction is a select
+; that shares the fast-math fcmp with the index select; the index reduction is the
+; decremented induction. The loop should vectorize with a vector fcmp/select
+; running minimum and a min-index reduction reconstructed from the base induction.
+define i32 @fp_argmin_decreasing(ptr %a, i32 %start, i64 %tc0) {
+; CHECK-LABEL: define i32 @fp_argmin_decreasing(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[START:%.*]], i64 [[TC0:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = sext i32 [[START]] to i64
+; CHECK-NEXT: [[MIN_P0_PHI_TRANS_INSERT:%.*]] = getelementptr [4 x i8], ptr [[A]], i64 [[TMP0]]
+; CHECK-NEXT: [[MIN_P_PHI_TRANS_INSERT:%.*]] = getelementptr i8, ptr [[MIN_P0_PHI_TRANS_INSERT]], i64 -4
+; CHECK-NEXT: [[MIN_V_PRE:%.*]] = load float, ptr [[MIN_P_PHI_TRANS_INSERT]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[TC0]], 1
+; CHECK-NEXT: [[SMIN:%.*]] = call i64 @llvm.smin.i64(i64 [[TC0]], i64 1)
+; CHECK-NEXT: [[TMP2:%.*]] = sub i64 [[TMP1]], [[SMIN]]
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP3:%.*]] = sub i64 [[TMP0]], [[N_VEC]]
+; CHECK-NEXT: [[TMP4:%.*]] = sub i64 [[TC0]], [[N_VEC]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[MIN_V_PRE]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ [[BROADCAST_SPLAT]], %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i64> [ poison, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP5:%.*]] = sub i64 [[TMP0]], [[INDEX]]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr [4 x i8], ptr [[A]], i64 [[TMP5]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[TMP6]], i64 -8
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr float, ptr [[TMP7]], i64 -3
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP8]], align 4
+; CHECK-NEXT: [[REVERSE:%.*]] = shufflevector <4 x float> [[WIDE_LOAD]], <4 x float> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: [[TMP9:%.*]] = fcmp fast olt <4 x float> [[REVERSE]], [[VEC_PHI]]
+; CHECK-NEXT: [[TMP10]] = select <4 x i1> [[TMP9]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI1]]
+; CHECK-NEXT: [[TMP11]] = select <4 x i1> [[TMP9]], <4 x float> [[REVERSE]], <4 x float> [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP13:%.*]] = call fast float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[TMP11]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <4 x float> poison, float [[TMP13]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT2]], <4 x float> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP14:%.*]] = fcmp oeq <4 x float> [[TMP11]], [[BROADCAST_SPLAT3]]
+; CHECK-NEXT: [[TMP15:%.*]] = select <4 x i1> [[TMP14]], <4 x i64> [[TMP10]], <4 x i64> splat (i64 -1)
+; CHECK-NEXT: [[TMP16:%.*]] = call i64 @llvm.vector.reduce.umin.v4i64(<4 x i64> [[TMP15]])
+; CHECK-NEXT: [[TMP17:%.*]] = sub i64 [[TMP0]], [[TMP16]]
+; CHECK-NEXT: [[TMP18:%.*]] = add i64 [[TMP17]], -1
+; CHECK-NEXT: [[TMP19:%.*]] = fcmp oeq float [[TMP13]], [[MIN_V_PRE]]
+; CHECK-NEXT: [[TMP20:%.*]] = select i1 [[TMP19]], i64 [[TMP0]], i64 [[TMP18]]
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ [[MIN_V_PRE]], %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ [[TMP0]], %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL4:%.*]] = phi i64 [ [[TMP4]], %[[MIDDLE_BLOCK]] ], [ [[TC0]], %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX5:%.*]] = phi i64 [ [[TMP20]], %[[MIDDLE_BLOCK]] ], [ [[TMP0]], %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[MIN_V:%.*]] = phi float [ [[TMP21:%.*]], %[[LOOP]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[CNT:%.*]] = phi i64 [ [[CNT_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL4]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[MIN_WIDE:%.*]] = phi i64 [ [[MIN_NEXT_WIDE:%.*]], %[[LOOP]] ], [ [[BC_MERGE_RDX5]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nsw i64 [[INDVARS_IV]], -1
+; CHECK-NEXT: [[SCAN_P0:%.*]] = getelementptr [4 x i8], ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[SCAN_P:%.*]] = getelementptr i8, ptr [[SCAN_P0]], i64 -8
+; CHECK-NEXT: [[SCAN_V:%.*]] = load float, ptr [[SCAN_P]], align 4
+; CHECK-NEXT: [[C:%.*]] = fcmp fast olt float [[SCAN_V]], [[MIN_V]]
+; CHECK-NEXT: [[MIN_NEXT_WIDE]] = select i1 [[C]], i64 [[INDVARS_IV_NEXT]], i64 [[MIN_WIDE]]
+; CHECK-NEXT: [[CNT_NEXT]] = add nsw i64 [[CNT]], -1
+; CHECK-NEXT: [[AGAIN:%.*]] = icmp sgt i64 [[CNT]], 1
+; CHECK-NEXT: [[TMP21]] = select i1 [[C]], float [[SCAN_V]], float [[MIN_V]]
+; CHECK-NEXT: br i1 [[AGAIN]], label %[[LOOP]], label %[[EXIT]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[MIN_NEXT_WIDE_LCSSA:%.*]] = phi i64 [ [[MIN_NEXT_WIDE]], %[[LOOP]] ], [ [[TMP20]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[TMP22:%.*]] = trunc nsw i64 [[MIN_NEXT_WIDE_LCSSA]] to i32
+; CHECK-NEXT: ret i32 [[TMP22]]
+;
+entry:
+ %0 = sext i32 %start to i64
+ %min.p0.phi.trans.insert = getelementptr [4 x i8], ptr %a, i64 %0
+ %min.p.phi.trans.insert = getelementptr i8, ptr %min.p0.phi.trans.insert, i64 -4
+ %min.v.pre = load float, ptr %min.p.phi.trans.insert, align 4
+ br label %loop
+
+loop:
+ %min.v = phi float [ %1, %loop ], [ %min.v.pre, %entry ]
+ %indvars.iv = phi i64 [ %indvars.iv.next, %loop ], [ %0, %entry ]
+ %cnt = phi i64 [ %cnt.next, %loop ], [ %tc0, %entry ]
+ %min.wide = phi i64 [ %min.next.wide, %loop ], [ %0, %entry ]
+ %indvars.iv.next = add nsw i64 %indvars.iv, -1
+ %scan.p0 = getelementptr [4 x i8], ptr %a, i64 %indvars.iv
+ %scan.p = getelementptr i8, ptr %scan.p0, i64 -8
+ %scan.v = load float, ptr %scan.p, align 4
+ %c = fcmp fast olt float %scan.v, %min.v
+ %min.next.wide = select i1 %c, i64 %indvars.iv.next, i64 %min.wide
+ %cnt.next = add nsw i64 %cnt, -1
+ %again = icmp sgt i64 %cnt, 1
+ %1 = select i1 %c, float %scan.v, float %min.v
+ br i1 %again, label %loop, label %exit
+
+exit:
+ %2 = trunc nsw i64 %min.next.wide to i32
+ ret i32 %2
+}
+;.
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
+;.
More information about the llvm-branch-commits
mailing list