[llvm] LV: Expand llvm.histogram intrinsic to support umax, umin, and uadd.sat operations (PR #127399)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Jun 23 08:02:10 PDT 2026
https://github.com/RonDahan101 updated https://github.com/llvm/llvm-project/pull/127399
>From 54c4fdbeb6898be6099ca0e0c84a390e971044fc Mon Sep 17 00:00:00 2001
From: rond <ron.dahan at mobileye.com>
Date: Sun, 31 May 2026 16:03:17 +0300
Subject: [PATCH 1/4] [LV] Extend histogram vectorization to support uadd.sat,
umax, and umin
Extend the Loop Vectorizer's histogram support to recognize and vectorize
loops with uadd.sat, umax, and umin update operations in addition to the
existing add/sub support.
This builds on the backend intrinsics added in PR #138447:
- llvm.experimental.vector.histogram.uadd.sat
- llvm.experimental.vector.histogram.umax
- llvm.experimental.vector.histogram.umin
Changes:
- LoopVectorizationLegality: Extend findHistogram() pattern matching to
recognize intrinsic-based update operations (uadd.sat, umax, umin) in
addition to binary add/sub.
- VPlan.h: Replace the raw 'unsigned Opcode' in VPHistogramRecipe with a
dedicated HistogramUpdateKind enum that cleanly represents all supported
operations without requiring artificial operands.
- VPlanRecipes.cpp: Update execute() to emit the correct histogram intrinsic
based on UpdateKind, add getHistogramIntrinsicID() and
getUpdateKindForInstruction() helpers, update cost model and print.
- LoopVectorize.cpp: Update widenIfHistogram() to use the new enum, and
extend the non-VPlan cost model to handle intrinsic histogram updates.
- Tests: Add new test functions for uadd.sat, umax, umin in sve2-histcnt.ll
and VPlan print verification in sve2-histcnt-vplan.ll.
Co-authored-by: Copilot <223556219+Copilot at users.noreply.github.com>
---
.../Vectorize/LoopVectorizationLegality.cpp | 38 ++--
.../Transforms/Vectorize/LoopVectorize.cpp | 54 +++++-
llvm/lib/Transforms/Vectorize/VPlan.h | 42 ++++-
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 105 +++++++++--
.../LoopVectorize/AArch64/sve2-histcnt.ll | 172 ++++++++++++++++++
.../VPlan/AArch64/sve2-histcnt-vplan.ll | 25 +++
6 files changed, 394 insertions(+), 42 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 214b5cff8a03a..ea2e5dbf1a8c3 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -1027,34 +1027,44 @@ bool LoopVectorizationLegality::canVectorizeInstr(Instruction &I) {
/// Find histogram operations that match high-level code in loops:
/// \code
-/// buckets[indices[i]]+=step;
+/// buckets[indices[i]] = UpdateOp(buckets[indices[i]], Val);
/// \endcode
+/// Where UpdateOp can be add, sub, uadd.sat, umax, or umin.
///
/// It matches a pattern starting from \p HSt, which Stores to the 'buckets'
-/// array the computed histogram. It uses a BinOp to sum all counts, storing
-/// them using a loop-variant index Load from the 'indices' input array.
+/// array the computed histogram. It uses an update instruction to update all
+/// counts, storing them using a loop-variant index Load from the 'indices'
+/// input array.
///
/// On successful matches it updates the STATISTIC 'HistogramsDetected',
/// regardless of hardware support. When there is support, it additionally
-/// stores the BinOp/Load pairs in \p HistogramCounts, as well the pointers
+/// stores the UpdateOp/Load pairs in \p HistogramCounts, as well the pointers
/// used to update histogram in \p HistogramPtrs.
static bool findHistogram(LoadInst *LI, StoreInst *HSt, Loop *TheLoop,
const PredicatedScalarEvolution &PSE,
SmallVectorImpl<HistogramInfo> &Histograms) {
- // Store value must come from a Binary Operation.
+ // Store value must come from an update operation (binary op or intrinsic).
Instruction *HPtrInstr = nullptr;
- BinaryOperator *HBinOp = nullptr;
- if (!match(HSt, m_Store(m_BinOp(HBinOp), m_Instruction(HPtrInstr))))
+ Instruction *HUpdateOp = nullptr;
+ if (!match(HSt, m_Store(m_Instruction(HUpdateOp), m_Instruction(HPtrInstr))))
return false;
- // BinOp must be an Add or a Sub modifying the bucket value by a
- // loop invariant amount.
+ // The update operation must modify the bucket value by a loop invariant
+ // amount. Supported operations: add, sub, uadd.sat, umax, umin.
// FIXME: We assume the loop invariant term is on the RHS.
// Fine for an immediate/constant, but maybe not a generic value?
Value *HIncVal = nullptr;
- if (!match(HBinOp, m_Add(m_Load(m_Specific(HPtrInstr)), m_Value(HIncVal))) &&
- !match(HBinOp, m_Sub(m_Load(m_Specific(HPtrInstr)), m_Value(HIncVal))))
+ if (!match(HUpdateOp,
+ m_Add(m_Load(m_Specific(HPtrInstr)), m_Value(HIncVal))) &&
+ !match(HUpdateOp,
+ m_Sub(m_Load(m_Specific(HPtrInstr)), m_Value(HIncVal))) &&
+ !match(HUpdateOp, m_Intrinsic<Intrinsic::uadd_sat>(
+ m_Load(m_Specific(HPtrInstr)), m_Value(HIncVal))) &&
+ !match(HUpdateOp, m_Intrinsic<Intrinsic::umax>(
+ m_Load(m_Specific(HPtrInstr)), m_Value(HIncVal))) &&
+ !match(HUpdateOp, m_Intrinsic<Intrinsic::umin>(
+ m_Load(m_Specific(HPtrInstr)), m_Value(HIncVal))))
return false;
// Make sure the increment value is loop invariant.
@@ -1096,15 +1106,15 @@ static bool findHistogram(LoadInst *LI, StoreInst *HSt, Loop *TheLoop,
// Ensure we'll have the same mask by checking that all parts of the histogram
// (gather load, update, scatter store) are in the same block.
- LoadInst *IndexedLoad = cast<LoadInst>(HBinOp->getOperand(0));
+ LoadInst *IndexedLoad = cast<LoadInst>(HUpdateOp->getOperand(0));
BasicBlock *LdBB = IndexedLoad->getParent();
- if (LdBB != HBinOp->getParent() || LdBB != HSt->getParent())
+ if (LdBB != HUpdateOp->getParent() || LdBB != HSt->getParent())
return false;
LLVM_DEBUG(dbgs() << "LV: Found histogram for: " << *HSt << "\n");
// Store the operations that make up the histogram.
- Histograms.emplace_back(IndexedLoad, HBinOp, HSt);
+ Histograms.emplace_back(IndexedLoad, HUpdateOp, HSt);
return true;
}
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 466cee091e7d0..08ee4004f3433 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -5335,8 +5335,50 @@ LoopVectorizationCostModel::getInstructionCost(Instruction *I,
return TTI.getCastInstrCost(Opcode, VectorTy, SrcVecTy, CCH,
Config.CostKind, I);
}
- case Instruction::Call:
+ case Instruction::Call: {
+ // Check if this is a histogram update operation (intrinsic like uadd.sat,
+ // umax, umin used as histogram bucket update).
+ auto Info = Legal->getHistogramInfo(I);
+ if (Info && VF.isVector()) {
+ const HistogramInfo *HGram = Info.value();
+ // Assume that a non-constant update value (or a constant != 1) requires
+ // a multiply, and add that into the cost.
+ InstructionCost MulCost = TTI::TCC_Free;
+ ConstantInt *RHS = dyn_cast<ConstantInt>(I->getOperand(1));
+ if (!RHS || RHS->getZExtValue() != 1)
+ MulCost = TTI.getArithmeticInstrCost(Instruction::Mul, VectorTy,
+ Config.CostKind);
+
+ // Find the cost of the histogram operation itself.
+ Type *PtrTy = VectorType::get(HGram->Load->getPointerOperandType(), VF);
+ Type *ScalarTy = I->getType();
+ Type *MaskTy = VectorType::get(Type::getInt1Ty(I->getContext()), VF);
+ auto *II = cast<IntrinsicInst>(I);
+ Intrinsic::ID HistID;
+ switch (II->getIntrinsicID()) {
+ case Intrinsic::uadd_sat:
+ HistID = Intrinsic::experimental_vector_histogram_uadd_sat;
+ break;
+ case Intrinsic::umax:
+ HistID = Intrinsic::experimental_vector_histogram_umax;
+ break;
+ case Intrinsic::umin:
+ HistID = Intrinsic::experimental_vector_histogram_umin;
+ break;
+ default:
+ llvm_unreachable("Unsupported histogram intrinsic");
+ }
+ IntrinsicCostAttributes ICA(HistID, Type::getVoidTy(I->getContext()),
+ {PtrTy, ScalarTy, MaskTy});
+
+ // Add the costs together with the update operation cost.
+ IntrinsicCostAttributes UpdateICA(II->getIntrinsicID(), VectorTy,
+ {VectorTy, VectorTy});
+ return TTI.getIntrinsicInstrCost(ICA, Config.CostKind) + MulCost +
+ TTI.getIntrinsicInstrCost(UpdateICA, Config.CostKind);
+ }
return getVectorCallCost(cast<CallInst>(I), VF);
+ }
case Instruction::ExtractValue:
return TTI.getInstructionCost(I, Config.CostKind);
case Instruction::Alloca:
@@ -6339,10 +6381,8 @@ VPHistogramRecipe *VPRecipeBuilder::widenIfHistogram(VPInstruction *VPI) {
return nullptr;
const HistogramInfo *HI = *HistInfo;
- // FIXME: Support other operations.
- unsigned Opcode = HI->Update->getOpcode();
- assert((Opcode == Instruction::Add || Opcode == Instruction::Sub) &&
- "Histogram update operation must be an Add or Sub");
+ auto UpdateKind = VPHistogramRecipe::getUpdateKindForInstruction(HI->Update);
+ assert(UpdateKind && "Unsupported histogram update operation");
SmallVector<VPValue *, 3> HGramOps;
// Bucket address.
@@ -6355,8 +6395,8 @@ VPHistogramRecipe *VPRecipeBuilder::widenIfHistogram(VPInstruction *VPI) {
if (CM.isMaskRequired(HI->Store))
HGramOps.push_back(VPI->getMask());
- return new VPHistogramRecipe(Opcode, HGramOps, cast<VPIRMetadata>(*VPI),
- VPI->getDebugLoc());
+ return new VPHistogramRecipe(*UpdateKind, HGramOps,
+ cast<VPIRMetadata>(*VPI), VPI->getDebugLoc());
}
bool VPRecipeBuilder::replaceWithFinalIfReductionStore(
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index cc2ff96b05be7..a149038f1e6e7 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -2158,23 +2158,35 @@ class LLVM_ABI_FOR_TEST VPWidenCallRecipe : public VPRecipeWithIRFlags,
/// A recipe representing a sequence of load -> update -> store as part of
/// a histogram operation. This means there may be aliasing between vector
/// lanes, which is handled by the llvm.experimental.vector.histogram family
-/// of intrinsics. The only update operations currently supported are
-/// 'add' and 'sub' where the other term is loop-invariant.
+/// of intrinsics. Supported update operations are: add, sub, uadd.sat, umax,
+/// and umin, where the other term is loop-invariant.
class VPHistogramRecipe : public VPRecipeBase, public VPIRMetadata {
- /// Opcode of the update operation, currently either add or sub.
- unsigned Opcode;
+public:
+ /// The kind of update operation performed on histogram buckets.
+ enum class HistogramUpdateKind {
+ Add,
+ Sub,
+ UAddSat,
+ UMax,
+ UMin,
+ };
+
+private:
+ /// The update operation kind for this histogram.
+ HistogramUpdateKind UpdateKind;
public:
- VPHistogramRecipe(unsigned Opcode, ArrayRef<VPValue *> Operands,
+ VPHistogramRecipe(HistogramUpdateKind UpdateKind,
+ ArrayRef<VPValue *> Operands,
const VPIRMetadata &Metadata = {},
DebugLoc DL = DebugLoc::getUnknown())
: VPRecipeBase(VPRecipeBase::VPHistogramSC, Operands, DL),
- VPIRMetadata(Metadata), Opcode(Opcode) {}
+ VPIRMetadata(Metadata), UpdateKind(UpdateKind) {}
~VPHistogramRecipe() override = default;
VPHistogramRecipe *clone() override {
- return new VPHistogramRecipe(Opcode, operands(), *this, getDebugLoc());
+ return new VPHistogramRecipe(UpdateKind, operands(), *this, getDebugLoc());
}
VP_CLASSOF_IMPL(VPRecipeBase::VPHistogramSC);
@@ -2186,7 +2198,16 @@ class VPHistogramRecipe : public VPRecipeBase, public VPIRMetadata {
InstructionCost computeCost(ElementCount VF,
VPCostContext &Ctx) const override;
- unsigned getOpcode() const { return Opcode; }
+ HistogramUpdateKind getUpdateKind() const { return UpdateKind; }
+
+ /// Return the histogram intrinsic ID for this recipe's update kind.
+ Intrinsic::ID getHistogramIntrinsicID() const;
+
+ /// Return true if the increment should be negated before passing to the
+ /// histogram intrinsic (only for Sub).
+ bool shouldNegateIncrement() const {
+ return UpdateKind == HistogramUpdateKind::Sub;
+ }
/// Return the mask operand if one was provided, or a null pointer if all
/// lanes should be executed unconditionally.
@@ -2194,6 +2215,11 @@ class VPHistogramRecipe : public VPRecipeBase, public VPIRMetadata {
return getNumOperands() == 3 ? getOperand(2) : nullptr;
}
+ /// Return the HistogramUpdateKind for the given update instruction, or
+ /// std::nullopt if the instruction is not a supported histogram update.
+ static std::optional<HistogramUpdateKind>
+ getUpdateKindForInstruction(Instruction *I);
+
protected:
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
/// Print the recipe
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 17e48075eb307..61b4d801464f5 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -2384,19 +2384,59 @@ void VPHistogramRecipe::execute(VPTransformState &State) {
Mask =
Builder.CreateVectorSplat(VTy->getElementCount(), Builder.getInt1(1));
- // If this is a subtract, we want to invert the increment amount. We may
- // add a separate intrinsic in future, but for now we'll try this.
- if (Opcode == Instruction::Sub)
+ // If this is a subtract, we want to invert the increment amount and use
+ // the histogram_add intrinsic.
+ if (shouldNegateIncrement())
IncAmt = Builder.CreateNeg(IncAmt);
- else
- assert(Opcode == Instruction::Add && "only add or sub supported for now");
Instruction *HistogramInst = State.Builder.CreateIntrinsicWithoutFolding(
- Intrinsic::experimental_vector_histogram_add, {VTy, IncAmt->getType()},
+ getHistogramIntrinsicID(), {VTy, IncAmt->getType()},
{Address, IncAmt, Mask});
applyMetadata(*HistogramInst);
}
+Intrinsic::ID VPHistogramRecipe::getHistogramIntrinsicID() const {
+ switch (UpdateKind) {
+ case HistogramUpdateKind::Add:
+ case HistogramUpdateKind::Sub:
+ return Intrinsic::experimental_vector_histogram_add;
+ case HistogramUpdateKind::UAddSat:
+ return Intrinsic::experimental_vector_histogram_uadd_sat;
+ case HistogramUpdateKind::UMax:
+ return Intrinsic::experimental_vector_histogram_umax;
+ case HistogramUpdateKind::UMin:
+ return Intrinsic::experimental_vector_histogram_umin;
+ }
+ llvm_unreachable("Unknown HistogramUpdateKind");
+}
+
+std::optional<VPHistogramRecipe::HistogramUpdateKind>
+VPHistogramRecipe::getUpdateKindForInstruction(Instruction *I) {
+ if (auto *BO = dyn_cast<BinaryOperator>(I)) {
+ switch (BO->getOpcode()) {
+ case Instruction::Add:
+ return HistogramUpdateKind::Add;
+ case Instruction::Sub:
+ return HistogramUpdateKind::Sub;
+ default:
+ return std::nullopt;
+ }
+ }
+ if (auto *II = dyn_cast<IntrinsicInst>(I)) {
+ switch (II->getIntrinsicID()) {
+ case Intrinsic::uadd_sat:
+ return HistogramUpdateKind::UAddSat;
+ case Intrinsic::umax:
+ return HistogramUpdateKind::UMax;
+ case Intrinsic::umin:
+ return HistogramUpdateKind::UMin;
+ default:
+ return std::nullopt;
+ }
+ }
+ return std::nullopt;
+}
+
InstructionCost VPHistogramRecipe::computeCost(ElementCount VF,
VPCostContext &Ctx) const {
// FIXME: Take the gather and scatter into account as well. For now we're
@@ -2420,13 +2460,41 @@ InstructionCost VPHistogramRecipe::computeCost(ElementCount VF,
// Find the cost of the histogram operation itself.
Type *PtrTy = VectorType::get(AddressTy, VF);
Type *MaskTy = VectorType::get(Type::getInt1Ty(Ctx.LLVMCtx), VF);
- IntrinsicCostAttributes ICA(Intrinsic::experimental_vector_histogram_add,
+ IntrinsicCostAttributes ICA(getHistogramIntrinsicID(),
Type::getVoidTy(Ctx.LLVMCtx),
{PtrTy, IncTy, MaskTy});
- // Add the costs together with the add/sub operation.
+ // Compute the cost of the update operation.
+ InstructionCost UpdateCost;
+ switch (UpdateKind) {
+ case HistogramUpdateKind::Add:
+ UpdateCost =
+ Ctx.TTI.getArithmeticInstrCost(Instruction::Add, VTy, Ctx.CostKind);
+ break;
+ case HistogramUpdateKind::Sub:
+ UpdateCost =
+ Ctx.TTI.getArithmeticInstrCost(Instruction::Sub, VTy, Ctx.CostKind);
+ break;
+ case HistogramUpdateKind::UAddSat: {
+ IntrinsicCostAttributes UpdateICA(Intrinsic::uadd_sat, VTy, {VTy, VTy});
+ UpdateCost = Ctx.TTI.getIntrinsicInstrCost(UpdateICA, Ctx.CostKind);
+ break;
+ }
+ case HistogramUpdateKind::UMax: {
+ IntrinsicCostAttributes UpdateICA(Intrinsic::umax, VTy, {VTy, VTy});
+ UpdateCost = Ctx.TTI.getIntrinsicInstrCost(UpdateICA, Ctx.CostKind);
+ break;
+ }
+ case HistogramUpdateKind::UMin: {
+ IntrinsicCostAttributes UpdateICA(Intrinsic::umin, VTy, {VTy, VTy});
+ UpdateCost = Ctx.TTI.getIntrinsicInstrCost(UpdateICA, Ctx.CostKind);
+ break;
+ }
+ }
+
+ // Add the costs together with the update operation.
return Ctx.TTI.getIntrinsicInstrCost(ICA, Ctx.CostKind) + MulCost +
- Ctx.TTI.getArithmeticInstrCost(Opcode, VTy, Ctx.CostKind);
+ UpdateCost;
}
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
@@ -2435,11 +2503,22 @@ void VPHistogramRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
O << Indent << "WIDEN-HISTOGRAM buckets: ";
getOperand(0)->printAsOperand(O, SlotTracker);
- if (Opcode == Instruction::Sub)
- O << ", dec: ";
- else {
- assert(Opcode == Instruction::Add);
+ switch (UpdateKind) {
+ case HistogramUpdateKind::Add:
O << ", inc: ";
+ break;
+ case HistogramUpdateKind::Sub:
+ O << ", dec: ";
+ break;
+ case HistogramUpdateKind::UAddSat:
+ O << ", saturated inc: ";
+ break;
+ case HistogramUpdateKind::UMax:
+ O << ", max: ";
+ break;
+ case HistogramUpdateKind::UMin:
+ O << ", min: ";
+ break;
}
getOperand(1)->printAsOperand(O, SlotTracker);
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
index 0f530be610a8b..d2a4a1230c0c8 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
@@ -887,3 +887,175 @@ attributes #0 = { "target-features"="+sve2" vscale_range(1,16) }
!11 = !{!10, !12}
!12 = distinct !{!12, !13, !"scope-bucket-2"}
!13 = distinct !{!13, !"scopes-buckets"}
+
+;; Test histogram with uadd.sat update operation
+define void @simple_histogram_uadd_sat(ptr noalias %buckets, ptr readonly %indices, i64 %N) #0 {
+; CHECK-LABEL: define void @simple_histogram_uadd_sat(
+; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i32> [[WIDE_LOAD]] to <4 x i64>
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP1]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP1]], i64 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP1]], i64 2
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP1]], i64 3
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP3]]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP4]]
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP5]]
+; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x ptr> poison, ptr [[TMP6]], i32 0
+; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x ptr> [[TMP10]], ptr [[TMP7]], i32 1
+; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x ptr> [[TMP11]], ptr [[TMP8]], i32 2
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x ptr> [[TMP12]], ptr [[TMP9]], i32 3
+; CHECK-NEXT: call void @llvm.experimental.vector.histogram.uadd.sat.v4p0.i32(<4 x ptr> [[TMP13]], i32 1, <4 x i1> splat (i1 true))
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK: scalar.ph:
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
+ %l.idx = load i32, ptr %gep.indices, align 4
+ %idxprom1 = zext i32 %l.idx to i64
+ %gep.bucket = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+ %l.bucket = load i32, ptr %gep.bucket, align 4
+ %inc = call i32 @llvm.uadd.sat.i32(i32 %l.bucket, i32 1)
+ store i32 %inc, ptr %gep.bucket, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, %N
+ br i1 %exitcond, label %for.exit, label %for.body, !llvm.loop !4
+
+for.exit:
+ ret void
+}
+
+;; Test histogram with umax update operation
+define void @simple_histogram_umax(ptr noalias %buckets, ptr readonly %indices, i64 %N) #0 {
+; CHECK-LABEL: define void @simple_histogram_umax(
+; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i32> [[WIDE_LOAD]] to <4 x i64>
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP1]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP1]], i64 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP1]], i64 2
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP1]], i64 3
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP3]]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP4]]
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP5]]
+; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x ptr> poison, ptr [[TMP6]], i32 0
+; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x ptr> [[TMP10]], ptr [[TMP7]], i32 1
+; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x ptr> [[TMP11]], ptr [[TMP8]], i32 2
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x ptr> [[TMP12]], ptr [[TMP9]], i32 3
+; CHECK-NEXT: call void @llvm.experimental.vector.histogram.umax.v4p0.i32(<4 x ptr> [[TMP13]], i32 120, <4 x i1> splat (i1 true))
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK: scalar.ph:
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
+ %l.idx = load i32, ptr %gep.indices, align 4
+ %idxprom1 = zext i32 %l.idx to i64
+ %gep.bucket = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+ %l.bucket = load i32, ptr %gep.bucket, align 4
+ %inc = call i32 @llvm.umax.i32(i32 %l.bucket, i32 120)
+ store i32 %inc, ptr %gep.bucket, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, %N
+ br i1 %exitcond, label %for.exit, label %for.body, !llvm.loop !4
+
+for.exit:
+ ret void
+}
+
+;; Test histogram with umin update operation
+define void @simple_histogram_umin(ptr noalias %buckets, ptr readonly %indices, i64 %N) #0 {
+; CHECK-LABEL: define void @simple_histogram_umin(
+; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i32> [[WIDE_LOAD]] to <4 x i64>
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP1]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP1]], i64 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP1]], i64 2
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP1]], i64 3
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP3]]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP4]]
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP5]]
+; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x ptr> poison, ptr [[TMP6]], i32 0
+; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x ptr> [[TMP10]], ptr [[TMP7]], i32 1
+; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x ptr> [[TMP11]], ptr [[TMP8]], i32 2
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x ptr> [[TMP12]], ptr [[TMP9]], i32 3
+; CHECK-NEXT: call void @llvm.experimental.vector.histogram.umin.v4p0.i32(<4 x ptr> [[TMP13]], i32 99, <4 x i1> splat (i1 true))
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP31:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK: scalar.ph:
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
+ %l.idx = load i32, ptr %gep.indices, align 4
+ %idxprom1 = zext i32 %l.idx to i64
+ %gep.bucket = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+ %l.bucket = load i32, ptr %gep.bucket, align 4
+ %inc = call i32 @llvm.umin.i32(i32 %l.bucket, i32 99)
+ store i32 %inc, ptr %gep.bucket, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, %N
+ br i1 %exitcond, label %for.exit, label %for.body, !llvm.loop !4
+
+for.exit:
+ ret void
+}
+
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve2-histcnt-vplan.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve2-histcnt-vplan.ll
index 7ce306d9101cb..8a35dd2392af1 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve2-histcnt-vplan.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve2-histcnt-vplan.ll
@@ -153,6 +153,31 @@ for.exit:
ret void
}
+;; Check that uadd.sat histogram gets a "saturated inc:" label in VPlan.
+; CHECK: VPlan 'Initial VPlan for VF={vscale x 2,vscale x 4},UF>=1' {
+; CHECK: WIDEN-HISTOGRAM buckets: {{.*}}, saturated inc: ir<1>
+
+define void @simple_histogram_uadd_sat(ptr noalias %buckets, ptr readonly %indices, i64 %N) {
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
+ %l.idx = load i32, ptr %gep.indices, align 4
+ %idxprom1 = zext i32 %l.idx to i64
+ %gep.bucket = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+ %l.bucket = load i32, ptr %gep.bucket, align 4
+ %inc = call i32 @llvm.uadd.sat.i32(i32 %l.bucket, i32 1)
+ store i32 %inc, ptr %gep.bucket, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, %N
+ br i1 %exitcond, label %for.exit, label %for.body
+
+for.exit:
+ ret void
+}
+
!0 = !{!1}
!1 = distinct !{!1, !2}
!2 = distinct !{!2}
>From 50510b637245e16f6a5eb68becaec477ba70a83f Mon Sep 17 00:00:00 2001
From: rond <ron.dahan at mobileye.com>
Date: Wed, 17 Jun 2026 12:13:58 +0300
Subject: [PATCH 2/4] Fix clang format
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 08ee4004f3433..6c6d8b7a87821 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6395,8 +6395,8 @@ VPHistogramRecipe *VPRecipeBuilder::widenIfHistogram(VPInstruction *VPI) {
if (CM.isMaskRequired(HI->Store))
HGramOps.push_back(VPI->getMask());
- return new VPHistogramRecipe(*UpdateKind, HGramOps,
- cast<VPIRMetadata>(*VPI), VPI->getDebugLoc());
+ return new VPHistogramRecipe(*UpdateKind, HGramOps, cast<VPIRMetadata>(*VPI),
+ VPI->getDebugLoc());
}
bool VPRecipeBuilder::replaceWithFinalIfReductionStore(
>From c69897e830bea5cf413e052e66cc76210acbb235 Mon Sep 17 00:00:00 2001
From: rond <ron.dahan at mobileye.com>
Date: Tue, 23 Jun 2026 16:26:35 +0300
Subject: [PATCH 3/4] Fix review
Co-authored-by: Copilot <223556219+Copilot at users.noreply.github.com>
---
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 17 +-
.../LoopVectorize/AArch64/sve2-histcnt.ll | 170 ---------------
.../VPlan/AArch64/sve2-histcnt-vplan.ll | 46 ++++-
.../Transforms/LoopVectorize/histograms.ll | 195 ++++++++++++++++++
4 files changed, 251 insertions(+), 177 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 61b4d801464f5..ce56e5ce34f71 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -2450,12 +2450,17 @@ InstructionCost VPHistogramRecipe::computeCost(ElementCount VF,
Type *IncTy = IncAmt->getScalarType();
VectorType *VTy = VectorType::get(IncTy, VF);
- // Assume that a non-constant update value (or a constant != 1) requires
- // a multiply, and add that into the cost.
- InstructionCost MulCost =
- Ctx.TTI.getArithmeticInstrCost(Instruction::Mul, VTy, Ctx.CostKind);
- if (match(IncAmt, m_One()))
- MulCost = TTI::TCC_Free;
+ // For umin/umax, there's no multiplication — the increment value is compared
+ // directly. For add/sub/uadd.sat, assume that a non-constant update value
+ // (or a constant != 1) requires a multiply.
+ InstructionCost MulCost = TTI::TCC_Free;
+ if (UpdateKind != HistogramUpdateKind::UMax &&
+ UpdateKind != HistogramUpdateKind::UMin) {
+ MulCost =
+ Ctx.TTI.getArithmeticInstrCost(Instruction::Mul, VTy, Ctx.CostKind);
+ if (match(IncAmt, m_One()))
+ MulCost = TTI::TCC_Free;
+ }
// Find the cost of the histogram operation itself.
Type *PtrTy = VectorType::get(AddressTy, VF);
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
index d2a4a1230c0c8..6af3a397f5569 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
@@ -888,174 +888,4 @@ attributes #0 = { "target-features"="+sve2" vscale_range(1,16) }
!12 = distinct !{!12, !13, !"scope-bucket-2"}
!13 = distinct !{!13, !"scopes-buckets"}
-;; Test histogram with uadd.sat update operation
-define void @simple_histogram_uadd_sat(ptr noalias %buckets, ptr readonly %indices, i64 %N) #0 {
-; CHECK-LABEL: define void @simple_histogram_uadd_sat(
-; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK: vector.ph:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i32> [[WIDE_LOAD]] to <4 x i64>
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP1]], i64 0
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP1]], i64 1
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP1]], i64 2
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP1]], i64 3
-; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP2]]
-; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP3]]
-; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP4]]
-; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP5]]
-; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x ptr> poison, ptr [[TMP6]], i32 0
-; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x ptr> [[TMP10]], ptr [[TMP7]], i32 1
-; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x ptr> [[TMP11]], ptr [[TMP8]], i32 2
-; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x ptr> [[TMP12]], ptr [[TMP9]], i32 3
-; CHECK-NEXT: call void @llvm.experimental.vector.histogram.uadd.sat.v4p0.i32(<4 x ptr> [[TMP13]], i32 1, <4 x i1> splat (i1 true))
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
-; CHECK: middle.block:
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
-; CHECK: scalar.ph:
-;
-entry:
- br label %for.body
-
-for.body:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
- %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
- %l.idx = load i32, ptr %gep.indices, align 4
- %idxprom1 = zext i32 %l.idx to i64
- %gep.bucket = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
- %l.bucket = load i32, ptr %gep.bucket, align 4
- %inc = call i32 @llvm.uadd.sat.i32(i32 %l.bucket, i32 1)
- store i32 %inc, ptr %gep.bucket, align 4
- %iv.next = add nuw nsw i64 %iv, 1
- %exitcond = icmp eq i64 %iv.next, %N
- br i1 %exitcond, label %for.exit, label %for.body, !llvm.loop !4
-
-for.exit:
- ret void
-}
-
-;; Test histogram with umax update operation
-define void @simple_histogram_umax(ptr noalias %buckets, ptr readonly %indices, i64 %N) #0 {
-; CHECK-LABEL: define void @simple_histogram_umax(
-; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK: vector.ph:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i32> [[WIDE_LOAD]] to <4 x i64>
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP1]], i64 0
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP1]], i64 1
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP1]], i64 2
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP1]], i64 3
-; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP2]]
-; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP3]]
-; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP4]]
-; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP5]]
-; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x ptr> poison, ptr [[TMP6]], i32 0
-; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x ptr> [[TMP10]], ptr [[TMP7]], i32 1
-; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x ptr> [[TMP11]], ptr [[TMP8]], i32 2
-; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x ptr> [[TMP12]], ptr [[TMP9]], i32 3
-; CHECK-NEXT: call void @llvm.experimental.vector.histogram.umax.v4p0.i32(<4 x ptr> [[TMP13]], i32 120, <4 x i1> splat (i1 true))
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]
-; CHECK: middle.block:
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
-; CHECK: scalar.ph:
-;
-entry:
- br label %for.body
-
-for.body:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
- %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
- %l.idx = load i32, ptr %gep.indices, align 4
- %idxprom1 = zext i32 %l.idx to i64
- %gep.bucket = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
- %l.bucket = load i32, ptr %gep.bucket, align 4
- %inc = call i32 @llvm.umax.i32(i32 %l.bucket, i32 120)
- store i32 %inc, ptr %gep.bucket, align 4
- %iv.next = add nuw nsw i64 %iv, 1
- %exitcond = icmp eq i64 %iv.next, %N
- br i1 %exitcond, label %for.exit, label %for.body, !llvm.loop !4
-
-for.exit:
- ret void
-}
-
-;; Test histogram with umin update operation
-define void @simple_histogram_umin(ptr noalias %buckets, ptr readonly %indices, i64 %N) #0 {
-; CHECK-LABEL: define void @simple_histogram_umin(
-; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK: vector.ph:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i32> [[WIDE_LOAD]] to <4 x i64>
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP1]], i64 0
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP1]], i64 1
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP1]], i64 2
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP1]], i64 3
-; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP2]]
-; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP3]]
-; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP4]]
-; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP5]]
-; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x ptr> poison, ptr [[TMP6]], i32 0
-; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x ptr> [[TMP10]], ptr [[TMP7]], i32 1
-; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x ptr> [[TMP11]], ptr [[TMP8]], i32 2
-; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x ptr> [[TMP12]], ptr [[TMP9]], i32 3
-; CHECK-NEXT: call void @llvm.experimental.vector.histogram.umin.v4p0.i32(<4 x ptr> [[TMP13]], i32 99, <4 x i1> splat (i1 true))
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP31:![0-9]+]]
-; CHECK: middle.block:
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
-; CHECK: scalar.ph:
-;
-entry:
- br label %for.body
-
-for.body:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
- %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
- %l.idx = load i32, ptr %gep.indices, align 4
- %idxprom1 = zext i32 %l.idx to i64
- %gep.bucket = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
- %l.bucket = load i32, ptr %gep.bucket, align 4
- %inc = call i32 @llvm.umin.i32(i32 %l.bucket, i32 99)
- store i32 %inc, ptr %gep.bucket, align 4
- %iv.next = add nuw nsw i64 %iv, 1
- %exitcond = icmp eq i64 %iv.next, %N
- br i1 %exitcond, label %for.exit, label %for.body, !llvm.loop !4
-
-for.exit:
- ret void
-}
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve2-histcnt-vplan.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve2-histcnt-vplan.ll
index 8a35dd2392af1..b414f232376c6 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve2-histcnt-vplan.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve2-histcnt-vplan.ll
@@ -155,7 +155,51 @@ for.exit:
;; Check that uadd.sat histogram gets a "saturated inc:" label in VPlan.
; CHECK: VPlan 'Initial VPlan for VF={vscale x 2,vscale x 4},UF>=1' {
-; CHECK: WIDEN-HISTOGRAM buckets: {{.*}}, saturated inc: ir<1>
+; CHECK-NEXT: Live-in [[VF3:.*]] = VF
+; CHECK-NEXT: Live-in [[VFxUF3:.*]] = VF * UF
+; CHECK-NEXT: Live-in [[VTC3:.*]] = vector-trip-count
+; CHECK-NEXT: Live-in [[OTC3:.*]] = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: [[IV3:.*]] = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: [[STEPS3:vp.*]] = SCALAR-STEPS [[IV3]], ir<1>, [[VF3]]
+; CHECK-NEXT: CLONE [[GEP_IDX3:.*]] = getelementptr inbounds ir<%indices>, [[STEPS3]]
+; CHECK-NEXT: [[VECP_IDX3:vp.*]] = vector-pointer inbounds [[GEP_IDX3]]
+; CHECK-NEXT: WIDEN [[IDX3:.*]] = load [[VECP_IDX3]]
+; CHECK-NEXT: WIDEN-CAST [[EXT_IDX3:.*]] = zext [[IDX3]] to i64
+; CHECK-NEXT: WIDEN-GEP [[GEP_BUCKET3:.*]] = getelementptr inbounds ir<%buckets>, [[EXT_IDX3]]
+; CHECK-NEXT: WIDEN-HISTOGRAM buckets: [[GEP_BUCKET3]], saturated inc: ir<1>
+; CHECK-NEXT: EMIT [[IV_NEXT3:.*]] = add nuw [[IV3]], [[VFxUF3]]
+; CHECK-NEXT: EMIT branch-on-count [[IV_NEXT3]], [[VTC3]]
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT [[TC_CHECK3:.*]] = icmp eq [[OTC3]], [[VTC3]]
+; CHECK-NEXT: EMIT branch-on-cond [[TC_CHECK3]]
+; CHECK-NEXT: Successor(s): ir-bb<for.exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<for.exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<[[RESUME3:%.+]]> = phi [ [[VTC3]], middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<for.body>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<for.body>:
+; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ] (extra operand: vp<[[RESUME3]]> from scalar.ph)
+; CHECK: IR %exitcond = icmp eq i64 %iv.next, %N
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
define void @simple_histogram_uadd_sat(ptr noalias %buckets, ptr readonly %indices, i64 %N) {
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/histograms.ll b/llvm/test/Transforms/LoopVectorize/histograms.ll
index 5850ac3195c39..128f9c854ce8e 100644
--- a/llvm/test/Transforms/LoopVectorize/histograms.ll
+++ b/llvm/test/Transforms/LoopVectorize/histograms.ll
@@ -65,3 +65,198 @@ for.body:
for.exit:
ret void
}
+
+define void @simple_histogram_uadd_sat(ptr noalias %buckets, ptr readonly %indices, i64 %N) {
+; CHECK-LABEL: define void @simple_histogram_uadd_sat(
+; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[N]], -2
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [4 x i8], ptr [[INDICES]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = zext <2 x i32> [[WIDE_LOAD]] to <2 x i64>
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i64> [[TMP1]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i64> [[TMP1]], i64 1
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[BUCKETS]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[BUCKETS]], i64 [[TMP3]]
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP4]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x ptr> [[TMP6]], ptr [[TMP5]], i64 1
+; CHECK-NEXT: call void @llvm.experimental.vector.histogram.uadd.sat.v2p0.i32(<2 x ptr> [[TMP7]], i32 1, <2 x i1> splat (i1 true))
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK: scalar.ph:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT: br label [[FOR_BODY:%.*]]
+; CHECK: for.body:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT: [[GEP_INDICES:%.*]] = getelementptr inbounds [4 x i8], ptr [[INDICES]], i64 [[IV]]
+; CHECK-NEXT: [[L_IDX:%.*]] = load i32, ptr [[GEP_INDICES]], align 4
+; CHECK-NEXT: [[IDXPROM1:%.*]] = zext i32 [[L_IDX]] to i64
+; CHECK-NEXT: [[GEP_BUCKET:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[BUCKETS]], i64 [[IDXPROM1]]
+; CHECK-NEXT: [[L_BUCKET:%.*]] = load i32, ptr [[GEP_BUCKET]], align 4
+; CHECK-NEXT: [[INC:%.*]] = call i32 @llvm.uadd.sat.i32(i32 [[L_BUCKET]], i32 1)
+; CHECK-NEXT: store i32 [[INC]], ptr [[GEP_BUCKET]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK: for.exit:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
+ %l.idx = load i32, ptr %gep.indices, align 4
+ %idxprom1 = zext i32 %l.idx to i64
+ %gep.bucket = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+ %l.bucket = load i32, ptr %gep.bucket, align 4
+ %inc = call i32 @llvm.uadd.sat.i32(i32 %l.bucket, i32 1)
+ store i32 %inc, ptr %gep.bucket, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, %N
+ br i1 %exitcond, label %for.exit, label %for.body
+
+for.exit:
+ ret void
+}
+
+define void @simple_histogram_umax(ptr noalias %buckets, ptr readonly %indices, i64 %N) {
+; CHECK-LABEL: define void @simple_histogram_umax(
+; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[N]], -2
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [4 x i8], ptr [[INDICES]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = zext <2 x i32> [[WIDE_LOAD]] to <2 x i64>
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i64> [[TMP1]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i64> [[TMP1]], i64 1
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[BUCKETS]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[BUCKETS]], i64 [[TMP3]]
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP4]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x ptr> [[TMP6]], ptr [[TMP5]], i64 1
+; CHECK-NEXT: call void @llvm.experimental.vector.histogram.umax.v2p0.i32(<2 x ptr> [[TMP7]], i32 120, <2 x i1> splat (i1 true))
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK: scalar.ph:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT: br label [[FOR_BODY:%.*]]
+; CHECK: for.body:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT: [[GEP_INDICES:%.*]] = getelementptr inbounds [4 x i8], ptr [[INDICES]], i64 [[IV]]
+; CHECK-NEXT: [[L_IDX:%.*]] = load i32, ptr [[GEP_INDICES]], align 4
+; CHECK-NEXT: [[IDXPROM1:%.*]] = zext i32 [[L_IDX]] to i64
+; CHECK-NEXT: [[GEP_BUCKET:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[BUCKETS]], i64 [[IDXPROM1]]
+; CHECK-NEXT: [[L_BUCKET:%.*]] = load i32, ptr [[GEP_BUCKET]], align 4
+; CHECK-NEXT: [[INC:%.*]] = call i32 @llvm.umax.i32(i32 [[L_BUCKET]], i32 120)
+; CHECK-NEXT: store i32 [[INC]], ptr [[GEP_BUCKET]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK: for.exit:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
+ %l.idx = load i32, ptr %gep.indices, align 4
+ %idxprom1 = zext i32 %l.idx to i64
+ %gep.bucket = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+ %l.bucket = load i32, ptr %gep.bucket, align 4
+ %inc = call i32 @llvm.umax.i32(i32 %l.bucket, i32 120)
+ store i32 %inc, ptr %gep.bucket, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, %N
+ br i1 %exitcond, label %for.exit, label %for.body
+
+for.exit:
+ ret void
+}
+
+define void @simple_histogram_umin(ptr noalias %buckets, ptr readonly %indices, i64 %N) {
+; CHECK-LABEL: define void @simple_histogram_umin(
+; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[N]], -2
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [4 x i8], ptr [[INDICES]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = zext <2 x i32> [[WIDE_LOAD]] to <2 x i64>
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i64> [[TMP1]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i64> [[TMP1]], i64 1
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[BUCKETS]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[BUCKETS]], i64 [[TMP3]]
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP4]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x ptr> [[TMP6]], ptr [[TMP5]], i64 1
+; CHECK-NEXT: call void @llvm.experimental.vector.histogram.umin.v2p0.i32(<2 x ptr> [[TMP7]], i32 99, <2 x i1> splat (i1 true))
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK: scalar.ph:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT: br label [[FOR_BODY:%.*]]
+; CHECK: for.body:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT: [[GEP_INDICES:%.*]] = getelementptr inbounds [4 x i8], ptr [[INDICES]], i64 [[IV]]
+; CHECK-NEXT: [[L_IDX:%.*]] = load i32, ptr [[GEP_INDICES]], align 4
+; CHECK-NEXT: [[IDXPROM1:%.*]] = zext i32 [[L_IDX]] to i64
+; CHECK-NEXT: [[GEP_BUCKET:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[BUCKETS]], i64 [[IDXPROM1]]
+; CHECK-NEXT: [[L_BUCKET:%.*]] = load i32, ptr [[GEP_BUCKET]], align 4
+; CHECK-NEXT: [[INC:%.*]] = call i32 @llvm.umin.i32(i32 [[L_BUCKET]], i32 99)
+; CHECK-NEXT: store i32 [[INC]], ptr [[GEP_BUCKET]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK: for.exit:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
+ %l.idx = load i32, ptr %gep.indices, align 4
+ %idxprom1 = zext i32 %l.idx to i64
+ %gep.bucket = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+ %l.bucket = load i32, ptr %gep.bucket, align 4
+ %inc = call i32 @llvm.umin.i32(i32 %l.bucket, i32 99)
+ store i32 %inc, ptr %gep.bucket, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, %N
+ br i1 %exitcond, label %for.exit, label %for.body
+
+for.exit:
+ ret void
+}
>From 22f8307d536dc129e4e8db4504d32d7c1fd57acc Mon Sep 17 00:00:00 2001
From: rond <ron.dahan at mobileye.com>
Date: Tue, 23 Jun 2026 16:52:03 +0300
Subject: [PATCH 4/4] Combine MulCost conditions in
VPHistogramRecipe::computeCost
Simplify the logic by combining the m_One() check with the UMax/UMin
checks into a single condition, avoiding redundant assignment of MulCost.
Co-authored-by: Copilot <223556219+Copilot at users.noreply.github.com>
---
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 7 ++-----
1 file changed, 2 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index ce56e5ce34f71..d200b61516834 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -2454,13 +2454,10 @@ InstructionCost VPHistogramRecipe::computeCost(ElementCount VF,
// directly. For add/sub/uadd.sat, assume that a non-constant update value
// (or a constant != 1) requires a multiply.
InstructionCost MulCost = TTI::TCC_Free;
- if (UpdateKind != HistogramUpdateKind::UMax &&
- UpdateKind != HistogramUpdateKind::UMin) {
+ if (!match(IncAmt, m_One()) && UpdateKind != HistogramUpdateKind::UMax &&
+ UpdateKind != HistogramUpdateKind::UMin)
MulCost =
Ctx.TTI.getArithmeticInstrCost(Instruction::Mul, VTy, Ctx.CostKind);
- if (match(IncAmt, m_One()))
- MulCost = TTI::TCC_Free;
- }
// Find the cost of the histogram operation itself.
Type *PtrTy = VectorType::get(AddressTy, VF);
More information about the llvm-commits
mailing list