[llvm] LV: Expand llvm.histogram intrinsic to support umax, umin, and uadd.sat operations (PR #127399)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Jul 9 08:24:02 PDT 2026
https://github.com/RonDahan101 updated https://github.com/llvm/llvm-project/pull/127399
>From e5ad0181c4cebe10eeb26e98cc74d728572a29b9 Mon Sep 17 00:00:00 2001
From: rond <ron.dahan at mobileye.com>
Date: Sun, 31 May 2026 16:03:17 +0300
Subject: [PATCH 1/5] [LV] Extend histogram vectorization to support uadd.sat,
umax, and umin
Extend the Loop Vectorizer's histogram support to recognize and vectorize
loops with uadd.sat, umax, and umin update operations in addition to the
existing add/sub support.
This builds on the backend intrinsics added in PR #138447:
- llvm.experimental.vector.histogram.uadd.sat
- llvm.experimental.vector.histogram.umax
- llvm.experimental.vector.histogram.umin
Changes:
- LoopVectorizationLegality: Extend findHistogram() pattern matching to
recognize intrinsic-based update operations (uadd.sat, umax, umin) in
addition to binary add/sub.
- VPlan.h: Replace the raw 'unsigned Opcode' in VPHistogramRecipe with a
dedicated HistogramUpdateKind enum that cleanly represents all supported
operations without requiring artificial operands.
- VPlanRecipes.cpp: Update execute() to emit the correct histogram intrinsic
based on UpdateKind, add getHistogramIntrinsicID() and
getUpdateKindForInstruction() helpers, update cost model and print.
- LoopVectorize.cpp: Update widenIfHistogram() to use the new enum, and
extend the non-VPlan cost model to handle intrinsic histogram updates.
- Tests: Add new test functions for uadd.sat, umax, umin in sve2-histcnt.ll
and VPlan print verification in sve2-histcnt-vplan.ll.
Co-authored-by: Copilot <223556219+Copilot at users.noreply.github.com>
---
.../Vectorize/LoopVectorizationLegality.cpp | 38 ++--
.../Transforms/Vectorize/LoopVectorize.cpp | 54 +++++-
llvm/lib/Transforms/Vectorize/VPlan.h | 42 ++++-
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 105 +++++++++--
.../LoopVectorize/AArch64/sve2-histcnt.ll | 172 ++++++++++++++++++
.../VPlan/AArch64/sve2-histcnt-vplan.ll | 25 +++
6 files changed, 394 insertions(+), 42 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 8d875b2b6e492..d6c538cc86fbd 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -1032,34 +1032,44 @@ bool LoopVectorizationLegality::canVectorizeInstr(Instruction &I) {
/// Find histogram operations that match high-level code in loops:
/// \code
-/// buckets[indices[i]]+=step;
+/// buckets[indices[i]] = UpdateOp(buckets[indices[i]], Val);
/// \endcode
+/// Where UpdateOp can be add, sub, uadd.sat, umax, or umin.
///
/// It matches a pattern starting from \p HSt, which Stores to the 'buckets'
-/// array the computed histogram. It uses a BinOp to sum all counts, storing
-/// them using a loop-variant index Load from the 'indices' input array.
+/// array the computed histogram. It uses an update instruction to update all
+/// counts, storing them using a loop-variant index Load from the 'indices'
+/// input array.
///
/// On successful matches it updates the STATISTIC 'HistogramsDetected',
/// regardless of hardware support. When there is support, it additionally
-/// stores the BinOp/Load pairs in \p HistogramCounts, as well the pointers
+/// stores the UpdateOp/Load pairs in \p HistogramCounts, as well the pointers
/// used to update histogram in \p HistogramPtrs.
static bool findHistogram(LoadInst *LI, StoreInst *HSt, Loop *TheLoop,
const PredicatedScalarEvolution &PSE,
SmallVectorImpl<HistogramInfo> &Histograms) {
- // Store value must come from a Binary Operation.
+ // Store value must come from an update operation (binary op or intrinsic).
Instruction *HPtrInstr = nullptr;
- BinaryOperator *HBinOp = nullptr;
- if (!match(HSt, m_Store(m_BinOp(HBinOp), m_Instruction(HPtrInstr))))
+ Instruction *HUpdateOp = nullptr;
+ if (!match(HSt, m_Store(m_Instruction(HUpdateOp), m_Instruction(HPtrInstr))))
return false;
- // BinOp must be an Add or a Sub modifying the bucket value by a
- // loop invariant amount.
+ // The update operation must modify the bucket value by a loop invariant
+ // amount. Supported operations: add, sub, uadd.sat, umax, umin.
// FIXME: We assume the loop invariant term is on the RHS.
// Fine for an immediate/constant, but maybe not a generic value?
Value *HIncVal = nullptr;
- if (!match(HBinOp, m_Add(m_Load(m_Specific(HPtrInstr)), m_Value(HIncVal))) &&
- !match(HBinOp, m_Sub(m_Load(m_Specific(HPtrInstr)), m_Value(HIncVal))))
+ if (!match(HUpdateOp,
+ m_Add(m_Load(m_Specific(HPtrInstr)), m_Value(HIncVal))) &&
+ !match(HUpdateOp,
+ m_Sub(m_Load(m_Specific(HPtrInstr)), m_Value(HIncVal))) &&
+ !match(HUpdateOp, m_Intrinsic<Intrinsic::uadd_sat>(
+ m_Load(m_Specific(HPtrInstr)), m_Value(HIncVal))) &&
+ !match(HUpdateOp, m_Intrinsic<Intrinsic::umax>(
+ m_Load(m_Specific(HPtrInstr)), m_Value(HIncVal))) &&
+ !match(HUpdateOp, m_Intrinsic<Intrinsic::umin>(
+ m_Load(m_Specific(HPtrInstr)), m_Value(HIncVal))))
return false;
// Make sure the increment value is loop invariant.
@@ -1101,15 +1111,15 @@ static bool findHistogram(LoadInst *LI, StoreInst *HSt, Loop *TheLoop,
// Ensure we'll have the same mask by checking that all parts of the histogram
// (gather load, update, scatter store) are in the same block.
- LoadInst *IndexedLoad = cast<LoadInst>(HBinOp->getOperand(0));
+ LoadInst *IndexedLoad = cast<LoadInst>(HUpdateOp->getOperand(0));
BasicBlock *LdBB = IndexedLoad->getParent();
- if (LdBB != HBinOp->getParent() || LdBB != HSt->getParent())
+ if (LdBB != HUpdateOp->getParent() || LdBB != HSt->getParent())
return false;
LLVM_DEBUG(dbgs() << "LV: Found histogram for: " << *HSt << "\n");
// Store the operations that make up the histogram.
- Histograms.emplace_back(IndexedLoad, HBinOp, HSt);
+ Histograms.emplace_back(IndexedLoad, HUpdateOp, HSt);
return true;
}
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 19b77e40a4103..611a265cd0d1d 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -5335,8 +5335,50 @@ LoopVectorizationCostModel::getInstructionCost(Instruction *I,
return TTI.getCastInstrCost(Opcode, VectorTy, SrcVecTy, CCH,
Config.CostKind, I);
}
- case Instruction::Call:
+ case Instruction::Call: {
+ // Check if this is a histogram update operation (intrinsic like uadd.sat,
+ // umax, umin used as histogram bucket update).
+ auto Info = Legal->getHistogramInfo(I);
+ if (Info && VF.isVector()) {
+ const HistogramInfo *HGram = Info.value();
+ // Assume that a non-constant update value (or a constant != 1) requires
+ // a multiply, and add that into the cost.
+ InstructionCost MulCost = TTI::TCC_Free;
+ ConstantInt *RHS = dyn_cast<ConstantInt>(I->getOperand(1));
+ if (!RHS || RHS->getZExtValue() != 1)
+ MulCost = TTI.getArithmeticInstrCost(Instruction::Mul, VectorTy,
+ Config.CostKind);
+
+ // Find the cost of the histogram operation itself.
+ Type *PtrTy = VectorType::get(HGram->Load->getPointerOperandType(), VF);
+ Type *ScalarTy = I->getType();
+ Type *MaskTy = VectorType::get(Type::getInt1Ty(I->getContext()), VF);
+ auto *II = cast<IntrinsicInst>(I);
+ Intrinsic::ID HistID;
+ switch (II->getIntrinsicID()) {
+ case Intrinsic::uadd_sat:
+ HistID = Intrinsic::experimental_vector_histogram_uadd_sat;
+ break;
+ case Intrinsic::umax:
+ HistID = Intrinsic::experimental_vector_histogram_umax;
+ break;
+ case Intrinsic::umin:
+ HistID = Intrinsic::experimental_vector_histogram_umin;
+ break;
+ default:
+ llvm_unreachable("Unsupported histogram intrinsic");
+ }
+ IntrinsicCostAttributes ICA(HistID, Type::getVoidTy(I->getContext()),
+ {PtrTy, ScalarTy, MaskTy});
+
+ // Add the costs together with the update operation cost.
+ IntrinsicCostAttributes UpdateICA(II->getIntrinsicID(), VectorTy,
+ {VectorTy, VectorTy});
+ return TTI.getIntrinsicInstrCost(ICA, Config.CostKind) + MulCost +
+ TTI.getIntrinsicInstrCost(UpdateICA, Config.CostKind);
+ }
return getVectorCallCost(cast<CallInst>(I), VF);
+ }
case Instruction::ExtractValue:
return TTI.getInstructionCost(I, Config.CostKind);
case Instruction::Alloca:
@@ -6329,10 +6371,8 @@ VPHistogramRecipe *VPRecipeBuilder::widenIfHistogram(VPInstruction *VPI) {
return nullptr;
const HistogramInfo *HI = *HistInfo;
- // FIXME: Support other operations.
- unsigned Opcode = HI->Update->getOpcode();
- assert((Opcode == Instruction::Add || Opcode == Instruction::Sub) &&
- "Histogram update operation must be an Add or Sub");
+ auto UpdateKind = VPHistogramRecipe::getUpdateKindForInstruction(HI->Update);
+ assert(UpdateKind && "Unsupported histogram update operation");
SmallVector<VPValue *, 3> HGramOps;
// Bucket address.
@@ -6345,8 +6385,8 @@ VPHistogramRecipe *VPRecipeBuilder::widenIfHistogram(VPInstruction *VPI) {
if (CM.isMaskRequired(HI->Store))
HGramOps.push_back(VPI->getMask());
- return new VPHistogramRecipe(Opcode, HGramOps, cast<VPIRMetadata>(*VPI),
- VPI->getDebugLoc());
+ return new VPHistogramRecipe(*UpdateKind, HGramOps,
+ cast<VPIRMetadata>(*VPI), VPI->getDebugLoc());
}
bool VPRecipeBuilder::replaceWithFinalIfReductionStore(
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 6f8ace6b81d6e..c01b847aaf33c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -2158,23 +2158,35 @@ class LLVM_ABI_FOR_TEST VPWidenCallRecipe : public VPRecipeWithIRFlags,
/// A recipe representing a sequence of load -> update -> store as part of
/// a histogram operation. This means there may be aliasing between vector
/// lanes, which is handled by the llvm.experimental.vector.histogram family
-/// of intrinsics. The only update operations currently supported are
-/// 'add' and 'sub' where the other term is loop-invariant.
+/// of intrinsics. Supported update operations are: add, sub, uadd.sat, umax,
+/// and umin, where the other term is loop-invariant.
class VPHistogramRecipe : public VPRecipeBase, public VPIRMetadata {
- /// Opcode of the update operation, currently either add or sub.
- unsigned Opcode;
+public:
+ /// The kind of update operation performed on histogram buckets.
+ enum class HistogramUpdateKind {
+ Add,
+ Sub,
+ UAddSat,
+ UMax,
+ UMin,
+ };
+
+private:
+ /// The update operation kind for this histogram.
+ HistogramUpdateKind UpdateKind;
public:
- VPHistogramRecipe(unsigned Opcode, ArrayRef<VPValue *> Operands,
+ VPHistogramRecipe(HistogramUpdateKind UpdateKind,
+ ArrayRef<VPValue *> Operands,
const VPIRMetadata &Metadata = {},
DebugLoc DL = DebugLoc::getUnknown())
: VPRecipeBase(VPRecipeBase::VPHistogramSC, Operands, DL),
- VPIRMetadata(Metadata), Opcode(Opcode) {}
+ VPIRMetadata(Metadata), UpdateKind(UpdateKind) {}
~VPHistogramRecipe() override = default;
VPHistogramRecipe *clone() override {
- return new VPHistogramRecipe(Opcode, operands(), *this, getDebugLoc());
+ return new VPHistogramRecipe(UpdateKind, operands(), *this, getDebugLoc());
}
VP_CLASSOF_IMPL(VPRecipeBase::VPHistogramSC);
@@ -2186,7 +2198,16 @@ class VPHistogramRecipe : public VPRecipeBase, public VPIRMetadata {
InstructionCost computeCost(ElementCount VF,
VPCostContext &Ctx) const override;
- unsigned getOpcode() const { return Opcode; }
+ HistogramUpdateKind getUpdateKind() const { return UpdateKind; }
+
+ /// Return the histogram intrinsic ID for this recipe's update kind.
+ Intrinsic::ID getHistogramIntrinsicID() const;
+
+ /// Return true if the increment should be negated before passing to the
+ /// histogram intrinsic (only for Sub).
+ bool shouldNegateIncrement() const {
+ return UpdateKind == HistogramUpdateKind::Sub;
+ }
/// Return the mask operand if one was provided, or a null pointer if all
/// lanes should be executed unconditionally.
@@ -2194,6 +2215,11 @@ class VPHistogramRecipe : public VPRecipeBase, public VPIRMetadata {
return getNumOperands() == 3 ? getOperand(2) : nullptr;
}
+ /// Return the HistogramUpdateKind for the given update instruction, or
+ /// std::nullopt if the instruction is not a supported histogram update.
+ static std::optional<HistogramUpdateKind>
+ getUpdateKindForInstruction(Instruction *I);
+
protected:
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
/// Print the recipe
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index fa67ac7246470..e34ae718b8e85 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -2402,19 +2402,59 @@ void VPHistogramRecipe::execute(VPTransformState &State) {
Mask =
Builder.CreateVectorSplat(VTy->getElementCount(), Builder.getInt1(1));
- // If this is a subtract, we want to invert the increment amount. We may
- // add a separate intrinsic in future, but for now we'll try this.
- if (Opcode == Instruction::Sub)
+ // If this is a subtract, we want to invert the increment amount and use
+ // the histogram_add intrinsic.
+ if (shouldNegateIncrement())
IncAmt = Builder.CreateNeg(IncAmt);
- else
- assert(Opcode == Instruction::Add && "only add or sub supported for now");
Instruction *HistogramInst = State.Builder.CreateIntrinsicWithoutFolding(
- Intrinsic::experimental_vector_histogram_add, {VTy, IncAmt->getType()},
+ getHistogramIntrinsicID(), {VTy, IncAmt->getType()},
{Address, IncAmt, Mask});
applyMetadata(*HistogramInst);
}
+Intrinsic::ID VPHistogramRecipe::getHistogramIntrinsicID() const {
+ switch (UpdateKind) {
+ case HistogramUpdateKind::Add:
+ case HistogramUpdateKind::Sub:
+ return Intrinsic::experimental_vector_histogram_add;
+ case HistogramUpdateKind::UAddSat:
+ return Intrinsic::experimental_vector_histogram_uadd_sat;
+ case HistogramUpdateKind::UMax:
+ return Intrinsic::experimental_vector_histogram_umax;
+ case HistogramUpdateKind::UMin:
+ return Intrinsic::experimental_vector_histogram_umin;
+ }
+ llvm_unreachable("Unknown HistogramUpdateKind");
+}
+
+std::optional<VPHistogramRecipe::HistogramUpdateKind>
+VPHistogramRecipe::getUpdateKindForInstruction(Instruction *I) {
+ if (auto *BO = dyn_cast<BinaryOperator>(I)) {
+ switch (BO->getOpcode()) {
+ case Instruction::Add:
+ return HistogramUpdateKind::Add;
+ case Instruction::Sub:
+ return HistogramUpdateKind::Sub;
+ default:
+ return std::nullopt;
+ }
+ }
+ if (auto *II = dyn_cast<IntrinsicInst>(I)) {
+ switch (II->getIntrinsicID()) {
+ case Intrinsic::uadd_sat:
+ return HistogramUpdateKind::UAddSat;
+ case Intrinsic::umax:
+ return HistogramUpdateKind::UMax;
+ case Intrinsic::umin:
+ return HistogramUpdateKind::UMin;
+ default:
+ return std::nullopt;
+ }
+ }
+ return std::nullopt;
+}
+
InstructionCost VPHistogramRecipe::computeCost(ElementCount VF,
VPCostContext &Ctx) const {
// FIXME: Take the gather and scatter into account as well. For now we're
@@ -2438,13 +2478,41 @@ InstructionCost VPHistogramRecipe::computeCost(ElementCount VF,
// Find the cost of the histogram operation itself.
Type *PtrTy = VectorType::get(AddressTy, VF);
Type *MaskTy = VectorType::get(Type::getInt1Ty(Ctx.LLVMCtx), VF);
- IntrinsicCostAttributes ICA(Intrinsic::experimental_vector_histogram_add,
+ IntrinsicCostAttributes ICA(getHistogramIntrinsicID(),
Type::getVoidTy(Ctx.LLVMCtx),
{PtrTy, IncTy, MaskTy});
- // Add the costs together with the add/sub operation.
+ // Compute the cost of the update operation.
+ InstructionCost UpdateCost;
+ switch (UpdateKind) {
+ case HistogramUpdateKind::Add:
+ UpdateCost =
+ Ctx.TTI.getArithmeticInstrCost(Instruction::Add, VTy, Ctx.CostKind);
+ break;
+ case HistogramUpdateKind::Sub:
+ UpdateCost =
+ Ctx.TTI.getArithmeticInstrCost(Instruction::Sub, VTy, Ctx.CostKind);
+ break;
+ case HistogramUpdateKind::UAddSat: {
+ IntrinsicCostAttributes UpdateICA(Intrinsic::uadd_sat, VTy, {VTy, VTy});
+ UpdateCost = Ctx.TTI.getIntrinsicInstrCost(UpdateICA, Ctx.CostKind);
+ break;
+ }
+ case HistogramUpdateKind::UMax: {
+ IntrinsicCostAttributes UpdateICA(Intrinsic::umax, VTy, {VTy, VTy});
+ UpdateCost = Ctx.TTI.getIntrinsicInstrCost(UpdateICA, Ctx.CostKind);
+ break;
+ }
+ case HistogramUpdateKind::UMin: {
+ IntrinsicCostAttributes UpdateICA(Intrinsic::umin, VTy, {VTy, VTy});
+ UpdateCost = Ctx.TTI.getIntrinsicInstrCost(UpdateICA, Ctx.CostKind);
+ break;
+ }
+ }
+
+ // Add the costs together with the update operation.
return Ctx.TTI.getIntrinsicInstrCost(ICA, Ctx.CostKind) + MulCost +
- Ctx.TTI.getArithmeticInstrCost(Opcode, VTy, Ctx.CostKind);
+ UpdateCost;
}
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
@@ -2453,11 +2521,22 @@ void VPHistogramRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
O << Indent << "WIDEN-HISTOGRAM buckets: ";
getOperand(0)->printAsOperand(O, SlotTracker);
- if (Opcode == Instruction::Sub)
- O << ", dec: ";
- else {
- assert(Opcode == Instruction::Add);
+ switch (UpdateKind) {
+ case HistogramUpdateKind::Add:
O << ", inc: ";
+ break;
+ case HistogramUpdateKind::Sub:
+ O << ", dec: ";
+ break;
+ case HistogramUpdateKind::UAddSat:
+ O << ", saturated inc: ";
+ break;
+ case HistogramUpdateKind::UMax:
+ O << ", max: ";
+ break;
+ case HistogramUpdateKind::UMin:
+ O << ", min: ";
+ break;
}
getOperand(1)->printAsOperand(O, SlotTracker);
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
index 43975ee68774e..aafdd7e6f3345 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
@@ -889,3 +889,175 @@ attributes #0 = { "target-features"="+sve2" vscale_range(1,16) }
!11 = !{!10, !12}
!12 = distinct !{!12, !13, !"scope-bucket-2"}
!13 = distinct !{!13, !"scopes-buckets"}
+
+;; Test histogram with uadd.sat update operation
+define void @simple_histogram_uadd_sat(ptr noalias %buckets, ptr readonly %indices, i64 %N) #0 {
+; CHECK-LABEL: define void @simple_histogram_uadd_sat(
+; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i32> [[WIDE_LOAD]] to <4 x i64>
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP1]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP1]], i64 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP1]], i64 2
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP1]], i64 3
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP3]]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP4]]
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP5]]
+; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x ptr> poison, ptr [[TMP6]], i32 0
+; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x ptr> [[TMP10]], ptr [[TMP7]], i32 1
+; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x ptr> [[TMP11]], ptr [[TMP8]], i32 2
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x ptr> [[TMP12]], ptr [[TMP9]], i32 3
+; CHECK-NEXT: call void @llvm.experimental.vector.histogram.uadd.sat.v4p0.i32(<4 x ptr> [[TMP13]], i32 1, <4 x i1> splat (i1 true))
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK: scalar.ph:
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
+ %l.idx = load i32, ptr %gep.indices, align 4
+ %idxprom1 = zext i32 %l.idx to i64
+ %gep.bucket = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+ %l.bucket = load i32, ptr %gep.bucket, align 4
+ %inc = call i32 @llvm.uadd.sat.i32(i32 %l.bucket, i32 1)
+ store i32 %inc, ptr %gep.bucket, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, %N
+ br i1 %exitcond, label %for.exit, label %for.body, !llvm.loop !4
+
+for.exit:
+ ret void
+}
+
+;; Test histogram with umax update operation
+define void @simple_histogram_umax(ptr noalias %buckets, ptr readonly %indices, i64 %N) #0 {
+; CHECK-LABEL: define void @simple_histogram_umax(
+; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i32> [[WIDE_LOAD]] to <4 x i64>
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP1]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP1]], i64 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP1]], i64 2
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP1]], i64 3
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP3]]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP4]]
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP5]]
+; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x ptr> poison, ptr [[TMP6]], i32 0
+; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x ptr> [[TMP10]], ptr [[TMP7]], i32 1
+; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x ptr> [[TMP11]], ptr [[TMP8]], i32 2
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x ptr> [[TMP12]], ptr [[TMP9]], i32 3
+; CHECK-NEXT: call void @llvm.experimental.vector.histogram.umax.v4p0.i32(<4 x ptr> [[TMP13]], i32 120, <4 x i1> splat (i1 true))
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK: scalar.ph:
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
+ %l.idx = load i32, ptr %gep.indices, align 4
+ %idxprom1 = zext i32 %l.idx to i64
+ %gep.bucket = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+ %l.bucket = load i32, ptr %gep.bucket, align 4
+ %inc = call i32 @llvm.umax.i32(i32 %l.bucket, i32 120)
+ store i32 %inc, ptr %gep.bucket, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, %N
+ br i1 %exitcond, label %for.exit, label %for.body, !llvm.loop !4
+
+for.exit:
+ ret void
+}
+
+;; Test histogram with umin update operation
+define void @simple_histogram_umin(ptr noalias %buckets, ptr readonly %indices, i64 %N) #0 {
+; CHECK-LABEL: define void @simple_histogram_umin(
+; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i32> [[WIDE_LOAD]] to <4 x i64>
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP1]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP1]], i64 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP1]], i64 2
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP1]], i64 3
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP3]]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP4]]
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP5]]
+; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x ptr> poison, ptr [[TMP6]], i32 0
+; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x ptr> [[TMP10]], ptr [[TMP7]], i32 1
+; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x ptr> [[TMP11]], ptr [[TMP8]], i32 2
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x ptr> [[TMP12]], ptr [[TMP9]], i32 3
+; CHECK-NEXT: call void @llvm.experimental.vector.histogram.umin.v4p0.i32(<4 x ptr> [[TMP13]], i32 99, <4 x i1> splat (i1 true))
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP31:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK: scalar.ph:
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
+ %l.idx = load i32, ptr %gep.indices, align 4
+ %idxprom1 = zext i32 %l.idx to i64
+ %gep.bucket = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+ %l.bucket = load i32, ptr %gep.bucket, align 4
+ %inc = call i32 @llvm.umin.i32(i32 %l.bucket, i32 99)
+ store i32 %inc, ptr %gep.bucket, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, %N
+ br i1 %exitcond, label %for.exit, label %for.body, !llvm.loop !4
+
+for.exit:
+ ret void
+}
+
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve2-histcnt-vplan.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve2-histcnt-vplan.ll
index 7ce306d9101cb..8a35dd2392af1 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve2-histcnt-vplan.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve2-histcnt-vplan.ll
@@ -153,6 +153,31 @@ for.exit:
ret void
}
+;; Check that uadd.sat histogram gets a "saturated inc:" label in VPlan.
+; CHECK: VPlan 'Initial VPlan for VF={vscale x 2,vscale x 4},UF>=1' {
+; CHECK: WIDEN-HISTOGRAM buckets: {{.*}}, saturated inc: ir<1>
+
+define void @simple_histogram_uadd_sat(ptr noalias %buckets, ptr readonly %indices, i64 %N) {
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
+ %l.idx = load i32, ptr %gep.indices, align 4
+ %idxprom1 = zext i32 %l.idx to i64
+ %gep.bucket = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+ %l.bucket = load i32, ptr %gep.bucket, align 4
+ %inc = call i32 @llvm.uadd.sat.i32(i32 %l.bucket, i32 1)
+ store i32 %inc, ptr %gep.bucket, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, %N
+ br i1 %exitcond, label %for.exit, label %for.body
+
+for.exit:
+ ret void
+}
+
!0 = !{!1}
!1 = distinct !{!1, !2}
!2 = distinct !{!2}
>From a171d7b7b7cf4055407539a46dec3b3347a36a77 Mon Sep 17 00:00:00 2001
From: rond <ron.dahan at mobileye.com>
Date: Wed, 17 Jun 2026 12:13:58 +0300
Subject: [PATCH 2/5] Fix clang format
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 611a265cd0d1d..dc3f3f16e8508 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6385,8 +6385,8 @@ VPHistogramRecipe *VPRecipeBuilder::widenIfHistogram(VPInstruction *VPI) {
if (CM.isMaskRequired(HI->Store))
HGramOps.push_back(VPI->getMask());
- return new VPHistogramRecipe(*UpdateKind, HGramOps,
- cast<VPIRMetadata>(*VPI), VPI->getDebugLoc());
+ return new VPHistogramRecipe(*UpdateKind, HGramOps, cast<VPIRMetadata>(*VPI),
+ VPI->getDebugLoc());
}
bool VPRecipeBuilder::replaceWithFinalIfReductionStore(
>From 8eb06823b6d0ea8c6dd12d86ab73c7f98ed4db7e Mon Sep 17 00:00:00 2001
From: rond <ron.dahan at mobileye.com>
Date: Tue, 23 Jun 2026 16:26:35 +0300
Subject: [PATCH 3/5] Fix review
Co-authored-by: Copilot <223556219+Copilot at users.noreply.github.com>
---
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 17 +-
.../LoopVectorize/AArch64/sve2-histcnt.ll | 170 ---------------
.../VPlan/AArch64/sve2-histcnt-vplan.ll | 46 ++++-
.../Transforms/LoopVectorize/histograms.ll | 195 ++++++++++++++++++
4 files changed, 251 insertions(+), 177 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index e34ae718b8e85..bfbb04a28822e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -2468,12 +2468,17 @@ InstructionCost VPHistogramRecipe::computeCost(ElementCount VF,
Type *IncTy = IncAmt->getScalarType();
VectorType *VTy = VectorType::get(IncTy, VF);
- // Assume that a non-constant update value (or a constant != 1) requires
- // a multiply, and add that into the cost.
- InstructionCost MulCost =
- Ctx.TTI.getArithmeticInstrCost(Instruction::Mul, VTy, Ctx.CostKind);
- if (match(IncAmt, m_One()))
- MulCost = TTI::TCC_Free;
+ // For umin/umax, there's no multiplication — the increment value is compared
+ // directly. For add/sub/uadd.sat, assume that a non-constant update value
+ // (or a constant != 1) requires a multiply.
+ InstructionCost MulCost = TTI::TCC_Free;
+ if (UpdateKind != HistogramUpdateKind::UMax &&
+ UpdateKind != HistogramUpdateKind::UMin) {
+ MulCost =
+ Ctx.TTI.getArithmeticInstrCost(Instruction::Mul, VTy, Ctx.CostKind);
+ if (match(IncAmt, m_One()))
+ MulCost = TTI::TCC_Free;
+ }
// Find the cost of the histogram operation itself.
Type *PtrTy = VectorType::get(AddressTy, VF);
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
index aafdd7e6f3345..131814730142f 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
@@ -890,174 +890,4 @@ attributes #0 = { "target-features"="+sve2" vscale_range(1,16) }
!12 = distinct !{!12, !13, !"scope-bucket-2"}
!13 = distinct !{!13, !"scopes-buckets"}
-;; Test histogram with uadd.sat update operation
-define void @simple_histogram_uadd_sat(ptr noalias %buckets, ptr readonly %indices, i64 %N) #0 {
-; CHECK-LABEL: define void @simple_histogram_uadd_sat(
-; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK: vector.ph:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i32> [[WIDE_LOAD]] to <4 x i64>
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP1]], i64 0
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP1]], i64 1
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP1]], i64 2
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP1]], i64 3
-; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP2]]
-; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP3]]
-; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP4]]
-; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP5]]
-; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x ptr> poison, ptr [[TMP6]], i32 0
-; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x ptr> [[TMP10]], ptr [[TMP7]], i32 1
-; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x ptr> [[TMP11]], ptr [[TMP8]], i32 2
-; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x ptr> [[TMP12]], ptr [[TMP9]], i32 3
-; CHECK-NEXT: call void @llvm.experimental.vector.histogram.uadd.sat.v4p0.i32(<4 x ptr> [[TMP13]], i32 1, <4 x i1> splat (i1 true))
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
-; CHECK: middle.block:
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
-; CHECK: scalar.ph:
-;
-entry:
- br label %for.body
-
-for.body:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
- %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
- %l.idx = load i32, ptr %gep.indices, align 4
- %idxprom1 = zext i32 %l.idx to i64
- %gep.bucket = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
- %l.bucket = load i32, ptr %gep.bucket, align 4
- %inc = call i32 @llvm.uadd.sat.i32(i32 %l.bucket, i32 1)
- store i32 %inc, ptr %gep.bucket, align 4
- %iv.next = add nuw nsw i64 %iv, 1
- %exitcond = icmp eq i64 %iv.next, %N
- br i1 %exitcond, label %for.exit, label %for.body, !llvm.loop !4
-
-for.exit:
- ret void
-}
-
-;; Test histogram with umax update operation
-define void @simple_histogram_umax(ptr noalias %buckets, ptr readonly %indices, i64 %N) #0 {
-; CHECK-LABEL: define void @simple_histogram_umax(
-; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK: vector.ph:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i32> [[WIDE_LOAD]] to <4 x i64>
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP1]], i64 0
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP1]], i64 1
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP1]], i64 2
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP1]], i64 3
-; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP2]]
-; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP3]]
-; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP4]]
-; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP5]]
-; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x ptr> poison, ptr [[TMP6]], i32 0
-; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x ptr> [[TMP10]], ptr [[TMP7]], i32 1
-; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x ptr> [[TMP11]], ptr [[TMP8]], i32 2
-; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x ptr> [[TMP12]], ptr [[TMP9]], i32 3
-; CHECK-NEXT: call void @llvm.experimental.vector.histogram.umax.v4p0.i32(<4 x ptr> [[TMP13]], i32 120, <4 x i1> splat (i1 true))
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]
-; CHECK: middle.block:
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
-; CHECK: scalar.ph:
-;
-entry:
- br label %for.body
-
-for.body:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
- %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
- %l.idx = load i32, ptr %gep.indices, align 4
- %idxprom1 = zext i32 %l.idx to i64
- %gep.bucket = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
- %l.bucket = load i32, ptr %gep.bucket, align 4
- %inc = call i32 @llvm.umax.i32(i32 %l.bucket, i32 120)
- store i32 %inc, ptr %gep.bucket, align 4
- %iv.next = add nuw nsw i64 %iv, 1
- %exitcond = icmp eq i64 %iv.next, %N
- br i1 %exitcond, label %for.exit, label %for.body, !llvm.loop !4
-
-for.exit:
- ret void
-}
-
-;; Test histogram with umin update operation
-define void @simple_histogram_umin(ptr noalias %buckets, ptr readonly %indices, i64 %N) #0 {
-; CHECK-LABEL: define void @simple_histogram_umin(
-; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK: vector.ph:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i32> [[WIDE_LOAD]] to <4 x i64>
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP1]], i64 0
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP1]], i64 1
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP1]], i64 2
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP1]], i64 3
-; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP2]]
-; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP3]]
-; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP4]]
-; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP5]]
-; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x ptr> poison, ptr [[TMP6]], i32 0
-; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x ptr> [[TMP10]], ptr [[TMP7]], i32 1
-; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x ptr> [[TMP11]], ptr [[TMP8]], i32 2
-; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x ptr> [[TMP12]], ptr [[TMP9]], i32 3
-; CHECK-NEXT: call void @llvm.experimental.vector.histogram.umin.v4p0.i32(<4 x ptr> [[TMP13]], i32 99, <4 x i1> splat (i1 true))
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP31:![0-9]+]]
-; CHECK: middle.block:
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
-; CHECK: scalar.ph:
-;
-entry:
- br label %for.body
-
-for.body:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
- %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
- %l.idx = load i32, ptr %gep.indices, align 4
- %idxprom1 = zext i32 %l.idx to i64
- %gep.bucket = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
- %l.bucket = load i32, ptr %gep.bucket, align 4
- %inc = call i32 @llvm.umin.i32(i32 %l.bucket, i32 99)
- store i32 %inc, ptr %gep.bucket, align 4
- %iv.next = add nuw nsw i64 %iv, 1
- %exitcond = icmp eq i64 %iv.next, %N
- br i1 %exitcond, label %for.exit, label %for.body, !llvm.loop !4
-
-for.exit:
- ret void
-}
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve2-histcnt-vplan.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve2-histcnt-vplan.ll
index 8a35dd2392af1..b414f232376c6 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve2-histcnt-vplan.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve2-histcnt-vplan.ll
@@ -155,7 +155,51 @@ for.exit:
;; Check that uadd.sat histogram gets a "saturated inc:" label in VPlan.
; CHECK: VPlan 'Initial VPlan for VF={vscale x 2,vscale x 4},UF>=1' {
-; CHECK: WIDEN-HISTOGRAM buckets: {{.*}}, saturated inc: ir<1>
+; CHECK-NEXT: Live-in [[VF3:.*]] = VF
+; CHECK-NEXT: Live-in [[VFxUF3:.*]] = VF * UF
+; CHECK-NEXT: Live-in [[VTC3:.*]] = vector-trip-count
+; CHECK-NEXT: Live-in [[OTC3:.*]] = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: [[IV3:.*]] = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: [[STEPS3:vp.*]] = SCALAR-STEPS [[IV3]], ir<1>, [[VF3]]
+; CHECK-NEXT: CLONE [[GEP_IDX3:.*]] = getelementptr inbounds ir<%indices>, [[STEPS3]]
+; CHECK-NEXT: [[VECP_IDX3:vp.*]] = vector-pointer inbounds [[GEP_IDX3]]
+; CHECK-NEXT: WIDEN [[IDX3:.*]] = load [[VECP_IDX3]]
+; CHECK-NEXT: WIDEN-CAST [[EXT_IDX3:.*]] = zext [[IDX3]] to i64
+; CHECK-NEXT: WIDEN-GEP [[GEP_BUCKET3:.*]] = getelementptr inbounds ir<%buckets>, [[EXT_IDX3]]
+; CHECK-NEXT: WIDEN-HISTOGRAM buckets: [[GEP_BUCKET3]], saturated inc: ir<1>
+; CHECK-NEXT: EMIT [[IV_NEXT3:.*]] = add nuw [[IV3]], [[VFxUF3]]
+; CHECK-NEXT: EMIT branch-on-count [[IV_NEXT3]], [[VTC3]]
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT [[TC_CHECK3:.*]] = icmp eq [[OTC3]], [[VTC3]]
+; CHECK-NEXT: EMIT branch-on-cond [[TC_CHECK3]]
+; CHECK-NEXT: Successor(s): ir-bb<for.exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<for.exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<[[RESUME3:%.+]]> = phi [ [[VTC3]], middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<for.body>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<for.body>:
+; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ] (extra operand: vp<[[RESUME3]]> from scalar.ph)
+; CHECK: IR %exitcond = icmp eq i64 %iv.next, %N
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
define void @simple_histogram_uadd_sat(ptr noalias %buckets, ptr readonly %indices, i64 %N) {
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/histograms.ll b/llvm/test/Transforms/LoopVectorize/histograms.ll
index f4988bb01dfa4..982ca0e12c753 100644
--- a/llvm/test/Transforms/LoopVectorize/histograms.ll
+++ b/llvm/test/Transforms/LoopVectorize/histograms.ll
@@ -65,3 +65,198 @@ for.body:
for.exit:
ret void
}
+
+define void @simple_histogram_uadd_sat(ptr noalias %buckets, ptr readonly %indices, i64 %N) {
+; CHECK-LABEL: define void @simple_histogram_uadd_sat(
+; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[N]], -2
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [4 x i8], ptr [[INDICES]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = zext <2 x i32> [[WIDE_LOAD]] to <2 x i64>
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i64> [[TMP1]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i64> [[TMP1]], i64 1
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[BUCKETS]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[BUCKETS]], i64 [[TMP3]]
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP4]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x ptr> [[TMP6]], ptr [[TMP5]], i64 1
+; CHECK-NEXT: call void @llvm.experimental.vector.histogram.uadd.sat.v2p0.i32(<2 x ptr> [[TMP7]], i32 1, <2 x i1> splat (i1 true))
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK: scalar.ph:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT: br label [[FOR_BODY:%.*]]
+; CHECK: for.body:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT: [[GEP_INDICES:%.*]] = getelementptr inbounds [4 x i8], ptr [[INDICES]], i64 [[IV]]
+; CHECK-NEXT: [[L_IDX:%.*]] = load i32, ptr [[GEP_INDICES]], align 4
+; CHECK-NEXT: [[IDXPROM1:%.*]] = zext i32 [[L_IDX]] to i64
+; CHECK-NEXT: [[GEP_BUCKET:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[BUCKETS]], i64 [[IDXPROM1]]
+; CHECK-NEXT: [[L_BUCKET:%.*]] = load i32, ptr [[GEP_BUCKET]], align 4
+; CHECK-NEXT: [[INC:%.*]] = call i32 @llvm.uadd.sat.i32(i32 [[L_BUCKET]], i32 1)
+; CHECK-NEXT: store i32 [[INC]], ptr [[GEP_BUCKET]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK: for.exit:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
+ %l.idx = load i32, ptr %gep.indices, align 4
+ %idxprom1 = zext i32 %l.idx to i64
+ %gep.bucket = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+ %l.bucket = load i32, ptr %gep.bucket, align 4
+ %inc = call i32 @llvm.uadd.sat.i32(i32 %l.bucket, i32 1)
+ store i32 %inc, ptr %gep.bucket, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, %N
+ br i1 %exitcond, label %for.exit, label %for.body
+
+for.exit:
+ ret void
+}
+
+define void @simple_histogram_umax(ptr noalias %buckets, ptr readonly %indices, i64 %N) {
+; CHECK-LABEL: define void @simple_histogram_umax(
+; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[N]], -2
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [4 x i8], ptr [[INDICES]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = zext <2 x i32> [[WIDE_LOAD]] to <2 x i64>
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i64> [[TMP1]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i64> [[TMP1]], i64 1
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[BUCKETS]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[BUCKETS]], i64 [[TMP3]]
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP4]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x ptr> [[TMP6]], ptr [[TMP5]], i64 1
+; CHECK-NEXT: call void @llvm.experimental.vector.histogram.umax.v2p0.i32(<2 x ptr> [[TMP7]], i32 120, <2 x i1> splat (i1 true))
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK: scalar.ph:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT: br label [[FOR_BODY:%.*]]
+; CHECK: for.body:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT: [[GEP_INDICES:%.*]] = getelementptr inbounds [4 x i8], ptr [[INDICES]], i64 [[IV]]
+; CHECK-NEXT: [[L_IDX:%.*]] = load i32, ptr [[GEP_INDICES]], align 4
+; CHECK-NEXT: [[IDXPROM1:%.*]] = zext i32 [[L_IDX]] to i64
+; CHECK-NEXT: [[GEP_BUCKET:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[BUCKETS]], i64 [[IDXPROM1]]
+; CHECK-NEXT: [[L_BUCKET:%.*]] = load i32, ptr [[GEP_BUCKET]], align 4
+; CHECK-NEXT: [[INC:%.*]] = call i32 @llvm.umax.i32(i32 [[L_BUCKET]], i32 120)
+; CHECK-NEXT: store i32 [[INC]], ptr [[GEP_BUCKET]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK: for.exit:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
+ %l.idx = load i32, ptr %gep.indices, align 4
+ %idxprom1 = zext i32 %l.idx to i64
+ %gep.bucket = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+ %l.bucket = load i32, ptr %gep.bucket, align 4
+ %inc = call i32 @llvm.umax.i32(i32 %l.bucket, i32 120)
+ store i32 %inc, ptr %gep.bucket, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, %N
+ br i1 %exitcond, label %for.exit, label %for.body
+
+for.exit:
+ ret void
+}
+
+define void @simple_histogram_umin(ptr noalias %buckets, ptr readonly %indices, i64 %N) {
+; CHECK-LABEL: define void @simple_histogram_umin(
+; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[N]], -2
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [4 x i8], ptr [[INDICES]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = zext <2 x i32> [[WIDE_LOAD]] to <2 x i64>
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i64> [[TMP1]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i64> [[TMP1]], i64 1
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[BUCKETS]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[BUCKETS]], i64 [[TMP3]]
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP4]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x ptr> [[TMP6]], ptr [[TMP5]], i64 1
+; CHECK-NEXT: call void @llvm.experimental.vector.histogram.umin.v2p0.i32(<2 x ptr> [[TMP7]], i32 99, <2 x i1> splat (i1 true))
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK: scalar.ph:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT: br label [[FOR_BODY:%.*]]
+; CHECK: for.body:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT: [[GEP_INDICES:%.*]] = getelementptr inbounds [4 x i8], ptr [[INDICES]], i64 [[IV]]
+; CHECK-NEXT: [[L_IDX:%.*]] = load i32, ptr [[GEP_INDICES]], align 4
+; CHECK-NEXT: [[IDXPROM1:%.*]] = zext i32 [[L_IDX]] to i64
+; CHECK-NEXT: [[GEP_BUCKET:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[BUCKETS]], i64 [[IDXPROM1]]
+; CHECK-NEXT: [[L_BUCKET:%.*]] = load i32, ptr [[GEP_BUCKET]], align 4
+; CHECK-NEXT: [[INC:%.*]] = call i32 @llvm.umin.i32(i32 [[L_BUCKET]], i32 99)
+; CHECK-NEXT: store i32 [[INC]], ptr [[GEP_BUCKET]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK: for.exit:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
+ %l.idx = load i32, ptr %gep.indices, align 4
+ %idxprom1 = zext i32 %l.idx to i64
+ %gep.bucket = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+ %l.bucket = load i32, ptr %gep.bucket, align 4
+ %inc = call i32 @llvm.umin.i32(i32 %l.bucket, i32 99)
+ store i32 %inc, ptr %gep.bucket, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, %N
+ br i1 %exitcond, label %for.exit, label %for.body
+
+for.exit:
+ ret void
+}
>From 509c24bd8f56b1768363c6aeeb85200069479965 Mon Sep 17 00:00:00 2001
From: rond <ron.dahan at mobileye.com>
Date: Tue, 23 Jun 2026 16:52:03 +0300
Subject: [PATCH 4/5] Combine MulCost conditions in
VPHistogramRecipe::computeCost
Simplify the logic by combining the m_One() check with the UMax/UMin
checks into a single condition, avoiding redundant assignment of MulCost.
Co-authored-by: Copilot <223556219+Copilot at users.noreply.github.com>
---
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 7 ++-----
1 file changed, 2 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index bfbb04a28822e..22fcbadcb95a0 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -2472,13 +2472,10 @@ InstructionCost VPHistogramRecipe::computeCost(ElementCount VF,
// directly. For add/sub/uadd.sat, assume that a non-constant update value
// (or a constant != 1) requires a multiply.
InstructionCost MulCost = TTI::TCC_Free;
- if (UpdateKind != HistogramUpdateKind::UMax &&
- UpdateKind != HistogramUpdateKind::UMin) {
+ if (!match(IncAmt, m_One()) && UpdateKind != HistogramUpdateKind::UMax &&
+ UpdateKind != HistogramUpdateKind::UMin)
MulCost =
Ctx.TTI.getArithmeticInstrCost(Instruction::Mul, VTy, Ctx.CostKind);
- if (match(IncAmt, m_One()))
- MulCost = TTI::TCC_Free;
- }
// Find the cost of the histogram operation itself.
Type *PtrTy = VectorType::get(AddressTy, VF);
>From 45504dd3a588c27292bb5da1d23880f88d11e81e Mon Sep 17 00:00:00 2001
From: rond <ron.dahan at mobileye.com>
Date: Thu, 9 Jul 2026 18:23:35 +0300
Subject: [PATCH 5/5] Answer fhahn's review
---
.../Transforms/Vectorize/LoopVectorize.cpp | 78 ++++++++-----------
llvm/lib/Transforms/Vectorize/VPlan.h | 5 --
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 27 -------
3 files changed, 34 insertions(+), 76 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index dc3f3f16e8508..edbf31853565e 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -5336,47 +5336,6 @@ LoopVectorizationCostModel::getInstructionCost(Instruction *I,
Config.CostKind, I);
}
case Instruction::Call: {
- // Check if this is a histogram update operation (intrinsic like uadd.sat,
- // umax, umin used as histogram bucket update).
- auto Info = Legal->getHistogramInfo(I);
- if (Info && VF.isVector()) {
- const HistogramInfo *HGram = Info.value();
- // Assume that a non-constant update value (or a constant != 1) requires
- // a multiply, and add that into the cost.
- InstructionCost MulCost = TTI::TCC_Free;
- ConstantInt *RHS = dyn_cast<ConstantInt>(I->getOperand(1));
- if (!RHS || RHS->getZExtValue() != 1)
- MulCost = TTI.getArithmeticInstrCost(Instruction::Mul, VectorTy,
- Config.CostKind);
-
- // Find the cost of the histogram operation itself.
- Type *PtrTy = VectorType::get(HGram->Load->getPointerOperandType(), VF);
- Type *ScalarTy = I->getType();
- Type *MaskTy = VectorType::get(Type::getInt1Ty(I->getContext()), VF);
- auto *II = cast<IntrinsicInst>(I);
- Intrinsic::ID HistID;
- switch (II->getIntrinsicID()) {
- case Intrinsic::uadd_sat:
- HistID = Intrinsic::experimental_vector_histogram_uadd_sat;
- break;
- case Intrinsic::umax:
- HistID = Intrinsic::experimental_vector_histogram_umax;
- break;
- case Intrinsic::umin:
- HistID = Intrinsic::experimental_vector_histogram_umin;
- break;
- default:
- llvm_unreachable("Unsupported histogram intrinsic");
- }
- IntrinsicCostAttributes ICA(HistID, Type::getVoidTy(I->getContext()),
- {PtrTy, ScalarTy, MaskTy});
-
- // Add the costs together with the update operation cost.
- IntrinsicCostAttributes UpdateICA(II->getIntrinsicID(), VectorTy,
- {VectorTy, VectorTy});
- return TTI.getIntrinsicInstrCost(ICA, Config.CostKind) + MulCost +
- TTI.getIntrinsicInstrCost(UpdateICA, Config.CostKind);
- }
return getVectorCallCost(cast<CallInst>(I), VF);
}
case Instruction::ExtractValue:
@@ -6361,6 +6320,37 @@ VPRecipeWithIRFlags *VPRecipeBuilder::tryToWiden(VPInstruction *VPI) {
};
}
+/// Return the HistogramUpdateKind for the given update instruction. The
+/// instruction must be one of the supported histogram update operations;
+/// callers are expected to have validated this via
+/// LoopVectorizationLegality::getHistogramInfo before invoking this helper.
+static VPHistogramRecipe::HistogramUpdateKind
+getHistogramUpdateKind(Instruction *I) {
+ using HistogramUpdateKind = VPHistogramRecipe::HistogramUpdateKind;
+ if (auto *BO = dyn_cast<BinaryOperator>(I)) {
+ switch (BO->getOpcode()) {
+ case Instruction::Add:
+ return HistogramUpdateKind::Add;
+ case Instruction::Sub:
+ return HistogramUpdateKind::Sub;
+ default:
+ break;
+ }
+ } else if (auto *II = dyn_cast<IntrinsicInst>(I)) {
+ switch (II->getIntrinsicID()) {
+ case Intrinsic::uadd_sat:
+ return HistogramUpdateKind::UAddSat;
+ case Intrinsic::umax:
+ return HistogramUpdateKind::UMax;
+ case Intrinsic::umin:
+ return HistogramUpdateKind::UMin;
+ default:
+ break;
+ }
+ }
+ llvm_unreachable("Unsupported histogram update operation");
+}
+
VPHistogramRecipe *VPRecipeBuilder::widenIfHistogram(VPInstruction *VPI) {
if (VPI->getOpcode() != Instruction::Store)
return nullptr;
@@ -6371,8 +6361,8 @@ VPHistogramRecipe *VPRecipeBuilder::widenIfHistogram(VPInstruction *VPI) {
return nullptr;
const HistogramInfo *HI = *HistInfo;
- auto UpdateKind = VPHistogramRecipe::getUpdateKindForInstruction(HI->Update);
- assert(UpdateKind && "Unsupported histogram update operation");
+ VPHistogramRecipe::HistogramUpdateKind UpdateKind =
+ getHistogramUpdateKind(HI->Update);
SmallVector<VPValue *, 3> HGramOps;
// Bucket address.
@@ -6385,7 +6375,7 @@ VPHistogramRecipe *VPRecipeBuilder::widenIfHistogram(VPInstruction *VPI) {
if (CM.isMaskRequired(HI->Store))
HGramOps.push_back(VPI->getMask());
- return new VPHistogramRecipe(*UpdateKind, HGramOps, cast<VPIRMetadata>(*VPI),
+ return new VPHistogramRecipe(UpdateKind, HGramOps, cast<VPIRMetadata>(*VPI),
VPI->getDebugLoc());
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index c01b847aaf33c..da5af746512ab 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -2215,11 +2215,6 @@ class VPHistogramRecipe : public VPRecipeBase, public VPIRMetadata {
return getNumOperands() == 3 ? getOperand(2) : nullptr;
}
- /// Return the HistogramUpdateKind for the given update instruction, or
- /// std::nullopt if the instruction is not a supported histogram update.
- static std::optional<HistogramUpdateKind>
- getUpdateKindForInstruction(Instruction *I);
-
protected:
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
/// Print the recipe
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 22fcbadcb95a0..bb60e67f9d5fb 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -2428,33 +2428,6 @@ Intrinsic::ID VPHistogramRecipe::getHistogramIntrinsicID() const {
llvm_unreachable("Unknown HistogramUpdateKind");
}
-std::optional<VPHistogramRecipe::HistogramUpdateKind>
-VPHistogramRecipe::getUpdateKindForInstruction(Instruction *I) {
- if (auto *BO = dyn_cast<BinaryOperator>(I)) {
- switch (BO->getOpcode()) {
- case Instruction::Add:
- return HistogramUpdateKind::Add;
- case Instruction::Sub:
- return HistogramUpdateKind::Sub;
- default:
- return std::nullopt;
- }
- }
- if (auto *II = dyn_cast<IntrinsicInst>(I)) {
- switch (II->getIntrinsicID()) {
- case Intrinsic::uadd_sat:
- return HistogramUpdateKind::UAddSat;
- case Intrinsic::umax:
- return HistogramUpdateKind::UMax;
- case Intrinsic::umin:
- return HistogramUpdateKind::UMin;
- default:
- return std::nullopt;
- }
- }
- return std::nullopt;
-}
-
InstructionCost VPHistogramRecipe::computeCost(ElementCount VF,
VPCostContext &Ctx) const {
// FIXME: Take the gather and scatter into account as well. For now we're
More information about the llvm-commits
mailing list