[llvm] LV: Expand llvm.histogram intrinsic to support umax, umin, and uadd.sat operations (PR #127399)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 4 01:21:13 PDT 2026
https://github.com/RonDahan101 updated https://github.com/llvm/llvm-project/pull/127399
>From 0c894f7f82531f59d19083ffba99f53004591d12 Mon Sep 17 00:00:00 2001
From: rond <ron.dahan at mobileye.com>
Date: Sun, 31 May 2026 16:03:17 +0300
Subject: [PATCH 1/7] [LV] Extend histogram vectorization to support uadd.sat,
umax, and umin
Extend the Loop Vectorizer's histogram support to recognize and vectorize
loops with uadd.sat, umax, and umin update operations in addition to the
existing add/sub support.
This builds on the backend intrinsics added in PR #138447:
- llvm.experimental.vector.histogram.uadd.sat
- llvm.experimental.vector.histogram.umax
- llvm.experimental.vector.histogram.umin
Changes:
- LoopVectorizationLegality: Extend findHistogram() pattern matching to
recognize intrinsic-based update operations (uadd.sat, umax, umin) in
addition to binary add/sub.
- VPlan.h: Replace the raw 'unsigned Opcode' in VPHistogramRecipe with a
dedicated HistogramUpdateKind enum that cleanly represents all supported
operations without requiring artificial operands.
- VPlanRecipes.cpp: Update execute() to emit the correct histogram intrinsic
based on UpdateKind, add getHistogramIntrinsicID() and
getUpdateKindForInstruction() helpers, update cost model and print.
- LoopVectorize.cpp: Update widenIfHistogram() to use the new enum, and
extend the non-VPlan cost model to handle intrinsic histogram updates.
- Tests: Add new test functions for uadd.sat, umax, umin in sve2-histcnt.ll
and VPlan print verification in sve2-histcnt-vplan.ll.
Co-authored-by: Copilot <223556219+Copilot at users.noreply.github.com>
---
.../Vectorize/LoopVectorizationLegality.cpp | 38 ++--
.../Transforms/Vectorize/LoopVectorize.cpp | 54 +++++-
llvm/lib/Transforms/Vectorize/VPlan.h | 42 ++++-
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 105 +++++++++--
.../LoopVectorize/AArch64/sve2-histcnt.ll | 172 ++++++++++++++++++
.../VPlan/AArch64/sve2-histcnt-vplan.ll | 25 +++
6 files changed, 394 insertions(+), 42 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 9086880599231..f79ed0c9c39bb 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -1040,34 +1040,44 @@ bool LoopVectorizationLegality::canVectorizeInstr(Instruction &I) {
/// Find histogram operations that match high-level code in loops:
/// \code
-/// buckets[indices[i]]+=step;
+/// buckets[indices[i]] = UpdateOp(buckets[indices[i]], Val);
/// \endcode
+/// Where UpdateOp can be add, sub, uadd.sat, umax, or umin.
///
/// It matches a pattern starting from \p HSt, which Stores to the 'buckets'
-/// array the computed histogram. It uses a BinOp to sum all counts, storing
-/// them using a loop-variant index Load from the 'indices' input array.
+/// array the computed histogram. It uses an update instruction to update all
+/// counts, storing them using a loop-variant index Load from the 'indices'
+/// input array.
///
/// On successful matches it updates the STATISTIC 'HistogramsDetected',
/// regardless of hardware support. When there is support, it additionally
-/// stores the BinOp/Load pairs in \p HistogramCounts, as well the pointers
+/// stores the UpdateOp/Load pairs in \p HistogramCounts, as well the pointers
/// used to update histogram in \p HistogramPtrs.
static bool findHistogram(LoadInst *LI, StoreInst *HSt, Loop *TheLoop,
const PredicatedScalarEvolution &PSE,
SmallVectorImpl<HistogramInfo> &Histograms) {
- // Store value must come from a Binary Operation.
+ // Store value must come from an update operation (binary op or intrinsic).
Instruction *HPtrInstr = nullptr;
- BinaryOperator *HBinOp = nullptr;
- if (!match(HSt, m_Store(m_BinOp(HBinOp), m_Instruction(HPtrInstr))))
+ Instruction *HUpdateOp = nullptr;
+ if (!match(HSt, m_Store(m_Instruction(HUpdateOp), m_Instruction(HPtrInstr))))
return false;
- // BinOp must be an Add or a Sub modifying the bucket value by a
- // loop invariant amount.
+ // The update operation must modify the bucket value by a loop invariant
+ // amount. Supported operations: add, sub, uadd.sat, umax, umin.
// FIXME: We assume the loop invariant term is on the RHS.
// Fine for an immediate/constant, but maybe not a generic value?
Value *HIncVal = nullptr;
- if (!match(HBinOp, m_Add(m_Load(m_Specific(HPtrInstr)), m_Value(HIncVal))) &&
- !match(HBinOp, m_Sub(m_Load(m_Specific(HPtrInstr)), m_Value(HIncVal))))
+ if (!match(HUpdateOp,
+ m_Add(m_Load(m_Specific(HPtrInstr)), m_Value(HIncVal))) &&
+ !match(HUpdateOp,
+ m_Sub(m_Load(m_Specific(HPtrInstr)), m_Value(HIncVal))) &&
+ !match(HUpdateOp, m_Intrinsic<Intrinsic::uadd_sat>(
+ m_Load(m_Specific(HPtrInstr)), m_Value(HIncVal))) &&
+ !match(HUpdateOp, m_Intrinsic<Intrinsic::umax>(
+ m_Load(m_Specific(HPtrInstr)), m_Value(HIncVal))) &&
+ !match(HUpdateOp, m_Intrinsic<Intrinsic::umin>(
+ m_Load(m_Specific(HPtrInstr)), m_Value(HIncVal))))
return false;
// Make sure the increment value is loop invariant.
@@ -1109,15 +1119,15 @@ static bool findHistogram(LoadInst *LI, StoreInst *HSt, Loop *TheLoop,
// Ensure we'll have the same mask by checking that all parts of the histogram
// (gather load, update, scatter store) are in the same block.
- LoadInst *IndexedLoad = cast<LoadInst>(HBinOp->getOperand(0));
+ LoadInst *IndexedLoad = cast<LoadInst>(HUpdateOp->getOperand(0));
BasicBlock *LdBB = IndexedLoad->getParent();
- if (LdBB != HBinOp->getParent() || LdBB != HSt->getParent())
+ if (LdBB != HUpdateOp->getParent() || LdBB != HSt->getParent())
return false;
LLVM_DEBUG(dbgs() << "LV: Found histogram for: " << *HSt << "\n");
// Store the operations that make up the histogram.
- Histograms.emplace_back(IndexedLoad, HBinOp, HSt);
+ Histograms.emplace_back(IndexedLoad, HUpdateOp, HSt);
return true;
}
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index de385ae0a4511..54c3bf11d62bb 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -5339,8 +5339,50 @@ LoopVectorizationCostModel::getInstructionCost(Instruction *I,
return TTI.getCastInstrCost(Opcode, VectorTy, SrcVecTy, CCH,
Config.CostKind, I);
}
- case Instruction::Call:
+ case Instruction::Call: {
+ // Check if this is a histogram update operation (intrinsic like uadd.sat,
+ // umax, umin used as histogram bucket update).
+ auto Info = Legal->getHistogramInfo(I);
+ if (Info && VF.isVector()) {
+ const HistogramInfo *HGram = Info.value();
+ // Assume that a non-constant update value (or a constant != 1) requires
+ // a multiply, and add that into the cost.
+ InstructionCost MulCost = TTI::TCC_Free;
+ ConstantInt *RHS = dyn_cast<ConstantInt>(I->getOperand(1));
+ if (!RHS || RHS->getZExtValue() != 1)
+ MulCost = TTI.getArithmeticInstrCost(Instruction::Mul, VectorTy,
+ Config.CostKind);
+
+ // Find the cost of the histogram operation itself.
+ Type *PtrTy = VectorType::get(HGram->Load->getPointerOperandType(), VF);
+ Type *ScalarTy = I->getType();
+ Type *MaskTy = VectorType::get(Type::getInt1Ty(I->getContext()), VF);
+ auto *II = cast<IntrinsicInst>(I);
+ Intrinsic::ID HistID;
+ switch (II->getIntrinsicID()) {
+ case Intrinsic::uadd_sat:
+ HistID = Intrinsic::experimental_vector_histogram_uadd_sat;
+ break;
+ case Intrinsic::umax:
+ HistID = Intrinsic::experimental_vector_histogram_umax;
+ break;
+ case Intrinsic::umin:
+ HistID = Intrinsic::experimental_vector_histogram_umin;
+ break;
+ default:
+ llvm_unreachable("Unsupported histogram intrinsic");
+ }
+ IntrinsicCostAttributes ICA(HistID, Type::getVoidTy(I->getContext()),
+ {PtrTy, ScalarTy, MaskTy});
+
+ // Add the costs together with the update operation cost.
+ IntrinsicCostAttributes UpdateICA(II->getIntrinsicID(), VectorTy,
+ {VectorTy, VectorTy});
+ return TTI.getIntrinsicInstrCost(ICA, Config.CostKind) + MulCost +
+ TTI.getIntrinsicInstrCost(UpdateICA, Config.CostKind);
+ }
return getVectorCallCost(cast<CallInst>(I), VF);
+ }
case Instruction::ExtractValue:
return TTI.getInstructionCost(I, Config.CostKind);
case Instruction::Alloca:
@@ -6368,10 +6410,8 @@ VPHistogramRecipe *VPRecipeBuilder::widenIfHistogram(VPInstruction *VPI) {
return nullptr;
const HistogramInfo *HI = *HistInfo;
- // FIXME: Support other operations.
- unsigned Opcode = HI->Update->getOpcode();
- assert((Opcode == Instruction::Add || Opcode == Instruction::Sub) &&
- "Histogram update operation must be an Add or Sub");
+ auto UpdateKind = VPHistogramRecipe::getUpdateKindForInstruction(HI->Update);
+ assert(UpdateKind && "Unsupported histogram update operation");
SmallVector<VPValue *, 3> HGramOps;
// Bucket address.
@@ -6384,8 +6424,8 @@ VPHistogramRecipe *VPRecipeBuilder::widenIfHistogram(VPInstruction *VPI) {
if (CM.isMaskRequired(HI->Store))
HGramOps.push_back(VPI->getMask());
- return new VPHistogramRecipe(Opcode, HGramOps, cast<VPIRMetadata>(*VPI),
- VPI->getDebugLoc());
+ return new VPHistogramRecipe(*UpdateKind, HGramOps,
+ cast<VPIRMetadata>(*VPI), VPI->getDebugLoc());
}
bool VPRecipeBuilder::replaceWithFinalIfReductionStore(
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 814b77a96e825..c8853c8999b5f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -2159,23 +2159,35 @@ class LLVM_ABI_FOR_TEST VPWidenCallRecipe : public VPRecipeWithIRFlags,
/// A recipe representing a sequence of load -> update -> store as part of
/// a histogram operation. This means there may be aliasing between vector
/// lanes, which is handled by the llvm.experimental.vector.histogram family
-/// of intrinsics. The only update operations currently supported are
-/// 'add' and 'sub' where the other term is loop-invariant.
+/// of intrinsics. Supported update operations are: add, sub, uadd.sat, umax,
+/// and umin, where the other term is loop-invariant.
class VPHistogramRecipe : public VPRecipeBase, public VPIRMetadata {
- /// Opcode of the update operation, currently either add or sub.
- unsigned Opcode;
+public:
+ /// The kind of update operation performed on histogram buckets.
+ enum class HistogramUpdateKind {
+ Add,
+ Sub,
+ UAddSat,
+ UMax,
+ UMin,
+ };
+
+private:
+ /// The update operation kind for this histogram.
+ HistogramUpdateKind UpdateKind;
public:
- VPHistogramRecipe(unsigned Opcode, ArrayRef<VPValue *> Operands,
+ VPHistogramRecipe(HistogramUpdateKind UpdateKind,
+ ArrayRef<VPValue *> Operands,
const VPIRMetadata &Metadata = {},
DebugLoc DL = DebugLoc::getUnknown())
: VPRecipeBase(VPRecipeBase::VPHistogramSC, Operands, DL),
- VPIRMetadata(Metadata), Opcode(Opcode) {}
+ VPIRMetadata(Metadata), UpdateKind(UpdateKind) {}
~VPHistogramRecipe() override = default;
VPHistogramRecipe *clone() override {
- return new VPHistogramRecipe(Opcode, operands(), *this, getDebugLoc());
+ return new VPHistogramRecipe(UpdateKind, operands(), *this, getDebugLoc());
}
VP_CLASSOF_IMPL(VPRecipeBase::VPHistogramSC);
@@ -2187,7 +2199,16 @@ class VPHistogramRecipe : public VPRecipeBase, public VPIRMetadata {
InstructionCost computeCost(ElementCount VF,
VPCostContext &Ctx) const override;
- unsigned getOpcode() const { return Opcode; }
+ HistogramUpdateKind getUpdateKind() const { return UpdateKind; }
+
+ /// Return the histogram intrinsic ID for this recipe's update kind.
+ Intrinsic::ID getHistogramIntrinsicID() const;
+
+ /// Return true if the increment should be negated before passing to the
+ /// histogram intrinsic (only for Sub).
+ bool shouldNegateIncrement() const {
+ return UpdateKind == HistogramUpdateKind::Sub;
+ }
/// Return the mask operand if one was provided, or a null pointer if all
/// lanes should be executed unconditionally.
@@ -2195,6 +2216,11 @@ class VPHistogramRecipe : public VPRecipeBase, public VPIRMetadata {
return getNumOperands() == 3 ? getOperand(2) : nullptr;
}
+ /// Return the HistogramUpdateKind for the given update instruction, or
+ /// std::nullopt if the instruction is not a supported histogram update.
+ static std::optional<HistogramUpdateKind>
+ getUpdateKindForInstruction(Instruction *I);
+
protected:
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
/// Print the recipe
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index ca63d1498316b..d6568ef012d2d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -2418,19 +2418,59 @@ void VPHistogramRecipe::execute(VPTransformState &State) {
Mask =
Builder.CreateVectorSplat(VTy->getElementCount(), Builder.getInt1(1));
- // If this is a subtract, we want to invert the increment amount. We may
- // add a separate intrinsic in future, but for now we'll try this.
- if (Opcode == Instruction::Sub)
+ // If this is a subtract, we want to invert the increment amount and use
+ // the histogram_add intrinsic.
+ if (shouldNegateIncrement())
IncAmt = Builder.CreateNeg(IncAmt);
- else
- assert(Opcode == Instruction::Add && "only add or sub supported for now");
Instruction *HistogramInst = State.Builder.CreateIntrinsicWithoutFolding(
- Intrinsic::experimental_vector_histogram_add, {VTy, IncAmt->getType()},
+ getHistogramIntrinsicID(), {VTy, IncAmt->getType()},
{Address, IncAmt, Mask});
applyMetadata(*HistogramInst);
}
+Intrinsic::ID VPHistogramRecipe::getHistogramIntrinsicID() const {
+ switch (UpdateKind) {
+ case HistogramUpdateKind::Add:
+ case HistogramUpdateKind::Sub:
+ return Intrinsic::experimental_vector_histogram_add;
+ case HistogramUpdateKind::UAddSat:
+ return Intrinsic::experimental_vector_histogram_uadd_sat;
+ case HistogramUpdateKind::UMax:
+ return Intrinsic::experimental_vector_histogram_umax;
+ case HistogramUpdateKind::UMin:
+ return Intrinsic::experimental_vector_histogram_umin;
+ }
+ llvm_unreachable("Unknown HistogramUpdateKind");
+}
+
+std::optional<VPHistogramRecipe::HistogramUpdateKind>
+VPHistogramRecipe::getUpdateKindForInstruction(Instruction *I) {
+ if (auto *BO = dyn_cast<BinaryOperator>(I)) {
+ switch (BO->getOpcode()) {
+ case Instruction::Add:
+ return HistogramUpdateKind::Add;
+ case Instruction::Sub:
+ return HistogramUpdateKind::Sub;
+ default:
+ return std::nullopt;
+ }
+ }
+ if (auto *II = dyn_cast<IntrinsicInst>(I)) {
+ switch (II->getIntrinsicID()) {
+ case Intrinsic::uadd_sat:
+ return HistogramUpdateKind::UAddSat;
+ case Intrinsic::umax:
+ return HistogramUpdateKind::UMax;
+ case Intrinsic::umin:
+ return HistogramUpdateKind::UMin;
+ default:
+ return std::nullopt;
+ }
+ }
+ return std::nullopt;
+}
+
InstructionCost VPHistogramRecipe::computeCost(ElementCount VF,
VPCostContext &Ctx) const {
// FIXME: Take the gather and scatter into account as well. For now we're
@@ -2454,13 +2494,41 @@ InstructionCost VPHistogramRecipe::computeCost(ElementCount VF,
// Find the cost of the histogram operation itself.
Type *PtrTy = VectorType::get(AddressTy, VF);
Type *MaskTy = VectorType::get(Type::getInt1Ty(Ctx.LLVMCtx), VF);
- IntrinsicCostAttributes ICA(Intrinsic::experimental_vector_histogram_add,
+ IntrinsicCostAttributes ICA(getHistogramIntrinsicID(),
Type::getVoidTy(Ctx.LLVMCtx),
{PtrTy, IncTy, MaskTy});
- // Add the costs together with the add/sub operation.
+ // Compute the cost of the update operation.
+ InstructionCost UpdateCost;
+ switch (UpdateKind) {
+ case HistogramUpdateKind::Add:
+ UpdateCost =
+ Ctx.TTI.getArithmeticInstrCost(Instruction::Add, VTy, Ctx.CostKind);
+ break;
+ case HistogramUpdateKind::Sub:
+ UpdateCost =
+ Ctx.TTI.getArithmeticInstrCost(Instruction::Sub, VTy, Ctx.CostKind);
+ break;
+ case HistogramUpdateKind::UAddSat: {
+ IntrinsicCostAttributes UpdateICA(Intrinsic::uadd_sat, VTy, {VTy, VTy});
+ UpdateCost = Ctx.TTI.getIntrinsicInstrCost(UpdateICA, Ctx.CostKind);
+ break;
+ }
+ case HistogramUpdateKind::UMax: {
+ IntrinsicCostAttributes UpdateICA(Intrinsic::umax, VTy, {VTy, VTy});
+ UpdateCost = Ctx.TTI.getIntrinsicInstrCost(UpdateICA, Ctx.CostKind);
+ break;
+ }
+ case HistogramUpdateKind::UMin: {
+ IntrinsicCostAttributes UpdateICA(Intrinsic::umin, VTy, {VTy, VTy});
+ UpdateCost = Ctx.TTI.getIntrinsicInstrCost(UpdateICA, Ctx.CostKind);
+ break;
+ }
+ }
+
+ // Add the costs together with the update operation.
return Ctx.TTI.getIntrinsicInstrCost(ICA, Ctx.CostKind) + MulCost +
- Ctx.TTI.getArithmeticInstrCost(Opcode, VTy, Ctx.CostKind);
+ UpdateCost;
}
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
@@ -2469,11 +2537,22 @@ void VPHistogramRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
O << Indent << "WIDEN-HISTOGRAM buckets: ";
getOperand(0)->printAsOperand(O, SlotTracker);
- if (Opcode == Instruction::Sub)
- O << ", dec: ";
- else {
- assert(Opcode == Instruction::Add);
+ switch (UpdateKind) {
+ case HistogramUpdateKind::Add:
O << ", inc: ";
+ break;
+ case HistogramUpdateKind::Sub:
+ O << ", dec: ";
+ break;
+ case HistogramUpdateKind::UAddSat:
+ O << ", saturated inc: ";
+ break;
+ case HistogramUpdateKind::UMax:
+ O << ", max: ";
+ break;
+ case HistogramUpdateKind::UMin:
+ O << ", min: ";
+ break;
}
getOperand(1)->printAsOperand(O, SlotTracker);
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
index 2ec48b25af5b5..d171ab28cde73 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
@@ -888,3 +888,175 @@ attributes #0 = { "target-features"="+sve2" vscale_range(1,16) }
!11 = !{!10, !12}
!12 = distinct !{!12, !13, !"scope-bucket-2"}
!13 = distinct !{!13, !"scopes-buckets"}
+
+;; Test histogram with uadd.sat update operation
+define void @simple_histogram_uadd_sat(ptr noalias %buckets, ptr readonly %indices, i64 %N) #0 {
+; CHECK-LABEL: define void @simple_histogram_uadd_sat(
+; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i32> [[WIDE_LOAD]] to <4 x i64>
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP1]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP1]], i64 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP1]], i64 2
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP1]], i64 3
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP3]]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP4]]
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP5]]
+; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x ptr> poison, ptr [[TMP6]], i32 0
+; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x ptr> [[TMP10]], ptr [[TMP7]], i32 1
+; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x ptr> [[TMP11]], ptr [[TMP8]], i32 2
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x ptr> [[TMP12]], ptr [[TMP9]], i32 3
+; CHECK-NEXT: call void @llvm.experimental.vector.histogram.uadd.sat.v4p0.i32(<4 x ptr> [[TMP13]], i32 1, <4 x i1> splat (i1 true))
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK: scalar.ph:
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
+ %l.idx = load i32, ptr %gep.indices, align 4
+ %idxprom1 = zext i32 %l.idx to i64
+ %gep.bucket = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+ %l.bucket = load i32, ptr %gep.bucket, align 4
+ %inc = call i32 @llvm.uadd.sat.i32(i32 %l.bucket, i32 1)
+ store i32 %inc, ptr %gep.bucket, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, %N
+ br i1 %exitcond, label %for.exit, label %for.body, !llvm.loop !4
+
+for.exit:
+ ret void
+}
+
+;; Test histogram with umax update operation
+define void @simple_histogram_umax(ptr noalias %buckets, ptr readonly %indices, i64 %N) #0 {
+; CHECK-LABEL: define void @simple_histogram_umax(
+; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i32> [[WIDE_LOAD]] to <4 x i64>
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP1]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP1]], i64 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP1]], i64 2
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP1]], i64 3
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP3]]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP4]]
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP5]]
+; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x ptr> poison, ptr [[TMP6]], i32 0
+; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x ptr> [[TMP10]], ptr [[TMP7]], i32 1
+; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x ptr> [[TMP11]], ptr [[TMP8]], i32 2
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x ptr> [[TMP12]], ptr [[TMP9]], i32 3
+; CHECK-NEXT: call void @llvm.experimental.vector.histogram.umax.v4p0.i32(<4 x ptr> [[TMP13]], i32 120, <4 x i1> splat (i1 true))
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK: scalar.ph:
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
+ %l.idx = load i32, ptr %gep.indices, align 4
+ %idxprom1 = zext i32 %l.idx to i64
+ %gep.bucket = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+ %l.bucket = load i32, ptr %gep.bucket, align 4
+ %inc = call i32 @llvm.umax.i32(i32 %l.bucket, i32 120)
+ store i32 %inc, ptr %gep.bucket, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, %N
+ br i1 %exitcond, label %for.exit, label %for.body, !llvm.loop !4
+
+for.exit:
+ ret void
+}
+
+;; Test histogram with umin update operation
+define void @simple_histogram_umin(ptr noalias %buckets, ptr readonly %indices, i64 %N) #0 {
+; CHECK-LABEL: define void @simple_histogram_umin(
+; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i32> [[WIDE_LOAD]] to <4 x i64>
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP1]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP1]], i64 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP1]], i64 2
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP1]], i64 3
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP3]]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP4]]
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP5]]
+; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x ptr> poison, ptr [[TMP6]], i32 0
+; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x ptr> [[TMP10]], ptr [[TMP7]], i32 1
+; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x ptr> [[TMP11]], ptr [[TMP8]], i32 2
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x ptr> [[TMP12]], ptr [[TMP9]], i32 3
+; CHECK-NEXT: call void @llvm.experimental.vector.histogram.umin.v4p0.i32(<4 x ptr> [[TMP13]], i32 99, <4 x i1> splat (i1 true))
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP31:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK: scalar.ph:
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
+ %l.idx = load i32, ptr %gep.indices, align 4
+ %idxprom1 = zext i32 %l.idx to i64
+ %gep.bucket = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+ %l.bucket = load i32, ptr %gep.bucket, align 4
+ %inc = call i32 @llvm.umin.i32(i32 %l.bucket, i32 99)
+ store i32 %inc, ptr %gep.bucket, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, %N
+ br i1 %exitcond, label %for.exit, label %for.body, !llvm.loop !4
+
+for.exit:
+ ret void
+}
+
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve2-histcnt-vplan.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve2-histcnt-vplan.ll
index bceb53bb131c8..73e26fa39da1b 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve2-histcnt-vplan.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve2-histcnt-vplan.ll
@@ -153,6 +153,31 @@ for.exit:
ret void
}
+;; Check that uadd.sat histogram gets a "saturated inc:" label in VPlan.
+; CHECK: VPlan 'Initial VPlan for VF={vscale x 2,vscale x 4},UF>=1' {
+; CHECK: WIDEN-HISTOGRAM buckets: {{.*}}, saturated inc: ir<1>
+
+define void @simple_histogram_uadd_sat(ptr noalias %buckets, ptr readonly %indices, i64 %N) {
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
+ %l.idx = load i32, ptr %gep.indices, align 4
+ %idxprom1 = zext i32 %l.idx to i64
+ %gep.bucket = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+ %l.bucket = load i32, ptr %gep.bucket, align 4
+ %inc = call i32 @llvm.uadd.sat.i32(i32 %l.bucket, i32 1)
+ store i32 %inc, ptr %gep.bucket, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, %N
+ br i1 %exitcond, label %for.exit, label %for.body
+
+for.exit:
+ ret void
+}
+
!0 = !{!1}
!1 = distinct !{!1, !2}
!2 = distinct !{!2}
>From eef1a74adc8243adf9f235622bd7354c9d41c2c0 Mon Sep 17 00:00:00 2001
From: rond <ron.dahan at mobileye.com>
Date: Wed, 17 Jun 2026 12:13:58 +0300
Subject: [PATCH 2/7] Fix clang format
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 54c3bf11d62bb..6361f326c572d 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6424,8 +6424,8 @@ VPHistogramRecipe *VPRecipeBuilder::widenIfHistogram(VPInstruction *VPI) {
if (CM.isMaskRequired(HI->Store))
HGramOps.push_back(VPI->getMask());
- return new VPHistogramRecipe(*UpdateKind, HGramOps,
- cast<VPIRMetadata>(*VPI), VPI->getDebugLoc());
+ return new VPHistogramRecipe(*UpdateKind, HGramOps, cast<VPIRMetadata>(*VPI),
+ VPI->getDebugLoc());
}
bool VPRecipeBuilder::replaceWithFinalIfReductionStore(
>From 484ba6c8a013dd99b8e9a269f8e5a6b5ec80b170 Mon Sep 17 00:00:00 2001
From: rond <ron.dahan at mobileye.com>
Date: Tue, 23 Jun 2026 16:26:35 +0300
Subject: [PATCH 3/7] Fix review
Co-authored-by: Copilot <223556219+Copilot at users.noreply.github.com>
---
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 17 +-
.../LoopVectorize/AArch64/sve2-histcnt.ll | 170 ---------------
.../VPlan/AArch64/sve2-histcnt-vplan.ll | 46 ++++-
.../Transforms/LoopVectorize/histograms.ll | 195 ++++++++++++++++++
4 files changed, 251 insertions(+), 177 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index d6568ef012d2d..63cdf7f3bab87 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -2484,12 +2484,17 @@ InstructionCost VPHistogramRecipe::computeCost(ElementCount VF,
Type *IncTy = IncAmt->getScalarType();
VectorType *VTy = VectorType::get(IncTy, VF);
- // Assume that a non-constant update value (or a constant != 1) requires
- // a multiply, and add that into the cost.
- InstructionCost MulCost =
- Ctx.TTI.getArithmeticInstrCost(Instruction::Mul, VTy, Ctx.CostKind);
- if (match(IncAmt, m_One()))
- MulCost = TTI::TCC_Free;
+ // For umin/umax, there's no multiplication — the increment value is compared
+ // directly. For add/sub/uadd.sat, assume that a non-constant update value
+ // (or a constant != 1) requires a multiply.
+ InstructionCost MulCost = TTI::TCC_Free;
+ if (UpdateKind != HistogramUpdateKind::UMax &&
+ UpdateKind != HistogramUpdateKind::UMin) {
+ MulCost =
+ Ctx.TTI.getArithmeticInstrCost(Instruction::Mul, VTy, Ctx.CostKind);
+ if (match(IncAmt, m_One()))
+ MulCost = TTI::TCC_Free;
+ }
// Find the cost of the histogram operation itself.
Type *PtrTy = VectorType::get(AddressTy, VF);
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
index d171ab28cde73..4baca89d0b769 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
@@ -889,174 +889,4 @@ attributes #0 = { "target-features"="+sve2" vscale_range(1,16) }
!12 = distinct !{!12, !13, !"scope-bucket-2"}
!13 = distinct !{!13, !"scopes-buckets"}
-;; Test histogram with uadd.sat update operation
-define void @simple_histogram_uadd_sat(ptr noalias %buckets, ptr readonly %indices, i64 %N) #0 {
-; CHECK-LABEL: define void @simple_histogram_uadd_sat(
-; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK: vector.ph:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i32> [[WIDE_LOAD]] to <4 x i64>
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP1]], i64 0
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP1]], i64 1
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP1]], i64 2
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP1]], i64 3
-; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP2]]
-; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP3]]
-; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP4]]
-; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP5]]
-; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x ptr> poison, ptr [[TMP6]], i32 0
-; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x ptr> [[TMP10]], ptr [[TMP7]], i32 1
-; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x ptr> [[TMP11]], ptr [[TMP8]], i32 2
-; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x ptr> [[TMP12]], ptr [[TMP9]], i32 3
-; CHECK-NEXT: call void @llvm.experimental.vector.histogram.uadd.sat.v4p0.i32(<4 x ptr> [[TMP13]], i32 1, <4 x i1> splat (i1 true))
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
-; CHECK: middle.block:
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
-; CHECK: scalar.ph:
-;
-entry:
- br label %for.body
-
-for.body:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
- %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
- %l.idx = load i32, ptr %gep.indices, align 4
- %idxprom1 = zext i32 %l.idx to i64
- %gep.bucket = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
- %l.bucket = load i32, ptr %gep.bucket, align 4
- %inc = call i32 @llvm.uadd.sat.i32(i32 %l.bucket, i32 1)
- store i32 %inc, ptr %gep.bucket, align 4
- %iv.next = add nuw nsw i64 %iv, 1
- %exitcond = icmp eq i64 %iv.next, %N
- br i1 %exitcond, label %for.exit, label %for.body, !llvm.loop !4
-
-for.exit:
- ret void
-}
-
-;; Test histogram with umax update operation
-define void @simple_histogram_umax(ptr noalias %buckets, ptr readonly %indices, i64 %N) #0 {
-; CHECK-LABEL: define void @simple_histogram_umax(
-; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK: vector.ph:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i32> [[WIDE_LOAD]] to <4 x i64>
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP1]], i64 0
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP1]], i64 1
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP1]], i64 2
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP1]], i64 3
-; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP2]]
-; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP3]]
-; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP4]]
-; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP5]]
-; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x ptr> poison, ptr [[TMP6]], i32 0
-; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x ptr> [[TMP10]], ptr [[TMP7]], i32 1
-; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x ptr> [[TMP11]], ptr [[TMP8]], i32 2
-; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x ptr> [[TMP12]], ptr [[TMP9]], i32 3
-; CHECK-NEXT: call void @llvm.experimental.vector.histogram.umax.v4p0.i32(<4 x ptr> [[TMP13]], i32 120, <4 x i1> splat (i1 true))
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]
-; CHECK: middle.block:
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
-; CHECK: scalar.ph:
-;
-entry:
- br label %for.body
-
-for.body:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
- %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
- %l.idx = load i32, ptr %gep.indices, align 4
- %idxprom1 = zext i32 %l.idx to i64
- %gep.bucket = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
- %l.bucket = load i32, ptr %gep.bucket, align 4
- %inc = call i32 @llvm.umax.i32(i32 %l.bucket, i32 120)
- store i32 %inc, ptr %gep.bucket, align 4
- %iv.next = add nuw nsw i64 %iv, 1
- %exitcond = icmp eq i64 %iv.next, %N
- br i1 %exitcond, label %for.exit, label %for.body, !llvm.loop !4
-
-for.exit:
- ret void
-}
-
-;; Test histogram with umin update operation
-define void @simple_histogram_umin(ptr noalias %buckets, ptr readonly %indices, i64 %N) #0 {
-; CHECK-LABEL: define void @simple_histogram_umin(
-; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK: vector.ph:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i32> [[WIDE_LOAD]] to <4 x i64>
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP1]], i64 0
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP1]], i64 1
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP1]], i64 2
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP1]], i64 3
-; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP2]]
-; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP3]]
-; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP4]]
-; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP5]]
-; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x ptr> poison, ptr [[TMP6]], i32 0
-; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x ptr> [[TMP10]], ptr [[TMP7]], i32 1
-; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x ptr> [[TMP11]], ptr [[TMP8]], i32 2
-; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x ptr> [[TMP12]], ptr [[TMP9]], i32 3
-; CHECK-NEXT: call void @llvm.experimental.vector.histogram.umin.v4p0.i32(<4 x ptr> [[TMP13]], i32 99, <4 x i1> splat (i1 true))
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP31:![0-9]+]]
-; CHECK: middle.block:
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
-; CHECK: scalar.ph:
-;
-entry:
- br label %for.body
-
-for.body:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
- %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
- %l.idx = load i32, ptr %gep.indices, align 4
- %idxprom1 = zext i32 %l.idx to i64
- %gep.bucket = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
- %l.bucket = load i32, ptr %gep.bucket, align 4
- %inc = call i32 @llvm.umin.i32(i32 %l.bucket, i32 99)
- store i32 %inc, ptr %gep.bucket, align 4
- %iv.next = add nuw nsw i64 %iv, 1
- %exitcond = icmp eq i64 %iv.next, %N
- br i1 %exitcond, label %for.exit, label %for.body, !llvm.loop !4
-
-for.exit:
- ret void
-}
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve2-histcnt-vplan.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve2-histcnt-vplan.ll
index 73e26fa39da1b..20c925cd55b76 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve2-histcnt-vplan.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve2-histcnt-vplan.ll
@@ -155,7 +155,51 @@ for.exit:
;; Check that uadd.sat histogram gets a "saturated inc:" label in VPlan.
; CHECK: VPlan 'Initial VPlan for VF={vscale x 2,vscale x 4},UF>=1' {
-; CHECK: WIDEN-HISTOGRAM buckets: {{.*}}, saturated inc: ir<1>
+; CHECK-NEXT: Live-in [[VF3:.*]] = VF
+; CHECK-NEXT: Live-in [[VFxUF3:.*]] = VF * UF
+; CHECK-NEXT: Live-in [[VTC3:.*]] = vector-trip-count
+; CHECK-NEXT: Live-in [[OTC3:.*]] = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: [[IV3:.*]] = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: [[STEPS3:vp.*]] = SCALAR-STEPS [[IV3]], ir<1>, [[VF3]]
+; CHECK-NEXT: CLONE [[GEP_IDX3:.*]] = getelementptr inbounds ir<%indices>, [[STEPS3]]
+; CHECK-NEXT: [[VECP_IDX3:vp.*]] = vector-pointer inbounds [[GEP_IDX3]]
+; CHECK-NEXT: WIDEN [[IDX3:.*]] = load [[VECP_IDX3]]
+; CHECK-NEXT: WIDEN-CAST [[EXT_IDX3:.*]] = zext [[IDX3]] to i64
+; CHECK-NEXT: WIDEN-GEP [[GEP_BUCKET3:.*]] = getelementptr inbounds ir<%buckets>, [[EXT_IDX3]]
+; CHECK-NEXT: WIDEN-HISTOGRAM buckets: [[GEP_BUCKET3]], saturated inc: ir<1>
+; CHECK-NEXT: EMIT [[IV_NEXT3:.*]] = add nuw [[IV3]], [[VFxUF3]]
+; CHECK-NEXT: EMIT branch-on-count [[IV_NEXT3]], [[VTC3]]
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT [[TC_CHECK3:.*]] = icmp eq [[OTC3]], [[VTC3]]
+; CHECK-NEXT: EMIT branch-on-cond [[TC_CHECK3]]
+; CHECK-NEXT: Successor(s): ir-bb<for.exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<for.exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<[[RESUME3:%.+]]> = phi [ [[VTC3]], middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<for.body>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<for.body>:
+; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ] (extra operand: vp<[[RESUME3]]> from scalar.ph)
+; CHECK: IR %exitcond = icmp eq i64 %iv.next, %N
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
define void @simple_histogram_uadd_sat(ptr noalias %buckets, ptr readonly %indices, i64 %N) {
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/histograms.ll b/llvm/test/Transforms/LoopVectorize/histograms.ll
index 8e51b4d1cc94c..88acf2ef7f037 100644
--- a/llvm/test/Transforms/LoopVectorize/histograms.ll
+++ b/llvm/test/Transforms/LoopVectorize/histograms.ll
@@ -66,3 +66,198 @@ for.body:
for.exit:
ret void
}
+
+define void @simple_histogram_uadd_sat(ptr noalias %buckets, ptr readonly %indices, i64 %N) {
+; CHECK-LABEL: define void @simple_histogram_uadd_sat(
+; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[N]], -2
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [4 x i8], ptr [[INDICES]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = zext <2 x i32> [[WIDE_LOAD]] to <2 x i64>
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i64> [[TMP1]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i64> [[TMP1]], i64 1
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[BUCKETS]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[BUCKETS]], i64 [[TMP3]]
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP4]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x ptr> [[TMP6]], ptr [[TMP5]], i64 1
+; CHECK-NEXT: call void @llvm.experimental.vector.histogram.uadd.sat.v2p0.i32(<2 x ptr> [[TMP7]], i32 1, <2 x i1> splat (i1 true))
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK: scalar.ph:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT: br label [[FOR_BODY:%.*]]
+; CHECK: for.body:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT: [[GEP_INDICES:%.*]] = getelementptr inbounds [4 x i8], ptr [[INDICES]], i64 [[IV]]
+; CHECK-NEXT: [[L_IDX:%.*]] = load i32, ptr [[GEP_INDICES]], align 4
+; CHECK-NEXT: [[IDXPROM1:%.*]] = zext i32 [[L_IDX]] to i64
+; CHECK-NEXT: [[GEP_BUCKET:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[BUCKETS]], i64 [[IDXPROM1]]
+; CHECK-NEXT: [[L_BUCKET:%.*]] = load i32, ptr [[GEP_BUCKET]], align 4
+; CHECK-NEXT: [[INC:%.*]] = call i32 @llvm.uadd.sat.i32(i32 [[L_BUCKET]], i32 1)
+; CHECK-NEXT: store i32 [[INC]], ptr [[GEP_BUCKET]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK: for.exit:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
+ %l.idx = load i32, ptr %gep.indices, align 4
+ %idxprom1 = zext i32 %l.idx to i64
+ %gep.bucket = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+ %l.bucket = load i32, ptr %gep.bucket, align 4
+ %inc = call i32 @llvm.uadd.sat.i32(i32 %l.bucket, i32 1)
+ store i32 %inc, ptr %gep.bucket, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, %N
+ br i1 %exitcond, label %for.exit, label %for.body
+
+for.exit:
+ ret void
+}
+
+define void @simple_histogram_umax(ptr noalias %buckets, ptr readonly %indices, i64 %N) {
+; CHECK-LABEL: define void @simple_histogram_umax(
+; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[N]], -2
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [4 x i8], ptr [[INDICES]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = zext <2 x i32> [[WIDE_LOAD]] to <2 x i64>
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i64> [[TMP1]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i64> [[TMP1]], i64 1
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[BUCKETS]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[BUCKETS]], i64 [[TMP3]]
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP4]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x ptr> [[TMP6]], ptr [[TMP5]], i64 1
+; CHECK-NEXT: call void @llvm.experimental.vector.histogram.umax.v2p0.i32(<2 x ptr> [[TMP7]], i32 120, <2 x i1> splat (i1 true))
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK: scalar.ph:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT: br label [[FOR_BODY:%.*]]
+; CHECK: for.body:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT: [[GEP_INDICES:%.*]] = getelementptr inbounds [4 x i8], ptr [[INDICES]], i64 [[IV]]
+; CHECK-NEXT: [[L_IDX:%.*]] = load i32, ptr [[GEP_INDICES]], align 4
+; CHECK-NEXT: [[IDXPROM1:%.*]] = zext i32 [[L_IDX]] to i64
+; CHECK-NEXT: [[GEP_BUCKET:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[BUCKETS]], i64 [[IDXPROM1]]
+; CHECK-NEXT: [[L_BUCKET:%.*]] = load i32, ptr [[GEP_BUCKET]], align 4
+; CHECK-NEXT: [[INC:%.*]] = call i32 @llvm.umax.i32(i32 [[L_BUCKET]], i32 120)
+; CHECK-NEXT: store i32 [[INC]], ptr [[GEP_BUCKET]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK: for.exit:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
+ %l.idx = load i32, ptr %gep.indices, align 4
+ %idxprom1 = zext i32 %l.idx to i64
+ %gep.bucket = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+ %l.bucket = load i32, ptr %gep.bucket, align 4
+ %inc = call i32 @llvm.umax.i32(i32 %l.bucket, i32 120)
+ store i32 %inc, ptr %gep.bucket, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, %N
+ br i1 %exitcond, label %for.exit, label %for.body
+
+for.exit:
+ ret void
+}
+
+define void @simple_histogram_umin(ptr noalias %buckets, ptr readonly %indices, i64 %N) {
+; CHECK-LABEL: define void @simple_histogram_umin(
+; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[N]], -2
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [4 x i8], ptr [[INDICES]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = zext <2 x i32> [[WIDE_LOAD]] to <2 x i64>
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i64> [[TMP1]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i64> [[TMP1]], i64 1
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[BUCKETS]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[BUCKETS]], i64 [[TMP3]]
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP4]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x ptr> [[TMP6]], ptr [[TMP5]], i64 1
+; CHECK-NEXT: call void @llvm.experimental.vector.histogram.umin.v2p0.i32(<2 x ptr> [[TMP7]], i32 99, <2 x i1> splat (i1 true))
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK: scalar.ph:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT: br label [[FOR_BODY:%.*]]
+; CHECK: for.body:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT: [[GEP_INDICES:%.*]] = getelementptr inbounds [4 x i8], ptr [[INDICES]], i64 [[IV]]
+; CHECK-NEXT: [[L_IDX:%.*]] = load i32, ptr [[GEP_INDICES]], align 4
+; CHECK-NEXT: [[IDXPROM1:%.*]] = zext i32 [[L_IDX]] to i64
+; CHECK-NEXT: [[GEP_BUCKET:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[BUCKETS]], i64 [[IDXPROM1]]
+; CHECK-NEXT: [[L_BUCKET:%.*]] = load i32, ptr [[GEP_BUCKET]], align 4
+; CHECK-NEXT: [[INC:%.*]] = call i32 @llvm.umin.i32(i32 [[L_BUCKET]], i32 99)
+; CHECK-NEXT: store i32 [[INC]], ptr [[GEP_BUCKET]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK: for.exit:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
+ %l.idx = load i32, ptr %gep.indices, align 4
+ %idxprom1 = zext i32 %l.idx to i64
+ %gep.bucket = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+ %l.bucket = load i32, ptr %gep.bucket, align 4
+ %inc = call i32 @llvm.umin.i32(i32 %l.bucket, i32 99)
+ store i32 %inc, ptr %gep.bucket, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, %N
+ br i1 %exitcond, label %for.exit, label %for.body
+
+for.exit:
+ ret void
+}
>From 3d922b634260c97f3d751a77e5c8e7ede30e91de Mon Sep 17 00:00:00 2001
From: rond <ron.dahan at mobileye.com>
Date: Tue, 23 Jun 2026 16:52:03 +0300
Subject: [PATCH 4/7] Combine MulCost conditions in
VPHistogramRecipe::computeCost
Simplify the logic by combining the m_One() check with the UMax/UMin
checks into a single condition, avoiding redundant assignment of MulCost.
Co-authored-by: Copilot <223556219+Copilot at users.noreply.github.com>
---
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 7 ++-----
1 file changed, 2 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 63cdf7f3bab87..6bf92546c3890 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -2488,13 +2488,10 @@ InstructionCost VPHistogramRecipe::computeCost(ElementCount VF,
// directly. For add/sub/uadd.sat, assume that a non-constant update value
// (or a constant != 1) requires a multiply.
InstructionCost MulCost = TTI::TCC_Free;
- if (UpdateKind != HistogramUpdateKind::UMax &&
- UpdateKind != HistogramUpdateKind::UMin) {
+ if (!match(IncAmt, m_One()) && UpdateKind != HistogramUpdateKind::UMax &&
+ UpdateKind != HistogramUpdateKind::UMin)
MulCost =
Ctx.TTI.getArithmeticInstrCost(Instruction::Mul, VTy, Ctx.CostKind);
- if (match(IncAmt, m_One()))
- MulCost = TTI::TCC_Free;
- }
// Find the cost of the histogram operation itself.
Type *PtrTy = VectorType::get(AddressTy, VF);
>From 82fb8db9ec51d5589b31f3e41b0592f3ed2d8c22 Mon Sep 17 00:00:00 2001
From: rond <ron.dahan at mobileye.com>
Date: Thu, 9 Jul 2026 18:23:35 +0300
Subject: [PATCH 5/7] Answer fhahn's review
---
.../Transforms/Vectorize/LoopVectorize.cpp | 78 ++++++++-----------
llvm/lib/Transforms/Vectorize/VPlan.h | 5 --
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 27 -------
3 files changed, 34 insertions(+), 76 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 6361f326c572d..80bd6f5bcb01f 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -5340,47 +5340,6 @@ LoopVectorizationCostModel::getInstructionCost(Instruction *I,
Config.CostKind, I);
}
case Instruction::Call: {
- // Check if this is a histogram update operation (intrinsic like uadd.sat,
- // umax, umin used as histogram bucket update).
- auto Info = Legal->getHistogramInfo(I);
- if (Info && VF.isVector()) {
- const HistogramInfo *HGram = Info.value();
- // Assume that a non-constant update value (or a constant != 1) requires
- // a multiply, and add that into the cost.
- InstructionCost MulCost = TTI::TCC_Free;
- ConstantInt *RHS = dyn_cast<ConstantInt>(I->getOperand(1));
- if (!RHS || RHS->getZExtValue() != 1)
- MulCost = TTI.getArithmeticInstrCost(Instruction::Mul, VectorTy,
- Config.CostKind);
-
- // Find the cost of the histogram operation itself.
- Type *PtrTy = VectorType::get(HGram->Load->getPointerOperandType(), VF);
- Type *ScalarTy = I->getType();
- Type *MaskTy = VectorType::get(Type::getInt1Ty(I->getContext()), VF);
- auto *II = cast<IntrinsicInst>(I);
- Intrinsic::ID HistID;
- switch (II->getIntrinsicID()) {
- case Intrinsic::uadd_sat:
- HistID = Intrinsic::experimental_vector_histogram_uadd_sat;
- break;
- case Intrinsic::umax:
- HistID = Intrinsic::experimental_vector_histogram_umax;
- break;
- case Intrinsic::umin:
- HistID = Intrinsic::experimental_vector_histogram_umin;
- break;
- default:
- llvm_unreachable("Unsupported histogram intrinsic");
- }
- IntrinsicCostAttributes ICA(HistID, Type::getVoidTy(I->getContext()),
- {PtrTy, ScalarTy, MaskTy});
-
- // Add the costs together with the update operation cost.
- IntrinsicCostAttributes UpdateICA(II->getIntrinsicID(), VectorTy,
- {VectorTy, VectorTy});
- return TTI.getIntrinsicInstrCost(ICA, Config.CostKind) + MulCost +
- TTI.getIntrinsicInstrCost(UpdateICA, Config.CostKind);
- }
return getVectorCallCost(cast<CallInst>(I), VF);
}
case Instruction::ExtractValue:
@@ -6400,6 +6359,37 @@ VPRecipeWithIRFlags *VPRecipeBuilder::tryToWiden(VPInstruction *VPI) {
};
}
+/// Return the HistogramUpdateKind for the given update instruction. The
+/// instruction must be one of the supported histogram update operations;
+/// callers are expected to have validated this via
+/// LoopVectorizationLegality::getHistogramInfo before invoking this helper.
+static VPHistogramRecipe::HistogramUpdateKind
+getHistogramUpdateKind(Instruction *I) {
+ using HistogramUpdateKind = VPHistogramRecipe::HistogramUpdateKind;
+ if (auto *BO = dyn_cast<BinaryOperator>(I)) {
+ switch (BO->getOpcode()) {
+ case Instruction::Add:
+ return HistogramUpdateKind::Add;
+ case Instruction::Sub:
+ return HistogramUpdateKind::Sub;
+ default:
+ break;
+ }
+ } else if (auto *II = dyn_cast<IntrinsicInst>(I)) {
+ switch (II->getIntrinsicID()) {
+ case Intrinsic::uadd_sat:
+ return HistogramUpdateKind::UAddSat;
+ case Intrinsic::umax:
+ return HistogramUpdateKind::UMax;
+ case Intrinsic::umin:
+ return HistogramUpdateKind::UMin;
+ default:
+ break;
+ }
+ }
+ llvm_unreachable("Unsupported histogram update operation");
+}
+
VPHistogramRecipe *VPRecipeBuilder::widenIfHistogram(VPInstruction *VPI) {
if (VPI->getOpcode() != Instruction::Store)
return nullptr;
@@ -6410,8 +6400,8 @@ VPHistogramRecipe *VPRecipeBuilder::widenIfHistogram(VPInstruction *VPI) {
return nullptr;
const HistogramInfo *HI = *HistInfo;
- auto UpdateKind = VPHistogramRecipe::getUpdateKindForInstruction(HI->Update);
- assert(UpdateKind && "Unsupported histogram update operation");
+ VPHistogramRecipe::HistogramUpdateKind UpdateKind =
+ getHistogramUpdateKind(HI->Update);
SmallVector<VPValue *, 3> HGramOps;
// Bucket address.
@@ -6424,7 +6414,7 @@ VPHistogramRecipe *VPRecipeBuilder::widenIfHistogram(VPInstruction *VPI) {
if (CM.isMaskRequired(HI->Store))
HGramOps.push_back(VPI->getMask());
- return new VPHistogramRecipe(*UpdateKind, HGramOps, cast<VPIRMetadata>(*VPI),
+ return new VPHistogramRecipe(UpdateKind, HGramOps, cast<VPIRMetadata>(*VPI),
VPI->getDebugLoc());
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index c8853c8999b5f..2aacfe2f64fc6 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -2216,11 +2216,6 @@ class VPHistogramRecipe : public VPRecipeBase, public VPIRMetadata {
return getNumOperands() == 3 ? getOperand(2) : nullptr;
}
- /// Return the HistogramUpdateKind for the given update instruction, or
- /// std::nullopt if the instruction is not a supported histogram update.
- static std::optional<HistogramUpdateKind>
- getUpdateKindForInstruction(Instruction *I);
-
protected:
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
/// Print the recipe
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 6bf92546c3890..68496b18c2084 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -2444,33 +2444,6 @@ Intrinsic::ID VPHistogramRecipe::getHistogramIntrinsicID() const {
llvm_unreachable("Unknown HistogramUpdateKind");
}
-std::optional<VPHistogramRecipe::HistogramUpdateKind>
-VPHistogramRecipe::getUpdateKindForInstruction(Instruction *I) {
- if (auto *BO = dyn_cast<BinaryOperator>(I)) {
- switch (BO->getOpcode()) {
- case Instruction::Add:
- return HistogramUpdateKind::Add;
- case Instruction::Sub:
- return HistogramUpdateKind::Sub;
- default:
- return std::nullopt;
- }
- }
- if (auto *II = dyn_cast<IntrinsicInst>(I)) {
- switch (II->getIntrinsicID()) {
- case Intrinsic::uadd_sat:
- return HistogramUpdateKind::UAddSat;
- case Intrinsic::umax:
- return HistogramUpdateKind::UMax;
- case Intrinsic::umin:
- return HistogramUpdateKind::UMin;
- default:
- return std::nullopt;
- }
- }
- return std::nullopt;
-}
-
InstructionCost VPHistogramRecipe::computeCost(ElementCount VF,
VPCostContext &Ctx) const {
// FIXME: Take the gather and scatter into account as well. For now we're
>From 3f2d4fe6132ece41ddfda0ffc512b091207a127c Mon Sep 17 00:00:00 2001
From: rond <ron.dahan at mobileye.com>
Date: Tue, 21 Jul 2026 15:15:18 +0300
Subject: [PATCH 6/7] Address review comments on histogram vectorization PR
- Remove braces from 'case Instruction::Call' (fhahn)
- Rename shouldNegateIncrement -> mustNegateIncrement (fhahn)
- Remove double-counted UpdateCost from computeCost; TTI histogram
intrinsic cost already includes the update operation (fhahn)
- Remove unrelated trailing blank lines in sve2-histcnt.ll (fhahn)
- Remove unused getUpdateKind() accessor (fhahn)
- Store UpdateOpcode in HistogramInfo during legality analysis to avoid
re-deriving it later in getHistogramUpdateKind (fhahn)
Co-authored-by: Copilot <223556219+Copilot at users.noreply.github.com>
---
.../Vectorize/LoopVectorizationLegality.h | 8 +++-
.../Vectorize/LoopVectorizationLegality.cpp | 9 +++-
.../Transforms/Vectorize/LoopVectorize.cpp | 47 +++++++------------
llvm/lib/Transforms/Vectorize/VPlan.h | 4 +-
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 36 ++------------
.../LoopVectorize/AArch64/sve2-histcnt.ll | 2 -
.../Transforms/LoopVectorize/histograms.ll | 6 +--
7 files changed, 40 insertions(+), 72 deletions(-)
diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
index 3e8db73fd79d2..ed55307a9a6ad 100644
--- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
+++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
@@ -248,9 +248,13 @@ struct HistogramInfo {
LoadInst *Load;
Instruction *Update;
StoreInst *Store;
+ /// The opcode (for BinaryOperator) or intrinsic ID (for IntrinsicInst) of
+ /// the update operation.
+ unsigned UpdateOpcode;
- HistogramInfo(LoadInst *Load, Instruction *Update, StoreInst *Store)
- : Load(Load), Update(Update), Store(Store) {}
+ HistogramInfo(LoadInst *Load, Instruction *Update, StoreInst *Store,
+ unsigned UpdateOpcode)
+ : Load(Load), Update(Update), Store(Store), UpdateOpcode(UpdateOpcode) {}
};
/// Indicates the characteristics of a loop with an uncountable exit.
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index f79ed0c9c39bb..3d5e01b82d670 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -1126,8 +1126,15 @@ static bool findHistogram(LoadInst *LI, StoreInst *HSt, Loop *TheLoop,
LLVM_DEBUG(dbgs() << "LV: Found histogram for: " << *HSt << "\n");
+ // Determine the update opcode.
+ unsigned UpdateOpcode;
+ if (auto *BO = dyn_cast<BinaryOperator>(HUpdateOp))
+ UpdateOpcode = BO->getOpcode();
+ else
+ UpdateOpcode = cast<IntrinsicInst>(HUpdateOp)->getIntrinsicID();
+
// Store the operations that make up the histogram.
- Histograms.emplace_back(IndexedLoad, HUpdateOp, HSt);
+ Histograms.emplace_back(IndexedLoad, HUpdateOp, HSt, UpdateOpcode);
return true;
}
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 80bd6f5bcb01f..3cc19e7690c47 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -5339,9 +5339,8 @@ LoopVectorizationCostModel::getInstructionCost(Instruction *I,
return TTI.getCastInstrCost(Opcode, VectorTy, SrcVecTy, CCH,
Config.CostKind, I);
}
- case Instruction::Call: {
+ case Instruction::Call:
return getVectorCallCost(cast<CallInst>(I), VF);
- }
case Instruction::ExtractValue:
return TTI.getInstructionCost(I, Config.CostKind);
case Instruction::Alloca:
@@ -6359,35 +6358,25 @@ VPRecipeWithIRFlags *VPRecipeBuilder::tryToWiden(VPInstruction *VPI) {
};
}
-/// Return the HistogramUpdateKind for the given update instruction. The
-/// instruction must be one of the supported histogram update operations;
-/// callers are expected to have validated this via
-/// LoopVectorizationLegality::getHistogramInfo before invoking this helper.
+/// Return the HistogramUpdateKind for the given update opcode (instruction
+/// opcode or intrinsic ID), as stored in HistogramInfo during legality.
static VPHistogramRecipe::HistogramUpdateKind
-getHistogramUpdateKind(Instruction *I) {
+getHistogramUpdateKind(unsigned Opcode) {
using HistogramUpdateKind = VPHistogramRecipe::HistogramUpdateKind;
- if (auto *BO = dyn_cast<BinaryOperator>(I)) {
- switch (BO->getOpcode()) {
- case Instruction::Add:
- return HistogramUpdateKind::Add;
- case Instruction::Sub:
- return HistogramUpdateKind::Sub;
- default:
- break;
- }
- } else if (auto *II = dyn_cast<IntrinsicInst>(I)) {
- switch (II->getIntrinsicID()) {
- case Intrinsic::uadd_sat:
- return HistogramUpdateKind::UAddSat;
- case Intrinsic::umax:
- return HistogramUpdateKind::UMax;
- case Intrinsic::umin:
- return HistogramUpdateKind::UMin;
- default:
- break;
- }
+ switch (Opcode) {
+ case Instruction::Add:
+ return HistogramUpdateKind::Add;
+ case Instruction::Sub:
+ return HistogramUpdateKind::Sub;
+ case Intrinsic::uadd_sat:
+ return HistogramUpdateKind::UAddSat;
+ case Intrinsic::umax:
+ return HistogramUpdateKind::UMax;
+ case Intrinsic::umin:
+ return HistogramUpdateKind::UMin;
+ default:
+ llvm_unreachable("Unsupported histogram update operation");
}
- llvm_unreachable("Unsupported histogram update operation");
}
VPHistogramRecipe *VPRecipeBuilder::widenIfHistogram(VPInstruction *VPI) {
@@ -6401,7 +6390,7 @@ VPHistogramRecipe *VPRecipeBuilder::widenIfHistogram(VPInstruction *VPI) {
const HistogramInfo *HI = *HistInfo;
VPHistogramRecipe::HistogramUpdateKind UpdateKind =
- getHistogramUpdateKind(HI->Update);
+ getHistogramUpdateKind(HI->UpdateOpcode);
SmallVector<VPValue *, 3> HGramOps;
// Bucket address.
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 2aacfe2f64fc6..c8d4aac0055e2 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -2199,14 +2199,12 @@ class VPHistogramRecipe : public VPRecipeBase, public VPIRMetadata {
InstructionCost computeCost(ElementCount VF,
VPCostContext &Ctx) const override;
- HistogramUpdateKind getUpdateKind() const { return UpdateKind; }
-
/// Return the histogram intrinsic ID for this recipe's update kind.
Intrinsic::ID getHistogramIntrinsicID() const;
/// Return true if the increment should be negated before passing to the
/// histogram intrinsic (only for Sub).
- bool shouldNegateIncrement() const {
+ bool mustNegateIncrement() const {
return UpdateKind == HistogramUpdateKind::Sub;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 68496b18c2084..9b606c6a48bd9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -2420,7 +2420,7 @@ void VPHistogramRecipe::execute(VPTransformState &State) {
// If this is a subtract, we want to invert the increment amount and use
// the histogram_add intrinsic.
- if (shouldNegateIncrement())
+ if (mustNegateIncrement())
IncAmt = Builder.CreateNeg(IncAmt);
Instruction *HistogramInst = State.Builder.CreateIntrinsicWithoutFolding(
@@ -2473,37 +2473,9 @@ InstructionCost VPHistogramRecipe::computeCost(ElementCount VF,
Type::getVoidTy(Ctx.LLVMCtx),
{PtrTy, IncTy, MaskTy});
- // Compute the cost of the update operation.
- InstructionCost UpdateCost;
- switch (UpdateKind) {
- case HistogramUpdateKind::Add:
- UpdateCost =
- Ctx.TTI.getArithmeticInstrCost(Instruction::Add, VTy, Ctx.CostKind);
- break;
- case HistogramUpdateKind::Sub:
- UpdateCost =
- Ctx.TTI.getArithmeticInstrCost(Instruction::Sub, VTy, Ctx.CostKind);
- break;
- case HistogramUpdateKind::UAddSat: {
- IntrinsicCostAttributes UpdateICA(Intrinsic::uadd_sat, VTy, {VTy, VTy});
- UpdateCost = Ctx.TTI.getIntrinsicInstrCost(UpdateICA, Ctx.CostKind);
- break;
- }
- case HistogramUpdateKind::UMax: {
- IntrinsicCostAttributes UpdateICA(Intrinsic::umax, VTy, {VTy, VTy});
- UpdateCost = Ctx.TTI.getIntrinsicInstrCost(UpdateICA, Ctx.CostKind);
- break;
- }
- case HistogramUpdateKind::UMin: {
- IntrinsicCostAttributes UpdateICA(Intrinsic::umin, VTy, {VTy, VTy});
- UpdateCost = Ctx.TTI.getIntrinsicInstrCost(UpdateICA, Ctx.CostKind);
- break;
- }
- }
-
- // Add the costs together with the update operation.
- return Ctx.TTI.getIntrinsicInstrCost(ICA, Ctx.CostKind) + MulCost +
- UpdateCost;
+ // The histogram intrinsic's TTI cost already includes the update operation
+ // (load + update + store), so no separate UpdateCost is needed.
+ return Ctx.TTI.getIntrinsicInstrCost(ICA, Ctx.CostKind) + MulCost;
}
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
index 4baca89d0b769..2ec48b25af5b5 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
@@ -888,5 +888,3 @@ attributes #0 = { "target-features"="+sve2" vscale_range(1,16) }
!11 = !{!10, !12}
!12 = distinct !{!12, !13, !"scope-bucket-2"}
!13 = distinct !{!13, !"scopes-buckets"}
-
-
diff --git a/llvm/test/Transforms/LoopVectorize/histograms.ll b/llvm/test/Transforms/LoopVectorize/histograms.ll
index 88acf2ef7f037..22d146283baa5 100644
--- a/llvm/test/Transforms/LoopVectorize/histograms.ll
+++ b/llvm/test/Transforms/LoopVectorize/histograms.ll
@@ -82,8 +82,8 @@ define void @simple_histogram_uadd_sat(ptr noalias %buckets, ptr readonly %indic
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP0]], align 4
; CHECK-NEXT: [[TMP1:%.*]] = zext <2 x i32> [[WIDE_LOAD]] to <2 x i64>
; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i64> [[TMP1]], i64 0
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i64> [[TMP1]], i64 1
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[BUCKETS]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i64> [[TMP1]], i64 1
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[BUCKETS]], i64 [[TMP3]]
; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP4]], i64 0
; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x ptr> [[TMP6]], ptr [[TMP5]], i64 1
@@ -147,8 +147,8 @@ define void @simple_histogram_umax(ptr noalias %buckets, ptr readonly %indices,
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP0]], align 4
; CHECK-NEXT: [[TMP1:%.*]] = zext <2 x i32> [[WIDE_LOAD]] to <2 x i64>
; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i64> [[TMP1]], i64 0
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i64> [[TMP1]], i64 1
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[BUCKETS]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i64> [[TMP1]], i64 1
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[BUCKETS]], i64 [[TMP3]]
; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP4]], i64 0
; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x ptr> [[TMP6]], ptr [[TMP5]], i64 1
@@ -212,8 +212,8 @@ define void @simple_histogram_umin(ptr noalias %buckets, ptr readonly %indices,
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP0]], align 4
; CHECK-NEXT: [[TMP1:%.*]] = zext <2 x i32> [[WIDE_LOAD]] to <2 x i64>
; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i64> [[TMP1]], i64 0
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i64> [[TMP1]], i64 1
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[BUCKETS]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i64> [[TMP1]], i64 1
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[BUCKETS]], i64 [[TMP3]]
; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP4]], i64 0
; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x ptr> [[TMP6]], ptr [[TMP5]], i64 1
>From 92f45ef09636ae0f4c93cb00276c9eb87b71cab1 Mon Sep 17 00:00:00 2001
From: rond <ron.dahan at mobileye.com>
Date: Tue, 4 Aug 2026 10:50:30 +0300
Subject: [PATCH 7/7] Address review comments and rebase onto current main
Move HistogramUpdateKind from VPHistogramRecipe to Analysis/VectorUtils.h so
LoopVectorizationLegality can name it, and derive the kind where the update
pattern is matched, storing it in HistogramInfo. This drops the UpdateOpcode
field, which mixed instruction opcodes and intrinsic IDs in a single unsigned,
and removes the duplicated lookup in VPRecipeBuilder.
Also regenerate the histogram check lines after rebasing onto current main.
Co-authored-by: Copilot <223556219+Copilot at users.noreply.github.com>
---
llvm/include/llvm/Analysis/VectorUtils.h | 10 ++++
.../Vectorize/LoopVectorizationLegality.h | 10 ++--
.../Vectorize/LoopVectorizationLegality.cpp | 33 ++++++------
.../Transforms/Vectorize/LoopVectorize.cpp | 27 +---------
llvm/lib/Transforms/Vectorize/VPlan.h | 11 ----
.../VPlan/AArch64/sve2-histcnt-vplan.ll | 2 +-
.../Transforms/LoopVectorize/histograms.ll | 51 ++++++++++---------
7 files changed, 60 insertions(+), 84 deletions(-)
diff --git a/llvm/include/llvm/Analysis/VectorUtils.h b/llvm/include/llvm/Analysis/VectorUtils.h
index f70ae3ff57f88..2a6ea9578a5a7 100644
--- a/llvm/include/llvm/Analysis/VectorUtils.h
+++ b/llvm/include/llvm/Analysis/VectorUtils.h
@@ -132,6 +132,16 @@ namespace Intrinsic {
typedef unsigned ID;
}
+/// The kind of update operation performed on the buckets of a histogram, i.e.
+/// buckets[indices[i]] = UpdateOp(buckets[indices[i]], Val).
+enum class HistogramUpdateKind {
+ Add,
+ Sub,
+ UAddSat,
+ UMax,
+ UMin,
+};
+
/// Identify if the intrinsic is trivially vectorizable.
/// This method returns true if the intrinsic's argument types are all scalars
/// for the scalar form of the intrinsic and all vectors (or scalars handled by
diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
index ed55307a9a6ad..d76406c62c272 100644
--- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
+++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
@@ -28,6 +28,7 @@
#include "llvm/ADT/MapVector.h"
#include "llvm/Analysis/LoopAccessAnalysis.h"
+#include "llvm/Analysis/VectorUtils.h"
#include "llvm/Support/TypeSize.h"
#include "llvm/Transforms/Utils/LoopUtils.h"
@@ -248,13 +249,12 @@ struct HistogramInfo {
LoadInst *Load;
Instruction *Update;
StoreInst *Store;
- /// The opcode (for BinaryOperator) or intrinsic ID (for IntrinsicInst) of
- /// the update operation.
- unsigned UpdateOpcode;
+ /// The update operation applied to the histogram buckets.
+ HistogramUpdateKind UpdateKind;
HistogramInfo(LoadInst *Load, Instruction *Update, StoreInst *Store,
- unsigned UpdateOpcode)
- : Load(Load), Update(Update), Store(Store), UpdateOpcode(UpdateOpcode) {}
+ HistogramUpdateKind UpdateKind)
+ : Load(Load), Update(Update), Store(Store), UpdateKind(UpdateKind) {}
};
/// Indicates the characteristics of a loop with an uncountable exit.
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 3d5e01b82d670..d5d4d91fc354c 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -1068,16 +1068,20 @@ static bool findHistogram(LoadInst *LI, StoreInst *HSt, Loop *TheLoop,
// FIXME: We assume the loop invariant term is on the RHS.
// Fine for an immediate/constant, but maybe not a generic value?
Value *HIncVal = nullptr;
- if (!match(HUpdateOp,
- m_Add(m_Load(m_Specific(HPtrInstr)), m_Value(HIncVal))) &&
- !match(HUpdateOp,
- m_Sub(m_Load(m_Specific(HPtrInstr)), m_Value(HIncVal))) &&
- !match(HUpdateOp, m_Intrinsic<Intrinsic::uadd_sat>(
- m_Load(m_Specific(HPtrInstr)), m_Value(HIncVal))) &&
- !match(HUpdateOp, m_Intrinsic<Intrinsic::umax>(
- m_Load(m_Specific(HPtrInstr)), m_Value(HIncVal))) &&
- !match(HUpdateOp, m_Intrinsic<Intrinsic::umin>(
- m_Load(m_Specific(HPtrInstr)), m_Value(HIncVal))))
+ HistogramUpdateKind UpdateKind;
+ auto Bucket = m_Load(m_Specific(HPtrInstr));
+ if (match(HUpdateOp, m_Add(Bucket, m_Value(HIncVal))))
+ UpdateKind = HistogramUpdateKind::Add;
+ else if (match(HUpdateOp, m_Sub(Bucket, m_Value(HIncVal))))
+ UpdateKind = HistogramUpdateKind::Sub;
+ else if (match(HUpdateOp,
+ m_Intrinsic<Intrinsic::uadd_sat>(Bucket, m_Value(HIncVal))))
+ UpdateKind = HistogramUpdateKind::UAddSat;
+ else if (match(HUpdateOp, m_UMax(Bucket, m_Value(HIncVal))))
+ UpdateKind = HistogramUpdateKind::UMax;
+ else if (match(HUpdateOp, m_UMin(Bucket, m_Value(HIncVal))))
+ UpdateKind = HistogramUpdateKind::UMin;
+ else
return false;
// Make sure the increment value is loop invariant.
@@ -1126,15 +1130,8 @@ static bool findHistogram(LoadInst *LI, StoreInst *HSt, Loop *TheLoop,
LLVM_DEBUG(dbgs() << "LV: Found histogram for: " << *HSt << "\n");
- // Determine the update opcode.
- unsigned UpdateOpcode;
- if (auto *BO = dyn_cast<BinaryOperator>(HUpdateOp))
- UpdateOpcode = BO->getOpcode();
- else
- UpdateOpcode = cast<IntrinsicInst>(HUpdateOp)->getIntrinsicID();
-
// Store the operations that make up the histogram.
- Histograms.emplace_back(IndexedLoad, HUpdateOp, HSt, UpdateOpcode);
+ Histograms.emplace_back(IndexedLoad, HUpdateOp, HSt, UpdateKind);
return true;
}
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 3cc19e7690c47..dadc78cef69c8 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6358,27 +6358,6 @@ VPRecipeWithIRFlags *VPRecipeBuilder::tryToWiden(VPInstruction *VPI) {
};
}
-/// Return the HistogramUpdateKind for the given update opcode (instruction
-/// opcode or intrinsic ID), as stored in HistogramInfo during legality.
-static VPHistogramRecipe::HistogramUpdateKind
-getHistogramUpdateKind(unsigned Opcode) {
- using HistogramUpdateKind = VPHistogramRecipe::HistogramUpdateKind;
- switch (Opcode) {
- case Instruction::Add:
- return HistogramUpdateKind::Add;
- case Instruction::Sub:
- return HistogramUpdateKind::Sub;
- case Intrinsic::uadd_sat:
- return HistogramUpdateKind::UAddSat;
- case Intrinsic::umax:
- return HistogramUpdateKind::UMax;
- case Intrinsic::umin:
- return HistogramUpdateKind::UMin;
- default:
- llvm_unreachable("Unsupported histogram update operation");
- }
-}
-
VPHistogramRecipe *VPRecipeBuilder::widenIfHistogram(VPInstruction *VPI) {
if (VPI->getOpcode() != Instruction::Store)
return nullptr;
@@ -6389,8 +6368,6 @@ VPHistogramRecipe *VPRecipeBuilder::widenIfHistogram(VPInstruction *VPI) {
return nullptr;
const HistogramInfo *HI = *HistInfo;
- VPHistogramRecipe::HistogramUpdateKind UpdateKind =
- getHistogramUpdateKind(HI->UpdateOpcode);
SmallVector<VPValue *, 3> HGramOps;
// Bucket address.
@@ -6403,8 +6380,8 @@ VPHistogramRecipe *VPRecipeBuilder::widenIfHistogram(VPInstruction *VPI) {
if (CM.isMaskRequired(HI->Store))
HGramOps.push_back(VPI->getMask());
- return new VPHistogramRecipe(UpdateKind, HGramOps, cast<VPIRMetadata>(*VPI),
- VPI->getDebugLoc());
+ return new VPHistogramRecipe(HI->UpdateKind, HGramOps,
+ cast<VPIRMetadata>(*VPI), VPI->getDebugLoc());
}
bool VPRecipeBuilder::replaceWithFinalIfReductionStore(
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index c8d4aac0055e2..44b925a8d9ee9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -2162,17 +2162,6 @@ class LLVM_ABI_FOR_TEST VPWidenCallRecipe : public VPRecipeWithIRFlags,
/// of intrinsics. Supported update operations are: add, sub, uadd.sat, umax,
/// and umin, where the other term is loop-invariant.
class VPHistogramRecipe : public VPRecipeBase, public VPIRMetadata {
-public:
- /// The kind of update operation performed on histogram buckets.
- enum class HistogramUpdateKind {
- Add,
- Sub,
- UAddSat,
- UMax,
- UMin,
- };
-
-private:
/// The update operation kind for this histogram.
HistogramUpdateKind UpdateKind;
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve2-histcnt-vplan.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve2-histcnt-vplan.ll
index 20c925cd55b76..c1942bc68304e 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve2-histcnt-vplan.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve2-histcnt-vplan.ll
@@ -172,7 +172,7 @@ for.exit:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: [[STEPS3:vp.*]] = SCALAR-STEPS [[IV3]], ir<1>, [[VF3]]
; CHECK-NEXT: CLONE [[GEP_IDX3:.*]] = getelementptr inbounds ir<%indices>, [[STEPS3]]
-; CHECK-NEXT: [[VECP_IDX3:vp.*]] = vector-pointer inbounds [[GEP_IDX3]]
+; CHECK-NEXT: [[VECP_IDX3:vp.*]] = vector-pointer inbounds i32, [[GEP_IDX3]]
; CHECK-NEXT: WIDEN [[IDX3:.*]] = load [[VECP_IDX3]]
; CHECK-NEXT: WIDEN-CAST [[EXT_IDX3:.*]] = zext [[IDX3]] to i64
; CHECK-NEXT: WIDEN-GEP [[GEP_BUCKET3:.*]] = getelementptr inbounds ir<%buckets>, [[EXT_IDX3]]
diff --git a/llvm/test/Transforms/LoopVectorize/histograms.ll b/llvm/test/Transforms/LoopVectorize/histograms.ll
index 22d146283baa5..a55367063c2a2 100644
--- a/llvm/test/Transforms/LoopVectorize/histograms.ll
+++ b/llvm/test/Transforms/LoopVectorize/histograms.ll
@@ -74,19 +74,20 @@ define void @simple_histogram_uadd_sat(ptr noalias %buckets, ptr readonly %indic
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
-; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[N]], -2
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 2
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [4 x i8], ptr [[INDICES]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP0]], align 4
; CHECK-NEXT: [[TMP1:%.*]] = zext <2 x i32> [[WIDE_LOAD]] to <2 x i64>
; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i64> [[TMP1]], i64 0
-; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[BUCKETS]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP2]]
; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i64> [[TMP1]], i64 1
-; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[BUCKETS]], i64 [[TMP3]]
-; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP4]], i64 0
-; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x ptr> [[TMP6]], ptr [[TMP5]], i64 1
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP3]]
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP4]], i32 0
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x ptr> [[TMP6]], ptr [[TMP5]], i32 1
; CHECK-NEXT: call void @llvm.experimental.vector.histogram.uadd.sat.v2p0.i32(<2 x ptr> [[TMP7]], i32 1, <2 x i1> splat (i1 true))
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -99,10 +100,10 @@ define void @simple_histogram_uadd_sat(ptr noalias %buckets, ptr readonly %indic
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
; CHECK: for.body:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT: [[GEP_INDICES:%.*]] = getelementptr inbounds [4 x i8], ptr [[INDICES]], i64 [[IV]]
+; CHECK-NEXT: [[GEP_INDICES:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV]]
; CHECK-NEXT: [[L_IDX:%.*]] = load i32, ptr [[GEP_INDICES]], align 4
; CHECK-NEXT: [[IDXPROM1:%.*]] = zext i32 [[L_IDX]] to i64
-; CHECK-NEXT: [[GEP_BUCKET:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[BUCKETS]], i64 [[IDXPROM1]]
+; CHECK-NEXT: [[GEP_BUCKET:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[IDXPROM1]]
; CHECK-NEXT: [[L_BUCKET:%.*]] = load i32, ptr [[GEP_BUCKET]], align 4
; CHECK-NEXT: [[INC:%.*]] = call i32 @llvm.uadd.sat.i32(i32 [[L_BUCKET]], i32 1)
; CHECK-NEXT: store i32 [[INC]], ptr [[GEP_BUCKET]], align 4
@@ -139,19 +140,20 @@ define void @simple_histogram_umax(ptr noalias %buckets, ptr readonly %indices,
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
-; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[N]], -2
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 2
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [4 x i8], ptr [[INDICES]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP0]], align 4
; CHECK-NEXT: [[TMP1:%.*]] = zext <2 x i32> [[WIDE_LOAD]] to <2 x i64>
; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i64> [[TMP1]], i64 0
-; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[BUCKETS]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP2]]
; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i64> [[TMP1]], i64 1
-; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[BUCKETS]], i64 [[TMP3]]
-; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP4]], i64 0
-; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x ptr> [[TMP6]], ptr [[TMP5]], i64 1
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP3]]
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP4]], i32 0
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x ptr> [[TMP6]], ptr [[TMP5]], i32 1
; CHECK-NEXT: call void @llvm.experimental.vector.histogram.umax.v2p0.i32(<2 x ptr> [[TMP7]], i32 120, <2 x i1> splat (i1 true))
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -164,10 +166,10 @@ define void @simple_histogram_umax(ptr noalias %buckets, ptr readonly %indices,
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
; CHECK: for.body:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT: [[GEP_INDICES:%.*]] = getelementptr inbounds [4 x i8], ptr [[INDICES]], i64 [[IV]]
+; CHECK-NEXT: [[GEP_INDICES:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV]]
; CHECK-NEXT: [[L_IDX:%.*]] = load i32, ptr [[GEP_INDICES]], align 4
; CHECK-NEXT: [[IDXPROM1:%.*]] = zext i32 [[L_IDX]] to i64
-; CHECK-NEXT: [[GEP_BUCKET:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[BUCKETS]], i64 [[IDXPROM1]]
+; CHECK-NEXT: [[GEP_BUCKET:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[IDXPROM1]]
; CHECK-NEXT: [[L_BUCKET:%.*]] = load i32, ptr [[GEP_BUCKET]], align 4
; CHECK-NEXT: [[INC:%.*]] = call i32 @llvm.umax.i32(i32 [[L_BUCKET]], i32 120)
; CHECK-NEXT: store i32 [[INC]], ptr [[GEP_BUCKET]], align 4
@@ -204,19 +206,20 @@ define void @simple_histogram_umin(ptr noalias %buckets, ptr readonly %indices,
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
-; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[N]], -2
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 2
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [4 x i8], ptr [[INDICES]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP0]], align 4
; CHECK-NEXT: [[TMP1:%.*]] = zext <2 x i32> [[WIDE_LOAD]] to <2 x i64>
; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i64> [[TMP1]], i64 0
-; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[BUCKETS]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP2]]
; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i64> [[TMP1]], i64 1
-; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[BUCKETS]], i64 [[TMP3]]
-; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP4]], i64 0
-; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x ptr> [[TMP6]], ptr [[TMP5]], i64 1
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[TMP3]]
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP4]], i32 0
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x ptr> [[TMP6]], ptr [[TMP5]], i32 1
; CHECK-NEXT: call void @llvm.experimental.vector.histogram.umin.v2p0.i32(<2 x ptr> [[TMP7]], i32 99, <2 x i1> splat (i1 true))
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -229,10 +232,10 @@ define void @simple_histogram_umin(ptr noalias %buckets, ptr readonly %indices,
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
; CHECK: for.body:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT: [[GEP_INDICES:%.*]] = getelementptr inbounds [4 x i8], ptr [[INDICES]], i64 [[IV]]
+; CHECK-NEXT: [[GEP_INDICES:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV]]
; CHECK-NEXT: [[L_IDX:%.*]] = load i32, ptr [[GEP_INDICES]], align 4
; CHECK-NEXT: [[IDXPROM1:%.*]] = zext i32 [[L_IDX]] to i64
-; CHECK-NEXT: [[GEP_BUCKET:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[BUCKETS]], i64 [[IDXPROM1]]
+; CHECK-NEXT: [[GEP_BUCKET:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[IDXPROM1]]
; CHECK-NEXT: [[L_BUCKET:%.*]] = load i32, ptr [[GEP_BUCKET]], align 4
; CHECK-NEXT: [[INC:%.*]] = call i32 @llvm.umin.i32(i32 [[L_BUCKET]], i32 99)
; CHECK-NEXT: store i32 [[INC]], ptr [[GEP_BUCKET]], align 4
More information about the llvm-commits
mailing list