[llvm] [LV] Vectorize histogram operations (PR #99851)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Jul 22 01:49:54 PDT 2024
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-transforms
Author: Graham Hunter (huntergr-arm)
<details>
<summary>Changes</summary>
This patch implements autovectorization support for the 'all-in-one' histogram intrinsic, which seems to have more support than the 'standalone' intrinsic. See https://discourse.llvm.org/t/rfc-vectorization-support-for-histogram-count-operations/74788/ for an overview of the work and my notes on the tradeoffs between the two approaches.
This is a new PR after the previous one (#<!-- -->91458) was reverted at maintainer request.
I have removed the changes to LoopAccessAnalysis in favor continuing analysis in LoopVectorizationLegality.
---
Patch is 57.53 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/99851.diff
11 Files Affected:
- (modified) llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h (+38)
- (modified) llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp (+109-1)
- (modified) llvm/lib/Transforms/Vectorize/LoopVectorize.cpp (+86-1)
- (modified) llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h (+8)
- (modified) llvm/lib/Transforms/Vectorize/VPlan.h (+35)
- (modified) llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp (+44)
- (modified) llvm/lib/Transforms/Vectorize/VPlanValue.h (+1)
- (added) llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-epilogue.ll (+76)
- (added) llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-no-scalar-interleave.ll (+53)
- (added) llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll (+514)
- (added) llvm/test/Transforms/LoopVectorize/histograms.ll (+44)
``````````diff
diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
index 2ff17bd2f7a71..372ac6ee3fb00 100644
--- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
+++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
@@ -224,6 +224,18 @@ class LoopVectorizationRequirements {
Instruction *ExactFPMathInst = nullptr;
};
+/// This holds details about a histogram operation -- a load -> update -> store
+/// sequence where each lane in a vector might be updating the same element as
+/// another lane.
+struct HistogramInfo {
+ LoadInst *Load;
+ Instruction *Update;
+ StoreInst *Store;
+
+ HistogramInfo(LoadInst *Load, Instruction *Update, StoreInst *Store)
+ : Load(Load), Update(Update), Store(Store) {}
+};
+
/// LoopVectorizationLegality checks if it is legal to vectorize a loop, and
/// to what vectorization factor.
/// This class does not look at the profitability of vectorization, only the
@@ -390,6 +402,22 @@ class LoopVectorizationLegality {
unsigned getNumStores() const { return LAI->getNumStores(); }
unsigned getNumLoads() const { return LAI->getNumLoads(); }
+ /// Returns a HistogramInfo* for the given instruction if it was determined
+ /// to be part of a load -> update -> store sequence where multiple lanes
+ /// may be working on the same memory address.
+ std::optional<const HistogramInfo *> getHistogramInfo(Instruction *I) const {
+ for (const HistogramInfo &HGram : Histograms)
+ if (HGram.Load == I || HGram.Update == I || HGram.Store == I)
+ return &HGram;
+
+ return std::nullopt;
+ }
+
+ /// Returns a list of all known histogram operations in the loop.
+ const SmallVectorImpl<HistogramInfo> &getHistograms() const {
+ return Histograms;
+ }
+
PredicatedScalarEvolution *getPredicatedScalarEvolution() const {
return &PSE;
}
@@ -438,6 +466,11 @@ class LoopVectorizationLegality {
/// Returns true if the loop is vectorizable
bool canVectorizeMemory();
+ /// If LAA cannot determine whether all dependences are safe, we may be able
+ /// to further analyse some unknown dependences and if they match a certain
+ /// pattern (like a histogram) then we may still be able to vectorize.
+ bool canVectorizeUnknownDependences();
+
/// Return true if we can vectorize this loop using the IF-conversion
/// transformation.
bool canVectorizeWithIfConvert();
@@ -542,6 +575,11 @@ class LoopVectorizationLegality {
/// conditional assumes.
SmallPtrSet<const Instruction *, 8> MaskedOp;
+ /// Contains all identified histogram operations, which are sequences of
+ /// load -> update -> store instructions where multiple lanes in a vector
+ /// may work on the same memory location.
+ SmallVector<HistogramInfo, 1> Histograms;
+
/// BFI and PSI are used to check for profile guided size optimizations.
BlockFrequencyInfo *BFI;
ProfileSummaryInfo *PSI;
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index cafec165f6d6f..e128c4124e1f8 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -78,6 +78,10 @@ static cl::opt<LoopVectorizeHints::ScalableForceKind>
"Scalable vectorization is available and favored when the "
"cost is inconclusive.")));
+static cl::opt<bool> EnableHistogramVectorization(
+ "enable-histogram-loop-vectorization", cl::init(false), cl::Hidden,
+ cl::desc("Enables autovectorization of some loops containing histograms"));
+
/// Maximum vectorization interleave count.
static const unsigned MaxInterleaveFactor = 16;
@@ -1054,6 +1058,110 @@ bool LoopVectorizationLegality::canVectorizeInstrs() {
return true;
}
+/// Find Histogram counts that match high-level code in loops:
+/// \code
+/// buckets[indices[i]]+=step;
+/// \endcode
+///
+/// It matches a pattern starting from \p HSt, which Stores to the 'buckets'
+/// array the computed histogram. It uses a BinOp to sum all counts, storing
+/// them using a loop-variant index Load from the 'indices' input array.
+///
+/// On successful matches it updates the STATISTIC 'HistogramsDetected',
+/// regardless of hardware support. When there is support, it additionally
+/// stores the BinOp/Load pairs in \p HistogramCounts, as well the pointers
+/// used to update histogram in \p HistogramPtrs.
+
+static bool findHistograms(LoadInst *LI, StoreInst *HSt, Loop *TheLoop,
+ const PredicatedScalarEvolution &PSE,
+ SmallVectorImpl<HistogramInfo> &Histograms) {
+
+ // Store value must come from a Binary Operation.
+ Instruction *HPtrInstr = nullptr;
+ BinaryOperator *HBinOp = nullptr;
+ if (!match(HSt, m_Store(m_BinOp(HBinOp), m_Instruction(HPtrInstr))))
+ return false;
+
+ // BinOp must be an Add or a Sub modifying the bucket value by a
+ // loop invariant amount.
+ // FIXME: We assume the loop invariant term is on the RHS.
+ // Fine for an immediate/constant, but maybe not a generic value?
+ Value *HIncVal = nullptr;
+ if (!match(HBinOp, m_Add(m_Load(m_Specific(HPtrInstr)), m_Value(HIncVal))) &&
+ !match(HBinOp, m_Sub(m_Load(m_Specific(HPtrInstr)), m_Value(HIncVal))))
+ return false;
+
+ // Make sure the increment value is loop invariant.
+ if (!TheLoop->isLoopInvariant(HIncVal))
+ return false;
+
+ // The address to store is calculated through a GEP Instruction.
+ // FIXME: Support GEPs with more operands.
+ GetElementPtrInst *HPtr = dyn_cast<GetElementPtrInst>(HPtrInstr);
+ if (!HPtr || HPtr->getNumOperands() > 2)
+ return false;
+
+ // Check that the index is calculated by loading from another array. Ignore
+ // any extensions.
+ // FIXME: Support indices from other sources that a linear load from memory?
+ Value *HIdx = HPtr->getOperand(1);
+ Instruction *IdxInst = nullptr;
+ if (!match(HIdx, m_ZExtOrSExtOrSelf(m_Instruction(IdxInst))))
+ return false;
+
+ // Currently restricting this to linear addressing when loading indices.
+ LoadInst *VLoad = dyn_cast<LoadInst>(IdxInst);
+ Value *VPtrVal;
+ if (!VLoad || !match(VLoad, m_Load(m_Value(VPtrVal))))
+ return false;
+
+ if (!isa<SCEVAddRecExpr>(PSE.getSE()->getSCEV(VPtrVal)))
+ return false;
+
+ // Ensure we'll have the same mask by checking that all parts of the histogram
+ // (gather load, update, scatter store) are in the same block.
+ LoadInst *IndexedLoad = cast<LoadInst>(HBinOp->getOperand(0));
+ BasicBlock *LdBB = IndexedLoad->getParent();
+ if (LdBB != HBinOp->getParent() || LdBB != HSt->getParent())
+ return false;
+
+ LLVM_DEBUG(dbgs() << "LV: Found histogram for: " << *HSt << "\n");
+
+ // Store the operations that make up the histogram.
+ Histograms.emplace_back(IndexedLoad, HBinOp, HSt);
+ return true;
+}
+
+bool LoopVectorizationLegality::canVectorizeUnknownDependences() {
+ // For now, we only support an unknown dependency that calculates a histogram
+ if (!EnableHistogramVectorization)
+ return false;
+
+ // FIXME: Support more than one unknown dependence, and check to see if some
+ // are handled by runtime checks before looking for histograms.
+ LAI = &LAIs.getInfo(*TheLoop);
+ const MemoryDepChecker &DepChecker = LAI->getDepChecker();
+ const auto *Deps = DepChecker.getDependences();
+ if (!Deps || Deps->size() > 1)
+ return false;
+
+ const MemoryDepChecker::Dependence &Dep = (*Deps).front();
+
+ // We're only interested in unknown dependences.
+ if (Dep.Type != MemoryDepChecker::Dependence::Unknown)
+ return false;
+
+ // For now only normal loads and stores are supported.
+ LoadInst *LI = dyn_cast<LoadInst>(Dep.getSource(DepChecker));
+ StoreInst *SI = dyn_cast<StoreInst>(Dep.getDestination(DepChecker));
+
+ if (!LI || !SI)
+ return false;
+
+ LLVM_DEBUG(dbgs() << "LV: Checking for a histogram on: " << *SI << "\n");
+ return findHistograms(LI, SI, TheLoop, LAI->getPSE(), Histograms);
+}
+
bool LoopVectorizationLegality::canVectorizeMemory() {
LAI = &LAIs.getInfo(*TheLoop);
const OptimizationRemarkAnalysis *LAR = LAI->getReport();
@@ -1065,7 +1173,7 @@ bool LoopVectorizationLegality::canVectorizeMemory() {
}
if (!LAI->canVectorizeMemory())
- return false;
+ return canVectorizeUnknownDependences();
if (LAI->hasLoadStoreDependenceInvolvingLoopInvariantAddress()) {
reportVectorizationFailure("We don't allow storing to uniform addresses",
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 6d28b8fabe42e..823c95d8bb4f9 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -4622,6 +4622,10 @@ bool LoopVectorizationPlanner::isCandidateForEpilogueVectorization(
if (OrigLoop->getExitingBlock() != OrigLoop->getLoopLatch())
return false;
+ // Loops containing histograms are not currently supported.
+ if (!Legal->getHistograms().empty())
+ return false;
+
return true;
}
@@ -6465,8 +6469,33 @@ LoopVectorizationCostModel::getInstructionCost(Instruction *I,
// We've proven all lanes safe to speculate, fall through.
[[fallthrough]];
case Instruction::Add:
+ case Instruction::Sub: {
+ auto Info = Legal->getHistogramInfo(I);
+ if (Info && VF.isVector()) {
+ const HistogramInfo *HGram = Info.value();
+ // Assume that a non-constant update value (or a constant != 1) requires
+ // a multiply, and add that into the cost.
+ InstructionCost MulCost = TTI::TCC_Free;
+ ConstantInt *RHS = dyn_cast<ConstantInt>(I->getOperand(1));
+ if (!RHS || RHS->getZExtValue() != 1)
+ MulCost = TTI.getArithmeticInstrCost(Instruction::Mul, VectorTy);
+
+ // Find the cost of the histogram operation itself.
+ Type *PtrTy = VectorType::get(HGram->Load->getPointerOperandType(), VF);
+ Type *ScalarTy = I->getType();
+ Type *MaskTy = VectorType::get(Type::getInt1Ty(I->getContext()), VF);
+ IntrinsicCostAttributes ICA(Intrinsic::experimental_vector_histogram_add,
+ Type::getVoidTy(I->getContext()),
+ {PtrTy, ScalarTy, MaskTy});
+
+ // Add the costs together with the add/sub operation.
+ return TTI.getIntrinsicInstrCost(
+ ICA, TargetTransformInfo::TCK_RecipThroughput) +
+ MulCost + TTI.getArithmeticInstrCost(I->getOpcode(), VectorTy);
+ }
+ [[fallthrough]];
+ }
case Instruction::FAdd:
- case Instruction::Sub:
case Instruction::FSub:
case Instruction::Mul:
case Instruction::FMul:
@@ -8173,6 +8202,36 @@ VPWidenRecipe *VPRecipeBuilder::tryToWiden(Instruction *I,
};
}
+VPHistogramRecipe *
+VPRecipeBuilder::tryToWidenHistogram(const HistogramInfo *HI,
+ ArrayRef<VPValue *> Operands) {
+ // FIXME: Support other operations.
+ unsigned Opcode = HI->Update->getOpcode();
+ assert((Opcode == Instruction::Add || Opcode == Instruction::Sub) &&
+ "Histogram update operation must be an Add or Sub");
+
+ SmallVector<VPValue *, 3> HGramOps;
+ // Bucket address.
+ HGramOps.push_back(Operands[1]);
+ // Increment value.
+ HGramOps.push_back(getVPValueOrAddLiveIn(HI->Update->getOperand(1), Plan));
+
+ // In case of predicated execution (due to tail-folding, or conditional
+ // execution, or both), pass the relevant mask. When there is no such mask,
+ // generate an all-true mask.
+ VPValue *Mask = nullptr;
+ if (Legal->isMaskRequired(HI->Store))
+ Mask = getBlockInMask(HI->Store->getParent());
+ else
+ Mask = Plan.getOrAddLiveIn(
+ ConstantInt::getTrue(IntegerType::getInt1Ty(HI->Load->getContext())));
+ HGramOps.push_back(Mask);
+
+ return new VPHistogramRecipe(HI, Opcode,
+ make_range(HGramOps.begin(), HGramOps.end()),
+ HI->Store->getDebugLoc());
+}
+
void VPRecipeBuilder::fixHeaderPhis() {
BasicBlock *OrigLatch = OrigLoop->getLoopLatch();
for (VPHeaderPHIRecipe *R : PhisToFix) {
@@ -8296,6 +8355,10 @@ VPRecipeBuilder::tryToCreateWidenRecipe(Instruction *Instr,
if (auto *CI = dyn_cast<CallInst>(Instr))
return tryToWidenCall(CI, Operands, Range);
+ if (StoreInst *SI = dyn_cast<StoreInst>(Instr))
+ if (auto HistInfo = Legal->getHistogramInfo(SI))
+ return tryToWidenHistogram(*HistInfo, Operands);
+
if (isa<LoadInst>(Instr) || isa<StoreInst>(Instr))
return tryToWidenMemory(Instr, Operands, Range);
@@ -8563,6 +8626,15 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
Operands = {OpRange.begin(), OpRange.end()};
}
+ // If this is a load instruction or a binop associated with a histogram,
+ // leave it until the store instruction to emit a combined intrinsic.
+ // Note that if the initial VF is scalar, we need to generate the normal
+ // clone recipe for these instructions. A histogram recipe will only be
+ // generated when minVF > 1.
+ if (Legal->getHistogramInfo(Instr) && !isa<StoreInst>(Instr) &&
+ !Range.Start.isScalar())
+ continue;
+
// Invariant stores inside loop will be deleted and a single store
// with the final reduction value will be added to the exit block
StoreInst *SI;
@@ -9890,6 +9962,19 @@ bool LoopVectorizePass::processLoop(Loop *L) {
InterleaveLoop = false;
}
+ // If there is a histogram in the loop, do not just interleave without
+ // vectorizing. The order of operations will be incorrect without the
+ // histogram intrinsics, which are only used for recipes with VF > 1.
+ if (!VectorizeLoop && InterleaveLoop && !LVL.getHistograms().empty()) {
+ LLVM_DEBUG(dbgs() << "LV: Not interleaving without vectorization due "
+ << "to histogram operations.\n");
+ IntDiagMsg = std::make_pair(
+ "HistogramPreventsScalarInterleaving",
+ "Unable to interleave without vectorization due to constraints on "
+ "the order of histogram operations");
+ InterleaveLoop = false;
+ }
+
// Override IC if user provided an interleave count.
IC = UserIC > 0 ? UserIC : IC;
diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
index b4c7ab02f928f..b3b25047820f1 100644
--- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
+++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
@@ -20,6 +20,7 @@ namespace llvm {
class LoopVectorizationLegality;
class LoopVectorizationCostModel;
class TargetLibraryInfo;
+struct HistogramInfo;
/// Helper class to create VPRecipies from IR instructions.
class VPRecipeBuilder {
@@ -102,6 +103,13 @@ class VPRecipeBuilder {
VPWidenRecipe *tryToWiden(Instruction *I, ArrayRef<VPValue *> Operands,
VPBasicBlock *VPBB);
+ /// Makes Histogram count operations safe for vectorization, by emitting a
+ /// llvm.experimental.vector.histogram.add intrinsic in place of the
+ /// Load + Add|Sub + Store operations that perform the histogram in the
+ /// original scalar loop.
+ VPHistogramRecipe *tryToWidenHistogram(const HistogramInfo *HI,
+ ArrayRef<VPValue *> Operands);
+
public:
VPRecipeBuilder(VPlan &Plan, Loop *OrigLoop, const TargetLibraryInfo *TLI,
LoopVectorizationLegality *Legal,
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 805d9d91fc186..7d41aba3f827d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -69,6 +69,7 @@ class LoopVectorizationCostModel;
class LoopVersioning;
struct VPCostContext;
+struct HistogramInfo;
namespace Intrinsic {
typedef unsigned ID;
@@ -937,6 +938,7 @@ class VPSingleDefRecipe : public VPRecipeBase, public VPValue {
case VPRecipeBase::VPWidenLoadSC:
case VPRecipeBase::VPWidenStoreEVLSC:
case VPRecipeBase::VPWidenStoreSC:
+ case VPRecipeBase::VPHistogramSC:
// TODO: Widened stores don't define a value, but widened loads do. Split
// the recipes to be able to make widened loads VPSingleDefRecipes.
return false;
@@ -1573,6 +1575,39 @@ class VPWidenCallRecipe : public VPSingleDefRecipe {
#endif
};
+class VPHistogramRecipe : public VPRecipeBase {
+ const HistogramInfo *Info;
+ unsigned Opcode;
+
+public:
+ template <typename IterT>
+ VPHistogramRecipe(const HistogramInfo *HI, unsigned Opcode,
+ iterator_range<IterT> Operands, DebugLoc DL = {})
+ : VPRecipeBase(VPDef::VPHistogramSC, Operands, DL), Info(HI),
+ Opcode(Opcode) {}
+
+ ~VPHistogramRecipe() override = default;
+
+ VPHistogramRecipe *clone() override {
+ llvm_unreachable("cloning not supported");
+ }
+
+ VP_CLASSOF_IMPL(VPDef::VPHistogramSC);
+
+ // Produce a histogram operation with widened ingredients
+ void execute(VPTransformState &State) override;
+
+ unsigned getOpcode() const { return Opcode; }
+
+ const HistogramInfo *getHistogramInfo() const { return Info; }
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+ /// Print the recipe
+ void print(raw_ostream &O, const Twine &Indent,
+ VPSlotTracker &SlotTracker) const override;
+#endif
+};
+
/// A recipe for widening select instructions.
struct VPWidenSelectRecipe : public VPSingleDefRecipe {
template <typename IterT>
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 1b787d0490672..7701300d9a273 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -21,6 +21,7 @@
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
+#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Value.h"
#include "llvm/Support/Casting.h"
@@ -30,6 +31,7 @@
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/Transforms/Utils/LoopUtils.h"
#include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"
+#include "llvm/Transforms/Vectorize/LoopVectorizationLegality.h"
#include <cassert>
using namespace llvm;
@@ -279,6 +281,10 @@ static Instruction *getInstructionForCost(const VPRecipeBase *R) {
return IG->getInsertPos();
if (auto *WidenMem = dyn_cast<VPWidenMemoryRecipe>(R))
return &WidenMem->getIngredient();
+ // FIXME: Override the cost method properly to take gather/scatter cost
+ // into account, instead of just the intrinsic via the legacy model.
+ if (auto *HG = dyn_cast<VPHistogramRecipe>(R))
+ return HG->getHistogramInfo()->Update;
return nullptr;
}
@@ -958,6 +964,44 @@ void VPWidenCallRecipe::print(raw_ostream &O, const Twine &Indent,
O << ")";
}
}
+#endif
+
+void VPHistogramRecipe::execute(VPTransformState &State) {
+ State.setDebugLocFrom(getDebugLoc());
+ IRBuilderBase &Builder = State.Builder;
+
+ for (unsigned Part = 0; Part < State.UF; ++Part) {
+ Value *Address = State.get(getOperand(0), Part);
+ Value *IncVec = State.get(getOperand(1), Part);
+ Value *Mask = State.get(getOperand(2), Part);
+
+ // Not sure how to make IncAmt stay scalar yet. For now just extract the
+ // first element and tidy up later.
+ // FIXME: Do we actually want this to be scalar? We just splat it in the
+ // backend anyway...
+ Value *IncAmt = Builder.CreateExtractElement(IncVec, Builder.getInt64(0));
+
+ // If this is a subtract, we want to invert the increment amount. We may
+ // add a separate intrinsic in future, but for now we'll try this.
+ if (Opcode == Instruction::Sub)
+ IncAmt = Builder.CreateNeg(IncAmt);
+
+ State.Builder.CreateIntrinsic(Intrinsic::experimental_vector_histogram_add,
+ {Address->getType(), IncAmt->getType()},
+ {Address, IncAmt, Mask});
+ }
+}
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+void VPHistogramRecipe::print(raw_ostream &O, const Twine &Indent,
+ VPSlotTracker &SlotTracker) const {
+ O << Indent << "WIDEN-HISTOGRAM buckets: ";
+ getOperand(0)->printAsOperand(O, SlotTracker);
+ O << ", inc: ";
+ getOperand(1)-...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/99851
More information about the llvm-commits
mailing list