[llvm] [LV] Vectorize histogram operations (PR #99851)
Graham Hunter via llvm-commits
llvm-commits at lists.llvm.org
Tue Sep 3 08:09:20 PDT 2024
https://github.com/huntergr-arm updated https://github.com/llvm/llvm-project/pull/99851
>From 9650054d7338c5ee09fecc3e07c97373386b1fa7 Mon Sep 17 00:00:00 2001
From: Graham Hunter <graham.hunter at arm.com>
Date: Tue, 11 Jun 2024 14:58:55 +0000
Subject: [PATCH 1/9] Initial tests for histogram autovec
---
.../LoopVectorize/AArch64/sve2-histcnt.ll | 275 ++++++++++++++++++
1 file changed, 275 insertions(+)
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
new file mode 100644
index 00000000000000..283baf61b73938
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
@@ -0,0 +1,275 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3
+; RUN: opt < %s -passes=loop-vectorize -force-vector-interleave=1 -sve-gather-overhead=2 -sve-scatter-overhead=2 -S | FileCheck %s
+
+target triple = "aarch64-unknown-linux-gnu"
+
+;; Based on the following C code:
+;;
+;; void simple_histogram(int *buckets, unsigned *indices, int N) {
+;; for (int i = 0; i < N; ++i)
+;; buckets[indices[i]]++;
+;; }
+
+define void @simple_histogram(ptr noalias %buckets, ptr readonly %indices, i64 %N) #0 {
+; CHECK-LABEL: define void @simple_histogram(
+; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br label [[FOR_BODY:%.*]]
+; CHECK: for.body:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV]]
+; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT: [[IDXPROM1:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[IDXPROM1]]
+; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT: [[INC:%.*]] = add nsw i32 [[TMP1]], 1
+; CHECK-NEXT: store i32 [[INC]], ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT:%.*]], label [[FOR_BODY]]
+; CHECK: for.exit:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds i32, ptr %indices, i64 %iv
+ %0 = load i32, ptr %arrayidx, align 4
+ %idxprom1 = zext i32 %0 to i64
+ %arrayidx2 = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+ %1 = load i32, ptr %arrayidx2, align 4
+ %inc = add nsw i32 %1, 1
+ store i32 %inc, ptr %arrayidx2, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, %N
+ br i1 %exitcond, label %for.exit, label %for.body
+
+for.exit:
+ ret void
+}
+
+define void @simple_histogram_sub(ptr noalias %buckets, ptr readonly %indices, i64 %N) #0 {
+; CHECK-LABEL: define void @simple_histogram_sub(
+; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br label [[FOR_BODY:%.*]]
+; CHECK: for.body:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV]]
+; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT: [[IDXPROM1:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[IDXPROM1]]
+; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT: [[INC:%.*]] = sub nsw i32 [[TMP1]], 1
+; CHECK-NEXT: store i32 [[INC]], ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT:%.*]], label [[FOR_BODY]]
+; CHECK: for.exit:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds i32, ptr %indices, i64 %iv
+ %0 = load i32, ptr %arrayidx, align 4
+ %idxprom1 = zext i32 %0 to i64
+ %arrayidx2 = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+ %1 = load i32, ptr %arrayidx2, align 4
+ %inc = sub nsw i32 %1, 1
+ store i32 %inc, ptr %arrayidx2, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, %N
+ br i1 %exitcond, label %for.exit, label %for.body
+
+for.exit:
+ ret void
+}
+
+define void @conditional_histogram(ptr noalias %buckets, ptr readonly %indices, ptr readonly %conds, i64 %N) #0 {
+; CHECK-LABEL: define void @conditional_histogram(
+; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], ptr readonly [[CONDS:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br label [[FOR_BODY:%.*]]
+; CHECK: for.body:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[NEXT:%.*]] ]
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV]]
+; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT: [[IDXPROM1:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[IDXPROM1]]
+; CHECK-NEXT: [[CONDIDX:%.*]] = getelementptr inbounds i32, ptr [[CONDS]], i64 [[IV]]
+; CHECK-NEXT: [[CONDDATA:%.*]] = load i32, ptr [[CONDIDX]], align 4
+; CHECK-NEXT: [[IFCOND:%.*]] = icmp sgt i32 [[CONDDATA]], 5100
+; CHECK-NEXT: br i1 [[IFCOND]], label [[IFTRUE:%.*]], label [[NEXT]]
+; CHECK: iftrue:
+; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT: [[INC:%.*]] = add nsw i32 [[TMP1]], 1
+; CHECK-NEXT: store i32 [[INC]], ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT: br label [[NEXT]]
+; CHECK: next:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT:%.*]], label [[FOR_BODY]]
+; CHECK: for.exit:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %next ]
+ %arrayidx = getelementptr inbounds i32, ptr %indices, i64 %iv
+ %0 = load i32, ptr %arrayidx, align 4
+ %idxprom1 = zext i32 %0 to i64
+ %arrayidx2 = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+ %condidx = getelementptr inbounds i32, ptr %conds, i64 %iv
+ %conddata = load i32, ptr %condidx, align 4
+ %ifcond = icmp sgt i32 %conddata, 5100
+ br i1 %ifcond, label %iftrue, label %next
+
+iftrue:
+ %1 = load i32, ptr %arrayidx2, align 4
+ %inc = add nsw i32 %1, 1
+ store i32 %inc, ptr %arrayidx2, align 4
+ br label %next
+
+next:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, %N
+ br i1 %exitcond, label %for.exit, label %for.body
+
+for.exit:
+ ret void
+}
+
+;; Need to support legalization of smaller int types.
+define void @histogram_8bit(ptr noalias %buckets, ptr readonly %indices, i64 %N) #0 {
+; CHECK-LABEL: define void @histogram_8bit(
+; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br label [[FOR_BODY:%.*]]
+; CHECK: for.body:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV]]
+; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT: [[IDXPROM1:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i8, ptr [[BUCKETS]], i64 [[IDXPROM1]]
+; CHECK-NEXT: [[TMP1:%.*]] = load i8, ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT: [[INC:%.*]] = add nsw i8 [[TMP1]], 1
+; CHECK-NEXT: store i8 [[INC]], ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT:%.*]], label [[FOR_BODY]]
+; CHECK: for.exit:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds i32, ptr %indices, i64 %iv
+ %0 = load i32, ptr %arrayidx, align 4
+ %idxprom1 = zext i32 %0 to i64
+ %arrayidx2 = getelementptr inbounds i8, ptr %buckets, i64 %idxprom1
+ %1 = load i8, ptr %arrayidx2, align 4
+ %inc = add nsw i8 %1, 1
+ store i8 %inc, ptr %arrayidx2, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, %N
+ br i1 %exitcond, label %for.exit, label %for.body
+
+for.exit:
+ ret void
+}
+
+;; We don't currently support floating point histograms.
+define void @histogram_float(ptr noalias %buckets, ptr readonly %indices, i64 %N) #0 {
+; CHECK-LABEL: define void @histogram_float(
+; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br label [[FOR_BODY:%.*]]
+; CHECK: for.body:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV]]
+; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT: [[IDXPROM1:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[BUCKETS]], i64 [[IDXPROM1]]
+; CHECK-NEXT: [[TMP1:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT: [[INC:%.*]] = fadd fast float [[TMP1]], 1.000000e+00
+; CHECK-NEXT: store float [[INC]], ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT:%.*]], label [[FOR_BODY]]
+; CHECK: for.exit:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds i32, ptr %indices, i64 %iv
+ %0 = load i32, ptr %arrayidx, align 4
+ %idxprom1 = zext i32 %0 to i64
+ %arrayidx2 = getelementptr inbounds float, ptr %buckets, i64 %idxprom1
+ %1 = load float, ptr %arrayidx2, align 4
+ %inc = fadd fast float %1, 1.0
+ store float %inc, ptr %arrayidx2, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, %N
+ br i1 %exitcond, label %for.exit, label %for.body
+
+for.exit:
+ ret void
+}
+
+define void @histogram_varying_increment(ptr noalias %buckets, ptr readonly %indices, ptr readonly %incvals, i64 %N) #0 {
+; CHECK-LABEL: define void @histogram_varying_increment(
+; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], ptr readonly [[INCVALS:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br label [[FOR_BODY:%.*]]
+; CHECK: for.body:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV]]
+; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT: [[IDXPROM1:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[IDXPROM1]]
+; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT: [[INCIDX:%.*]] = getelementptr inbounds i32, ptr [[INCVALS]], i64 [[IV]]
+; CHECK-NEXT: [[INCVAL:%.*]] = load i32, ptr [[INCIDX]], align 4
+; CHECK-NEXT: [[INC:%.*]] = add nsw i32 [[TMP1]], [[INCVAL]]
+; CHECK-NEXT: store i32 [[INC]], ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT:%.*]], label [[FOR_BODY]]
+; CHECK: for.exit:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds i32, ptr %indices, i64 %iv
+ %0 = load i32, ptr %arrayidx, align 4
+ %idxprom1 = zext i32 %0 to i64
+ %arrayidx2 = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+ %1 = load i32, ptr %arrayidx2, align 4
+ %incidx = getelementptr inbounds i32, ptr %incvals, i64 %iv
+ %incval = load i32, ptr %incidx, align 4
+ %inc = add nsw i32 %1, %incval
+ store i32 %inc, ptr %arrayidx2, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, %N
+ br i1 %exitcond, label %for.exit, label %for.body
+
+for.exit:
+ ret void
+}
+
+attributes #0 = { "target-features"="+sve2" vscale_range(1,16) }
>From 87601153ca9c9e609acee4ecb5177f47758bf127 Mon Sep 17 00:00:00 2001
From: Graham Hunter <graham.hunter at arm.com>
Date: Thu, 27 Jun 2024 10:20:36 +0000
Subject: [PATCH 2/9] LV Changes
---
.../Vectorize/LoopVectorizationLegality.h | 38 +++
.../Vectorize/LoopVectorizationLegality.cpp | 110 ++++++-
.../Transforms/Vectorize/LoopVectorize.cpp | 87 +++++-
.../Transforms/Vectorize/VPRecipeBuilder.h | 8 +
llvm/lib/Transforms/Vectorize/VPlan.h | 35 +++
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 44 +++
llvm/lib/Transforms/Vectorize/VPlanValue.h | 1 +
.../LoopVectorize/AArch64/sve2-histcnt.ll | 285 ++++++++++++++++--
8 files changed, 583 insertions(+), 25 deletions(-)
diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
index 0f4d1355dd2bfe..341cbe15280609 100644
--- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
+++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
@@ -224,6 +224,18 @@ class LoopVectorizationRequirements {
Instruction *ExactFPMathInst = nullptr;
};
+/// This holds details about a histogram operation -- a load -> update -> store
+/// sequence where each lane in a vector might be updating the same element as
+/// another lane.
+struct HistogramInfo {
+ LoadInst *Load;
+ Instruction *Update;
+ StoreInst *Store;
+
+ HistogramInfo(LoadInst *Load, Instruction *Update, StoreInst *Store)
+ : Load(Load), Update(Update), Store(Store) {}
+};
+
/// LoopVectorizationLegality checks if it is legal to vectorize a loop, and
/// to what vectorization factor.
/// This class does not look at the profitability of vectorization, only the
@@ -390,6 +402,22 @@ class LoopVectorizationLegality {
unsigned getNumStores() const { return LAI->getNumStores(); }
unsigned getNumLoads() const { return LAI->getNumLoads(); }
+ /// Returns a HistogramInfo* for the given instruction if it was determined
+ /// to be part of a load -> update -> store sequence where multiple lanes
+ /// may be working on the same memory address.
+ std::optional<const HistogramInfo *> getHistogramInfo(Instruction *I) const {
+ for (const HistogramInfo &HGram : Histograms)
+ if (HGram.Load == I || HGram.Update == I || HGram.Store == I)
+ return &HGram;
+
+ return std::nullopt;
+ }
+
+ /// Returns a list of all known histogram operations in the loop.
+ const SmallVectorImpl<HistogramInfo> &getHistograms() const {
+ return Histograms;
+ }
+
PredicatedScalarEvolution *getPredicatedScalarEvolution() const {
return &PSE;
}
@@ -438,6 +466,11 @@ class LoopVectorizationLegality {
/// Returns true if the loop is vectorizable
bool canVectorizeMemory();
+ /// If LAA cannot determine whether all dependences are safe, we may be able
+ /// to further analyse some unknown dependences and if they match a certain
+ /// pattern (like a histogram) then we may still be able to vectorize.
+ bool canVectorizeUnknownDependences();
+
/// Return true if we can vectorize this loop using the IF-conversion
/// transformation.
bool canVectorizeWithIfConvert();
@@ -542,6 +575,11 @@ class LoopVectorizationLegality {
/// conditional assumes.
SmallPtrSet<const Instruction *, 8> MaskedOp;
+ /// Contains all identified histogram operations, which are sequences of
+ /// load -> update -> store instructions where multiple lanes in a vector
+ /// may work on the same memory location.
+ SmallVector<HistogramInfo, 1> Histograms;
+
/// BFI and PSI are used to check for profile guided size optimizations.
BlockFrequencyInfo *BFI;
ProfileSummaryInfo *PSI;
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 66a779da8c25bc..a94d0ddec06ab8 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -79,6 +79,10 @@ static cl::opt<LoopVectorizeHints::ScalableForceKind>
"Scalable vectorization is available and favored when the "
"cost is inconclusive.")));
+static cl::opt<bool> EnableHistogramVectorization(
+ "enable-histogram-loop-vectorization", cl::init(false), cl::Hidden,
+ cl::desc("Enables autovectorization of some loops containing histograms"));
+
/// Maximum vectorization interleave count.
static const unsigned MaxInterleaveFactor = 16;
@@ -1048,6 +1052,110 @@ bool LoopVectorizationLegality::canVectorizeInstrs() {
return true;
}
+/// Find Histogram counts that match high-level code in loops:
+/// \code
+/// buckets[indices[i]]+=step;
+/// \endcode
+///
+/// It matches a pattern starting from \p HSt, which Stores to the 'buckets'
+/// array the computed histogram. It uses a BinOp to sum all counts, storing
+/// them using a loop-variant index Load from the 'indices' input array.
+///
+/// On successful matches it updates the STATISTIC 'HistogramsDetected',
+/// regardless of hardware support. When there is support, it additionally
+/// stores the BinOp/Load pairs in \p HistogramCounts, as well the pointers
+/// used to update histogram in \p HistogramPtrs.
+
+static bool findHistograms(LoadInst *LI, StoreInst *HSt, Loop *TheLoop,
+ const PredicatedScalarEvolution &PSE,
+ SmallVectorImpl<HistogramInfo> &Histograms) {
+
+ // Store value must come from a Binary Operation.
+ Instruction *HPtrInstr = nullptr;
+ BinaryOperator *HBinOp = nullptr;
+ if (!match(HSt, m_Store(m_BinOp(HBinOp), m_Instruction(HPtrInstr))))
+ return false;
+
+ // BinOp must be an Add or a Sub modifying the bucket value by a
+ // loop invariant amount.
+ // FIXME: We assume the loop invariant term is on the RHS.
+ // Fine for an immediate/constant, but maybe not a generic value?
+ Value *HIncVal = nullptr;
+ if (!match(HBinOp, m_Add(m_Load(m_Specific(HPtrInstr)), m_Value(HIncVal))) &&
+ !match(HBinOp, m_Sub(m_Load(m_Specific(HPtrInstr)), m_Value(HIncVal))))
+ return false;
+
+ // Make sure the increment value is loop invariant.
+ if (!TheLoop->isLoopInvariant(HIncVal))
+ return false;
+
+ // The address to store is calculated through a GEP Instruction.
+ // FIXME: Support GEPs with more operands.
+ GetElementPtrInst *HPtr = dyn_cast<GetElementPtrInst>(HPtrInstr);
+ if (!HPtr || HPtr->getNumOperands() > 2)
+ return false;
+
+ // Check that the index is calculated by loading from another array. Ignore
+ // any extensions.
+ // FIXME: Support indices from other sources that a linear load from memory?
+ Value *HIdx = HPtr->getOperand(1);
+ Instruction *IdxInst = nullptr;
+ if (!match(HIdx, m_ZExtOrSExtOrSelf(m_Instruction(IdxInst))))
+ return false;
+
+ // Currently restricting this to linear addressing when loading indices.
+ LoadInst *VLoad = dyn_cast<LoadInst>(IdxInst);
+ Value *VPtrVal;
+ if (!VLoad || !match(VLoad, m_Load(m_Value(VPtrVal))))
+ return false;
+
+ if (!isa<SCEVAddRecExpr>(PSE.getSE()->getSCEV(VPtrVal)))
+ return false;
+
+ // Ensure we'll have the same mask by checking that all parts of the histogram
+ // (gather load, update, scatter store) are in the same block.
+ LoadInst *IndexedLoad = cast<LoadInst>(HBinOp->getOperand(0));
+ BasicBlock *LdBB = IndexedLoad->getParent();
+ if (LdBB != HBinOp->getParent() || LdBB != HSt->getParent())
+ return false;
+
+ LLVM_DEBUG(dbgs() << "LV: Found histogram for: " << *HSt << "\n");
+
+ // Store the operations that make up the histogram.
+ Histograms.emplace_back(IndexedLoad, HBinOp, HSt);
+ return true;
+}
+
+bool LoopVectorizationLegality::canVectorizeUnknownDependences() {
+ // For now, we only support an unknown dependency that calculates a histogram
+ if (!EnableHistogramVectorization)
+ return false;
+
+ // FIXME: Support more than one unknown dependence, and check to see if some
+ // are handled by runtime checks before looking for histograms.
+ LAI = &LAIs.getInfo(*TheLoop);
+ const MemoryDepChecker &DepChecker = LAI->getDepChecker();
+ const auto *Deps = DepChecker.getDependences();
+ if (!Deps || Deps->size() > 1)
+ return false;
+
+ const MemoryDepChecker::Dependence &Dep = (*Deps).front();
+
+ // We're only interested in unknown dependences.
+ if (Dep.Type != MemoryDepChecker::Dependence::Unknown)
+ return false;
+
+ // For now only normal loads and stores are supported.
+ LoadInst *LI = dyn_cast<LoadInst>(Dep.getSource(DepChecker));
+ StoreInst *SI = dyn_cast<StoreInst>(Dep.getDestination(DepChecker));
+
+ if (!LI || !SI)
+ return false;
+
+ LLVM_DEBUG(dbgs() << "LV: Checking for a histogram on: " << *SI << "\n");
+ return findHistograms(LI, SI, TheLoop, LAI->getPSE(), Histograms);
+}
+
bool LoopVectorizationLegality::canVectorizeMemory() {
LAI = &LAIs.getInfo(*TheLoop);
const OptimizationRemarkAnalysis *LAR = LAI->getReport();
@@ -1059,7 +1167,7 @@ bool LoopVectorizationLegality::canVectorizeMemory() {
}
if (!LAI->canVectorizeMemory())
- return false;
+ return canVectorizeUnknownDependences();
if (LAI->hasLoadStoreDependenceInvolvingLoopInvariantAddress()) {
reportVectorizationFailure("We don't allow storing to uniform addresses",
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 4cc75e2e754603..7f1fff2e1cd685 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -4652,6 +4652,10 @@ bool LoopVectorizationPlanner::isCandidateForEpilogueVectorization(
if (OrigLoop->getExitingBlock() != OrigLoop->getLoopLatch())
return false;
+ // Loops containing histograms are not currently supported.
+ if (!Legal->getHistograms().empty())
+ return false;
+
return true;
}
@@ -6501,8 +6505,33 @@ LoopVectorizationCostModel::getInstructionCost(Instruction *I,
// We've proven all lanes safe to speculate, fall through.
[[fallthrough]];
case Instruction::Add:
+ case Instruction::Sub: {
+ auto Info = Legal->getHistogramInfo(I);
+ if (Info && VF.isVector()) {
+ const HistogramInfo *HGram = Info.value();
+ // Assume that a non-constant update value (or a constant != 1) requires
+ // a multiply, and add that into the cost.
+ InstructionCost MulCost = TTI::TCC_Free;
+ ConstantInt *RHS = dyn_cast<ConstantInt>(I->getOperand(1));
+ if (!RHS || RHS->getZExtValue() != 1)
+ MulCost = TTI.getArithmeticInstrCost(Instruction::Mul, VectorTy);
+
+ // Find the cost of the histogram operation itself.
+ Type *PtrTy = VectorType::get(HGram->Load->getPointerOperandType(), VF);
+ Type *ScalarTy = I->getType();
+ Type *MaskTy = VectorType::get(Type::getInt1Ty(I->getContext()), VF);
+ IntrinsicCostAttributes ICA(Intrinsic::experimental_vector_histogram_add,
+ Type::getVoidTy(I->getContext()),
+ {PtrTy, ScalarTy, MaskTy});
+
+ // Add the costs together with the add/sub operation.
+ return TTI.getIntrinsicInstrCost(
+ ICA, TargetTransformInfo::TCK_RecipThroughput) +
+ MulCost + TTI.getArithmeticInstrCost(I->getOpcode(), VectorTy);
+ }
+ [[fallthrough]];
+ }
case Instruction::FAdd:
- case Instruction::Sub:
case Instruction::FSub:
case Instruction::Mul:
case Instruction::FMul:
@@ -8393,6 +8422,36 @@ VPWidenRecipe *VPRecipeBuilder::tryToWiden(Instruction *I,
};
}
+VPHistogramRecipe *
+VPRecipeBuilder::tryToWidenHistogram(const HistogramInfo *HI,
+ ArrayRef<VPValue *> Operands) {
+ // FIXME: Support other operations.
+ unsigned Opcode = HI->Update->getOpcode();
+ assert((Opcode == Instruction::Add || Opcode == Instruction::Sub) &&
+ "Histogram update operation must be an Add or Sub");
+
+ SmallVector<VPValue *, 3> HGramOps;
+ // Bucket address.
+ HGramOps.push_back(Operands[1]);
+ // Increment value.
+ HGramOps.push_back(getVPValueOrAddLiveIn(HI->Update->getOperand(1), Plan));
+
+ // In case of predicated execution (due to tail-folding, or conditional
+ // execution, or both), pass the relevant mask. When there is no such mask,
+ // generate an all-true mask.
+ VPValue *Mask = nullptr;
+ if (Legal->isMaskRequired(HI->Store))
+ Mask = getBlockInMask(HI->Store->getParent());
+ else
+ Mask = Plan.getOrAddLiveIn(
+ ConstantInt::getTrue(IntegerType::getInt1Ty(HI->Load->getContext())));
+ HGramOps.push_back(Mask);
+
+ return new VPHistogramRecipe(HI, Opcode,
+ make_range(HGramOps.begin(), HGramOps.end()),
+ HI->Store->getDebugLoc());
+}
+
void VPRecipeBuilder::fixHeaderPhis() {
BasicBlock *OrigLatch = OrigLoop->getLoopLatch();
for (VPHeaderPHIRecipe *R : PhisToFix) {
@@ -8516,6 +8575,10 @@ VPRecipeBuilder::tryToCreateWidenRecipe(Instruction *Instr,
if (auto *CI = dyn_cast<CallInst>(Instr))
return tryToWidenCall(CI, Operands, Range);
+ if (StoreInst *SI = dyn_cast<StoreInst>(Instr))
+ if (auto HistInfo = Legal->getHistogramInfo(SI))
+ return tryToWidenHistogram(*HistInfo, Operands);
+
if (isa<LoadInst>(Instr) || isa<StoreInst>(Instr))
return tryToWidenMemory(Instr, Operands, Range);
@@ -8937,6 +9000,15 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
Operands = {OpRange.begin(), OpRange.end()};
}
+ // If this is a load instruction or a binop associated with a histogram,
+ // leave it until the store instruction to emit a combined intrinsic.
+ // Note that if the initial VF is scalar, we need to generate the normal
+ // clone recipe for these instructions. A histogram recipe will only be
+ // generated when minVF > 1.
+ if (Legal->getHistogramInfo(Instr) && !isa<StoreInst>(Instr) &&
+ !Range.Start.isScalar())
+ continue;
+
// Invariant stores inside loop will be deleted and a single store
// with the final reduction value will be added to the exit block
StoreInst *SI;
@@ -9999,6 +10071,19 @@ bool LoopVectorizePass::processLoop(Loop *L) {
InterleaveLoop = false;
}
+ // If there is a histogram in the loop, do not just interleave without
+ // vectorizing. The order of operations will be incorrect without the
+ // histogram intrinsics, which are only used for recipes with VF > 1.
+ if (!VectorizeLoop && InterleaveLoop && !LVL.getHistograms().empty()) {
+ LLVM_DEBUG(dbgs() << "LV: Not interleaving without vectorization due "
+ << "to histogram operations.\n");
+ IntDiagMsg = std::make_pair(
+ "HistogramPreventsScalarInterleaving",
+ "Unable to interleave without vectorization due to constraints on "
+ "the order of histogram operations");
+ InterleaveLoop = false;
+ }
+
// Override IC if user provided an interleave count.
IC = UserIC > 0 ? UserIC : IC;
diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
index 0fbdeeddaf4a17..af7b3925f10f02 100644
--- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
+++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
@@ -21,6 +21,7 @@ namespace llvm {
class LoopVectorizationLegality;
class LoopVectorizationCostModel;
class TargetLibraryInfo;
+struct HistogramInfo;
/// Helper class to create VPRecipies from IR instructions.
class VPRecipeBuilder {
@@ -103,6 +104,13 @@ class VPRecipeBuilder {
VPWidenRecipe *tryToWiden(Instruction *I, ArrayRef<VPValue *> Operands,
VPBasicBlock *VPBB);
+ /// Makes Histogram count operations safe for vectorization, by emitting a
+ /// llvm.experimental.vector.histogram.add intrinsic in place of the
+ /// Load + Add|Sub + Store operations that perform the histogram in the
+ /// original scalar loop.
+ VPHistogramRecipe *tryToWidenHistogram(const HistogramInfo *HI,
+ ArrayRef<VPValue *> Operands);
+
public:
VPRecipeBuilder(VPlan &Plan, Loop *OrigLoop, const TargetLibraryInfo *TLI,
LoopVectorizationLegality *Legal,
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index e7ea5cb23b90d3..e0007048d5e41a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -69,6 +69,7 @@ class LoopVectorizationCostModel;
class LoopVersioning;
struct VPCostContext;
+struct HistogramInfo;
namespace Intrinsic {
typedef unsigned ID;
@@ -941,6 +942,7 @@ class VPSingleDefRecipe : public VPRecipeBase, public VPValue {
case VPRecipeBase::VPWidenLoadSC:
case VPRecipeBase::VPWidenStoreEVLSC:
case VPRecipeBase::VPWidenStoreSC:
+ case VPRecipeBase::VPHistogramSC:
// TODO: Widened stores don't define a value, but widened loads do. Split
// the recipes to be able to make widened loads VPSingleDefRecipes.
return false;
@@ -1583,6 +1585,39 @@ class VPWidenCallRecipe : public VPSingleDefRecipe {
#endif
};
+class VPHistogramRecipe : public VPRecipeBase {
+ const HistogramInfo *Info;
+ unsigned Opcode;
+
+public:
+ template <typename IterT>
+ VPHistogramRecipe(const HistogramInfo *HI, unsigned Opcode,
+ iterator_range<IterT> Operands, DebugLoc DL = {})
+ : VPRecipeBase(VPDef::VPHistogramSC, Operands, DL), Info(HI),
+ Opcode(Opcode) {}
+
+ ~VPHistogramRecipe() override = default;
+
+ VPHistogramRecipe *clone() override {
+ llvm_unreachable("cloning not supported");
+ }
+
+ VP_CLASSOF_IMPL(VPDef::VPHistogramSC);
+
+ // Produce a histogram operation with widened ingredients
+ void execute(VPTransformState &State) override;
+
+ unsigned getOpcode() const { return Opcode; }
+
+ const HistogramInfo *getHistogramInfo() const { return Info; }
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+ /// Print the recipe
+ void print(raw_ostream &O, const Twine &Indent,
+ VPSlotTracker &SlotTracker) const override;
+#endif
+};
+
/// A recipe for widening select instructions.
struct VPWidenSelectRecipe : public VPSingleDefRecipe {
template <typename IterT>
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index f84317ba51257a..5b292a8858f75e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -22,6 +22,7 @@
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
+#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Value.h"
#include "llvm/Support/Casting.h"
@@ -31,6 +32,7 @@
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/Transforms/Utils/LoopUtils.h"
#include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"
+#include "llvm/Transforms/Vectorize/LoopVectorizationLegality.h"
#include <cassert>
using namespace llvm;
@@ -274,6 +276,10 @@ static Instruction *getInstructionForCost(const VPRecipeBase *R) {
return IG->getInsertPos();
if (auto *WidenMem = dyn_cast<VPWidenMemoryRecipe>(R))
return &WidenMem->getIngredient();
+ // FIXME: Override the cost method properly to take gather/scatter cost
+ // into account, instead of just the intrinsic via the legacy model.
+ if (auto *HG = dyn_cast<VPHistogramRecipe>(R))
+ return HG->getHistogramInfo()->Update;
return nullptr;
}
@@ -953,6 +959,44 @@ void VPWidenCallRecipe::print(raw_ostream &O, const Twine &Indent,
O << ")";
}
}
+#endif
+
+void VPHistogramRecipe::execute(VPTransformState &State) {
+ State.setDebugLocFrom(getDebugLoc());
+ IRBuilderBase &Builder = State.Builder;
+
+ for (unsigned Part = 0; Part < State.UF; ++Part) {
+ Value *Address = State.get(getOperand(0), Part);
+ Value *IncVec = State.get(getOperand(1), Part);
+ Value *Mask = State.get(getOperand(2), Part);
+
+ // Not sure how to make IncAmt stay scalar yet. For now just extract the
+ // first element and tidy up later.
+ // FIXME: Do we actually want this to be scalar? We just splat it in the
+ // backend anyway...
+ Value *IncAmt = Builder.CreateExtractElement(IncVec, Builder.getInt64(0));
+
+ // If this is a subtract, we want to invert the increment amount. We may
+ // add a separate intrinsic in future, but for now we'll try this.
+ if (Opcode == Instruction::Sub)
+ IncAmt = Builder.CreateNeg(IncAmt);
+
+ State.Builder.CreateIntrinsic(Intrinsic::experimental_vector_histogram_add,
+ {Address->getType(), IncAmt->getType()},
+ {Address, IncAmt, Mask});
+ }
+}
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+void VPHistogramRecipe::print(raw_ostream &O, const Twine &Indent,
+ VPSlotTracker &SlotTracker) const {
+ O << Indent << "WIDEN-HISTOGRAM buckets: ";
+ getOperand(0)->printAsOperand(O, SlotTracker);
+ O << ", inc: ";
+ getOperand(1)->printAsOperand(O, SlotTracker);
+ O << ", mask: ";
+ getOperand(2)->printAsOperand(O, SlotTracker);
+}
void VPWidenSelectRecipe::print(raw_ostream &O, const Twine &Indent,
VPSlotTracker &SlotTracker) const {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h
index 452c977106a773..5358712a1d41a0 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanValue.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h
@@ -358,6 +358,7 @@ class VPDef {
VPWidenSC,
VPWidenSelectSC,
VPBlendSC,
+ VPHistogramSC,
// START: Phi-like recipes. Need to be kept together.
VPWidenPHISC,
VPPredInstPHISC,
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
index 283baf61b73938..0e077ccefad77c 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3
-; RUN: opt < %s -passes=loop-vectorize -force-vector-interleave=1 -sve-gather-overhead=2 -sve-scatter-overhead=2 -S | FileCheck %s
+; RUN: opt < %s -passes=loop-vectorize,instcombine -enable-histogram-loop-vectorization -sve-gather-overhead=2 -sve-scatter-overhead=2 -debug-only=loop-vectorize -S 2>&1 | FileCheck %s
target triple = "aarch64-unknown-linux-gnu"
@@ -10,23 +10,129 @@ target triple = "aarch64-unknown-linux-gnu"
;; buckets[indices[i]]++;
;; }
+; CHECK-LABEL: Checking a loop in 'simple_histogram'
+
+; CHECK: LV: Checking for a histogram on: store i32 %inc, ptr %arrayidx2, align 4
+; CHECK: LV: Found histogram for: store i32 %inc, ptr %arrayidx2, align 4
+
+;; Check that the scalar plan contains the original instructions.
+; CHECK: VPlan 'Initial VPlan for VF={1},UF>=1' {
+; CHECK-NEXT: Live-in vp<%0> = VF * UF
+; CHECK-NEXT: Live-in vp<%1> = vector-trip-count
+; CHECK-NEXT: Live-in ir<%N> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: EMIT vp<%2> = CANONICAL-INDUCTION ir<0>, vp<%4>
+; CHECK-NEXT: vp<%3> = SCALAR-STEPS vp<%2>, ir<1>
+; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%indices>, vp<%3>
+; CHECK-NEXT: CLONE ir<%0> = load ir<%arrayidx>
+; CHECK-NEXT: CLONE ir<%idxprom1> = zext ir<%0>
+; CHECK-NEXT: CLONE ir<%arrayidx2> = getelementptr inbounds ir<%buckets>, ir<%idxprom1>
+; CHECK-NEXT: CLONE ir<%1> = load ir<%arrayidx2>
+; CHECK-NEXT: CLONE ir<%inc> = add nsw ir<%1>, ir<1>
+; CHECK-NEXT: CLONE store ir<%inc>, ir<%arrayidx2>
+; CHECK-NEXT: EMIT vp<%4> = add nuw vp<%2>, vp<%0>
+; CHECK-NEXT: EMIT branch-on-count vp<%4>, vp<%1>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<%6> = icmp eq ir<%N>, vp<%1>
+; CHECK-NEXT: EMIT branch-on-cond vp<%6>
+; CHECK-NEXT: Successor(s): ir-bb<for.exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<for.exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+
+;; Check that the vectorized plan contains a histogram recipe instead.
+; CHECK: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF>=1' {
+; CHECK-NEXT: Live-in vp<%0> = VF * UF
+; CHECK-NEXT: Live-in vp<%1> = vector-trip-count
+; CHECK-NEXT: Live-in ir<%N> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: EMIT vp<%2> = CANONICAL-INDUCTION ir<0>, vp<%5>
+; CHECK-NEXT: vp<%3> = SCALAR-STEPS vp<%2>, ir<1>
+; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%indices>, vp<%3>
+; CHECK-NEXT: vp<%4> = vector-pointer ir<%arrayidx>
+; CHECK-NEXT: WIDEN ir<%0> = load vp<%4>
+; CHECK-NEXT: WIDEN-CAST ir<%idxprom1> = zext ir<%0> to i64
+; CHECK-NEXT: WIDEN-GEP Inv[Var] ir<%arrayidx2> = getelementptr inbounds ir<%buckets>, ir<%idxprom1>
+; CHECK-NEXT: WIDEN-HISTOGRAM buckets: ir<%arrayidx2>, inc: ir<1>, mask: ir<true>
+; CHECK-NEXT: EMIT vp<%5> = add nuw vp<%2>, vp<%0>
+; CHECK-NEXT: EMIT branch-on-count vp<%5>, vp<%1>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<%7> = icmp eq ir<%N>, vp<%1>
+; CHECK-NEXT: EMIT branch-on-cond vp<%7>
+; CHECK-NEXT: Successor(s): ir-bb<for.exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<for.exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+
define void @simple_histogram(ptr noalias %buckets, ptr readonly %indices, i64 %N) #0 {
; CHECK-LABEL: define void @simple_histogram(
; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 2
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ugt i64 [[TMP1]], [[N]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[DOTNEG:%.*]] = mul nsw i64 [[TMP2]], -4
+; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[DOTNEG]], [[N]]
+; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP5:%.*]] = shl nuw nsw i64 [[TMP4]], 2
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP8]], align 4
+; CHECK-NEXT: [[TMP9:%.*]] = zext <vscale x 4 x i32> [[WIDE_LOAD]] to <vscale x 4 x i64>
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], <vscale x 4 x i64> [[TMP9]]
+; CHECK-NEXT: call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr> [[TMP10]], i32 1, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[N]]
+; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK: scalar.ph:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
; CHECK: for.body:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV]]
-; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
-; CHECK-NEXT: [[IDXPROM1:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-NEXT: [[TMP12:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT: [[IDXPROM1:%.*]] = zext i32 [[TMP12]] to i64
; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[IDXPROM1]]
-; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
-; CHECK-NEXT: [[INC:%.*]] = add nsw i32 [[TMP1]], 1
+; CHECK-NEXT: [[TMP13:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT: [[INC:%.*]] = add nsw i32 [[TMP13]], 1
; CHECK-NEXT: store i32 [[INC]], ptr [[ARRAYIDX2]], align 4
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT:%.*]], label [[FOR_BODY]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: for.exit:
; CHECK-NEXT: ret void
;
@@ -54,19 +160,45 @@ define void @simple_histogram_sub(ptr noalias %buckets, ptr readonly %indices, i
; CHECK-LABEL: define void @simple_histogram_sub(
; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 2
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ugt i64 [[TMP1]], [[N]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[DOTNEG:%.*]] = mul nsw i64 [[TMP2]], -4
+; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[DOTNEG]], [[N]]
+; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP5:%.*]] = shl nuw nsw i64 [[TMP4]], 2
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP8]], align 4
+; CHECK-NEXT: [[TMP9:%.*]] = zext <vscale x 4 x i32> [[WIDE_LOAD]] to <vscale x 4 x i64>
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], <vscale x 4 x i64> [[TMP9]]
+; CHECK-NEXT: call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr> [[TMP10]], i32 -1, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[N]]
+; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK: scalar.ph:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
; CHECK: for.body:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV]]
-; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
-; CHECK-NEXT: [[IDXPROM1:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-NEXT: [[TMP12:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT: [[IDXPROM1:%.*]] = zext i32 [[TMP12]] to i64
; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[IDXPROM1]]
-; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
-; CHECK-NEXT: [[INC:%.*]] = sub nsw i32 [[TMP1]], 1
+; CHECK-NEXT: [[TMP13:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT: [[INC:%.*]] = add nsw i32 [[TMP13]], -1
; CHECK-NEXT: store i32 [[INC]], ptr [[ARRAYIDX2]], align 4
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT:%.*]], label [[FOR_BODY]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: for.exit:
; CHECK-NEXT: ret void
;
@@ -94,26 +226,55 @@ define void @conditional_histogram(ptr noalias %buckets, ptr readonly %indices,
; CHECK-LABEL: define void @conditional_histogram(
; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], ptr readonly [[CONDS:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP3:%.*]] = shl nuw nsw i64 [[TMP6]], 2
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ugt i64 [[TMP3]], [[N]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[DOTNEG:%.*]] = mul nsw i64 [[TMP2]], -4
+; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[DOTNEG]], [[N]]
+; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP5:%.*]] = shl nuw nsw i64 [[TMP4]], 2
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP8]], align 4
+; CHECK-NEXT: [[TMP9:%.*]] = zext <vscale x 4 x i32> [[WIDE_LOAD]] to <vscale x 4 x i64>
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], <vscale x 4 x i64> [[TMP9]]
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[CONDS]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 4 x i32>, ptr [[TMP12]], align 4
+; CHECK-NEXT: [[TMP13:%.*]] = icmp sgt <vscale x 4 x i32> [[WIDE_LOAD1]], shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 5100, i64 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
+; CHECK-NEXT: call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr> [[TMP10]], i32 1, <vscale x 4 x i1> [[TMP13]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[N]]
+; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK: scalar.ph:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT: br label [[FOR_BODY1:%.*]]
; CHECK: for.body:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[NEXT:%.*]] ]
-; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV]]
-; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
-; CHECK-NEXT: [[IDXPROM1:%.*]] = zext i32 [[TMP0]] to i64
-; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[IDXPROM1]]
-; CHECK-NEXT: [[CONDIDX:%.*]] = getelementptr inbounds i32, ptr [[CONDS]], i64 [[IV]]
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[NEXT:%.*]] ]
+; CHECK-NEXT: [[CONDIDX:%.*]] = getelementptr inbounds i32, ptr [[CONDS]], i64 [[IV1]]
; CHECK-NEXT: [[CONDDATA:%.*]] = load i32, ptr [[CONDIDX]], align 4
; CHECK-NEXT: [[IFCOND:%.*]] = icmp sgt i32 [[CONDDATA]], 5100
; CHECK-NEXT: br i1 [[IFCOND]], label [[IFTRUE:%.*]], label [[NEXT]]
; CHECK: iftrue:
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV1]]
; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
-; CHECK-NEXT: [[INC:%.*]] = add nsw i32 [[TMP1]], 1
-; CHECK-NEXT: store i32 [[INC]], ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT: [[IDXPROM1:%.*]] = zext i32 [[TMP1]] to i64
+; CHECK-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[IDXPROM1]]
+; CHECK-NEXT: [[TMP15:%.*]] = load i32, ptr [[ARRAYIDX3]], align 4
+; CHECK-NEXT: [[INC:%.*]] = add nsw i32 [[TMP15]], 1
+; CHECK-NEXT: store i32 [[INC]], ptr [[ARRAYIDX3]], align 4
; CHECK-NEXT: br label [[NEXT]]
; CHECK: next:
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT:%.*]], label [[FOR_BODY]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT]], label [[FOR_BODY1]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK: for.exit:
; CHECK-NEXT: ret void
;
@@ -228,6 +389,7 @@ for.exit:
ret void
}
+;; We don't support histograms with a update value that's not loop-invariant.
define void @histogram_varying_increment(ptr noalias %buckets, ptr readonly %indices, ptr readonly %incvals, i64 %N) #0 {
; CHECK-LABEL: define void @histogram_varying_increment(
; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], ptr readonly [[INCVALS:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
@@ -272,4 +434,81 @@ for.exit:
ret void
}
+;; Test that interleaving works when vectorizing.
+define void @simple_histogram_user_interleave(ptr noalias %buckets, ptr readonly %indices, i64 %N) #0 {
+; CHECK-LABEL: define void @simple_histogram_user_interleave(
+; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 3
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ugt i64 [[TMP1]], [[N]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[DOTNEG:%.*]] = mul nsw i64 [[TMP2]], -8
+; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[DOTNEG]], [[N]]
+; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP5:%.*]] = shl nuw nsw i64 [[TMP4]], 3
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[DOTIDX:%.*]] = shl nuw nsw i64 [[TMP15]], 4
+; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[TMP8]], i64 [[DOTIDX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP8]], align 4
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 4 x i32>, ptr [[TMP17]], align 4
+; CHECK-NEXT: [[TMP9:%.*]] = zext <vscale x 4 x i32> [[WIDE_LOAD]] to <vscale x 4 x i64>
+; CHECK-NEXT: [[TMP19:%.*]] = zext <vscale x 4 x i32> [[WIDE_LOAD1]] to <vscale x 4 x i64>
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], <vscale x 4 x i64> [[TMP9]]
+; CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], <vscale x 4 x i64> [[TMP19]]
+; CHECK-NEXT: call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr> [[TMP10]], i32 1, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; CHECK-NEXT: call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr> [[TMP21]], i32 1, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[N]]
+; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK: scalar.ph:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT: br label [[FOR_BODY:%.*]]
+; CHECK: for.body:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV]]
+; CHECK-NEXT: [[TMP12:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT: [[IDXPROM1:%.*]] = zext i32 [[TMP12]] to i64
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[IDXPROM1]]
+; CHECK-NEXT: [[TMP13:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT: [[INC:%.*]] = add nsw i32 [[TMP13]], 1
+; CHECK-NEXT: store i32 [[INC]], ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK: for.exit:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds i32, ptr %indices, i64 %iv
+ %0 = load i32, ptr %arrayidx, align 4
+ %idxprom1 = zext i32 %0 to i64
+ %arrayidx2 = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+ %1 = load i32, ptr %arrayidx2, align 4
+ %inc = add nsw i32 %1, 1
+ store i32 %inc, ptr %arrayidx2, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, %N
+ br i1 %exitcond, label %for.exit, label %for.body, !llvm.loop !0
+
+for.exit:
+ ret void
+}
+
attributes #0 = { "target-features"="+sve2" vscale_range(1,16) }
+
+!0 = distinct !{!0, !1}
+!1 = !{!"llvm.loop.interleave.count", i32 2}
>From 1e13e5c6402e973153bb8c16ddff77ea99a27040 Mon Sep 17 00:00:00 2001
From: Graham Hunter <graham.hunter at arm.com>
Date: Fri, 19 Jul 2024 13:39:46 +0000
Subject: [PATCH 3/9] Extra tests
---
.../AArch64/sve2-histcnt-epilogue.ll | 76 +++++++++++++++++++
.../sve2-histcnt-no-scalar-interleave.ll | 53 +++++++++++++
.../Transforms/LoopVectorize/histograms.ll | 44 +++++++++++
3 files changed, 173 insertions(+)
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-epilogue.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-no-scalar-interleave.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/histograms.ll
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-epilogue.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-epilogue.ll
new file mode 100644
index 00000000000000..2c03b69c6534cf
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-epilogue.ll
@@ -0,0 +1,76 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3
+; RUN: opt < %s -passes=loop-vectorize,instcombine -enable-histogram-loop-vectorization -sve-gather-overhead=2 -sve-scatter-overhead=2 -epilogue-vectorization-minimum-VF=4 -debug-only=loop-vectorize -S 2>&1 | FileCheck %s
+
+target triple = "aarch64-unknown-linux-gnu"
+
+;; We don't currently support histograms in epilogue vectorization, so check for the debug message.
+; CHECK-LABEL: LV: Checking a loop in 'simple_histogram'
+; CHECK: LEV: Unable to vectorize epilogue because the loop is not a supported candidate.
+
+define void @simple_histogram(ptr noalias %buckets, ptr readonly %indices, i64 %N) #0 {
+; CHECK-LABEL: define void @simple_histogram(
+; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 2
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ugt i64 [[TMP1]], [[N]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[DOTNEG:%.*]] = mul nsw i64 [[TMP2]], -4
+; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[DOTNEG]], [[N]]
+; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP5:%.*]] = shl nuw nsw i64 [[TMP4]], 2
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP8]], align 4
+; CHECK-NEXT: [[TMP9:%.*]] = zext <vscale x 4 x i32> [[WIDE_LOAD]] to <vscale x 4 x i64>
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], <vscale x 4 x i64> [[TMP9]]
+; CHECK-NEXT: call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr> [[TMP10]], i32 1, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[N]]
+; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK: scalar.ph:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT: br label [[FOR_BODY:%.*]]
+; CHECK: for.body:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV]]
+; CHECK-NEXT: [[TMP12:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT: [[IDXPROM1:%.*]] = zext i32 [[TMP12]] to i64
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[IDXPROM1]]
+; CHECK-NEXT: [[TMP13:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT: [[INC:%.*]] = add nsw i32 [[TMP13]], 1
+; CHECK-NEXT: store i32 [[INC]], ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: for.exit:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds i32, ptr %indices, i64 %iv
+ %0 = load i32, ptr %arrayidx, align 4
+ %idxprom1 = zext i32 %0 to i64
+ %arrayidx2 = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+ %1 = load i32, ptr %arrayidx2, align 4
+ %inc = add nsw i32 %1, 1
+ store i32 %inc, ptr %arrayidx2, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, %N
+ br i1 %exitcond, label %for.exit, label %for.body
+
+for.exit:
+ ret void
+}
+
+attributes #0 = { "target-features"="+sve2" vscale_range(1,16) }
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-no-scalar-interleave.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-no-scalar-interleave.ll
new file mode 100644
index 00000000000000..606e0352dad464
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-no-scalar-interleave.ll
@@ -0,0 +1,53 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3
+; RUN: opt < %s -passes=loop-vectorize,instcombine -enable-histogram-loop-vectorization -sve-gather-overhead=2 -sve-scatter-overhead=2 -debug-only=loop-vectorize -S 2>&1 | FileCheck %s
+
+;; Make sure we don't interleave a histogram when vectorization is disabled.
+
+; CHECK-LABEL: LV: Checking a loop in 'simple_histogram_forced_scalar_interleave'
+; CHECK: LV: Not interleaving without vectorization due to histogram operations.
+
+define void @simple_histogram_forced_scalar_interleave(ptr noalias %buckets, ptr readonly %indices, i64 %N) #0 {
+; CHECK-LABEL: define void @simple_histogram_forced_scalar_interleave(
+; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br label [[FOR_BODY:%.*]]
+; CHECK: for.body:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV]]
+; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT: [[IDXPROM1:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[IDXPROM1]]
+; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT: [[INC:%.*]] = add nsw i32 [[TMP1]], 1
+; CHECK-NEXT: store i32 [[INC]], ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: for.exit:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds i32, ptr %indices, i64 %iv
+ %0 = load i32, ptr %arrayidx, align 4
+ %idxprom1 = zext i32 %0 to i64
+ %arrayidx2 = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+ %1 = load i32, ptr %arrayidx2, align 4
+ %inc = add nsw i32 %1, 1
+ store i32 %inc, ptr %arrayidx2, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, %N
+ br i1 %exitcond, label %for.exit, label %for.body, !llvm.loop !0
+
+for.exit:
+ ret void
+}
+
+attributes #0 = { "target-features"="+sve2" vscale_range(1,16) }
+
+!0 = distinct !{!0, !1, !2}
+!1 = !{!"llvm.loop.interleave.count", i32 2}
+!2 = !{!"llvm.loop.vectorize.width", i32 1}
diff --git a/llvm/test/Transforms/LoopVectorize/histograms.ll b/llvm/test/Transforms/LoopVectorize/histograms.ll
new file mode 100644
index 00000000000000..7d308308ebede0
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/histograms.ll
@@ -0,0 +1,44 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3
+; RUN: opt < %s -passes=loop-vectorize,instcombine -enable-histogram-loop-vectorization -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+define void @simple_histogram(ptr noalias %buckets, ptr readonly %indices, i64 %N) #0 {
+; CHECK-LABEL: define void @simple_histogram(
+; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br label [[FOR_BODY:%.*]]
+; CHECK: for.body:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV]]
+; CHECK-NEXT: [[TMP12:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT: [[IDXPROM1:%.*]] = zext i32 [[TMP12]] to i64
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[IDXPROM1]]
+; CHECK-NEXT: [[TMP13:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT: [[INC:%.*]] = add nsw i32 [[TMP13]], 1
+; CHECK-NEXT: store i32 [[INC]], ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT:%.*]], label [[FOR_BODY]]
+; CHECK: for.exit:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds i32, ptr %indices, i64 %iv
+ %0 = load i32, ptr %arrayidx, align 4
+ %idxprom1 = zext i32 %0 to i64
+ %arrayidx2 = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+ %1 = load i32, ptr %arrayidx2, align 4
+ %inc = add nsw i32 %1, 1
+ store i32 %inc, ptr %arrayidx2, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, %N
+ br i1 %exitcond, label %for.exit, label %for.body
+
+for.exit:
+ ret void
+}
>From 8585e4a911164eb24be0746b8ed101e030ec8da5 Mon Sep 17 00:00:00 2001
From: Graham Hunter <graham.hunter at arm.com>
Date: Fri, 26 Jul 2024 14:27:36 +0000
Subject: [PATCH 4/9] Addressing comments * Mask is optional for vplan recipe *
Left load+update to DCE * Documentation for new recipe * Stored histogram
info by reference * Allow GEPs with more than one index value, provided that
all but the last are constant. * Cleanup to remove some unnecessary local
variables. * Minor nits addressed.
---
.../Vectorize/LoopVectorizationLegality.cpp | 39 +--
.../Transforms/Vectorize/LoopVectorize.cpp | 26 +-
llvm/lib/Transforms/Vectorize/VPlan.h | 21 +-
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 33 ++-
.../LoopVectorize/AArch64/sve2-histcnt.ll | 270 +++++++++++++++++-
.../Transforms/LoopVectorize/histograms.ll | 2 +-
6 files changed, 332 insertions(+), 59 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index a94d0ddec06ab8..280031878e13a7 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -1052,7 +1052,7 @@ bool LoopVectorizationLegality::canVectorizeInstrs() {
return true;
}
-/// Find Histogram counts that match high-level code in loops:
+/// Find histogram operations that match high-level code in loops:
/// \code
/// buckets[indices[i]]+=step;
/// \endcode
@@ -1065,10 +1065,9 @@ bool LoopVectorizationLegality::canVectorizeInstrs() {
/// regardless of hardware support. When there is support, it additionally
/// stores the BinOp/Load pairs in \p HistogramCounts, as well the pointers
/// used to update histogram in \p HistogramPtrs.
-
-static bool findHistograms(LoadInst *LI, StoreInst *HSt, Loop *TheLoop,
- const PredicatedScalarEvolution &PSE,
- SmallVectorImpl<HistogramInfo> &Histograms) {
+static bool findHistogram(LoadInst *LI, StoreInst *HSt, Loop *TheLoop,
+ const PredicatedScalarEvolution &PSE,
+ SmallVectorImpl<HistogramInfo> &Histograms) {
// Store value must come from a Binary Operation.
Instruction *HPtrInstr = nullptr;
@@ -1090,23 +1089,27 @@ static bool findHistograms(LoadInst *LI, StoreInst *HSt, Loop *TheLoop,
return false;
// The address to store is calculated through a GEP Instruction.
- // FIXME: Support GEPs with more operands.
- GetElementPtrInst *HPtr = dyn_cast<GetElementPtrInst>(HPtrInstr);
- if (!HPtr || HPtr->getNumOperands() > 2)
+ GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(HPtrInstr);
+ if (!GEP)
+ return false;
+
+ // Restrict address calculation to constant indices except for the last term.
+ Value *HIdx = nullptr;
+ for (Value *Index : GEP->indices()) {
+ if (HIdx)
+ return false;
+ if (!isa<ConstantInt>(Index))
+ HIdx = Index;
+ }
+
+ if (!HIdx)
return false;
// Check that the index is calculated by loading from another array. Ignore
// any extensions.
// FIXME: Support indices from other sources that a linear load from memory?
- Value *HIdx = HPtr->getOperand(1);
- Instruction *IdxInst = nullptr;
- if (!match(HIdx, m_ZExtOrSExtOrSelf(m_Instruction(IdxInst))))
- return false;
-
- // Currently restricting this to linear addressing when loading indices.
- LoadInst *VLoad = dyn_cast<LoadInst>(IdxInst);
Value *VPtrVal;
- if (!VLoad || !match(VLoad, m_Load(m_Value(VPtrVal))))
+ if (!match(HIdx, m_ZExtOrSExtOrSelf(m_Load(m_Value(VPtrVal)))))
return false;
if (!isa<SCEVAddRecExpr>(PSE.getSE()->getSCEV(VPtrVal)))
@@ -1136,7 +1139,7 @@ bool LoopVectorizationLegality::canVectorizeUnknownDependences() {
LAI = &LAIs.getInfo(*TheLoop);
const MemoryDepChecker &DepChecker = LAI->getDepChecker();
const auto *Deps = DepChecker.getDependences();
- if (!Deps || Deps->size() > 1)
+ if (!Deps || Deps->size() != 1)
return false;
const MemoryDepChecker::Dependence &Dep = (*Deps).front();
@@ -1153,7 +1156,7 @@ bool LoopVectorizationLegality::canVectorizeUnknownDependences() {
return false;
LLVM_DEBUG(dbgs() << "LV: Checking for a histogram on: " << *SI << "\n");
- return findHistograms(LI, SI, TheLoop, LAI->getPSE(), Histograms);
+ return findHistogram(LI, SI, TheLoop, LAI->getPSE(), Histograms);
}
bool LoopVectorizationLegality::canVectorizeMemory() {
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 7f1fff2e1cd685..e2840003206cf8 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -4653,10 +4653,7 @@ bool LoopVectorizationPlanner::isCandidateForEpilogueVectorization(
return false;
// Loops containing histograms are not currently supported.
- if (!Legal->getHistograms().empty())
- return false;
-
- return true;
+ return Legal->getHistograms().empty();
}
bool LoopVectorizationCostModel::isEpilogueVectorizationProfitable(
@@ -8437,17 +8434,11 @@ VPRecipeBuilder::tryToWidenHistogram(const HistogramInfo *HI,
HGramOps.push_back(getVPValueOrAddLiveIn(HI->Update->getOperand(1), Plan));
// In case of predicated execution (due to tail-folding, or conditional
- // execution, or both), pass the relevant mask. When there is no such mask,
- // generate an all-true mask.
- VPValue *Mask = nullptr;
+ // execution, or both), pass the relevant mask.
if (Legal->isMaskRequired(HI->Store))
- Mask = getBlockInMask(HI->Store->getParent());
- else
- Mask = Plan.getOrAddLiveIn(
- ConstantInt::getTrue(IntegerType::getInt1Ty(HI->Load->getContext())));
- HGramOps.push_back(Mask);
+ HGramOps.push_back(getBlockInMask(HI->Store->getParent()));
- return new VPHistogramRecipe(HI, Opcode,
+ return new VPHistogramRecipe(*HI, Opcode,
make_range(HGramOps.begin(), HGramOps.end()),
HI->Store->getDebugLoc());
}
@@ -9000,15 +8991,6 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
Operands = {OpRange.begin(), OpRange.end()};
}
- // If this is a load instruction or a binop associated with a histogram,
- // leave it until the store instruction to emit a combined intrinsic.
- // Note that if the initial VF is scalar, we need to generate the normal
- // clone recipe for these instructions. A histogram recipe will only be
- // generated when minVF > 1.
- if (Legal->getHistogramInfo(Instr) && !isa<StoreInst>(Instr) &&
- !Range.Start.isScalar())
- continue;
-
// Invariant stores inside loop will be deleted and a single store
// with the final reduction value will be added to the exit block
StoreInst *SI;
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index e0007048d5e41a..a6e489d1b6e1e4 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1585,13 +1585,18 @@ class VPWidenCallRecipe : public VPSingleDefRecipe {
#endif
};
+/// A recipe representing a sequence of load -> update -> store as part of
+/// a histogram operation. This means there may be aliasing between vector
+/// lanes, which is handled by the llvm.experimental.vector.histogram family
+/// of intrinsics. The only update operations currently supported are
+/// 'add' and 'sub' where the other term is loop-invariant.
class VPHistogramRecipe : public VPRecipeBase {
- const HistogramInfo *Info;
+ const HistogramInfo &Info;
unsigned Opcode;
public:
template <typename IterT>
- VPHistogramRecipe(const HistogramInfo *HI, unsigned Opcode,
+ VPHistogramRecipe(const HistogramInfo &HI, unsigned Opcode,
iterator_range<IterT> Operands, DebugLoc DL = {})
: VPRecipeBase(VPDef::VPHistogramSC, Operands, DL), Info(HI),
Opcode(Opcode) {}
@@ -1604,12 +1609,20 @@ class VPHistogramRecipe : public VPRecipeBase {
VP_CLASSOF_IMPL(VPDef::VPHistogramSC);
- // Produce a histogram operation with widened ingredients
+ /// Produce a vectorized histogram operation.
void execute(VPTransformState &State) override;
unsigned getOpcode() const { return Opcode; }
- const HistogramInfo *getHistogramInfo() const { return Info; }
+ const HistogramInfo &getHistogramInfo() const { return Info; }
+
+ /// Return the mask operand if one was provided, or a null pointer if all
+ /// lanes should be executed unconditionally.
+ VPValue *getMask() const {
+ if (getNumOperands() == 3)
+ return getOperand(2);
+ return nullptr;
+ }
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
/// Print the recipe
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 5b292a8858f75e..d75dc8cded413c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -279,7 +279,7 @@ static Instruction *getInstructionForCost(const VPRecipeBase *R) {
// FIXME: Override the cost method properly to take gather/scatter cost
// into account, instead of just the intrinsic via the legacy model.
if (auto *HG = dyn_cast<VPHistogramRecipe>(R))
- return HG->getHistogramInfo()->Update;
+ return HG->getHistogramInfo().Update;
return nullptr;
}
@@ -968,7 +968,17 @@ void VPHistogramRecipe::execute(VPTransformState &State) {
for (unsigned Part = 0; Part < State.UF; ++Part) {
Value *Address = State.get(getOperand(0), Part);
Value *IncVec = State.get(getOperand(1), Part);
- Value *Mask = State.get(getOperand(2), Part);
+ VectorType *VTy = cast<VectorType>(Address->getType());
+
+ // The histogram intrinsic requires a mask even if the recipe doesn't;
+ // if the mask operand was omitted then all lanes should be executed and
+ // we just need to synthesize an all-true mask.
+ Value *Mask = nullptr;
+ if (VPValue *VPMask = getMask())
+ Mask = State.get(VPMask, Part);
+ else
+ Mask = Builder.CreateVectorSplat(
+ VTy->getElementCount(), ConstantInt::getTrue(Builder.getInt1Ty()));
// Not sure how to make IncAmt stay scalar yet. For now just extract the
// first element and tidy up later.
@@ -980,9 +990,11 @@ void VPHistogramRecipe::execute(VPTransformState &State) {
// add a separate intrinsic in future, but for now we'll try this.
if (Opcode == Instruction::Sub)
IncAmt = Builder.CreateNeg(IncAmt);
+ else
+ assert(Opcode == Instruction::Add);
State.Builder.CreateIntrinsic(Intrinsic::experimental_vector_histogram_add,
- {Address->getType(), IncAmt->getType()},
+ {VTy, IncAmt->getType()},
{Address, IncAmt, Mask});
}
}
@@ -992,10 +1004,19 @@ void VPHistogramRecipe::print(raw_ostream &O, const Twine &Indent,
VPSlotTracker &SlotTracker) const {
O << Indent << "WIDEN-HISTOGRAM buckets: ";
getOperand(0)->printAsOperand(O, SlotTracker);
- O << ", inc: ";
+
+ if (Opcode == Instruction::Sub)
+ O << ", dec: ";
+ else {
+ assert(Opcode == Instruction::Add);
+ O << ", inc: ";
+ }
getOperand(1)->printAsOperand(O, SlotTracker);
- O << ", mask: ";
- getOperand(2)->printAsOperand(O, SlotTracker);
+
+ if (VPValue *Mask = getMask()) {
+ O << ", mask: ";
+ Mask->printAsOperand(O, SlotTracker);
+ }
}
void VPWidenSelectRecipe::print(raw_ostream &O, const Twine &Indent,
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
index 0e077ccefad77c..cac5e055d496ae 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
@@ -54,7 +54,7 @@ target triple = "aarch64-unknown-linux-gnu"
; CHECK-NEXT: }
;; Check that the vectorized plan contains a histogram recipe instead.
-; CHECK: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF>=1' {
+; CHECK: VPlan 'Initial VPlan for VF={vscale x 2,vscale x 4},UF>=1' {
; CHECK-NEXT: Live-in vp<%0> = VF * UF
; CHECK-NEXT: Live-in vp<%1> = vector-trip-count
; CHECK-NEXT: Live-in ir<%N> = original trip-count
@@ -71,7 +71,7 @@ target triple = "aarch64-unknown-linux-gnu"
; CHECK-NEXT: WIDEN ir<%0> = load vp<%4>
; CHECK-NEXT: WIDEN-CAST ir<%idxprom1> = zext ir<%0> to i64
; CHECK-NEXT: WIDEN-GEP Inv[Var] ir<%arrayidx2> = getelementptr inbounds ir<%buckets>, ir<%idxprom1>
-; CHECK-NEXT: WIDEN-HISTOGRAM buckets: ir<%arrayidx2>, inc: ir<1>, mask: ir<true>
+; CHECK-NEXT: WIDEN-HISTOGRAM buckets: ir<%arrayidx2>, inc: ir<1>
; CHECK-NEXT: EMIT vp<%5> = add nuw vp<%2>, vp<%0>
; CHECK-NEXT: EMIT branch-on-count vp<%5>, vp<%1>
; CHECK-NEXT: No successors
@@ -90,6 +90,124 @@ target triple = "aarch64-unknown-linux-gnu"
; CHECK-NEXT: No successors
; CHECK-NEXT: }
+;; Check that a non-constant but loop invariant inc value is correctly represented
+; CHECK-LABEL: LV: Checking a loop in 'simple_histogram_inc_param'
+; CHECK: VPlan 'Initial VPlan for VF={vscale x 2,vscale x 4},UF>=1' {
+; CHECK-NEXT: Live-in vp<%0> = VF * UF
+; CHECK-NEXT: Live-in vp<%1> = vector-trip-count
+; CHECK-NEXT: Live-in ir<%N> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: EMIT vp<%2> = CANONICAL-INDUCTION ir<0>, vp<%5>
+; CHECK-NEXT: vp<%3> = SCALAR-STEPS vp<%2>, ir<1>
+; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%indices>, vp<%3>
+; CHECK-NEXT: vp<%4> = vector-pointer ir<%arrayidx>
+; CHECK-NEXT: WIDEN ir<%0> = load vp<%4>
+; CHECK-NEXT: WIDEN-CAST ir<%idxprom1> = zext ir<%0> to i64
+; CHECK-NEXT: WIDEN-GEP Inv[Var] ir<%arrayidx2> = getelementptr inbounds ir<%buckets>, ir<%idxprom1>
+; CHECK-NEXT: WIDEN-HISTOGRAM buckets: ir<%arrayidx2>, inc: ir<%incval>
+; CHECK-NEXT: EMIT vp<%5> = add nuw vp<%2>, vp<%0>
+; CHECK-NEXT: EMIT branch-on-count vp<%5>, vp<%1>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<%7> = icmp eq ir<%N>, vp<%1>
+; CHECK-NEXT: EMIT branch-on-cond vp<%7>
+; CHECK-NEXT: Successor(s): ir-bb<for.exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<for.exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+
+;; Check that a sub operation results in a dec parameter in the recipe
+; CHECK-LABEL: LV: Checking a loop in 'simple_histogram_sub'
+; CHECK: VPlan 'Initial VPlan for VF={vscale x 2,vscale x 4},UF>=1' {
+; CHECK-NEXT: Live-in vp<%0> = VF * UF
+; CHECK-NEXT: Live-in vp<%1> = vector-trip-count
+; CHECK-NEXT: Live-in ir<%N> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: EMIT vp<%2> = CANONICAL-INDUCTION ir<0>, vp<%5>
+; CHECK-NEXT: vp<%3> = SCALAR-STEPS vp<%2>, ir<1>
+; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%indices>, vp<%3>
+; CHECK-NEXT: vp<%4> = vector-pointer ir<%arrayidx>
+; CHECK-NEXT: WIDEN ir<%0> = load vp<%4>
+; CHECK-NEXT: WIDEN-CAST ir<%idxprom1> = zext ir<%0> to i64
+; CHECK-NEXT: WIDEN-GEP Inv[Var] ir<%arrayidx2> = getelementptr inbounds ir<%buckets>, ir<%idxprom1>
+; CHECK-NEXT: WIDEN-HISTOGRAM buckets: ir<%arrayidx2>, dec: ir<1>
+; CHECK-NEXT: EMIT vp<%5> = add nuw vp<%2>, vp<%0>
+; CHECK-NEXT: EMIT branch-on-count vp<%5>, vp<%1>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<%7> = icmp eq ir<%N>, vp<%1>
+; CHECK-NEXT: EMIT branch-on-cond vp<%7>
+; CHECK-NEXT: Successor(s): ir-bb<for.exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<for.exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+
+;; Check that masks are represented
+; CHECK-LABEL: LV: Checking a loop in 'conditional_histogram'
+; CHECK: VPlan 'Initial VPlan for VF={vscale x 2,vscale x 4},UF>=1' {
+; CHECK-NEXT: Live-in vp<%0> = VF * UF
+; CHECK-NEXT: Live-in vp<%1> = vector-trip-count
+; CHECK-NEXT: Live-in ir<%N> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: EMIT vp<%2> = CANONICAL-INDUCTION ir<0>, vp<%6>
+; CHECK-NEXT: vp<%3> = SCALAR-STEPS vp<%2>, ir<1>
+; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%indices>, vp<%3>
+; CHECK-NEXT: vp<%4> = vector-pointer ir<%arrayidx>
+; CHECK-NEXT: WIDEN ir<%0> = load vp<%4>
+; CHECK-NEXT: WIDEN-CAST ir<%idxprom1> = zext ir<%0> to i64
+; CHECK-NEXT: WIDEN-GEP Inv[Var] ir<%arrayidx2> = getelementptr inbounds ir<%buckets>, ir<%idxprom1>
+; CHECK-NEXT: CLONE ir<%condidx> = getelementptr inbounds ir<%conds>, vp<%3>
+; CHECK-NEXT: vp<%5> = vector-pointer ir<%condidx>
+; CHECK-NEXT: WIDEN ir<%conddata> = load vp<%5>
+; CHECK-NEXT: WIDEN ir<%ifcond> = icmp sgt ir<%conddata>, ir<5100>
+; CHECK-NEXT: WIDEN-HISTOGRAM buckets: ir<%arrayidx2>, inc: ir<1>, mask: ir<%ifcond>
+; CHECK-NEXT: EMIT vp<%6> = add nuw vp<%2>, vp<%0>
+; CHECK-NEXT: EMIT branch-on-count vp<%6>, vp<%1>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<%8> = icmp eq ir<%N>, vp<%1>
+; CHECK-NEXT: EMIT branch-on-cond vp<%8>
+; CHECK-NEXT: Successor(s): ir-bb<for.exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<for.exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+
define void @simple_histogram(ptr noalias %buckets, ptr readonly %indices, i64 %N) #0 {
; CHECK-LABEL: define void @simple_histogram(
; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
@@ -156,6 +274,72 @@ for.exit:
ret void
}
+define void @simple_histogram_inc_param(ptr noalias %buckets, ptr readonly %indices, i64 %N, i32 %incval) #0 {
+; CHECK-LABEL: define void @simple_histogram_inc_param(
+; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], i64 [[N:%.*]], i32 [[INCVAL:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 2
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ugt i64 [[TMP1]], [[N]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[DOTNEG:%.*]] = mul nsw i64 [[TMP2]], -4
+; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[DOTNEG]], [[N]]
+; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP5:%.*]] = shl nuw nsw i64 [[TMP4]], 2
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP8]], align 4
+; CHECK-NEXT: [[TMP9:%.*]] = zext <vscale x 4 x i32> [[WIDE_LOAD]] to <vscale x 4 x i64>
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], <vscale x 4 x i64> [[TMP9]]
+; CHECK-NEXT: call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr> [[TMP10]], i32 [[INCVAL]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[N]]
+; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK: scalar.ph:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT: br label [[FOR_BODY:%.*]]
+; CHECK: for.body:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV]]
+; CHECK-NEXT: [[TMP12:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT: [[IDXPROM1:%.*]] = zext i32 [[TMP12]] to i64
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[IDXPROM1]]
+; CHECK-NEXT: [[TMP13:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT: [[INC:%.*]] = add nsw i32 [[TMP13]], [[INCVAL]]
+; CHECK-NEXT: store i32 [[INC]], ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK: for.exit:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds i32, ptr %indices, i64 %iv
+ %0 = load i32, ptr %arrayidx, align 4
+ %idxprom1 = zext i32 %0 to i64
+ %arrayidx2 = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+ %1 = load i32, ptr %arrayidx2, align 4
+ %inc = add nsw i32 %1, %incval
+ store i32 %inc, ptr %arrayidx2, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, %N
+ br i1 %exitcond, label %for.exit, label %for.body
+
+for.exit:
+ ret void
+}
+
define void @simple_histogram_sub(ptr noalias %buckets, ptr readonly %indices, i64 %N) #0 {
; CHECK-LABEL: define void @simple_histogram_sub(
; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
@@ -180,7 +364,7 @@ define void @simple_histogram_sub(ptr noalias %buckets, ptr readonly %indices, i
; CHECK-NEXT: call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr> [[TMP10]], i32 -1, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: middle.block:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[N]]
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
@@ -198,7 +382,7 @@ define void @simple_histogram_sub(ptr noalias %buckets, ptr readonly %indices, i
; CHECK-NEXT: store i32 [[INC]], ptr [[ARRAYIDX2]], align 4
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK: for.exit:
; CHECK-NEXT: ret void
;
@@ -249,7 +433,7 @@ define void @conditional_histogram(ptr noalias %buckets, ptr readonly %indices,
; CHECK-NEXT: call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr> [[TMP10]], i32 1, <vscale x 4 x i1> [[TMP13]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK: middle.block:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[N]]
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
@@ -274,7 +458,7 @@ define void @conditional_histogram(ptr noalias %buckets, ptr readonly %indices,
; CHECK: next:
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT]], label [[FOR_BODY1]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT]], label [[FOR_BODY1]], !llvm.loop [[LOOP9:![0-9]+]]
; CHECK: for.exit:
; CHECK-NEXT: ret void
;
@@ -466,7 +650,7 @@ define void @simple_histogram_user_interleave(ptr noalias %buckets, ptr readonly
; CHECK-NEXT: call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr> [[TMP21]], i32 1, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK: middle.block:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[N]]
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
@@ -484,7 +668,7 @@ define void @simple_histogram_user_interleave(ptr noalias %buckets, ptr readonly
; CHECK-NEXT: store i32 [[INC]], ptr [[ARRAYIDX2]], align 4
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
; CHECK: for.exit:
; CHECK-NEXT: ret void
;
@@ -508,6 +692,76 @@ for.exit:
ret void
}
+;; Test that we can handle more than one GEP index.
+ at idx_array = dso_local local_unnamed_addr global [1048576 x i32] zeroinitializer, align 4
+ at data_array = dso_local local_unnamed_addr global [1048576 x i32] zeroinitializer, align 4
+
+define void @histogram_array_3op_gep(i64 noundef %N) #0 {
+; CHECK-LABEL: define void @histogram_array_3op_gep(
+; CHECK-SAME: i64 noundef [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 2
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ugt i64 [[TMP1]], [[N]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[DOTNEG:%.*]] = mul nsw i64 [[TMP2]], -4
+; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[DOTNEG]], [[N]]
+; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 2
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds [1048576 x i32], ptr @idx_array, i64 0, i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP5]], align 4
+; CHECK-NEXT: [[TMP6:%.*]] = sext <vscale x 4 x i32> [[WIDE_LOAD]] to <vscale x 4 x i64>
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds [1048576 x i32], ptr @data_array, i64 0, <vscale x 4 x i64> [[TMP6]]
+; CHECK-NEXT: call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr> [[TMP7]], i32 1, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
+; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[N]]
+; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK: scalar.ph:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT: br label [[FOR_BODY:%.*]]
+; CHECK: for.body:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [1048576 x i32], ptr @idx_array, i64 0, i64 [[IV]]
+; CHECK-NEXT: [[TMP9:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT: [[IDXPROM5:%.*]] = sext i32 [[TMP9]] to i64
+; CHECK-NEXT: [[ARRAYIDX6:%.*]] = getelementptr inbounds [1048576 x i32], ptr @data_array, i64 0, i64 [[IDXPROM5]]
+; CHECK-NEXT: [[TMP10:%.*]] = load i32, ptr [[ARRAYIDX6]], align 4
+; CHECK-NEXT: [[INC:%.*]] = add nsw i32 [[TMP10]], 1
+; CHECK-NEXT: store i32 [[INC]], ptr [[ARRAYIDX6]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK: for.exit:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds [1048576 x i32], ptr @idx_array, i64 0, i64 %iv
+ %2 = load i32, ptr %arrayidx, align 4
+ %idxprom5 = sext i32 %2 to i64
+ %arrayidx6 = getelementptr inbounds [1048576 x i32], ptr @data_array, i64 0, i64 %idxprom5
+ %3 = load i32, ptr %arrayidx6, align 4
+ %inc = add nsw i32 %3, 1
+ store i32 %inc, ptr %arrayidx6, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, %N
+ br i1 %exitcond, label %for.exit, label %for.body
+
+for.exit:
+ ret void
+}
+
attributes #0 = { "target-features"="+sve2" vscale_range(1,16) }
!0 = distinct !{!0, !1}
diff --git a/llvm/test/Transforms/LoopVectorize/histograms.ll b/llvm/test/Transforms/LoopVectorize/histograms.ll
index 7d308308ebede0..5a439c4b485331 100644
--- a/llvm/test/Transforms/LoopVectorize/histograms.ll
+++ b/llvm/test/Transforms/LoopVectorize/histograms.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3
-; RUN: opt < %s -passes=loop-vectorize,instcombine -enable-histogram-loop-vectorization -S | FileCheck %s
+; RUN: opt < %s -passes=loop-vectorize,instcombine -enable-histogram-loop-vectorization -force-vector-width=2 -S | FileCheck %s
target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
>From 7e7be735ad0916f2d1c5cc981bc8cb422074dc5e Mon Sep 17 00:00:00 2001
From: Graham Hunter <graham.hunter at arm.com>
Date: Fri, 26 Jul 2024 15:29:18 +0000
Subject: [PATCH 5/9] Deal with change in dep type to IndirectUnsafe
---
.../Transforms/Vectorize/LoopVectorizationLegality.h | 2 +-
.../Vectorize/LoopVectorizationLegality.cpp | 11 ++++++-----
2 files changed, 7 insertions(+), 6 deletions(-)
diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
index 341cbe15280609..18e8e87d93ed35 100644
--- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
+++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
@@ -469,7 +469,7 @@ class LoopVectorizationLegality {
/// If LAA cannot determine whether all dependences are safe, we may be able
/// to further analyse some unknown dependences and if they match a certain
/// pattern (like a histogram) then we may still be able to vectorize.
- bool canVectorizeUnknownDependences();
+ bool canVectorizeUncheckedDependences();
/// Return true if we can vectorize this loop using the IF-conversion
/// transformation.
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 280031878e13a7..c30b198b19dc80 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -1129,12 +1129,12 @@ static bool findHistogram(LoadInst *LI, StoreInst *HSt, Loop *TheLoop,
return true;
}
-bool LoopVectorizationLegality::canVectorizeUnknownDependences() {
+bool LoopVectorizationLegality::canVectorizeUncheckedDependences() {
// For now, we only support an unknown dependency that calculates a histogram
if (!EnableHistogramVectorization)
return false;
- // FIXME: Support more than one unknown dependence, and check to see if some
+ // FIXME: Support more than one unchecked dependence, and check to see if some
// are handled by runtime checks before looking for histograms.
LAI = &LAIs.getInfo(*TheLoop);
const MemoryDepChecker &DepChecker = LAI->getDepChecker();
@@ -1144,8 +1144,9 @@ bool LoopVectorizationLegality::canVectorizeUnknownDependences() {
const MemoryDepChecker::Dependence &Dep = (*Deps).front();
- // We're only interested in unknown dependences.
- if (Dep.Type != MemoryDepChecker::Dependence::Unknown)
+ // We're only interested in Unknown or IndirectUnsafe dependences.
+ if (Dep.Type != MemoryDepChecker::Dependence::Unknown &&
+ Dep.Type != MemoryDepChecker::Dependence::IndirectUnsafe)
return false;
// For now only normal loads and stores are supported.
@@ -1170,7 +1171,7 @@ bool LoopVectorizationLegality::canVectorizeMemory() {
}
if (!LAI->canVectorizeMemory())
- return canVectorizeUnknownDependences();
+ return canVectorizeUncheckedDependences();
if (LAI->hasLoadStoreDependenceInvolvingLoopInvariantAddress()) {
reportVectorizationFailure("We don't allow storing to uniform addresses",
>From f3232c6d979fc00e9b27d6af39e4a2c2ac0eb9ee Mon Sep 17 00:00:00 2001
From: Graham Hunter <graham.hunter at arm.com>
Date: Mon, 29 Jul 2024 16:02:16 +0000
Subject: [PATCH 6/9] Add tailfolding histogram test case
---
.../LoopVectorize/AArch64/sve2-histcnt.ll | 57 +++++++++++++++++++
1 file changed, 57 insertions(+)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
index cac5e055d496ae..5aa07195a487f3 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
@@ -762,7 +762,64 @@ for.exit:
ret void
}
+;; Make sure the histogram intrinsic uses the active lane mask when tail folding.
+define void @simple_histogram_tailfold(ptr noalias %buckets, ptr readonly %indices, i64 %N) #0 {
+; CHECK-LABEL: define void @simple_histogram_tailfold(
+; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP2]], 2
+; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP5:%.*]] = shl nuw nsw i64 [[TMP4]], 2
+; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[N]], i64 [[TMP5]])
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr [[TMP8]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison)
+; CHECK-NEXT: [[TMP9:%.*]] = zext <vscale x 4 x i32> [[WIDE_LOAD]] to <vscale x 4 x i64>
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], <vscale x 4 x i64> [[TMP9]]
+; CHECK-NEXT: call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr> [[TMP10]], i32 1, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX]], i64 [[TMP6]])
+; CHECK-NEXT: [[TMP11:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-NEXT: br i1 [[TMP11]], label [[VECTOR_BODY]], label [[MIDDLE_BLOCK:%.*]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: br i1 true, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK: scalar.ph:
+; CHECK-NEXT: br label [[FOR_BODY:%.*]]
+; CHECK: for.body:
+; CHECK-NEXT: br i1 poison, label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK: for.exit:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds i32, ptr %indices, i64 %iv
+ %0 = load i32, ptr %arrayidx, align 4
+ %idxprom1 = zext i32 %0 to i64
+ %arrayidx2 = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+ %1 = load i32, ptr %arrayidx2, align 4
+ %inc = add nsw i32 %1, 1
+ store i32 %inc, ptr %arrayidx2, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, %N
+ br i1 %exitcond, label %for.exit, label %for.body, !llvm.loop !2
+
+for.exit:
+ ret void
+}
+
attributes #0 = { "target-features"="+sve2" vscale_range(1,16) }
!0 = distinct !{!0, !1}
!1 = !{!"llvm.loop.interleave.count", i32 2}
+!2 = distinct !{!2, !3}
+!3 = !{!"llvm.loop.vectorize.predicate.enable", i1 true}
>From 6d7bb66f56e2f2597fca96b05ec061c89c49dabc Mon Sep 17 00:00:00 2001
From: Graham Hunter <graham.hunter at arm.com>
Date: Wed, 14 Aug 2024 12:38:40 +0000
Subject: [PATCH 7/9] Address comments, add tests for additional dependences.
---
.../Vectorize/LoopVectorizationLegality.h | 10 +-
.../Vectorize/LoopVectorizationLegality.cpp | 35 +++--
.../Transforms/Vectorize/LoopVectorize.cpp | 4 +-
llvm/lib/Transforms/Vectorize/VPlan.h | 4 +-
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 10 +-
.../LoopVectorize/AArch64/sve2-histcnt.ll | 145 ++++++++++++++++++
6 files changed, 176 insertions(+), 32 deletions(-)
diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
index 18e8e87d93ed35..3a4151b454d609 100644
--- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
+++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
@@ -414,9 +414,7 @@ class LoopVectorizationLegality {
}
/// Returns a list of all known histogram operations in the loop.
- const SmallVectorImpl<HistogramInfo> &getHistograms() const {
- return Histograms;
- }
+ bool hasHistograms() const { return !Histograms.empty(); }
PredicatedScalarEvolution *getPredicatedScalarEvolution() const {
return &PSE;
@@ -467,9 +465,9 @@ class LoopVectorizationLegality {
bool canVectorizeMemory();
/// If LAA cannot determine whether all dependences are safe, we may be able
- /// to further analyse some unknown dependences and if they match a certain
- /// pattern (like a histogram) then we may still be able to vectorize.
- bool canVectorizeUncheckedDependences();
+ /// to further analyse some IndirectUnsafe dependences and if they match a
+ /// certain pattern (like a histogram) then we may still be able to vectorize.
+ bool canVectorizeIndirectUnsafeDependences();
/// Return true if we can vectorize this loop using the IF-conversion
/// transformation.
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index c30b198b19dc80..e682db530b3fe0 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -1129,29 +1129,38 @@ static bool findHistogram(LoadInst *LI, StoreInst *HSt, Loop *TheLoop,
return true;
}
-bool LoopVectorizationLegality::canVectorizeUncheckedDependences() {
- // For now, we only support an unknown dependency that calculates a histogram
+bool LoopVectorizationLegality::canVectorizeIndirectUnsafeDependences() {
+ // For now, we only support an IndirectUnsafe dependency that calculates
+ // a histogram
if (!EnableHistogramVectorization)
return false;
- // FIXME: Support more than one unchecked dependence, and check to see if some
- // are handled by runtime checks before looking for histograms.
+ // Find a single IndirectUnsafe dependency.
LAI = &LAIs.getInfo(*TheLoop);
+ const MemoryDepChecker::Dependence *IUDep = nullptr;
const MemoryDepChecker &DepChecker = LAI->getDepChecker();
const auto *Deps = DepChecker.getDependences();
- if (!Deps || Deps->size() != 1)
- return false;
+ for (const MemoryDepChecker::Dependence &Dep : *Deps) {
+ // Ignore dependencies that are either known to be safe or can be
+ // checked at runtime.
+ if (MemoryDepChecker::Dependence::isSafeForVectorization(Dep.Type) !=
+ MemoryDepChecker::VectorizationSafetyStatus::Unsafe)
+ continue;
- const MemoryDepChecker::Dependence &Dep = (*Deps).front();
+ // We're only interested in IndirectUnsafe dependencies here, where the
+ // address might come from a load from memory. We also only want to handle
+ // one such dependency, at least for now.
+ if (Dep.Type != MemoryDepChecker::Dependence::IndirectUnsafe || IUDep)
+ return false;
- // We're only interested in Unknown or IndirectUnsafe dependences.
- if (Dep.Type != MemoryDepChecker::Dependence::Unknown &&
- Dep.Type != MemoryDepChecker::Dependence::IndirectUnsafe)
+ IUDep = &Dep;
+ }
+ if (!IUDep)
return false;
// For now only normal loads and stores are supported.
- LoadInst *LI = dyn_cast<LoadInst>(Dep.getSource(DepChecker));
- StoreInst *SI = dyn_cast<StoreInst>(Dep.getDestination(DepChecker));
+ LoadInst *LI = dyn_cast<LoadInst>(IUDep->getSource(DepChecker));
+ StoreInst *SI = dyn_cast<StoreInst>(IUDep->getDestination(DepChecker));
if (!LI || !SI)
return false;
@@ -1171,7 +1180,7 @@ bool LoopVectorizationLegality::canVectorizeMemory() {
}
if (!LAI->canVectorizeMemory())
- return canVectorizeUncheckedDependences();
+ return canVectorizeIndirectUnsafeDependences();
if (LAI->hasLoadStoreDependenceInvolvingLoopInvariantAddress()) {
reportVectorizationFailure("We don't allow storing to uniform addresses",
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index e2840003206cf8..96b92b42a6138f 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -4653,7 +4653,7 @@ bool LoopVectorizationPlanner::isCandidateForEpilogueVectorization(
return false;
// Loops containing histograms are not currently supported.
- return Legal->getHistograms().empty();
+ return !Legal->hasHistograms();
}
bool LoopVectorizationCostModel::isEpilogueVectorizationProfitable(
@@ -10056,7 +10056,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
// If there is a histogram in the loop, do not just interleave without
// vectorizing. The order of operations will be incorrect without the
// histogram intrinsics, which are only used for recipes with VF > 1.
- if (!VectorizeLoop && InterleaveLoop && !LVL.getHistograms().empty()) {
+ if (!VectorizeLoop && InterleaveLoop && LVL.hasHistograms()) {
LLVM_DEBUG(dbgs() << "LV: Not interleaving without vectorization due "
<< "to histogram operations.\n");
IntDiagMsg = std::make_pair(
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index a6e489d1b6e1e4..6d62986bce1479 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1619,9 +1619,7 @@ class VPHistogramRecipe : public VPRecipeBase {
/// Return the mask operand if one was provided, or a null pointer if all
/// lanes should be executed unconditionally.
VPValue *getMask() const {
- if (getNumOperands() == 3)
- return getOperand(2);
- return nullptr;
+ return getNumOperands() == 3 ? getOperand(2) : nullptr;
}
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index d75dc8cded413c..9bda04ee9e7244 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -967,7 +967,7 @@ void VPHistogramRecipe::execute(VPTransformState &State) {
for (unsigned Part = 0; Part < State.UF; ++Part) {
Value *Address = State.get(getOperand(0), Part);
- Value *IncVec = State.get(getOperand(1), Part);
+ Value *IncAmt = State.get(getOperand(1), Part, /*IsScalar=*/true);
VectorType *VTy = cast<VectorType>(Address->getType());
// The histogram intrinsic requires a mask even if the recipe doesn't;
@@ -980,18 +980,12 @@ void VPHistogramRecipe::execute(VPTransformState &State) {
Mask = Builder.CreateVectorSplat(
VTy->getElementCount(), ConstantInt::getTrue(Builder.getInt1Ty()));
- // Not sure how to make IncAmt stay scalar yet. For now just extract the
- // first element and tidy up later.
- // FIXME: Do we actually want this to be scalar? We just splat it in the
- // backend anyway...
- Value *IncAmt = Builder.CreateExtractElement(IncVec, Builder.getInt64(0));
-
// If this is a subtract, we want to invert the increment amount. We may
// add a separate intrinsic in future, but for now we'll try this.
if (Opcode == Instruction::Sub)
IncAmt = Builder.CreateNeg(IncAmt);
else
- assert(Opcode == Instruction::Add);
+ assert(Opcode == Instruction::Add && "only add or sub supported for now");
State.Builder.CreateIntrinsic(Intrinsic::experimental_vector_histogram_add,
{VTy, IncAmt->getType()},
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
index 5aa07195a487f3..df6839c3ba435b 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
@@ -208,6 +208,17 @@ target triple = "aarch64-unknown-linux-gnu"
; CHECK-NEXT: No successors
; CHECK-NEXT: }
+;; Confirm cost calculation for runtime checks
+; CHECK-LABEL: LV: Checking a loop in 'simple_histogram_rtdepcheck'
+; CHECK: Calculating cost of runtime checks:
+; CHECK: Total cost of runtime checks:
+; CHECK: LV: Minimum required TC for runtime checks to be profitable:
+
+;; Confirm inability to vectorize with potential alias to buckets
+; CHECK-LABEL: LV: Checking a loop in 'simple_histogram_unsafe_alias'
+; CHECK: LV: Can't vectorize due to memory conflicts
+; CHECK-NEXT: LV: Not vectorizing: Cannot prove legality.
+
define void @simple_histogram(ptr noalias %buckets, ptr readonly %indices, i64 %N) #0 {
; CHECK-LABEL: define void @simple_histogram(
; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
@@ -817,6 +828,140 @@ for.exit:
ret void
}
+;; Check that we can still vectorize a histogram when LAA found another dependency
+;; that doesn't conflict with the buckets.
+define void @simple_histogram_rtdepcheck(ptr noalias %buckets, ptr %array, ptr %indices, i64 %N) #0 {
+; CHECK-LABEL: define void @simple_histogram_rtdepcheck(
+; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr [[ARRAY:%.*]], ptr [[INDICES:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 2
+; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 8)
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ugt i64 [[TMP2]], [[N]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
+; CHECK: vector.memcheck:
+; CHECK-NEXT: [[ARRAY1:%.*]] = ptrtoint ptr [[ARRAY]] to i64
+; CHECK-NEXT: [[INDICES2:%.*]] = ptrtoint ptr [[INDICES]] to i64
+; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 4
+; CHECK-NEXT: [[TMP5:%.*]] = sub i64 [[ARRAY1]], [[INDICES2]]
+; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP5]], [[TMP4]]
+; CHECK-NEXT: br i1 [[DIFF_CHECK]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[DOTNEG:%.*]] = mul nsw i64 [[TMP6]], -4
+; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[DOTNEG]], [[N]]
+; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP8:%.*]] = shl nuw nsw i64 [[TMP7]], 2
+; CHECK-NEXT: [[TMP9:%.*]] = call <vscale x 4 x i32> @llvm.experimental.stepvector.nxv4i32()
+; CHECK-NEXT: [[TMP10:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT: [[TMP11:%.*]] = shl nuw nsw i32 [[TMP10]], 2
+; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP11]], i64 0
+; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[DOTSPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i32> [ [[TMP9]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP12]], align 4
+; CHECK-NEXT: [[TMP13:%.*]] = zext <vscale x 4 x i32> [[WIDE_LOAD]] to <vscale x 4 x i64>
+; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], <vscale x 4 x i64> [[TMP13]]
+; CHECK-NEXT: call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr> [[TMP14]], i32 1, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[ARRAY]], i64 [[INDEX]]
+; CHECK-NEXT: store <vscale x 4 x i32> [[VEC_IND]], ptr [[TMP15]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 4 x i32> [[VEC_IND]], [[DOTSPLAT]]
+; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[N]]
+; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK: scalar.ph:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT: br label [[FOR_BODY:%.*]]
+; CHECK: for.body:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV]]
+; CHECK-NEXT: [[TMP17:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT: [[IDXPROM1:%.*]] = zext i32 [[TMP17]] to i64
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[IDXPROM1]]
+; CHECK-NEXT: [[TMP18:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT: [[INC:%.*]] = add nsw i32 [[TMP18]], 1
+; CHECK-NEXT: store i32 [[INC]], ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT: [[IDX_ADDR:%.*]] = getelementptr inbounds i32, ptr [[ARRAY]], i64 [[IV]]
+; CHECK-NEXT: [[IV_TRUNC:%.*]] = trunc i64 [[IV]] to i32
+; CHECK-NEXT: store i32 [[IV_TRUNC]], ptr [[IDX_ADDR]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK: for.exit:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds i32, ptr %indices, i64 %iv
+ %0 = load i32, ptr %arrayidx, align 4
+ %idxprom1 = zext i32 %0 to i64
+ %arrayidx2 = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+ %1 = load i32, ptr %arrayidx2, align 4
+ %inc = add nsw i32 %1, 1
+ store i32 %inc, ptr %arrayidx2, align 4
+ %idx.addr = getelementptr inbounds i32, ptr %array, i64 %iv
+ %iv.trunc = trunc i64 %iv to i32
+ store i32 %iv.trunc, ptr %idx.addr, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, %N
+ br i1 %exitcond, label %for.exit, label %for.body
+
+for.exit:
+ ret void
+}
+
+;; Make sure we don't vectorize if there's a potential alias between buckets
+;; and indices.
+define void @simple_histogram_unsafe_alias(ptr %buckets, ptr %indices, i64 %N) #0 {
+; CHECK-LABEL: define void @simple_histogram_unsafe_alias(
+; CHECK-SAME: ptr [[BUCKETS:%.*]], ptr [[INDICES:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br label [[FOR_BODY:%.*]]
+; CHECK: for.body:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV]]
+; CHECK-NEXT: [[TMP12:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT: [[IDXPROM1:%.*]] = zext i32 [[TMP12]] to i64
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[IDXPROM1]]
+; CHECK-NEXT: [[TMP13:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT: [[INC:%.*]] = add nsw i32 [[TMP13]], 1
+; CHECK-NEXT: store i32 [[INC]], ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT:%.*]], label [[FOR_BODY]]
+; CHECK: for.exit:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds i32, ptr %indices, i64 %iv
+ %0 = load i32, ptr %arrayidx, align 4
+ %idxprom1 = zext i32 %0 to i64
+ %arrayidx2 = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+ %1 = load i32, ptr %arrayidx2, align 4
+ %inc = add nsw i32 %1, 1
+ store i32 %inc, ptr %arrayidx2, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, %N
+ br i1 %exitcond, label %for.exit, label %for.body
+
+for.exit:
+ ret void
+}
+
attributes #0 = { "target-features"="+sve2" vscale_range(1,16) }
!0 = distinct !{!0, !1}
>From 199862b2e1dd716a29d3bdedfdb5d9409992c0c8 Mon Sep 17 00:00:00 2001
From: Graham Hunter <graham.hunter at arm.com>
Date: Thu, 29 Aug 2024 13:33:25 +0000
Subject: [PATCH 8/9] Rebase, require asserts for tests using debug output
---
.../Transforms/Vectorize/LoopVectorize.cpp | 2 +-
.../AArch64/sve2-histcnt-epilogue.ll | 20 ++--
.../sve2-histcnt-no-scalar-interleave.ll | 1 +
.../LoopVectorize/AArch64/sve2-histcnt.ll | 93 ++++++++++---------
4 files changed, 62 insertions(+), 54 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 96b92b42a6138f..b7b13ef9b0fadd 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8431,7 +8431,7 @@ VPRecipeBuilder::tryToWidenHistogram(const HistogramInfo *HI,
// Bucket address.
HGramOps.push_back(Operands[1]);
// Increment value.
- HGramOps.push_back(getVPValueOrAddLiveIn(HI->Update->getOperand(1), Plan));
+ HGramOps.push_back(getVPValueOrAddLiveIn(HI->Update->getOperand(1)));
// In case of predicated execution (due to tail-folding, or conditional
// execution, or both), pass the relevant mask.
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-epilogue.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-epilogue.ll
index 2c03b69c6534cf..e435a96ed081a1 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-epilogue.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-epilogue.ll
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3
; RUN: opt < %s -passes=loop-vectorize,instcombine -enable-histogram-loop-vectorization -sve-gather-overhead=2 -sve-scatter-overhead=2 -epilogue-vectorization-minimum-VF=4 -debug-only=loop-vectorize -S 2>&1 | FileCheck %s
+; REQUIRES: asserts
target triple = "aarch64-unknown-linux-gnu"
@@ -13,27 +14,27 @@ define void @simple_histogram(ptr noalias %buckets, ptr readonly %indices, i64 %
; CHECK-NEXT: entry:
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 2
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ugt i64 [[TMP1]], [[N]]
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[DOTNEG:%.*]] = mul nsw i64 [[TMP2]], -4
-; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[DOTNEG]], [[N]]
+; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[N]], [[DOTNEG]]
; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP5:%.*]] = shl nuw nsw i64 [[TMP4]], 2
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP8]], align 4
-; CHECK-NEXT: [[TMP9:%.*]] = zext <vscale x 4 x i32> [[WIDE_LOAD]] to <vscale x 4 x i64>
-; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], <vscale x 4 x i64> [[TMP9]]
-; CHECK-NEXT: call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr> [[TMP10]], i32 1, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 4 x i32>, ptr [[TMP8]], align 4
+; CHECK-NEXT: [[TMP14:%.*]] = zext <vscale x 4 x i32> [[WIDE_LOAD1]] to <vscale x 4 x i64>
+; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], <vscale x 4 x i64> [[TMP14]]
+; CHECK-NEXT: call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr> [[TMP15]], i32 1, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: middle.block:
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[N]]
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
; CHECK: scalar.ph:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
@@ -67,10 +68,13 @@ for.body:
store i32 %inc, ptr %arrayidx2, align 4
%iv.next = add nuw nsw i64 %iv, 1
%exitcond = icmp eq i64 %iv.next, %N
- br i1 %exitcond, label %for.exit, label %for.body
+ br i1 %exitcond, label %for.exit, label %for.body, !llvm.loop !0
for.exit:
ret void
}
attributes #0 = { "target-features"="+sve2" vscale_range(1,16) }
+
+!0 = distinct !{!0, !1}
+!1 = !{!"llvm.loop.interleave.count", i32 1}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-no-scalar-interleave.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-no-scalar-interleave.ll
index 606e0352dad464..74c0e93ba07c64 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-no-scalar-interleave.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-no-scalar-interleave.ll
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3
; RUN: opt < %s -passes=loop-vectorize,instcombine -enable-histogram-loop-vectorization -sve-gather-overhead=2 -sve-scatter-overhead=2 -debug-only=loop-vectorize -S 2>&1 | FileCheck %s
+; REQUIRES: asserts
;; Make sure we don't interleave a histogram when vectorization is disabled.
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
index df6839c3ba435b..3fd8d0d45072b2 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3
; RUN: opt < %s -passes=loop-vectorize,instcombine -enable-histogram-loop-vectorization -sve-gather-overhead=2 -sve-scatter-overhead=2 -debug-only=loop-vectorize -S 2>&1 | FileCheck %s
+; REQUIRES: asserts
target triple = "aarch64-unknown-linux-gnu"
@@ -225,12 +226,12 @@ define void @simple_histogram(ptr noalias %buckets, ptr readonly %indices, i64 %
; CHECK-NEXT: entry:
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 2
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ugt i64 [[TMP1]], [[N]]
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[DOTNEG:%.*]] = mul nsw i64 [[TMP2]], -4
-; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[DOTNEG]], [[N]]
+; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[N]], [[DOTNEG]]
; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP5:%.*]] = shl nuw nsw i64 [[TMP4]], 2
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
@@ -245,7 +246,7 @@ define void @simple_histogram(ptr noalias %buckets, ptr readonly %indices, i64 %
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: middle.block:
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[N]]
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
; CHECK: scalar.ph:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
@@ -279,7 +280,7 @@ for.body:
store i32 %inc, ptr %arrayidx2, align 4
%iv.next = add nuw nsw i64 %iv, 1
%exitcond = icmp eq i64 %iv.next, %N
- br i1 %exitcond, label %for.exit, label %for.body
+ br i1 %exitcond, label %for.exit, label %for.body, !llvm.loop !4
for.exit:
ret void
@@ -291,12 +292,12 @@ define void @simple_histogram_inc_param(ptr noalias %buckets, ptr readonly %indi
; CHECK-NEXT: entry:
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 2
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ugt i64 [[TMP1]], [[N]]
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[DOTNEG:%.*]] = mul nsw i64 [[TMP2]], -4
-; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[DOTNEG]], [[N]]
+; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[N]], [[DOTNEG]]
; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP5:%.*]] = shl nuw nsw i64 [[TMP4]], 2
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
@@ -311,7 +312,7 @@ define void @simple_histogram_inc_param(ptr noalias %buckets, ptr readonly %indi
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: middle.block:
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[N]]
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
; CHECK: scalar.ph:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
@@ -345,7 +346,7 @@ for.body:
store i32 %inc, ptr %arrayidx2, align 4
%iv.next = add nuw nsw i64 %iv, 1
%exitcond = icmp eq i64 %iv.next, %N
- br i1 %exitcond, label %for.exit, label %for.body
+ br i1 %exitcond, label %for.exit, label %for.body, !llvm.loop !4
for.exit:
ret void
@@ -357,12 +358,12 @@ define void @simple_histogram_sub(ptr noalias %buckets, ptr readonly %indices, i
; CHECK-NEXT: entry:
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 2
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ugt i64 [[TMP1]], [[N]]
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[DOTNEG:%.*]] = mul nsw i64 [[TMP2]], -4
-; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[DOTNEG]], [[N]]
+; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[N]], [[DOTNEG]]
; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP5:%.*]] = shl nuw nsw i64 [[TMP4]], 2
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
@@ -377,7 +378,7 @@ define void @simple_histogram_sub(ptr noalias %buckets, ptr readonly %indices, i
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: middle.block:
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[N]]
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
; CHECK: scalar.ph:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
@@ -411,7 +412,7 @@ for.body:
store i32 %inc, ptr %arrayidx2, align 4
%iv.next = add nuw nsw i64 %iv, 1
%exitcond = icmp eq i64 %iv.next, %N
- br i1 %exitcond, label %for.exit, label %for.body
+ br i1 %exitcond, label %for.exit, label %for.body, !llvm.loop !4
for.exit:
ret void
@@ -423,12 +424,12 @@ define void @conditional_histogram(ptr noalias %buckets, ptr readonly %indices,
; CHECK-NEXT: entry:
; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP3:%.*]] = shl nuw nsw i64 [[TMP6]], 2
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ugt i64 [[TMP3]], [[N]]
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP3]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[DOTNEG:%.*]] = mul nsw i64 [[TMP2]], -4
-; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[DOTNEG]], [[N]]
+; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[N]], [[DOTNEG]]
; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP5:%.*]] = shl nuw nsw i64 [[TMP4]], 2
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
@@ -446,7 +447,7 @@ define void @conditional_histogram(ptr noalias %buckets, ptr readonly %indices,
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK: middle.block:
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[N]]
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
; CHECK: scalar.ph:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
@@ -496,7 +497,7 @@ iftrue:
next:
%iv.next = add nuw nsw i64 %iv, 1
%exitcond = icmp eq i64 %iv.next, %N
- br i1 %exitcond, label %for.exit, label %for.body
+ br i1 %exitcond, label %for.exit, label %for.body, !llvm.loop !4
for.exit:
ret void
@@ -519,7 +520,7 @@ define void @histogram_8bit(ptr noalias %buckets, ptr readonly %indices, i64 %N)
; CHECK-NEXT: store i8 [[INC]], ptr [[ARRAYIDX2]], align 4
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT:%.*]], label [[FOR_BODY]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK: for.exit:
; CHECK-NEXT: ret void
;
@@ -537,7 +538,7 @@ for.body:
store i8 %inc, ptr %arrayidx2, align 4
%iv.next = add nuw nsw i64 %iv, 1
%exitcond = icmp eq i64 %iv.next, %N
- br i1 %exitcond, label %for.exit, label %for.body
+ br i1 %exitcond, label %for.exit, label %for.body, !llvm.loop !4
for.exit:
ret void
@@ -560,7 +561,7 @@ define void @histogram_float(ptr noalias %buckets, ptr readonly %indices, i64 %N
; CHECK-NEXT: store float [[INC]], ptr [[ARRAYIDX2]], align 4
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT:%.*]], label [[FOR_BODY]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP10]]
; CHECK: for.exit:
; CHECK-NEXT: ret void
;
@@ -578,7 +579,7 @@ for.body:
store float %inc, ptr %arrayidx2, align 4
%iv.next = add nuw nsw i64 %iv, 1
%exitcond = icmp eq i64 %iv.next, %N
- br i1 %exitcond, label %for.exit, label %for.body
+ br i1 %exitcond, label %for.exit, label %for.body, !llvm.loop !4
for.exit:
ret void
@@ -603,7 +604,7 @@ define void @histogram_varying_increment(ptr noalias %buckets, ptr readonly %ind
; CHECK-NEXT: store i32 [[INC]], ptr [[ARRAYIDX2]], align 4
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT:%.*]], label [[FOR_BODY]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP10]]
; CHECK: for.exit:
; CHECK-NEXT: ret void
;
@@ -623,7 +624,7 @@ for.body:
store i32 %inc, ptr %arrayidx2, align 4
%iv.next = add nuw nsw i64 %iv, 1
%exitcond = icmp eq i64 %iv.next, %N
- br i1 %exitcond, label %for.exit, label %for.body
+ br i1 %exitcond, label %for.exit, label %for.body, !llvm.loop !4
for.exit:
ret void
@@ -636,12 +637,12 @@ define void @simple_histogram_user_interleave(ptr noalias %buckets, ptr readonly
; CHECK-NEXT: entry:
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 3
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ugt i64 [[TMP1]], [[N]]
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[DOTNEG:%.*]] = mul nsw i64 [[TMP2]], -8
-; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[DOTNEG]], [[N]]
+; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[N]], [[DOTNEG]]
; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP5:%.*]] = shl nuw nsw i64 [[TMP4]], 3
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
@@ -661,9 +662,9 @@ define void @simple_histogram_user_interleave(ptr noalias %buckets, ptr readonly
; CHECK-NEXT: call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr> [[TMP21]], i32 1, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
; CHECK: middle.block:
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[N]]
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
; CHECK: scalar.ph:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
@@ -679,7 +680,7 @@ define void @simple_histogram_user_interleave(ptr noalias %buckets, ptr readonly
; CHECK-NEXT: store i32 [[INC]], ptr [[ARRAYIDX2]], align 4
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
; CHECK: for.exit:
; CHECK-NEXT: ret void
;
@@ -713,27 +714,27 @@ define void @histogram_array_3op_gep(i64 noundef %N) #0 {
; CHECK-NEXT: entry:
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 2
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ugt i64 [[TMP1]], [[N]]
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[DOTNEG:%.*]] = mul nsw i64 [[TMP2]], -4
-; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[DOTNEG]], [[N]]
+; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[N]], [[DOTNEG]]
; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 2
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds [1048576 x i32], ptr @idx_array, i64 0, i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP5]], align 4
-; CHECK-NEXT: [[TMP6:%.*]] = sext <vscale x 4 x i32> [[WIDE_LOAD]] to <vscale x 4 x i64>
-; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds [1048576 x i32], ptr @data_array, i64 0, <vscale x 4 x i64> [[TMP6]]
-; CHECK-NEXT: call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr> [[TMP7]], i32 1, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 4 x i32>, ptr [[TMP5]], align 4
+; CHECK-NEXT: [[TMP14:%.*]] = sext <vscale x 4 x i32> [[WIDE_LOAD1]] to <vscale x 4 x i64>
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds [1048576 x i32], ptr @data_array, i64 0, <vscale x 4 x i64> [[TMP14]]
+; CHECK-NEXT: call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr> [[TMP11]], i32 1, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
; CHECK: middle.block:
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[N]]
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
; CHECK: scalar.ph:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
@@ -749,7 +750,7 @@ define void @histogram_array_3op_gep(i64 noundef %N) #0 {
; CHECK-NEXT: store i32 [[INC]], ptr [[ARRAYIDX6]], align 4
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
; CHECK: for.exit:
; CHECK-NEXT: ret void
;
@@ -767,7 +768,7 @@ for.body:
store i32 %inc, ptr %arrayidx6, align 4
%iv.next = add nuw nsw i64 %iv, 1
%exitcond = icmp eq i64 %iv.next, %N
- br i1 %exitcond, label %for.exit, label %for.body
+ br i1 %exitcond, label %for.exit, label %for.body, !llvm.loop !4
for.exit:
ret void
@@ -798,13 +799,13 @@ define void @simple_histogram_tailfold(ptr noalias %buckets, ptr readonly %indic
; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX]], i64 [[TMP6]])
; CHECK-NEXT: [[TMP11:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
-; CHECK-NEXT: br i1 [[TMP11]], label [[VECTOR_BODY]], label [[MIDDLE_BLOCK:%.*]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP11]], label [[VECTOR_BODY]], label [[MIDDLE_BLOCK:%.*]], !llvm.loop [[LOOP16:![0-9]+]]
; CHECK: middle.block:
; CHECK-NEXT: br i1 true, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
; CHECK: scalar.ph:
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
; CHECK: for.body:
-; CHECK-NEXT: br i1 poison, label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK-NEXT: br i1 poison, label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
; CHECK: for.exit:
; CHECK-NEXT: ret void
;
@@ -837,7 +838,7 @@ define void @simple_histogram_rtdepcheck(ptr noalias %buckets, ptr %array, ptr %
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 2
; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 8)
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ugt i64 [[TMP2]], [[N]]
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP2]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
; CHECK: vector.memcheck:
; CHECK-NEXT: [[ARRAY1:%.*]] = ptrtoint ptr [[ARRAY]] to i64
@@ -850,10 +851,10 @@ define void @simple_histogram_rtdepcheck(ptr noalias %buckets, ptr %array, ptr %
; CHECK: vector.ph:
; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[DOTNEG:%.*]] = mul nsw i64 [[TMP6]], -4
-; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[DOTNEG]], [[N]]
+; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[N]], [[DOTNEG]]
; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP8:%.*]] = shl nuw nsw i64 [[TMP7]], 2
-; CHECK-NEXT: [[TMP9:%.*]] = call <vscale x 4 x i32> @llvm.experimental.stepvector.nxv4i32()
+; CHECK-NEXT: [[TMP9:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
; CHECK-NEXT: [[TMP10:%.*]] = call i32 @llvm.vscale.i32()
; CHECK-NEXT: [[TMP11:%.*]] = shl nuw nsw i32 [[TMP10]], 2
; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP11]], i64 0
@@ -872,9 +873,9 @@ define void @simple_histogram_rtdepcheck(ptr noalias %buckets, ptr %array, ptr %
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]]
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 4 x i32> [[VEC_IND]], [[DOTSPLAT]]
; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
; CHECK: middle.block:
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[N]]
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
; CHECK: scalar.ph:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
@@ -893,7 +894,7 @@ define void @simple_histogram_rtdepcheck(ptr noalias %buckets, ptr %array, ptr %
; CHECK-NEXT: store i32 [[IV_TRUNC]], ptr [[IDX_ADDR]], align 4
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
; CHECK: for.exit:
; CHECK-NEXT: ret void
;
@@ -968,3 +969,5 @@ attributes #0 = { "target-features"="+sve2" vscale_range(1,16) }
!1 = !{!"llvm.loop.interleave.count", i32 2}
!2 = distinct !{!2, !3}
!3 = !{!"llvm.loop.vectorize.predicate.enable", i1 true}
+!4 = distinct !{!4, !5}
+!5 = !{!"llvm.loop.interleave.count", i32 1}
>From 1c63f0516cb49f267bb308bdeeb22504b6fda6fc Mon Sep 17 00:00:00 2001
From: Graham Hunter <graham.hunter at arm.com>
Date: Tue, 3 Sep 2024 11:53:27 +0000
Subject: [PATCH 9/9] Addressing review comments: * Improve test ir names *
Explain current limitation on indices * Check index address is not
loop-invariant (with test) * Check for null pointer on dependences (with
test) * Remove HistogramInfo from recipe, remove legality header from
VPlanRecipes.cpp * Implement overridden computeCost for VPHistogramRecipe *
Separated VPlan checks from main test file, used regex matching * Added a
4-op GEP test featuring a struct * Added 64b histogram test without index
extension * Added comment to target-generic test file explaining it doesn't
currently vectorize
---
.../Vectorize/LoopVectorizationLegality.cpp | 14 +-
.../Transforms/Vectorize/LoopVectorize.cpp | 2 +-
llvm/lib/Transforms/Vectorize/VPlan.h | 14 +-
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 38 +-
.../AArch64/sve2-histcnt-epilogue.ll | 14 +-
.../sve2-histcnt-no-scalar-interleave.ll | 14 +-
.../sve2-histcnt-outerloop-scevaddrec.ll | 42 ++
.../AArch64/sve2-histcnt-too-many-deps.ll | 156 ++++++
.../AArch64/sve2-histcnt-vplan.ll | 112 ++++
.../LoopVectorize/AArch64/sve2-histcnt.ll | 516 ++++++++----------
.../Transforms/LoopVectorize/histograms.ll | 16 +-
11 files changed, 614 insertions(+), 324 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-outerloop-scevaddrec.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-too-many-deps.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-vplan.ll
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index e682db530b3fe0..d3a7c1fd007d45 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -1107,12 +1107,17 @@ static bool findHistogram(LoadInst *LI, StoreInst *HSt, Loop *TheLoop,
// Check that the index is calculated by loading from another array. Ignore
// any extensions.
- // FIXME: Support indices from other sources that a linear load from memory?
+ // FIXME: Support indices from other sources than a linear load from memory?
+ // We're currently trying to match an operation looping over an array
+ // of indices, but there could be additional levels of indirection
+ // in place, or possibly some additional calculation to form the index
+ // from the loaded data.
Value *VPtrVal;
if (!match(HIdx, m_ZExtOrSExtOrSelf(m_Load(m_Value(VPtrVal)))))
return false;
- if (!isa<SCEVAddRecExpr>(PSE.getSE()->getSCEV(VPtrVal)))
+ if (!isa<SCEVAddRecExpr>(PSE.getSE()->getSCEV(VPtrVal)) ||
+ TheLoop->isLoopInvariant(VPtrVal))
return false;
// Ensure we'll have the same mask by checking that all parts of the histogram
@@ -1140,6 +1145,11 @@ bool LoopVectorizationLegality::canVectorizeIndirectUnsafeDependences() {
const MemoryDepChecker::Dependence *IUDep = nullptr;
const MemoryDepChecker &DepChecker = LAI->getDepChecker();
const auto *Deps = DepChecker.getDependences();
+ // If there were too many dependences, LAA abandons recording them. We can't
+ // proceed safely if we don't know what the dependences are.
+ if (!Deps)
+ return false;
+
for (const MemoryDepChecker::Dependence &Dep : *Deps) {
// Ignore dependencies that are either known to be safe or can be
// checked at runtime.
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index b7b13ef9b0fadd..7c4265ed20b182 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8438,7 +8438,7 @@ VPRecipeBuilder::tryToWidenHistogram(const HistogramInfo *HI,
if (Legal->isMaskRequired(HI->Store))
HGramOps.push_back(getBlockInMask(HI->Store->getParent()));
- return new VPHistogramRecipe(*HI, Opcode,
+ return new VPHistogramRecipe(Opcode,
make_range(HGramOps.begin(), HGramOps.end()),
HI->Store->getDebugLoc());
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 6d62986bce1479..7f51695c8b4db4 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1591,15 +1591,13 @@ class VPWidenCallRecipe : public VPSingleDefRecipe {
/// of intrinsics. The only update operations currently supported are
/// 'add' and 'sub' where the other term is loop-invariant.
class VPHistogramRecipe : public VPRecipeBase {
- const HistogramInfo &Info;
unsigned Opcode;
public:
template <typename IterT>
- VPHistogramRecipe(const HistogramInfo &HI, unsigned Opcode,
- iterator_range<IterT> Operands, DebugLoc DL = {})
- : VPRecipeBase(VPDef::VPHistogramSC, Operands, DL), Info(HI),
- Opcode(Opcode) {}
+ VPHistogramRecipe(unsigned Opcode, iterator_range<IterT> Operands,
+ DebugLoc DL = {})
+ : VPRecipeBase(VPDef::VPHistogramSC, Operands, DL), Opcode(Opcode) {}
~VPHistogramRecipe() override = default;
@@ -1612,9 +1610,11 @@ class VPHistogramRecipe : public VPRecipeBase {
/// Produce a vectorized histogram operation.
void execute(VPTransformState &State) override;
- unsigned getOpcode() const { return Opcode; }
+ /// Return the cost of this VPHistogramRecipe.
+ InstructionCost computeCost(ElementCount VF,
+ VPCostContext &Ctx) const override;
- const HistogramInfo &getHistogramInfo() const { return Info; }
+ unsigned getOpcode() const { return Opcode; }
/// Return the mask operand if one was provided, or a null pointer if all
/// lanes should be executed unconditionally.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 9bda04ee9e7244..02f63cec09daaf 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -32,7 +32,6 @@
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/Transforms/Utils/LoopUtils.h"
#include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"
-#include "llvm/Transforms/Vectorize/LoopVectorizationLegality.h"
#include <cassert>
using namespace llvm;
@@ -276,10 +275,6 @@ static Instruction *getInstructionForCost(const VPRecipeBase *R) {
return IG->getInsertPos();
if (auto *WidenMem = dyn_cast<VPWidenMemoryRecipe>(R))
return &WidenMem->getIngredient();
- // FIXME: Override the cost method properly to take gather/scatter cost
- // into account, instead of just the intrinsic via the legacy model.
- if (auto *HG = dyn_cast<VPHistogramRecipe>(R))
- return HG->getHistogramInfo().Update;
return nullptr;
}
@@ -993,6 +988,39 @@ void VPHistogramRecipe::execute(VPTransformState &State) {
}
}
+InstructionCost VPHistogramRecipe::computeCost(ElementCount VF,
+ VPCostContext &Ctx) const {
+ // FIXME: Take the gather and scatter into account as well. For now we're
+ // generating the same cost as the fallback path, but we'll likely
+ // need to create a new TTI method for determining the cost, including
+ // whether we can use base + vec-of-smaller-indices or just
+ // vec-of-pointers.
+ assert(VF.isVector() && "Invalid VF for histogram cost");
+ Value *Address = getOperand(0)->getUnderlyingValue();
+ Value *IncAmt = getOperand(1)->getUnderlyingValue();
+ Type *IncTy = IncAmt->getType();
+ VectorType *VTy = VectorType::get(IncTy, VF);
+
+ // Assume that a non-constant update value (or a constant != 1) requires
+ // a multiply, and add that into the cost.
+ InstructionCost MulCost = TTI::TCC_Free;
+ ConstantInt *RHS = dyn_cast<ConstantInt>(IncAmt);
+ if (!RHS || RHS->getZExtValue() != 1)
+ MulCost = Ctx.TTI.getArithmeticInstrCost(Instruction::Mul, VTy);
+
+ // Find the cost of the histogram operation itself.
+ Type *PtrTy = VectorType::get(Address->getType(), VF);
+ Type *MaskTy = VectorType::get(Type::getInt1Ty(Ctx.LLVMCtx), VF);
+ IntrinsicCostAttributes ICA(Intrinsic::experimental_vector_histogram_add,
+ Type::getVoidTy(Ctx.LLVMCtx),
+ {PtrTy, IncTy, MaskTy});
+
+ // Add the costs together with the add/sub operation.
+ return Ctx.TTI.getIntrinsicInstrCost(
+ ICA, TargetTransformInfo::TCK_RecipThroughput) +
+ MulCost + Ctx.TTI.getArithmeticInstrCost(Opcode, VTy);
+}
+
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
void VPHistogramRecipe::print(raw_ostream &O, const Twine &Indent,
VPSlotTracker &SlotTracker) const {
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-epilogue.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-epilogue.ll
index e435a96ed081a1..27c0eaf0ad0513 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-epilogue.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-epilogue.ll
@@ -59,13 +59,13 @@ entry:
for.body:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
- %arrayidx = getelementptr inbounds i32, ptr %indices, i64 %iv
- %0 = load i32, ptr %arrayidx, align 4
- %idxprom1 = zext i32 %0 to i64
- %arrayidx2 = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
- %1 = load i32, ptr %arrayidx2, align 4
- %inc = add nsw i32 %1, 1
- store i32 %inc, ptr %arrayidx2, align 4
+ %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
+ %l.idx = load i32, ptr %gep.indices, align 4
+ %idxprom1 = zext i32 %l.idx to i64
+ %gep.bucket = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+ %l.bucket = load i32, ptr %gep.bucket, align 4
+ %inc = add nsw i32 %l.bucket, 1
+ store i32 %inc, ptr %gep.bucket, align 4
%iv.next = add nuw nsw i64 %iv, 1
%exitcond = icmp eq i64 %iv.next, %N
br i1 %exitcond, label %for.exit, label %for.body, !llvm.loop !0
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-no-scalar-interleave.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-no-scalar-interleave.ll
index 74c0e93ba07c64..2460c015b77ba9 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-no-scalar-interleave.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-no-scalar-interleave.ll
@@ -32,13 +32,13 @@ entry:
for.body:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
- %arrayidx = getelementptr inbounds i32, ptr %indices, i64 %iv
- %0 = load i32, ptr %arrayidx, align 4
- %idxprom1 = zext i32 %0 to i64
- %arrayidx2 = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
- %1 = load i32, ptr %arrayidx2, align 4
- %inc = add nsw i32 %1, 1
- store i32 %inc, ptr %arrayidx2, align 4
+ %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
+ %l.idx = load i32, ptr %gep.indices, align 4
+ %idxprom1 = zext i32 %l.idx to i64
+ %gep.bucket = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+ %l.bucket = load i32, ptr %gep.bucket, align 4
+ %inc = add nsw i32 %l.bucket, 1
+ store i32 %inc, ptr %gep.bucket, align 4
%iv.next = add nuw nsw i64 %iv, 1
%exitcond = icmp eq i64 %iv.next, %N
br i1 %exitcond, label %for.exit, label %for.body, !llvm.loop !0
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-outerloop-scevaddrec.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-outerloop-scevaddrec.ll
new file mode 100644
index 00000000000000..646937889551c1
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-outerloop-scevaddrec.ll
@@ -0,0 +1,42 @@
+; RUN: opt < %s -passes=loop-vectorize,instcombine -enable-histogram-loop-vectorization -sve-gather-overhead=2 -sve-scatter-overhead=2 -debug-only=loop-vectorize -S 2>&1 | FileCheck %s
+; REQUIRES: asserts
+
+target triple = "aarch64-unknown-linux-gnu"
+
+;; Make sure we don't detect a histogram operation if the index address is
+;; loop invariant.
+; CHECK: LV: Checking for a histogram on: store i32 %inc, ptr %gep.bucket, align 4
+; CHECK-NEXT: LV: Can't vectorize due to memory conflicts
+; CHECK-NEXT: LV: Not vectorizing: Cannot prove legality.
+
+define void @outer_loop_scevaddrec(ptr noalias %buckets, ptr readonly %indices, i64 %N, i64 %M) #0 {
+entry:
+ br label %outer.header
+
+outer.header:
+ %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
+ %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %outer.iv
+ br label %inner.body
+
+inner.body:
+ %iv = phi i64 [ 0, %outer.header ], [ %iv.next, %inner.body ]
+ %l.idx = load i32, ptr %gep.indices, align 4
+ %idxprom1 = zext i32 %l.idx to i64
+ %gep.bucket = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+ %l.bucket = load i32, ptr %gep.bucket, align 4
+ %inc = add nsw i32 %l.bucket, 1
+ store i32 %inc, ptr %gep.bucket, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, %N
+ br i1 %exitcond, label %outer.latch, label %inner.body
+
+outer.latch:
+ %outer.iv.next = add nuw nsw i64 %outer.iv, 1
+ %outer.exitcond = icmp eq i64 %outer.iv.next, %M
+ br i1 %outer.exitcond, label %outer.exit, label %outer.header
+
+outer.exit:
+ ret void
+}
+
+attributes #0 = { "target-features"="+sve2" vscale_range(1,16) }
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-too-many-deps.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-too-many-deps.ll
new file mode 100644
index 00000000000000..e42afcb4ed014f
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-too-many-deps.ll
@@ -0,0 +1,156 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3
+; RUN: opt < %s -passes=loop-vectorize,instcombine -enable-histogram-loop-vectorization -sve-gather-overhead=2 -sve-scatter-overhead=2 -max-dependences=2 -debug-only=loop-vectorize,loop-accesses -S 2>&1 | FileCheck %s
+; RUN: opt < %s -passes=loop-vectorize,instcombine -enable-histogram-loop-vectorization -sve-gather-overhead=2 -sve-scatter-overhead=2 -debug-only=loop-vectorize,loop-accesses -S 2>&1 | FileCheck %s --check-prefix=NORMAL_DEP_LIMIT
+; REQUIRES: asserts
+
+target triple = "aarch64-unknown-linux-gnu"
+
+;; Check that we don't crash if LAA gives up on recording dependences and
+;; returns a null pointer.
+
+; CHECK-LABEL: LAA: Checking a loop in 'many_deps'
+; CHECK: Too many dependences, stopped recording
+; CHECK: LV: Can't vectorize due to memory conflicts
+; CHECK: LV: Not vectorizing: Cannot prove legality.
+
+define void @many_deps(ptr noalias %buckets, ptr %array, ptr %indices, ptr %other, i64 %N) #0 {
+; CHECK-LABEL: define void @many_deps(
+; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr [[ARRAY:%.*]], ptr [[INDICES:%.*]], ptr [[OTHER:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br label [[FOR_BODY:%.*]]
+; CHECK: for.body:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT: [[GEP_INDICES:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV]]
+; CHECK-NEXT: [[L_IDX:%.*]] = load i32, ptr [[GEP_INDICES]], align 4
+; CHECK-NEXT: [[IDXPROM1:%.*]] = zext i32 [[L_IDX]] to i64
+; CHECK-NEXT: [[GEP_BUCKET:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[IDXPROM1]]
+; CHECK-NEXT: [[L_BUCKET:%.*]] = load i32, ptr [[GEP_BUCKET]], align 4
+; CHECK-NEXT: [[INC:%.*]] = add nsw i32 [[L_BUCKET]], 1
+; CHECK-NEXT: store i32 [[INC]], ptr [[GEP_BUCKET]], align 4
+; CHECK-NEXT: [[IDX_ADDR:%.*]] = getelementptr inbounds i32, ptr [[ARRAY]], i64 [[IV]]
+; CHECK-NEXT: [[IV_TRUNC:%.*]] = trunc i64 [[IV]] to i32
+; CHECK-NEXT: store i32 [[IV_TRUNC]], ptr [[IDX_ADDR]], align 4
+; CHECK-NEXT: [[GEP_OTHER:%.*]] = getelementptr inbounds i32, ptr [[OTHER]], i64 [[IV]]
+; CHECK-NEXT: [[L_OTHER:%.*]] = load i32, ptr [[GEP_OTHER]], align 4
+; CHECK-NEXT: [[ADD_OTHER:%.*]] = add i32 [[L_OTHER]], [[IV_TRUNC]]
+; CHECK-NEXT: store i32 [[ADD_OTHER]], ptr [[GEP_OTHER]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: for.exit:
+; CHECK-NEXT: ret void
+;
+; NORMAL_DEP_LIMIT-LABEL: define void @many_deps(
+; NORMAL_DEP_LIMIT-SAME: ptr noalias [[BUCKETS:%.*]], ptr [[ARRAY:%.*]], ptr [[INDICES:%.*]], ptr [[OTHER:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; NORMAL_DEP_LIMIT-NEXT: entry:
+; NORMAL_DEP_LIMIT-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; NORMAL_DEP_LIMIT-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 2
+; NORMAL_DEP_LIMIT-NEXT: [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 8)
+; NORMAL_DEP_LIMIT-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP2]]
+; NORMAL_DEP_LIMIT-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
+; NORMAL_DEP_LIMIT: vector.memcheck:
+; NORMAL_DEP_LIMIT-NEXT: [[TMP3:%.*]] = shl i64 [[N]], 2
+; NORMAL_DEP_LIMIT-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[ARRAY]], i64 [[TMP3]]
+; NORMAL_DEP_LIMIT-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[OTHER]], i64 [[TMP3]]
+; NORMAL_DEP_LIMIT-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[INDICES]], i64 [[TMP3]]
+; NORMAL_DEP_LIMIT-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[ARRAY]], [[SCEVGEP1]]
+; NORMAL_DEP_LIMIT-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[OTHER]], [[SCEVGEP]]
+; NORMAL_DEP_LIMIT-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; NORMAL_DEP_LIMIT-NEXT: [[BOUND03:%.*]] = icmp ult ptr [[ARRAY]], [[SCEVGEP2]]
+; NORMAL_DEP_LIMIT-NEXT: [[BOUND14:%.*]] = icmp ult ptr [[INDICES]], [[SCEVGEP]]
+; NORMAL_DEP_LIMIT-NEXT: [[FOUND_CONFLICT5:%.*]] = and i1 [[BOUND03]], [[BOUND14]]
+; NORMAL_DEP_LIMIT-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT5]]
+; NORMAL_DEP_LIMIT-NEXT: [[BOUND06:%.*]] = icmp ult ptr [[OTHER]], [[SCEVGEP2]]
+; NORMAL_DEP_LIMIT-NEXT: [[BOUND17:%.*]] = icmp ult ptr [[INDICES]], [[SCEVGEP1]]
+; NORMAL_DEP_LIMIT-NEXT: [[FOUND_CONFLICT8:%.*]] = and i1 [[BOUND06]], [[BOUND17]]
+; NORMAL_DEP_LIMIT-NEXT: [[CONFLICT_RDX9:%.*]] = or i1 [[CONFLICT_RDX]], [[FOUND_CONFLICT8]]
+; NORMAL_DEP_LIMIT-NEXT: br i1 [[CONFLICT_RDX9]], label [[SCALAR_PH]], label [[ENTRY:%.*]]
+; NORMAL_DEP_LIMIT: vector.ph:
+; NORMAL_DEP_LIMIT-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; NORMAL_DEP_LIMIT-NEXT: [[DOTNEG:%.*]] = mul nsw i64 [[TMP4]], -4
+; NORMAL_DEP_LIMIT-NEXT: [[N_VEC:%.*]] = and i64 [[N]], [[DOTNEG]]
+; NORMAL_DEP_LIMIT-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
+; NORMAL_DEP_LIMIT-NEXT: [[TMP6:%.*]] = shl nuw nsw i64 [[TMP5]], 2
+; NORMAL_DEP_LIMIT-NEXT: [[TMP7:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
+; NORMAL_DEP_LIMIT-NEXT: [[TMP8:%.*]] = call i32 @llvm.vscale.i32()
+; NORMAL_DEP_LIMIT-NEXT: [[TMP9:%.*]] = shl nuw nsw i32 [[TMP8]], 2
+; NORMAL_DEP_LIMIT-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP9]], i64 0
+; NORMAL_DEP_LIMIT-NEXT: [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[DOTSPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; NORMAL_DEP_LIMIT-NEXT: br label [[FOR_BODY:%.*]]
+; NORMAL_DEP_LIMIT: vector.body:
+; NORMAL_DEP_LIMIT-NEXT: [[IV:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; NORMAL_DEP_LIMIT-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i32> [ [[TMP7]], [[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], [[FOR_BODY]] ]
+; NORMAL_DEP_LIMIT-NEXT: [[GEP_INDICES:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV]]
+; NORMAL_DEP_LIMIT-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[GEP_INDICES]], align 4, !alias.scope [[META0:![0-9]+]]
+; NORMAL_DEP_LIMIT-NEXT: [[TMP11:%.*]] = zext <vscale x 4 x i32> [[WIDE_LOAD]] to <vscale x 4 x i64>
+; NORMAL_DEP_LIMIT-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], <vscale x 4 x i64> [[TMP11]]
+; NORMAL_DEP_LIMIT-NEXT: call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr> [[TMP12]], i32 1, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; NORMAL_DEP_LIMIT-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[ARRAY]], i64 [[IV]]
+; NORMAL_DEP_LIMIT-NEXT: store <vscale x 4 x i32> [[VEC_IND]], ptr [[TMP13]], align 4, !alias.scope [[META3:![0-9]+]], !noalias [[META5:![0-9]+]]
+; NORMAL_DEP_LIMIT-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[OTHER]], i64 [[IV]]
+; NORMAL_DEP_LIMIT-NEXT: [[WIDE_LOAD10:%.*]] = load <vscale x 4 x i32>, ptr [[TMP14]], align 4, !alias.scope [[META7:![0-9]+]], !noalias [[META0]]
+; NORMAL_DEP_LIMIT-NEXT: [[TMP15:%.*]] = add <vscale x 4 x i32> [[WIDE_LOAD10]], [[VEC_IND]]
+; NORMAL_DEP_LIMIT-NEXT: store <vscale x 4 x i32> [[TMP15]], ptr [[TMP14]], align 4, !alias.scope [[META7]], !noalias [[META0]]
+; NORMAL_DEP_LIMIT-NEXT: [[IV_NEXT]] = add nuw i64 [[IV]], [[TMP6]]
+; NORMAL_DEP_LIMIT-NEXT: [[VEC_IND_NEXT]] = add <vscale x 4 x i32> [[VEC_IND]], [[DOTSPLAT]]
+; NORMAL_DEP_LIMIT-NEXT: [[TMP16:%.*]] = icmp eq i64 [[IV_NEXT]], [[N_VEC]]
+; NORMAL_DEP_LIMIT-NEXT: br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; NORMAL_DEP_LIMIT: middle.block:
+; NORMAL_DEP_LIMIT-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; NORMAL_DEP_LIMIT-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; NORMAL_DEP_LIMIT: scalar.ph:
+; NORMAL_DEP_LIMIT-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
+; NORMAL_DEP_LIMIT-NEXT: br label [[FOR_BODY1:%.*]]
+; NORMAL_DEP_LIMIT: for.body:
+; NORMAL_DEP_LIMIT-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT1:%.*]], [[FOR_BODY1]] ]
+; NORMAL_DEP_LIMIT-NEXT: [[GEP_INDICES1:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV1]]
+; NORMAL_DEP_LIMIT-NEXT: [[L_IDX:%.*]] = load i32, ptr [[GEP_INDICES1]], align 4
+; NORMAL_DEP_LIMIT-NEXT: [[IDXPROM1:%.*]] = zext i32 [[L_IDX]] to i64
+; NORMAL_DEP_LIMIT-NEXT: [[GEP_BUCKET:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[IDXPROM1]]
+; NORMAL_DEP_LIMIT-NEXT: [[L_BUCKET:%.*]] = load i32, ptr [[GEP_BUCKET]], align 4
+; NORMAL_DEP_LIMIT-NEXT: [[INC:%.*]] = add nsw i32 [[L_BUCKET]], 1
+; NORMAL_DEP_LIMIT-NEXT: store i32 [[INC]], ptr [[GEP_BUCKET]], align 4
+; NORMAL_DEP_LIMIT-NEXT: [[IDX_ADDR:%.*]] = getelementptr inbounds i32, ptr [[ARRAY]], i64 [[IV1]]
+; NORMAL_DEP_LIMIT-NEXT: [[IV_TRUNC:%.*]] = trunc i64 [[IV1]] to i32
+; NORMAL_DEP_LIMIT-NEXT: store i32 [[IV_TRUNC]], ptr [[IDX_ADDR]], align 4
+; NORMAL_DEP_LIMIT-NEXT: [[GEP_OTHER:%.*]] = getelementptr inbounds i32, ptr [[OTHER]], i64 [[IV1]]
+; NORMAL_DEP_LIMIT-NEXT: [[L_OTHER:%.*]] = load i32, ptr [[GEP_OTHER]], align 4
+; NORMAL_DEP_LIMIT-NEXT: [[ADD_OTHER:%.*]] = add i32 [[L_OTHER]], [[IV_TRUNC]]
+; NORMAL_DEP_LIMIT-NEXT: store i32 [[ADD_OTHER]], ptr [[GEP_OTHER]], align 4
+; NORMAL_DEP_LIMIT-NEXT: [[IV_NEXT1]] = add nuw nsw i64 [[IV1]], 1
+; NORMAL_DEP_LIMIT-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT1]], [[N]]
+; NORMAL_DEP_LIMIT-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT]], label [[FOR_BODY1]], !llvm.loop [[LOOP11:![0-9]+]]
+; NORMAL_DEP_LIMIT: for.exit:
+; NORMAL_DEP_LIMIT-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
+ %l.idx = load i32, ptr %gep.indices, align 4
+ %idxprom1 = zext i32 %l.idx to i64
+ %gep.bucket = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+ %l.bucket = load i32, ptr %gep.bucket, align 4
+ %inc = add nsw i32 %l.bucket, 1
+ store i32 %inc, ptr %gep.bucket, align 4
+ %idx.addr = getelementptr inbounds i32, ptr %array, i64 %iv
+ %iv.trunc = trunc i64 %iv to i32
+ store i32 %iv.trunc, ptr %idx.addr, align 4
+ %gep.other = getelementptr inbounds i32, ptr %other, i64 %iv
+ %l.other = load i32, ptr %gep.other, align 4
+ %add.other = add i32 %l.other, %iv.trunc
+ store i32 %add.other, ptr %gep.other, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, %N
+ br i1 %exitcond, label %for.exit, label %for.body, !llvm.loop !0
+
+for.exit:
+ ret void
+}
+
+attributes #0 = { "target-features"="+sve2" vscale_range(1,16) }
+
+!0 = distinct !{!0, !1}
+!1 = !{!"llvm.loop.interleave.count", i32 1}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-vplan.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-vplan.ll
new file mode 100644
index 00000000000000..1da8913ee2c7ee
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-vplan.ll
@@ -0,0 +1,112 @@
+; RUN: opt < %s -passes=loop-vectorize,instcombine -enable-histogram-loop-vectorization -sve-gather-overhead=2 -sve-scatter-overhead=2 -debug-only=loop-vectorize -S 2>&1 | FileCheck %s
+; REQUIRES: asserts
+
+target triple = "aarch64-unknown-linux-gnu"
+
+;; Based on the following C code:
+;;
+;; void simple_histogram(int *buckets, unsigned *indices, int N) {
+;; for (int i = 0; i < N; ++i)
+;; buckets[indices[i]]++;
+;; }
+
+;; Check that the scalar plan contains the original instructions.
+; CHECK: VPlan 'Initial VPlan for VF={1},UF>=1' {
+; CHECK-NEXT: Live-in [[VFxUF:.*]] = VF * UF
+; CHECK-NEXT: Live-in [[VTC:.*]] = vector-trip-count
+; CHECK-NEXT: Live-in [[OTC:.*]] = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: EMIT [[IV:.*]] = CANONICAL-INDUCTION ir<0>, [[IV_NEXT:.*]]
+; CHECK-NEXT: [[STEPS:vp.*]] = SCALAR-STEPS [[IV]], ir<1>
+; CHECK-NEXT: CLONE [[GEP_IDX:.*]] = getelementptr inbounds ir<%indices>, [[STEPS]]
+; CHECK-NEXT: CLONE [[IDX:.*]] = load [[GEP_IDX]]
+; CHECK-NEXT: CLONE [[EXT_IDX:.*]] = zext [[IDX]]
+; CHECK-NEXT: CLONE [[GEP_BUCKET:.*]] = getelementptr inbounds ir<%buckets>, [[EXT_IDX]]
+; CHECK-NEXT: CLONE [[HISTVAL:.*]] = load [[GEP_BUCKET]]
+; CHECK-NEXT: CLONE [[UPDATE:.*]] = add nsw [[HISTVAL]], ir<1>
+; CHECK-NEXT: CLONE store [[UPDATE]], [[GEP_BUCKET]]
+; CHECK-NEXT: EMIT [[IV_NEXT]] = add nuw [[IV]], [[VFxUF]]
+; CHECK-NEXT: EMIT branch-on-count [[IV_NEXT]], [[VTC]]
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT [[TC_CHECK:.*]] = icmp eq [[OTC:.*]], [[VTC]]
+; CHECK-NEXT: EMIT branch-on-cond [[TC_CHECK]]
+; CHECK-NEXT: Successor(s): ir-bb<for.exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<for.exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+
+;; Check that the vectorized plan contains a histogram recipe instead.
+; CHECK: VPlan 'Initial VPlan for VF={vscale x 2,vscale x 4},UF>=1' {
+; CHECK-NEXT: Live-in [[VFxUF:.*]] = VF * UF
+; CHECK-NEXT: Live-in [[VTC:.*]] = vector-trip-count
+; CHECK-NEXT: Live-in [[OTC:.*]] = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: EMIT [[IV:.*]] = CANONICAL-INDUCTION ir<0>, [[IV_NEXT:.*]]
+; CHECK-NEXT: [[STEPS:vp.*]] = SCALAR-STEPS [[IV]], ir<1>
+; CHECK-NEXT: CLONE [[GEP_IDX:.*]] = getelementptr inbounds ir<%indices>, [[STEPS]]
+; CHECK-NEXT: [[VECP_IDX:vp.*]] = vector-pointer [[GEP_IDX]]
+; CHECK-NEXT: WIDEN [[IDX:.*]] = load [[VECP_IDX]]
+; CHECK-NEXT: WIDEN-CAST [[EXT_IDX:.*]] = zext [[IDX]] to i64
+; CHECK-NEXT: WIDEN-GEP Inv[Var] [[GEP_BUCKET:.*]] = getelementptr inbounds ir<%buckets>, [[EXT_IDX]]
+; CHECK-NEXT: WIDEN-HISTOGRAM buckets: [[GEP_BUCKET]], inc: ir<1>
+; CHECK-NEXT: EMIT [[IV_NEXT]] = add nuw [[IV]], [[VFxUF]]
+; CHECK-NEXT: EMIT branch-on-count [[IV_NEXT]], [[VTC]]
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT [[TC_CHECK:.*]] = icmp eq [[OTC]], [[VTC]]
+; CHECK-NEXT: EMIT branch-on-cond [[TC_CHECK]]
+; CHECK-NEXT: Successor(s): ir-bb<for.exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<for.exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+
+define void @simple_histogram(ptr noalias %buckets, ptr readonly %indices, i64 %N) #0 {
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
+ %l.idx = load i32, ptr %gep.indices, align 4
+ %idxprom1 = zext i32 %l.idx to i64
+ %gep.bucket = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+ %l.bucket = load i32, ptr %gep.bucket, align 4
+ %inc = add nsw i32 %l.bucket, 1
+ store i32 %inc, ptr %gep.bucket, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, %N
+ br i1 %exitcond, label %for.exit, label %for.body, !llvm.loop !0
+
+for.exit:
+ ret void
+}
+
+attributes #0 = { "target-features"="+sve2" vscale_range(1,16) }
+
+!0 = distinct !{!0, !1}
+!1 = !{!"llvm.loop.interleave.count", i32 1}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
index 3fd8d0d45072b2..da90208c1a278a 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
@@ -11,203 +11,10 @@ target triple = "aarch64-unknown-linux-gnu"
;; buckets[indices[i]]++;
;; }
+;; Confirm finding a histogram operation
; CHECK-LABEL: Checking a loop in 'simple_histogram'
-
-; CHECK: LV: Checking for a histogram on: store i32 %inc, ptr %arrayidx2, align 4
-; CHECK: LV: Found histogram for: store i32 %inc, ptr %arrayidx2, align 4
-
-;; Check that the scalar plan contains the original instructions.
-; CHECK: VPlan 'Initial VPlan for VF={1},UF>=1' {
-; CHECK-NEXT: Live-in vp<%0> = VF * UF
-; CHECK-NEXT: Live-in vp<%1> = vector-trip-count
-; CHECK-NEXT: Live-in ir<%N> = original trip-count
-; CHECK-EMPTY:
-; CHECK-NEXT: vector.ph:
-; CHECK-NEXT: Successor(s): vector loop
-; CHECK-EMPTY:
-; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vector.body:
-; CHECK-NEXT: EMIT vp<%2> = CANONICAL-INDUCTION ir<0>, vp<%4>
-; CHECK-NEXT: vp<%3> = SCALAR-STEPS vp<%2>, ir<1>
-; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%indices>, vp<%3>
-; CHECK-NEXT: CLONE ir<%0> = load ir<%arrayidx>
-; CHECK-NEXT: CLONE ir<%idxprom1> = zext ir<%0>
-; CHECK-NEXT: CLONE ir<%arrayidx2> = getelementptr inbounds ir<%buckets>, ir<%idxprom1>
-; CHECK-NEXT: CLONE ir<%1> = load ir<%arrayidx2>
-; CHECK-NEXT: CLONE ir<%inc> = add nsw ir<%1>, ir<1>
-; CHECK-NEXT: CLONE store ir<%inc>, ir<%arrayidx2>
-; CHECK-NEXT: EMIT vp<%4> = add nuw vp<%2>, vp<%0>
-; CHECK-NEXT: EMIT branch-on-count vp<%4>, vp<%1>
-; CHECK-NEXT: No successors
-; CHECK-NEXT: }
-; CHECK-NEXT: Successor(s): middle.block
-; CHECK-EMPTY:
-; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<%6> = icmp eq ir<%N>, vp<%1>
-; CHECK-NEXT: EMIT branch-on-cond vp<%6>
-; CHECK-NEXT: Successor(s): ir-bb<for.exit>, scalar.ph
-; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<for.exit>:
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
-; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: No successors
-; CHECK-NEXT: }
-
-;; Check that the vectorized plan contains a histogram recipe instead.
-; CHECK: VPlan 'Initial VPlan for VF={vscale x 2,vscale x 4},UF>=1' {
-; CHECK-NEXT: Live-in vp<%0> = VF * UF
-; CHECK-NEXT: Live-in vp<%1> = vector-trip-count
-; CHECK-NEXT: Live-in ir<%N> = original trip-count
-; CHECK-EMPTY:
-; CHECK-NEXT: vector.ph:
-; CHECK-NEXT: Successor(s): vector loop
-; CHECK-EMPTY:
-; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vector.body:
-; CHECK-NEXT: EMIT vp<%2> = CANONICAL-INDUCTION ir<0>, vp<%5>
-; CHECK-NEXT: vp<%3> = SCALAR-STEPS vp<%2>, ir<1>
-; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%indices>, vp<%3>
-; CHECK-NEXT: vp<%4> = vector-pointer ir<%arrayidx>
-; CHECK-NEXT: WIDEN ir<%0> = load vp<%4>
-; CHECK-NEXT: WIDEN-CAST ir<%idxprom1> = zext ir<%0> to i64
-; CHECK-NEXT: WIDEN-GEP Inv[Var] ir<%arrayidx2> = getelementptr inbounds ir<%buckets>, ir<%idxprom1>
-; CHECK-NEXT: WIDEN-HISTOGRAM buckets: ir<%arrayidx2>, inc: ir<1>
-; CHECK-NEXT: EMIT vp<%5> = add nuw vp<%2>, vp<%0>
-; CHECK-NEXT: EMIT branch-on-count vp<%5>, vp<%1>
-; CHECK-NEXT: No successors
-; CHECK-NEXT: }
-; CHECK-NEXT: Successor(s): middle.block
-; CHECK-EMPTY:
-; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<%7> = icmp eq ir<%N>, vp<%1>
-; CHECK-NEXT: EMIT branch-on-cond vp<%7>
-; CHECK-NEXT: Successor(s): ir-bb<for.exit>, scalar.ph
-; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<for.exit>:
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
-; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: No successors
-; CHECK-NEXT: }
-
-;; Check that a non-constant but loop invariant inc value is correctly represented
-; CHECK-LABEL: LV: Checking a loop in 'simple_histogram_inc_param'
-; CHECK: VPlan 'Initial VPlan for VF={vscale x 2,vscale x 4},UF>=1' {
-; CHECK-NEXT: Live-in vp<%0> = VF * UF
-; CHECK-NEXT: Live-in vp<%1> = vector-trip-count
-; CHECK-NEXT: Live-in ir<%N> = original trip-count
-; CHECK-EMPTY:
-; CHECK-NEXT: vector.ph:
-; CHECK-NEXT: Successor(s): vector loop
-; CHECK-EMPTY:
-; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vector.body:
-; CHECK-NEXT: EMIT vp<%2> = CANONICAL-INDUCTION ir<0>, vp<%5>
-; CHECK-NEXT: vp<%3> = SCALAR-STEPS vp<%2>, ir<1>
-; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%indices>, vp<%3>
-; CHECK-NEXT: vp<%4> = vector-pointer ir<%arrayidx>
-; CHECK-NEXT: WIDEN ir<%0> = load vp<%4>
-; CHECK-NEXT: WIDEN-CAST ir<%idxprom1> = zext ir<%0> to i64
-; CHECK-NEXT: WIDEN-GEP Inv[Var] ir<%arrayidx2> = getelementptr inbounds ir<%buckets>, ir<%idxprom1>
-; CHECK-NEXT: WIDEN-HISTOGRAM buckets: ir<%arrayidx2>, inc: ir<%incval>
-; CHECK-NEXT: EMIT vp<%5> = add nuw vp<%2>, vp<%0>
-; CHECK-NEXT: EMIT branch-on-count vp<%5>, vp<%1>
-; CHECK-NEXT: No successors
-; CHECK-NEXT: }
-; CHECK-NEXT: Successor(s): middle.block
-; CHECK-EMPTY:
-; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<%7> = icmp eq ir<%N>, vp<%1>
-; CHECK-NEXT: EMIT branch-on-cond vp<%7>
-; CHECK-NEXT: Successor(s): ir-bb<for.exit>, scalar.ph
-; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<for.exit>:
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
-; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: No successors
-; CHECK-NEXT: }
-
-;; Check that a sub operation results in a dec parameter in the recipe
-; CHECK-LABEL: LV: Checking a loop in 'simple_histogram_sub'
-; CHECK: VPlan 'Initial VPlan for VF={vscale x 2,vscale x 4},UF>=1' {
-; CHECK-NEXT: Live-in vp<%0> = VF * UF
-; CHECK-NEXT: Live-in vp<%1> = vector-trip-count
-; CHECK-NEXT: Live-in ir<%N> = original trip-count
-; CHECK-EMPTY:
-; CHECK-NEXT: vector.ph:
-; CHECK-NEXT: Successor(s): vector loop
-; CHECK-EMPTY:
-; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vector.body:
-; CHECK-NEXT: EMIT vp<%2> = CANONICAL-INDUCTION ir<0>, vp<%5>
-; CHECK-NEXT: vp<%3> = SCALAR-STEPS vp<%2>, ir<1>
-; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%indices>, vp<%3>
-; CHECK-NEXT: vp<%4> = vector-pointer ir<%arrayidx>
-; CHECK-NEXT: WIDEN ir<%0> = load vp<%4>
-; CHECK-NEXT: WIDEN-CAST ir<%idxprom1> = zext ir<%0> to i64
-; CHECK-NEXT: WIDEN-GEP Inv[Var] ir<%arrayidx2> = getelementptr inbounds ir<%buckets>, ir<%idxprom1>
-; CHECK-NEXT: WIDEN-HISTOGRAM buckets: ir<%arrayidx2>, dec: ir<1>
-; CHECK-NEXT: EMIT vp<%5> = add nuw vp<%2>, vp<%0>
-; CHECK-NEXT: EMIT branch-on-count vp<%5>, vp<%1>
-; CHECK-NEXT: No successors
-; CHECK-NEXT: }
-; CHECK-NEXT: Successor(s): middle.block
-; CHECK-EMPTY:
-; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<%7> = icmp eq ir<%N>, vp<%1>
-; CHECK-NEXT: EMIT branch-on-cond vp<%7>
-; CHECK-NEXT: Successor(s): ir-bb<for.exit>, scalar.ph
-; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<for.exit>:
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
-; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: No successors
-; CHECK-NEXT: }
-
-;; Check that masks are represented
-; CHECK-LABEL: LV: Checking a loop in 'conditional_histogram'
-; CHECK: VPlan 'Initial VPlan for VF={vscale x 2,vscale x 4},UF>=1' {
-; CHECK-NEXT: Live-in vp<%0> = VF * UF
-; CHECK-NEXT: Live-in vp<%1> = vector-trip-count
-; CHECK-NEXT: Live-in ir<%N> = original trip-count
-; CHECK-EMPTY:
-; CHECK-NEXT: vector.ph:
-; CHECK-NEXT: Successor(s): vector loop
-; CHECK-EMPTY:
-; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vector.body:
-; CHECK-NEXT: EMIT vp<%2> = CANONICAL-INDUCTION ir<0>, vp<%6>
-; CHECK-NEXT: vp<%3> = SCALAR-STEPS vp<%2>, ir<1>
-; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%indices>, vp<%3>
-; CHECK-NEXT: vp<%4> = vector-pointer ir<%arrayidx>
-; CHECK-NEXT: WIDEN ir<%0> = load vp<%4>
-; CHECK-NEXT: WIDEN-CAST ir<%idxprom1> = zext ir<%0> to i64
-; CHECK-NEXT: WIDEN-GEP Inv[Var] ir<%arrayidx2> = getelementptr inbounds ir<%buckets>, ir<%idxprom1>
-; CHECK-NEXT: CLONE ir<%condidx> = getelementptr inbounds ir<%conds>, vp<%3>
-; CHECK-NEXT: vp<%5> = vector-pointer ir<%condidx>
-; CHECK-NEXT: WIDEN ir<%conddata> = load vp<%5>
-; CHECK-NEXT: WIDEN ir<%ifcond> = icmp sgt ir<%conddata>, ir<5100>
-; CHECK-NEXT: WIDEN-HISTOGRAM buckets: ir<%arrayidx2>, inc: ir<1>, mask: ir<%ifcond>
-; CHECK-NEXT: EMIT vp<%6> = add nuw vp<%2>, vp<%0>
-; CHECK-NEXT: EMIT branch-on-count vp<%6>, vp<%1>
-; CHECK-NEXT: No successors
-; CHECK-NEXT: }
-; CHECK-NEXT: Successor(s): middle.block
-; CHECK-EMPTY:
-; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<%8> = icmp eq ir<%N>, vp<%1>
-; CHECK-NEXT: EMIT branch-on-cond vp<%8>
-; CHECK-NEXT: Successor(s): ir-bb<for.exit>, scalar.ph
-; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<for.exit>:
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
-; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: No successors
-; CHECK-NEXT: }
+; CHECK: LV: Checking for a histogram on: store i32 %inc, ptr %gep.bucket, align 4
+; CHECK: LV: Found histogram for: store i32 %inc, ptr %gep.bucket, align 4
;; Confirm cost calculation for runtime checks
; CHECK-LABEL: LV: Checking a loop in 'simple_histogram_rtdepcheck'
@@ -271,13 +78,13 @@ entry:
for.body:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
- %arrayidx = getelementptr inbounds i32, ptr %indices, i64 %iv
- %0 = load i32, ptr %arrayidx, align 4
- %idxprom1 = zext i32 %0 to i64
- %arrayidx2 = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
- %1 = load i32, ptr %arrayidx2, align 4
- %inc = add nsw i32 %1, 1
- store i32 %inc, ptr %arrayidx2, align 4
+ %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
+ %l.idx = load i32, ptr %gep.indices, align 4
+ %idxprom1 = zext i32 %l.idx to i64
+ %gep.bucket = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+ %l.bucket = load i32, ptr %gep.bucket, align 4
+ %inc = add nsw i32 %l.bucket, 1
+ store i32 %inc, ptr %gep.bucket, align 4
%iv.next = add nuw nsw i64 %iv, 1
%exitcond = icmp eq i64 %iv.next, %N
br i1 %exitcond, label %for.exit, label %for.body, !llvm.loop !4
@@ -337,13 +144,13 @@ entry:
for.body:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
- %arrayidx = getelementptr inbounds i32, ptr %indices, i64 %iv
- %0 = load i32, ptr %arrayidx, align 4
- %idxprom1 = zext i32 %0 to i64
- %arrayidx2 = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
- %1 = load i32, ptr %arrayidx2, align 4
- %inc = add nsw i32 %1, %incval
- store i32 %inc, ptr %arrayidx2, align 4
+ %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
+ %l.idx = load i32, ptr %gep.indices, align 4
+ %idxprom1 = zext i32 %l.idx to i64
+ %gep.bucket = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+ %l.bucket = load i32, ptr %gep.bucket, align 4
+ %inc = add nsw i32 %l.bucket, %incval
+ store i32 %inc, ptr %gep.bucket, align 4
%iv.next = add nuw nsw i64 %iv, 1
%exitcond = icmp eq i64 %iv.next, %N
br i1 %exitcond, label %for.exit, label %for.body, !llvm.loop !4
@@ -371,7 +178,7 @@ define void @simple_histogram_sub(ptr noalias %buckets, ptr readonly %indices, i
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP8]], align 4
-; CHECK-NEXT: [[TMP9:%.*]] = zext <vscale x 4 x i32> [[WIDE_LOAD]] to <vscale x 4 x i64>
+; CHECK-NEXT: [[TMP9:%.*]] = sext <vscale x 4 x i32> [[WIDE_LOAD]] to <vscale x 4 x i64>
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], <vscale x 4 x i64> [[TMP9]]
; CHECK-NEXT: call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr> [[TMP10]], i32 -1, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
@@ -387,7 +194,7 @@ define void @simple_histogram_sub(ptr noalias %buckets, ptr readonly %indices, i
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV]]
; CHECK-NEXT: [[TMP12:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
-; CHECK-NEXT: [[IDXPROM1:%.*]] = zext i32 [[TMP12]] to i64
+; CHECK-NEXT: [[IDXPROM1:%.*]] = sext i32 [[TMP12]] to i64
; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[IDXPROM1]]
; CHECK-NEXT: [[TMP13:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
; CHECK-NEXT: [[INC:%.*]] = add nsw i32 [[TMP13]], -1
@@ -403,13 +210,13 @@ entry:
for.body:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
- %arrayidx = getelementptr inbounds i32, ptr %indices, i64 %iv
- %0 = load i32, ptr %arrayidx, align 4
- %idxprom1 = zext i32 %0 to i64
- %arrayidx2 = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
- %1 = load i32, ptr %arrayidx2, align 4
- %inc = sub nsw i32 %1, 1
- store i32 %inc, ptr %arrayidx2, align 4
+ %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
+ %l.idx = load i32, ptr %gep.indices, align 4
+ %idxprom1 = sext i32 %l.idx to i64
+ %gep.bucket = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+ %l.bucket = load i32, ptr %gep.bucket, align 4
+ %inc = sub nsw i32 %l.bucket, 1
+ store i32 %inc, ptr %gep.bucket, align 4
%iv.next = add nuw nsw i64 %iv, 1
%exitcond = icmp eq i64 %iv.next, %N
br i1 %exitcond, label %for.exit, label %for.body, !llvm.loop !4
@@ -479,19 +286,19 @@ entry:
for.body:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %next ]
- %arrayidx = getelementptr inbounds i32, ptr %indices, i64 %iv
- %0 = load i32, ptr %arrayidx, align 4
- %idxprom1 = zext i32 %0 to i64
- %arrayidx2 = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+ %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
+ %l.idx = load i32, ptr %gep.indices, align 4
+ %idxprom1 = zext i32 %l.idx to i64
+ %gep.bucket = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
%condidx = getelementptr inbounds i32, ptr %conds, i64 %iv
%conddata = load i32, ptr %condidx, align 4
%ifcond = icmp sgt i32 %conddata, 5100
br i1 %ifcond, label %iftrue, label %next
iftrue:
- %1 = load i32, ptr %arrayidx2, align 4
- %inc = add nsw i32 %1, 1
- store i32 %inc, ptr %arrayidx2, align 4
+ %l.bucket = load i32, ptr %gep.bucket, align 4
+ %inc = add nsw i32 %l.bucket, 1
+ store i32 %inc, ptr %gep.bucket, align 4
br label %next
next:
@@ -529,13 +336,13 @@ entry:
for.body:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
- %arrayidx = getelementptr inbounds i32, ptr %indices, i64 %iv
- %0 = load i32, ptr %arrayidx, align 4
- %idxprom1 = zext i32 %0 to i64
- %arrayidx2 = getelementptr inbounds i8, ptr %buckets, i64 %idxprom1
- %1 = load i8, ptr %arrayidx2, align 4
- %inc = add nsw i8 %1, 1
- store i8 %inc, ptr %arrayidx2, align 4
+ %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
+ %l.idx = load i32, ptr %gep.indices, align 4
+ %idxprom1 = zext i32 %l.idx to i64
+ %gep.bucket = getelementptr inbounds i8, ptr %buckets, i64 %idxprom1
+ %l.bucket = load i8, ptr %gep.bucket, align 4
+ %inc = add nsw i8 %l.bucket, 1
+ store i8 %inc, ptr %gep.bucket, align 4
%iv.next = add nuw nsw i64 %iv, 1
%exitcond = icmp eq i64 %iv.next, %N
br i1 %exitcond, label %for.exit, label %for.body, !llvm.loop !4
@@ -570,13 +377,13 @@ entry:
for.body:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
- %arrayidx = getelementptr inbounds i32, ptr %indices, i64 %iv
- %0 = load i32, ptr %arrayidx, align 4
- %idxprom1 = zext i32 %0 to i64
- %arrayidx2 = getelementptr inbounds float, ptr %buckets, i64 %idxprom1
- %1 = load float, ptr %arrayidx2, align 4
- %inc = fadd fast float %1, 1.0
- store float %inc, ptr %arrayidx2, align 4
+ %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
+ %l.idx = load i32, ptr %gep.indices, align 4
+ %idxprom1 = zext i32 %l.idx to i64
+ %gep.bucket = getelementptr inbounds float, ptr %buckets, i64 %idxprom1
+ %l.bucket = load float, ptr %gep.bucket, align 4
+ %inc = fadd fast float %l.bucket, 1.0
+ store float %inc, ptr %gep.bucket, align 4
%iv.next = add nuw nsw i64 %iv, 1
%exitcond = icmp eq i64 %iv.next, %N
br i1 %exitcond, label %for.exit, label %for.body, !llvm.loop !4
@@ -613,15 +420,15 @@ entry:
for.body:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
- %arrayidx = getelementptr inbounds i32, ptr %indices, i64 %iv
- %0 = load i32, ptr %arrayidx, align 4
- %idxprom1 = zext i32 %0 to i64
- %arrayidx2 = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
- %1 = load i32, ptr %arrayidx2, align 4
- %incidx = getelementptr inbounds i32, ptr %incvals, i64 %iv
- %incval = load i32, ptr %incidx, align 4
- %inc = add nsw i32 %1, %incval
- store i32 %inc, ptr %arrayidx2, align 4
+ %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
+ %l.idx = load i32, ptr %gep.indices, align 4
+ %idxprom1 = zext i32 %l.idx to i64
+ %gep.bucket = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+ %l.bucket = load i32, ptr %gep.bucket, align 4
+ %gep.incvals = getelementptr inbounds i32, ptr %incvals, i64 %iv
+ %l.incval = load i32, ptr %gep.incvals, align 4
+ %inc = add nsw i32 %l.bucket, %l.incval
+ store i32 %inc, ptr %gep.bucket, align 4
%iv.next = add nuw nsw i64 %iv, 1
%exitcond = icmp eq i64 %iv.next, %N
br i1 %exitcond, label %for.exit, label %for.body, !llvm.loop !4
@@ -689,13 +496,13 @@ entry:
for.body:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
- %arrayidx = getelementptr inbounds i32, ptr %indices, i64 %iv
- %0 = load i32, ptr %arrayidx, align 4
- %idxprom1 = zext i32 %0 to i64
- %arrayidx2 = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
- %1 = load i32, ptr %arrayidx2, align 4
- %inc = add nsw i32 %1, 1
- store i32 %inc, ptr %arrayidx2, align 4
+ %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
+ %l.idx = load i32, ptr %gep.indices, align 4
+ %idxprom1 = zext i32 %l.idx to i64
+ %gep.bucket = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+ %l.bucket = load i32, ptr %gep.bucket, align 4
+ %inc = add nsw i32 %l.bucket, 1
+ store i32 %inc, ptr %gep.bucket, align 4
%iv.next = add nuw nsw i64 %iv, 1
%exitcond = icmp eq i64 %iv.next, %N
br i1 %exitcond, label %for.exit, label %for.body, !llvm.loop !0
@@ -759,13 +566,83 @@ entry:
for.body:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
- %arrayidx = getelementptr inbounds [1048576 x i32], ptr @idx_array, i64 0, i64 %iv
- %2 = load i32, ptr %arrayidx, align 4
- %idxprom5 = sext i32 %2 to i64
- %arrayidx6 = getelementptr inbounds [1048576 x i32], ptr @data_array, i64 0, i64 %idxprom5
- %3 = load i32, ptr %arrayidx6, align 4
- %inc = add nsw i32 %3, 1
- store i32 %inc, ptr %arrayidx6, align 4
+ %gep.indices = getelementptr inbounds [1048576 x i32], ptr @idx_array, i64 0, i64 %iv
+ %l.idx = load i32, ptr %gep.indices, align 4
+ %idxprom5 = sext i32 %l.idx to i64
+ %gep.bucket = getelementptr inbounds [1048576 x i32], ptr @data_array, i64 0, i64 %idxprom5
+ %l.bucket = load i32, ptr %gep.bucket, align 4
+ %inc = add nsw i32 %l.bucket, 1
+ store i32 %inc, ptr %gep.bucket, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, %N
+ br i1 %exitcond, label %for.exit, label %for.body, !llvm.loop !4
+
+for.exit:
+ ret void
+}
+
+;; Add a struct into the mix, use a different constant index.
+;; { unused, buckets }
+%somestruct = type { [1048576 x i32], [1048576 x i32] }
+
+define void @histogram_array_4op_gep_nonzero_const_idx(i64 noundef %N, ptr readonly %indices, ptr noalias %data.struct) #0 {
+; CHECK-LABEL: define void @histogram_array_4op_gep_nonzero_const_idx(
+; CHECK-SAME: i64 noundef [[N:%.*]], ptr readonly [[INDICES:%.*]], ptr noalias [[DATA_STRUCT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 2
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[DOTNEG:%.*]] = mul nsw i64 [[TMP2]], -4
+; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[N]], [[DOTNEG]]
+; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 2
+; CHECK-NEXT: br label [[FOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP5]], align 4
+; CHECK-NEXT: [[TMP6:%.*]] = sext <vscale x 4 x i32> [[WIDE_LOAD]] to <vscale x 4 x i64>
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds [[SOMESTRUCT:%.*]], ptr [[DATA_STRUCT]], i64 1, i32 0, <vscale x 4 x i64> [[TMP6]]
+; CHECK-NEXT: call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr> [[TMP7]], i32 1, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; CHECK-NEXT: [[IV_NEXT]] = add nuw i64 [[IV]], [[TMP4]]
+; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[IV_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK: scalar.ph:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1:%.*]] ]
+; CHECK-NEXT: br label [[FOR_BODY1:%.*]]
+; CHECK: for.body:
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT1:%.*]], [[FOR_BODY1]] ]
+; CHECK-NEXT: [[GEP_INDICES:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV1]]
+; CHECK-NEXT: [[L_IDX:%.*]] = load i32, ptr [[GEP_INDICES]], align 4
+; CHECK-NEXT: [[IDXPROM5:%.*]] = sext i32 [[L_IDX]] to i64
+; CHECK-NEXT: [[GEP_BUCKET:%.*]] = getelementptr inbounds [[SOMESTRUCT]], ptr [[DATA_STRUCT]], i64 1, i32 0, i64 [[IDXPROM5]]
+; CHECK-NEXT: [[L_BUCKET:%.*]] = load i32, ptr [[GEP_BUCKET]], align 4
+; CHECK-NEXT: [[INC:%.*]] = add nsw i32 [[L_BUCKET]], 1
+; CHECK-NEXT: store i32 [[INC]], ptr [[GEP_BUCKET]], align 4
+; CHECK-NEXT: [[IV_NEXT1]] = add nuw nsw i64 [[IV1]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT1]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT]], label [[FOR_BODY1]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK: for.exit:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
+ %l.idx = load i32, ptr %gep.indices, align 4
+ %idxprom5 = sext i32 %l.idx to i64
+ %gep.bucket = getelementptr inbounds %somestruct, ptr %data.struct, i32 1, i32 0, i64 %idxprom5
+ %l.bucket = load i32, ptr %gep.bucket, align 4
+ %inc = add nsw i32 %l.bucket, 1
+ store i32 %inc, ptr %gep.bucket, align 4
%iv.next = add nuw nsw i64 %iv, 1
%exitcond = icmp eq i64 %iv.next, %N
br i1 %exitcond, label %for.exit, label %for.body, !llvm.loop !4
@@ -799,13 +676,13 @@ define void @simple_histogram_tailfold(ptr noalias %buckets, ptr readonly %indic
; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX]], i64 [[TMP6]])
; CHECK-NEXT: [[TMP11:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
-; CHECK-NEXT: br i1 [[TMP11]], label [[VECTOR_BODY]], label [[MIDDLE_BLOCK:%.*]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP11]], label [[VECTOR_BODY]], label [[MIDDLE_BLOCK:%.*]], !llvm.loop [[LOOP18:![0-9]+]]
; CHECK: middle.block:
; CHECK-NEXT: br i1 true, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
; CHECK: scalar.ph:
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
; CHECK: for.body:
-; CHECK-NEXT: br i1 poison, label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK-NEXT: br i1 poison, label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
; CHECK: for.exit:
; CHECK-NEXT: ret void
;
@@ -814,13 +691,13 @@ entry:
for.body:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
- %arrayidx = getelementptr inbounds i32, ptr %indices, i64 %iv
- %0 = load i32, ptr %arrayidx, align 4
- %idxprom1 = zext i32 %0 to i64
- %arrayidx2 = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
- %1 = load i32, ptr %arrayidx2, align 4
- %inc = add nsw i32 %1, 1
- store i32 %inc, ptr %arrayidx2, align 4
+ %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
+ %l.idx = load i32, ptr %gep.indices, align 4
+ %idxprom1 = zext i32 %l.idx to i64
+ %gep.bucket = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+ %l.bucket = load i32, ptr %gep.bucket, align 4
+ %inc = add nsw i32 %l.bucket, 1
+ store i32 %inc, ptr %gep.bucket, align 4
%iv.next = add nuw nsw i64 %iv, 1
%exitcond = icmp eq i64 %iv.next, %N
br i1 %exitcond, label %for.exit, label %for.body, !llvm.loop !2
@@ -873,7 +750,7 @@ define void @simple_histogram_rtdepcheck(ptr noalias %buckets, ptr %array, ptr %
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]]
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 4 x i32> [[VEC_IND]], [[DOTSPLAT]]
; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
; CHECK: middle.block:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
@@ -894,7 +771,7 @@ define void @simple_histogram_rtdepcheck(ptr noalias %buckets, ptr %array, ptr %
; CHECK-NEXT: store i32 [[IV_TRUNC]], ptr [[IDX_ADDR]], align 4
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
; CHECK: for.exit:
; CHECK-NEXT: ret void
;
@@ -903,13 +780,13 @@ entry:
for.body:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
- %arrayidx = getelementptr inbounds i32, ptr %indices, i64 %iv
- %0 = load i32, ptr %arrayidx, align 4
- %idxprom1 = zext i32 %0 to i64
- %arrayidx2 = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
- %1 = load i32, ptr %arrayidx2, align 4
- %inc = add nsw i32 %1, 1
- store i32 %inc, ptr %arrayidx2, align 4
+ %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
+ %l.idx = load i32, ptr %gep.indices, align 4
+ %idxprom1 = zext i32 %l.idx to i64
+ %gep.bucket = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+ %l.bucket = load i32, ptr %gep.bucket, align 4
+ %inc = add nsw i32 %l.bucket, 1
+ store i32 %inc, ptr %gep.bucket, align 4
%idx.addr = getelementptr inbounds i32, ptr %array, i64 %iv
%iv.trunc = trunc i64 %iv to i32
store i32 %iv.trunc, ptr %idx.addr, align 4
@@ -948,13 +825,13 @@ entry:
for.body:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
- %arrayidx = getelementptr inbounds i32, ptr %indices, i64 %iv
- %0 = load i32, ptr %arrayidx, align 4
- %idxprom1 = zext i32 %0 to i64
- %arrayidx2 = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
- %1 = load i32, ptr %arrayidx2, align 4
- %inc = add nsw i32 %1, 1
- store i32 %inc, ptr %arrayidx2, align 4
+ %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
+ %l.idx = load i32, ptr %gep.indices, align 4
+ %idxprom1 = zext i32 %l.idx to i64
+ %gep.bucket = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+ %l.bucket = load i32, ptr %gep.bucket, align 4
+ %inc = add nsw i32 %l.bucket, 1
+ store i32 %inc, ptr %gep.bucket, align 4
%iv.next = add nuw nsw i64 %iv, 1
%exitcond = icmp eq i64 %iv.next, %N
br i1 %exitcond, label %for.exit, label %for.body
@@ -963,6 +840,69 @@ for.exit:
ret void
}
+define void @simple_histogram_64b(ptr noalias %buckets, ptr readonly %indices, i64 %N) #0 {
+; CHECK-LABEL: define void @simple_histogram_64b(
+; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 1
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[DOTNEG:%.*]] = mul nsw i64 [[TMP2]], -2
+; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[N]], [[DOTNEG]]
+; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 1
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[INDICES]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 2 x i64>, ptr [[TMP5]], align 4
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[BUCKETS]], <vscale x 2 x i64> [[WIDE_LOAD]]
+; CHECK-NEXT: call void @llvm.experimental.vector.histogram.add.nxv2p0.i64(<vscale x 2 x ptr> [[TMP6]], i64 1, <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK: scalar.ph:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT: br label [[FOR_BODY:%.*]]
+; CHECK: for.body:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT: [[GEP_INDICES:%.*]] = getelementptr inbounds i64, ptr [[INDICES]], i64 [[IV]]
+; CHECK-NEXT: [[L_IDX:%.*]] = load i64, ptr [[GEP_INDICES]], align 4
+; CHECK-NEXT: [[GEP_BUCKET:%.*]] = getelementptr inbounds i64, ptr [[BUCKETS]], i64 [[L_IDX]]
+; CHECK-NEXT: [[L_BUCKET:%.*]] = load i64, ptr [[GEP_BUCKET]], align 4
+; CHECK-NEXT: [[INC:%.*]] = add nsw i64 [[L_BUCKET]], 1
+; CHECK-NEXT: store i64 [[INC]], ptr [[GEP_BUCKET]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
+; CHECK: for.exit:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %gep.indices = getelementptr inbounds i64, ptr %indices, i64 %iv
+ %l.idx = load i64, ptr %gep.indices, align 4
+ %gep.bucket = getelementptr inbounds i64, ptr %buckets, i64 %l.idx
+ %l.bucket = load i64, ptr %gep.bucket, align 4
+ %inc = add nsw i64 %l.bucket, 1
+ store i64 %inc, ptr %gep.bucket, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, %N
+ br i1 %exitcond, label %for.exit, label %for.body, !llvm.loop !4
+
+for.exit:
+ ret void
+}
+
attributes #0 = { "target-features"="+sve2" vscale_range(1,16) }
!0 = distinct !{!0, !1}
diff --git a/llvm/test/Transforms/LoopVectorize/histograms.ll b/llvm/test/Transforms/LoopVectorize/histograms.ll
index 5a439c4b485331..b43a3e94f41d87 100644
--- a/llvm/test/Transforms/LoopVectorize/histograms.ll
+++ b/llvm/test/Transforms/LoopVectorize/histograms.ll
@@ -3,6 +3,8 @@
target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+;; Currently we don't expect this to vectorize, since the generic cost model returns
+;; invalid for the histogram intrinsic.
define void @simple_histogram(ptr noalias %buckets, ptr readonly %indices, i64 %N) #0 {
; CHECK-LABEL: define void @simple_histogram(
; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], i64 [[N:%.*]]) {
@@ -28,13 +30,13 @@ entry:
for.body:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
- %arrayidx = getelementptr inbounds i32, ptr %indices, i64 %iv
- %0 = load i32, ptr %arrayidx, align 4
- %idxprom1 = zext i32 %0 to i64
- %arrayidx2 = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
- %1 = load i32, ptr %arrayidx2, align 4
- %inc = add nsw i32 %1, 1
- store i32 %inc, ptr %arrayidx2, align 4
+ %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
+ %l.idx = load i32, ptr %gep.indices, align 4
+ %idxprom1 = zext i32 %l.idx to i64
+ %gep.bucket = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+ %l.bucket = load i32, ptr %gep.bucket, align 4
+ %inc = add nsw i32 %l.bucket, 1
+ store i32 %inc, ptr %gep.bucket, align 4
%iv.next = add nuw nsw i64 %iv, 1
%exitcond = icmp eq i64 %iv.next, %N
br i1 %exitcond, label %for.exit, label %for.body
More information about the llvm-commits
mailing list