[llvm] [LV] Fix missing MetaData for histogram instructions (PR #134241)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Jun 1 05:54:01 PDT 2026
https://github.com/RonDahan101 updated https://github.com/llvm/llvm-project/pull/134241
>From 8458599d01987a50e3cf0d493978e51554692bcf Mon Sep 17 00:00:00 2001
From: rond <ron.dahan at mobileye.com>
Date: Sun, 31 May 2026 13:57:17 +0300
Subject: [PATCH 1/2] [LV] Propagate metadata to histogram instructions via
VPIRMetadata
The histogram intrinsic generated by the loop vectorizer was missing
metadata (alias.scope, noalias, etc.) from the original store
instruction. Fix this by having VPHistogramRecipe inherit from
VPIRMetadata and propagate metadata from the original store's
VPInstruction when the recipe is created.
This follows the established pattern used by VPWidenIntrinsicRecipe,
VPWidenCastRecipe, and other VPlan recipes that carry metadata
explicitly using the VPIRMetadata infrastructure (introduced in
Co-authored-by: Copilot <223556219+Copilot at users.noreply.github.com>
---
.../Transforms/Vectorize/LoopVectorize.cpp | 4 +-
llvm/lib/Transforms/Vectorize/VPlan.h | 15 ++---
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 7 ++-
.../LoopVectorize/AArch64/sve2-histcnt.ll | 58 +++++++++++++++++++
4 files changed, 73 insertions(+), 11 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 30456819602b2..96bfb2af6d5ce 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6569,7 +6569,9 @@ VPHistogramRecipe *VPRecipeBuilder::widenIfHistogram(VPInstruction *VPI) {
if (CM.isMaskRequired(HI->Store))
HGramOps.push_back(VPI->getMask());
- return new VPHistogramRecipe(Opcode, HGramOps, VPI->getDebugLoc());
+ return new VPHistogramRecipe(Opcode, HGramOps,
+ static_cast<const VPIRMetadata &>(*VPI),
+ VPI->getDebugLoc());
}
bool VPRecipeBuilder::replaceWithFinalIfReductionStore(
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index d07b9896514d7..56a2d36e148f9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -2143,20 +2143,21 @@ class LLVM_ABI_FOR_TEST VPWidenCallRecipe : public VPRecipeWithIRFlags,
/// lanes, which is handled by the llvm.experimental.vector.histogram family
/// of intrinsics. The only update operations currently supported are
/// 'add' and 'sub' where the other term is loop-invariant.
-class VPHistogramRecipe : public VPRecipeBase {
+class VPHistogramRecipe : public VPRecipeBase, public VPIRMetadata {
/// Opcode of the update operation, currently either add or sub.
unsigned Opcode;
public:
VPHistogramRecipe(unsigned Opcode, ArrayRef<VPValue *> Operands,
+ const VPIRMetadata &Metadata = {},
DebugLoc DL = DebugLoc::getUnknown())
: VPRecipeBase(VPRecipeBase::VPHistogramSC, Operands, DL),
- Opcode(Opcode) {}
+ VPIRMetadata(Metadata), Opcode(Opcode) {}
~VPHistogramRecipe() override = default;
VPHistogramRecipe *clone() override {
- return new VPHistogramRecipe(Opcode, operands(), getDebugLoc());
+ return new VPHistogramRecipe(Opcode, operands(), *this, getDebugLoc());
}
VP_CLASSOF_IMPL(VPRecipeBase::VPHistogramSC);
@@ -4274,10 +4275,10 @@ struct CastInfo<VPWidenMemoryRecipe, const VPRecipeBase *>
/// Support casting from VPRecipeBase -> VPIRMetadata.
template <>
struct CastInfo<VPIRMetadata, VPRecipeBase *>
- : vpdetail::CastInfoMixinImpl<VPIRMetadata, VPInstruction, VPWidenRecipe,
- VPWidenCastRecipe, VPWidenIntrinsicRecipe,
- VPWidenCallRecipe, VPReplicateRecipe,
- VPInterleaveBase, VPWidenMemoryRecipe> {};
+ : vpdetail::CastInfoMixinImpl<
+ VPIRMetadata, VPInstruction, VPWidenRecipe, VPWidenCastRecipe,
+ VPWidenIntrinsicRecipe, VPWidenCallRecipe, VPReplicateRecipe,
+ VPInterleaveBase, VPWidenMemoryRecipe, VPHistogramRecipe> {};
template <>
struct CastInfo<VPIRMetadata, const VPRecipeBase *>
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 59084bbaf41b3..0bedc226ec650 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -2251,9 +2251,10 @@ void VPHistogramRecipe::execute(VPTransformState &State) {
else
assert(Opcode == Instruction::Add && "only add or sub supported for now");
- State.Builder.CreateIntrinsic(Intrinsic::experimental_vector_histogram_add,
- {VTy, IncAmt->getType()},
- {Address, IncAmt, Mask});
+ auto *HistogramInst = State.Builder.CreateIntrinsic(
+ Intrinsic::experimental_vector_histogram_add, {VTy, IncAmt->getType()},
+ {Address, IncAmt, Mask});
+ applyMetadata(*HistogramInst);
}
InstructionCost VPHistogramRecipe::computeCost(ElementCount VF,
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
index ab4a3bed1caa6..7c0195cafb45a 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
@@ -825,9 +825,67 @@ for.exit:
attributes #0 = { "target-features"="+sve2" vscale_range(1,16) }
+define void @simple_histogram_metadata(ptr noalias %buckets, ptr readonly %indices, i64 %N) #0 {
+; CHECK-LABEL: define void @simple_histogram_metadata(
+; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 2
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP2]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP3]], align 4, !alias.scope [[META27:![0-9]+]], !noalias [[META30:![0-9]+]]
+; CHECK-NEXT: [[TMP4:%.*]] = zext <vscale x 4 x i32> [[WIDE_LOAD]] to <vscale x 4 x i64>
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], <vscale x 4 x i64> [[TMP4]]
+; CHECK-NEXT: call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr> [[TMP5]], i32 1, <vscale x 4 x i1> splat (i1 true)), !alias.scope [[META32:![0-9]+]], !noalias [[META27]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP35:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK: scalar.ph:
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
+ %l.idx = load i32, ptr %gep.indices, align 4, !alias.scope !6, !noalias !9
+ %idxprom1 = zext i32 %l.idx to i64
+ %gep.bucket = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+ %l.bucket = load i32, ptr %gep.bucket, align 4, !alias.scope !11, !noalias !6
+ %inc = add nsw i32 %l.bucket, 1
+ store i32 %inc, ptr %gep.bucket, align 4, !alias.scope !11, !noalias !6
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, %N
+ br i1 %exitcond, label %for.exit, label %for.body, !llvm.loop !4
+
+for.exit:
+ ret void
+}
+
+attributes #0 = { "target-features"="+sve2" vscale_range(1,16) }
+
!0 = distinct !{!0, !1}
!1 = !{!"llvm.loop.interleave.count", i32 2}
!2 = distinct !{!2, !3}
!3 = !{!"llvm.loop.vectorize.predicate.enable", i1 true}
!4 = distinct !{!4, !5}
!5 = !{!"llvm.loop.interleave.count", i32 1}
+!6 = !{!7}
+!7 = distinct !{!7, !8, !"scope-index"}
+!8 = distinct !{!8, !"scopes-indices"}
+!9 = !{!10}
+!10 = distinct !{!10, !8, !"scope-bucket"}
+!11 = !{!10, !12}
+!12 = distinct !{!12, !13, !"scope-bucket-2"}
+!13 = distinct !{!13, !"scopes-buckets"}
>From ac54ce073ae7c5f499495067b75670a7644028f6 Mon Sep 17 00:00:00 2001
From: rond <ron.dahan at mobileye.com>
Date: Mon, 1 Jun 2026 15:52:58 +0300
Subject: [PATCH 2/2] Fix review
---
.../Transforms/Vectorize/LoopVectorize.cpp | 3 +-
.../LoopVectorize/AArch64/sve2-histcnt.ll | 2 --
.../VPlan/AArch64/sve2-histcnt-vplan.ll | 32 +++++++++++++++++++
3 files changed, 33 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 96bfb2af6d5ce..7213269c08434 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6569,8 +6569,7 @@ VPHistogramRecipe *VPRecipeBuilder::widenIfHistogram(VPInstruction *VPI) {
if (CM.isMaskRequired(HI->Store))
HGramOps.push_back(VPI->getMask());
- return new VPHistogramRecipe(Opcode, HGramOps,
- static_cast<const VPIRMetadata &>(*VPI),
+ return new VPHistogramRecipe(Opcode, HGramOps, cast<VPIRMetadata>(*VPI),
VPI->getDebugLoc());
}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
index 7c0195cafb45a..0f530be610a8b 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
@@ -823,8 +823,6 @@ for.exit:
ret void
}
-attributes #0 = { "target-features"="+sve2" vscale_range(1,16) }
-
define void @simple_histogram_metadata(ptr noalias %buckets, ptr readonly %indices, i64 %N) #0 {
; CHECK-LABEL: define void @simple_histogram_metadata(
; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve2-histcnt-vplan.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve2-histcnt-vplan.ll
index 00d74e27918ac..046ffd3f45f47 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve2-histcnt-vplan.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/sve2-histcnt-vplan.ll
@@ -126,3 +126,35 @@ for.body:
for.exit:
ret void
}
+
+;; Check that metadata is preserved in the WIDEN-HISTOGRAM recipe when the
+;; histogram store carries alias.scope/noalias metadata.
+; CHECK: VPlan 'Initial VPlan for VF={vscale x 2,vscale x 4},UF>=1' {
+; CHECK: WIDEN-HISTOGRAM buckets: {{.*}}, inc: ir<1> (!alias.scope {{![0-9]+}}, !noalias {{![0-9]+}})
+
+define void @simple_histogram_metadata(ptr noalias %buckets, ptr readonly %indices, i64 %N) {
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
+ %l.idx = load i32, ptr %gep.indices, align 4, !alias.scope !0, !noalias !3
+ %idxprom1 = zext i32 %l.idx to i64
+ %gep.bucket = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+ %l.bucket = load i32, ptr %gep.bucket, align 4, !alias.scope !3, !noalias !0
+ %inc = add nsw i32 %l.bucket, 1
+ store i32 %inc, ptr %gep.bucket, align 4, !alias.scope !3, !noalias !0
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, %N
+ br i1 %exitcond, label %for.exit, label %for.body
+
+for.exit:
+ ret void
+}
+
+!0 = !{!1}
+!1 = distinct !{!1, !2}
+!2 = distinct !{!2}
+!3 = !{!4}
+!4 = distinct !{!4, !2}
More information about the llvm-commits
mailing list