[llvm] [AArch64][SVE][SelectionDAG] Lower umin/umax vector histogram intrinsics (PR #216626)
via llvm-commits
llvm-commits at lists.llvm.org
Sun Aug 16 19:50:23 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-selectiondag
Author: Jerry Dang (kuroyukiasuna)
<details>
<summary>Changes</summary>
Both intrinsics are now routed through `visitVectorHistogram` and lowered on AArch64 as gather -> umin/umax -> scatter.
No histcnt needed. The increment is lane-uniform and umin/umax are idempotent, so colliding lanes all compute the same value and the ordering of the scatter's conflicting writes does not matter.
Fixes #<!-- -->215242
Fixes #<!-- -->215243
---
Full diff: https://github.com/llvm/llvm-project/pull/216626.diff
3 Files Affected:
- (modified) llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp (+6-2)
- (modified) llvm/lib/Target/AArch64/AArch64ISelLowering.cpp (+49-23)
- (added) llvm/test/CodeGen/AArch64/sve-histogram-minmax.ll (+109)
``````````diff
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index b3e22ff50d4bd..9e05d552a3f84 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -6607,7 +6607,9 @@ void SelectionDAGBuilder::visitVectorHistogram(const CallInst &I,
unsigned IntrinsicID) {
// For now, we're only lowering an 'add' histogram.
// We can add others later, e.g. saturating adds, min/max.
- assert(IntrinsicID == Intrinsic::experimental_vector_histogram_add &&
+ assert((IntrinsicID == Intrinsic::experimental_vector_histogram_add ||
+ IntrinsicID == Intrinsic::experimental_vector_histogram_umin ||
+ IntrinsicID == Intrinsic::experimental_vector_histogram_umax) &&
"Tried to lower unsupported histogram type");
SDLoc sdl = getCurSDLoc();
Value *Ptr = I.getOperand(0);
@@ -8604,7 +8606,9 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
case Intrinsic::experimental_convergence_loop:
visitConvergenceControl(I, Intrinsic);
return;
- case Intrinsic::experimental_vector_histogram_add: {
+ case Intrinsic::experimental_vector_histogram_add:
+ case Intrinsic::experimental_vector_histogram_umin:
+ case Intrinsic::experimental_vector_histogram_umax: {
visitVectorHistogram(I, Intrinsic);
return;
}
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 46ffc9287dc62..4433a83222920 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -2281,13 +2281,12 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
setOperationAction(ISD::MSTORE, VT, Custom);
}
- // Histcnt is SVE2 only
- if (Subtarget->hasSVE2()) {
- setOperationAction(ISD::EXPERIMENTAL_VECTOR_HISTOGRAM, MVT::nxv4i32,
- Custom);
- setOperationAction(ISD::EXPERIMENTAL_VECTOR_HISTOGRAM, MVT::nxv2i64,
- Custom);
+ setOperationAction(ISD::EXPERIMENTAL_VECTOR_HISTOGRAM, MVT::nxv4i32,
+ Custom);
+ setOperationAction(ISD::EXPERIMENTAL_VECTOR_HISTOGRAM, MVT::nxv2i64,
+ Custom);
+ if (Subtarget->hasSVE2()) {
static const unsigned MLAOps[] = {ISD::PARTIAL_REDUCE_SMLA,
ISD::PARTIAL_REDUCE_UMLA};
// Must be lowered to SVE instructions.
@@ -34807,10 +34806,16 @@ SDValue AArch64TargetLowering::LowerVECTOR_HISTOGRAM(SDValue Op,
SDValue IntID = HG->getIntID();
// The Intrinsic ID determines the type of update operation.
- [[maybe_unused]] ConstantSDNode *CID = cast<ConstantSDNode>(IntID.getNode());
- // Right now, we only support 'add' as an update.
- assert(CID->getZExtValue() == Intrinsic::experimental_vector_histogram_add &&
- "Unexpected histogram update operation");
+ ConstantSDNode *CID = cast<ConstantSDNode>(IntID.getNode());
+ Intrinsic::ID IID = static_cast<Intrinsic::ID>(CID->getZExtValue());
+
+ // HISTCNT is SVE2-only, so 'add' has no lowering without it.
+ if (IID == Intrinsic::experimental_vector_histogram_add &&
+ !Subtarget->hasSVE2())
+ return SDValue();
+
+ bool IsMinMax = IID == Intrinsic::experimental_vector_histogram_umin ||
+ IID == Intrinsic::experimental_vector_histogram_umax;
EVT IndexVT = Index.getValueType();
LLVMContext &Ctx = *DAG.getContext();
@@ -34823,8 +34828,11 @@ SDValue AArch64TargetLowering::LowerVECTOR_HISTOGRAM(SDValue Op,
SDValue Zero = DAG.getConstant(0, DL, MVT::i64);
SDValue PassThru = DAG.getSplatVector(IncSplatVT, DL, Zero);
- SDValue IncSplat = DAG.getSplatVector(
- IncSplatVT, DL, DAG.getAnyExtOrTrunc(Inc, DL, IncExtVT));
+ SDValue IncScalar = DAG.getAnyExtOrTrunc(Inc, DL, IncExtVT);
+ if (ExtTrunc && IsMinMax) {
+ IncScalar = DAG.getZeroExtendInReg(IncScalar, DL, HG->getMemoryVT());
+ }
+ SDValue IncSplat = DAG.getSplatVector(IncSplatVT, DL, IncScalar);
SDValue Ops[] = {Chain, PassThru, Mask, Ptr, Index, Scale};
MachineMemOperand *MMO = HG->getMemOperand();
@@ -34833,26 +34841,44 @@ SDValue AArch64TargetLowering::LowerVECTOR_HISTOGRAM(SDValue Op,
MMO->getPointerInfo(), MachineMemOperand::MOLoad, MMO->getSize(),
MMO->getAlign(), MMO->getAAInfo());
ISD::MemIndexType IndexType = HG->getIndexType();
- SDValue Gather = DAG.getMaskedGather(
- DAG.getVTList(IncSplatVT, MVT::Other), MemVT, DL, Ops, GMMO, IndexType,
- ExtTrunc ? ISD::EXTLOAD : ISD::NON_EXTLOAD);
+ ISD::LoadExtType ExtType = !ExtTrunc ? ISD::NON_EXTLOAD
+ : IsMinMax ? ISD::ZEXTLOAD
+ : ISD::EXTLOAD;
+ SDValue Gather =
+ DAG.getMaskedGather(DAG.getVTList(IncSplatVT, MVT::Other), MemVT, DL, Ops,
+ GMMO, IndexType, ExtType);
SDValue GChain = Gather.getValue(1);
- // Perform the histcnt, multiply by inc, add to bucket data.
- SDValue ID =
- DAG.getTargetConstant(Intrinsic::aarch64_sve_histcnt, DL, IncExtVT);
- SDValue HistCnt =
- DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, IndexVT, ID, Mask, Index, Index);
- SDValue Mul = DAG.getNode(ISD::MUL, DL, IncSplatVT, HistCnt, IncSplat);
- SDValue Add = DAG.getNode(ISD::ADD, DL, IncSplatVT, Gather, Mul);
+ SDValue Update;
+ switch (IID) {
+ case Intrinsic::experimental_vector_histogram_add: {
+ // Perform the histcnt, multiply by inc, add to bucket data.
+ SDValue ID =
+ DAG.getTargetConstant(Intrinsic::aarch64_sve_histcnt, DL, IncExtVT);
+ SDValue HistCnt = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, IndexVT, ID,
+ Mask, Index, Index);
+ SDValue Mul = DAG.getNode(ISD::MUL, DL, IncSplatVT, HistCnt, IncSplat);
+ Update = DAG.getNode(ISD::ADD, DL, IncSplatVT, Gather, Mul);
+ break;
+ }
+ case Intrinsic::experimental_vector_histogram_umin:
+ Update = DAG.getNode(ISD::UMIN, DL, IncSplatVT, Gather, IncSplat);
+ break;
+ case Intrinsic::experimental_vector_histogram_umax:
+ Update = DAG.getNode(ISD::UMAX, DL, IncSplatVT, Gather, IncSplat);
+ break;
+ // TODO: uadd_sat needs a saturating multiply by the histcnt
+ default:
+ llvm_unreachable("Unexpected histogram update operation");
+ }
// Create an MMO for the scatter, without load|store flags.
MachineMemOperand *SMMO = DAG.getMachineFunction().getMachineMemOperand(
MMO->getPointerInfo(), MachineMemOperand::MOStore, MMO->getSize(),
MMO->getAlign(), MMO->getAAInfo());
- SDValue ScatterOps[] = {GChain, Add, Mask, Ptr, Index, Scale};
+ SDValue ScatterOps[] = {GChain, Update, Mask, Ptr, Index, Scale};
SDValue Scatter = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), MemVT, DL,
ScatterOps, SMMO, IndexType, ExtTrunc);
return Scatter;
diff --git a/llvm/test/CodeGen/AArch64/sve-histogram-minmax.ll b/llvm/test/CodeGen/AArch64/sve-histogram-minmax.ll
new file mode 100644
index 0000000000000..1628163b7097c
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-histogram-minmax.ll
@@ -0,0 +1,109 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64 -mattr=+sve -verify-machineinstrs < %s -o - | FileCheck %s
+; RUN: llc -mtriple=aarch64 -mattr=+sve2 -verify-machineinstrs < %s -o - | FileCheck %s
+
+define void @histogram_umin_nxv2i64(<vscale x 2 x ptr> %buckets, i64 %inc, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: histogram_umin_nxv2i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ld1d { z1.d }, p0/z, [z0.d]
+; CHECK-NEXT: mov z2.d, x0
+; CHECK-NEXT: ptrue p1.d
+; CHECK-NEXT: umin z1.d, p1/m, z1.d, z2.d
+; CHECK-NEXT: st1d { z1.d }, p0, [z0.d]
+; CHECK-NEXT: ret
+ call void @llvm.experimental.vector.histogram.umin.nxv2p0.i64(<vscale x 2 x ptr> %buckets, i64 %inc, <vscale x 2 x i1> %mask)
+ ret void
+}
+
+define void @histogram_umin_nxv4i32(<vscale x 4 x ptr> %buckets, i32 %inc, <vscale x 4 x i1> %mask) {
+; CHECK-LABEL: histogram_umin_nxv4i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: punpklo p1.h, p0.b
+; CHECK-NEXT: mov w8, w0
+; CHECK-NEXT: mov z3.d, x8
+; CHECK-NEXT: ptrue p2.d
+; CHECK-NEXT: ld1w { z2.d }, p1/z, [z0.d]
+; CHECK-NEXT: punpkhi p0.h, p0.b
+; CHECK-NEXT: umin z2.d, p2/m, z2.d, z3.d
+; CHECK-NEXT: st1w { z2.d }, p1, [z0.d]
+; CHECK-NEXT: ld1w { z0.d }, p0/z, [z1.d]
+; CHECK-NEXT: umin z0.d, p2/m, z0.d, z3.d
+; CHECK-NEXT: st1w { z0.d }, p0, [z1.d]
+; CHECK-NEXT: ret
+ call void @llvm.experimental.vector.histogram.umin.nxv4p0.i32(<vscale x 4 x ptr> %buckets, i32 %inc, <vscale x 4 x i1> %mask)
+ ret void
+}
+
+
+define void @histogram_umax_nxv2i64(<vscale x 2 x ptr> %buckets, i64 %inc, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: histogram_umax_nxv2i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ld1d { z1.d }, p0/z, [z0.d]
+; CHECK-NEXT: mov z2.d, x0
+; CHECK-NEXT: ptrue p1.d
+; CHECK-NEXT: umax z1.d, p1/m, z1.d, z2.d
+; CHECK-NEXT: st1d { z1.d }, p0, [z0.d]
+; CHECK-NEXT: ret
+ call void @llvm.experimental.vector.histogram.umax.nxv2p0.i64(<vscale x 2 x ptr> %buckets, i64 %inc, <vscale x 2 x i1> %mask)
+ ret void
+}
+
+define void @histogram_umax_nxv4i32(<vscale x 4 x ptr> %buckets, i32 %inc, <vscale x 4 x i1> %mask) {
+; CHECK-LABEL: histogram_umax_nxv4i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: punpklo p1.h, p0.b
+; CHECK-NEXT: mov w8, w0
+; CHECK-NEXT: mov z3.d, x8
+; CHECK-NEXT: ptrue p2.d
+; CHECK-NEXT: ld1w { z2.d }, p1/z, [z0.d]
+; CHECK-NEXT: punpkhi p0.h, p0.b
+; CHECK-NEXT: umax z2.d, p2/m, z2.d, z3.d
+; CHECK-NEXT: st1w { z2.d }, p1, [z0.d]
+; CHECK-NEXT: ld1w { z0.d }, p0/z, [z1.d]
+; CHECK-NEXT: umax z0.d, p2/m, z0.d, z3.d
+; CHECK-NEXT: st1w { z0.d }, p0, [z1.d]
+; CHECK-NEXT: ret
+ call void @llvm.experimental.vector.histogram.umax.nxv4p0.i32(<vscale x 4 x ptr> %buckets, i32 %inc, <vscale x 4 x i1> %mask)
+ ret void
+}
+
+;
+; Buckets narrower than the SVE container. The gather must zero-extend and the
+; increment must be masked to the bucket width, because umin/umax compare the
+; full container. Any-extension would let undefined high bits decide the result.
+;
+
+define void @histogram_umin_nxv2i8(<vscale x 2 x ptr> %buckets, i8 %inc, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: histogram_umin_nxv2i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ld1b { z1.d }, p0/z, [z0.d]
+; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT: and x8, x0, #0xff
+; CHECK-NEXT: ptrue p1.d
+; CHECK-NEXT: mov z2.d, x8
+; CHECK-NEXT: umin z1.d, p1/m, z1.d, z2.d
+; CHECK-NEXT: st1b { z1.d }, p0, [z0.d]
+; CHECK-NEXT: ret
+ call void @llvm.experimental.vector.histogram.umin.nxv2p0.i8(<vscale x 2 x ptr> %buckets, i8 %inc, <vscale x 2 x i1> %mask)
+ ret void
+}
+
+define void @histogram_umin_nxv4i8(<vscale x 4 x ptr> %buckets, i8 %inc, <vscale x 4 x i1> %mask) {
+; CHECK-LABEL: histogram_umin_nxv4i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: punpklo p1.h, p0.b
+; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT: and x8, x0, #0xff
+; CHECK-NEXT: mov z3.d, x8
+; CHECK-NEXT: ptrue p2.d
+; CHECK-NEXT: ld1b { z2.d }, p1/z, [z0.d]
+; CHECK-NEXT: punpkhi p0.h, p0.b
+; CHECK-NEXT: umin z2.d, p2/m, z2.d, z3.d
+; CHECK-NEXT: st1b { z2.d }, p1, [z0.d]
+; CHECK-NEXT: ld1b { z0.d }, p0/z, [z1.d]
+; CHECK-NEXT: umin z0.d, p2/m, z0.d, z3.d
+; CHECK-NEXT: st1b { z0.d }, p0, [z1.d]
+; CHECK-NEXT: ret
+ call void @llvm.experimental.vector.histogram.umin.nxv4p0.i8(<vscale x 4 x ptr> %buckets, i8 %inc, <vscale x 4 x i1> %mask)
+ ret void
+}
``````````
</details>
https://github.com/llvm/llvm-project/pull/216626
More information about the llvm-commits
mailing list