[llvm] [SystemZ] Custom lowering for vector min/max reductions (PR #229382)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Oct 6 04:49:55 PDT 2026
https://github.com/MarkVeerasingam updated https://github.com/llvm/llvm-project/pull/229382
>From 1b7f88c1adea261e0d8d88f9456297b7a8dfc681 Mon Sep 17 00:00:00 2001
From: MarkVeerasingam <markveer70 at gmail.com>
Date: Mon, 5 Oct 2026 09:12:47 -0500
Subject: [PATCH 1/3] [SystemZ] Custom lower vector min/max reductions
---
.../Target/SystemZ/SystemZISelLowering.cpp | 47 +++++++++++++++++++
llvm/lib/Target/SystemZ/SystemZISelLowering.h | 1 +
.../SystemZ/SystemZTargetTransformInfo.cpp | 15 +++++-
3 files changed, 62 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
index a6cc3e94b7e7d..e83085e8f99f1 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
@@ -495,6 +495,14 @@ SystemZTargetLowering::SystemZTargetLowering(const TargetMachine &TM,
// it with VZERO+VSUM
setOperationAction(ISD::VECREDUCE_ADD, VT, Custom);
+ // Custom-lower VECREDUCE_{SMIN,SMAX,UMIN,UMAX} to a halving tree of
+ // native VMN/VMX instructions, avoiding the generic expander's
+ // insert/extract subvector sequence on the illegal v2i32/v1i32 types.
+ setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
+ setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
+ setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom);
+ setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
+
// Map SETCCs onto one of VCE, VCH or VCHL, swapping the operands
// and inverting the result as necessary.
setOperationAction(ISD::SETCC, VT, Custom);
@@ -7270,6 +7278,11 @@ SDValue SystemZTargetLowering::LowerOperation(SDValue Op,
return lowerCTPOP(Op, DAG);
case ISD::VECREDUCE_ADD:
return lowerVECREDUCE_ADD(Op, DAG);
+ case ISD::VECREDUCE_SMIN:
+ case ISD::VECREDUCE_SMAX:
+ case ISD::VECREDUCE_UMIN:
+ case ISD::VECREDUCE_UMAX:
+ return lowerVECREDUCE_MINMAX(Op, DAG);
case ISD::ATOMIC_FENCE:
return lowerATOMIC_FENCE(Op, DAG);
case ISD::ATOMIC_SWAP:
@@ -11465,6 +11478,40 @@ SDValue SystemZTargetLowering::lowerVECREDUCE_ADD(SDValue Op,
DAG.getConstant(OpVT.getVectorNumElements() - 1, DL, MVT::i32));
}
+SDValue SystemZTargetLowering::lowerVECREDUCE_MINMAX(SDValue Op,
+ SelectionDAG &DAG) const {
+ unsigned Opcode = Op.getOpcode();
+ EVT VT = Op.getValueType();
+ SDValue V = Op.getOperand(0);
+ EVT VecVT = V.getValueType();
+ SDLoc DL(Op);
+
+ unsigned EltOp;
+ switch (Opcode) {
+ case ISD::VECREDUCE_SMIN: EltOp = ISD::SMIN; break;
+ case ISD::VECREDUCE_SMAX: EltOp = ISD::SMAX; break;
+ case ISD::VECREDUCE_UMIN: EltOp = ISD::UMIN; break;
+ case ISD::VECREDUCE_UMAX: EltOp = ISD::UMAX; break;
+ default: llvm_unreachable("Unexpected opcode.");
+ }
+
+ unsigned EltBytes = VecVT.getScalarSizeInBits() / 8;
+ unsigned NumElts = VecVT.getVectorNumElements();
+
+ for (unsigned Step = NumElts / 2; Step >= 1; Step /= 2) {
+ unsigned ByteShift = Step * EltBytes;
+ SDValue V16i8 = DAG.getBitcast(MVT::v16i8, V);
+ SDValue Shifted16i8 = DAG.getNode(
+ SystemZISD::SHL_DOUBLE, DL, MVT::v16i8, V16i8, V16i8,
+ DAG.getTargetConstant(ByteShift, DL, MVT::i32));
+ SDValue Shifted = DAG.getBitcast(VecVT, Shifted16i8);
+ V = DAG.getNode(EltOp, DL, VecVT, V, Shifted);
+ }
+
+ return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, V,
+ DAG.getConstant(0, DL, MVT::i32));
+}
+
static void printFunctionArgExts(const Function *F, raw_fd_ostream &OS) {
FunctionType *FT = F->getFunctionType();
const AttributeList &Attrs = F->getAttributes();
diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.h b/llvm/lib/Target/SystemZ/SystemZISelLowering.h
index 6d99fef8b568b..5a5f8504d6bac 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.h
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.h
@@ -370,6 +370,7 @@ class SystemZTargetLowering : public TargetLowering {
SDValue lowerOR(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerCTPOP(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerVECREDUCE_ADD(SDValue Op, SelectionDAG &DAG) const;
+ SDValue lowerVECREDUCE_MINMAX(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerATOMIC_LOAD(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerATOMIC_STORE(SDValue Op, SelectionDAG &DAG) const;
diff --git a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
index 06f84c78a504d..2153605bbc449 100644
--- a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
@@ -1619,7 +1619,7 @@ bool SystemZTTIImpl::shouldExpandReduction(const IntrinsicInst *II) const {
default:
return true;
// Do not expand vector.reduce.add...
- case Intrinsic::vector_reduce_add:
+ case Intrinsic::vector_reduce_add: {
auto *VType = cast<FixedVectorType>(II->getOperand(0)->getType());
// ...unless the scalar size is i64 or larger,
// or the operand vector is not full, since the
@@ -1627,4 +1627,17 @@ bool SystemZTTIImpl::shouldExpandReduction(const IntrinsicInst *II) const {
return VType->getScalarSizeInBits() >= 64 ||
VType->getPrimitiveSizeInBits() < SystemZ::VectorBits;
}
+ // Do not expand vector.reduce.{smin,smax,umin,umax}, so they reach ISel
+ // and are custom-lowered by lowerVECREDUCE_MINMAX...
+ case Intrinsic::vector_reduce_smin:
+ case Intrinsic::vector_reduce_smax:
+ case Intrinsic::vector_reduce_umin:
+ case Intrinsic::vector_reduce_umax: {
+ auto *VType = cast<FixedVectorType>(II->getOperand(0)->getType());
+ // ...unless the scalar size is i64 (the generic expansion is already
+ // a single vmrlg + vmn/vmx), or the operand vector is not full.
+ return VType->getScalarSizeInBits() >= 64 ||
+ VType->getPrimitiveSizeInBits() < SystemZ::VectorBits;
+ }
+ }
}
>From 3ad066ee8e84e7abc7e968f3eeceb0c4b1eba093 Mon Sep 17 00:00:00 2001
From: MarkVeerasingam <markveer70 at gmail.com>
Date: Tue, 6 Oct 2026 06:17:41 -0500
Subject: [PATCH 2/3] [SystemZ] added regression test for vector reduction
lowering
---
.../test/CodeGen/SystemZ/vec-reduce-minmax.ll | 76 +++++++++++++++++++
1 file changed, 76 insertions(+)
create mode 100644 llvm/test/CodeGen/SystemZ/vec-reduce-minmax.ll
diff --git a/llvm/test/CodeGen/SystemZ/vec-reduce-minmax.ll b/llvm/test/CodeGen/SystemZ/vec-reduce-minmax.ll
new file mode 100644
index 0000000000000..b978aa9fbceed
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/vec-reduce-minmax.ll
@@ -0,0 +1,76 @@
+; Test vector min/max reductions on SystemZ.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+define i8 @f_smin_v16i8(<16 x i8> %v) {
+; CHECK-LABEL: f_smin_v16i8:
+; CHECK-NOT: vpk
+; CHECK-NOT: vrep
+; CHECK: vsldb
+; CHECK: vmnb
+; CHECK: vlgvb %r2, {{%v[0-9]+}}, 0
+ %r = call i8 @llvm.vector.reduce.smin.v16i8(<16 x i8> %v)
+ ret i8 %r
+}
+
+define i8 @f_umax_v16i8(<16 x i8> %v) {
+; CHECK-LABEL: f_umax_v16i8:
+; CHECK-NOT: vpk
+; CHECK-NOT: vrep
+; CHECK: vsldb
+; CHECK: vmxlb
+; CHECK: vlgvb %r2, {{%v[0-9]+}}, 0
+ %r = call i8 @llvm.vector.reduce.umax.v16i8(<16 x i8> %v)
+ ret i8 %r
+}
+
+define i16 @f_smax_v8i16(<8 x i16> %v) {
+; CHECK-LABEL: f_smax_v8i16:
+; CHECK-NOT: vpk
+; CHECK-NOT: vrep
+; CHECK: vsldb
+; CHECK: vmxh
+; CHECK: vlgvh %r2, {{%v[0-9]+}}, 0
+ %r = call i16 @llvm.vector.reduce.smax.v8i16(<8 x i16> %v)
+ ret i16 %r
+}
+
+define i16 @f_umin_v8i16(<8 x i16> %v) {
+; CHECK-LABEL: f_umin_v8i16:
+; CHECK-NOT: vpk
+; CHECK-NOT: vrep
+; CHECK: vsldb
+; CHECK: vmnlh
+; CHECK: vlgvh %r2, {{%v[0-9]+}}, 0
+ %r = call i16 @llvm.vector.reduce.umin.v8i16(<8 x i16> %v)
+ ret i16 %r
+}
+
+define i32 @f_smin_v4i32(<4 x i32> %v) {
+; CHECK-LABEL: f_smin_v4i32:
+; CHECK-NOT: vpk
+; CHECK-NOT: vrep
+; CHECK: vsldb
+; CHECK: vmnf
+; CHECK: vlgvf %r2, {{%v[0-9]+}}, 0
+ %r = call i32 @llvm.vector.reduce.smin.v4i32(<4 x i32> %v)
+ ret i32 %r
+}
+
+define i32 @f_umax_v4i32(<4 x i32> %v) {
+; CHECK-LABEL: f_umax_v4i32:
+; CHECK-NOT: vpk
+; CHECK-NOT: vrep
+; CHECK: vsldb
+; CHECK: vmxlf
+; CHECK: vlgvf %r2, {{%v[0-9]+}}, 0
+ %r = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> %v)
+ ret i32 %r
+}
+
+declare i8 @llvm.vector.reduce.smin.v16i8(<16 x i8>)
+declare i8 @llvm.vector.reduce.umax.v16i8(<16 x i8>)
+declare i16 @llvm.vector.reduce.smax.v8i16(<8 x i16>)
+declare i16 @llvm.vector.reduce.umin.v8i16(<8 x i16>)
+declare i32 @llvm.vector.reduce.smin.v4i32(<4 x i32>)
+declare i32 @llvm.vector.reduce.umax.v4i32(<4 x i32>)
>From 4ed24779078a2b4085899a1ee4f698621c726691 Mon Sep 17 00:00:00 2001
From: MarkVeerasingam <markveer70 at gmail.com>
Date: Tue, 6 Oct 2026 06:24:24 -0500
Subject: [PATCH 3/3] [SystemZ] added comments to vector reduce min/max
lowering halving tree
---
.../Target/SystemZ/SystemZISelLowering.cpp | 35 ++++++++++++++-----
1 file changed, 26 insertions(+), 9 deletions(-)
diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
index e83085e8f99f1..6eb245cdcc3bf 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
@@ -11479,35 +11479,52 @@ SDValue SystemZTargetLowering::lowerVECREDUCE_ADD(SDValue Op,
}
SDValue SystemZTargetLowering::lowerVECREDUCE_MINMAX(SDValue Op,
- SelectionDAG &DAG) const {
+ SelectionDAG &DAG) const {
unsigned Opcode = Op.getOpcode();
EVT VT = Op.getValueType();
SDValue V = Op.getOperand(0);
EVT VecVT = V.getValueType();
SDLoc DL(Op);
+ // Pick the elementwise min/max matching the reduction.
unsigned EltOp;
switch (Opcode) {
- case ISD::VECREDUCE_SMIN: EltOp = ISD::SMIN; break;
- case ISD::VECREDUCE_SMAX: EltOp = ISD::SMAX; break;
- case ISD::VECREDUCE_UMIN: EltOp = ISD::UMIN; break;
- case ISD::VECREDUCE_UMAX: EltOp = ISD::UMAX; break;
- default: llvm_unreachable("Unexpected opcode.");
+ case ISD::VECREDUCE_SMIN:
+ EltOp = ISD::SMIN;
+ break;
+ case ISD::VECREDUCE_SMAX:
+ EltOp = ISD::SMAX;
+ break;
+ case ISD::VECREDUCE_UMIN:
+ EltOp = ISD::UMIN;
+ break;
+ case ISD::VECREDUCE_UMAX:
+ EltOp = ISD::UMAX;
+ break;
+ default:
+ llvm_unreachable("Unexpected opcode.");
}
unsigned EltBytes = VecVT.getScalarSizeInBits() / 8;
unsigned NumElts = VecVT.getVectorNumElements();
+ // Halve the number of live lanes each step: combine lane I with lane
+ // I + Step. Lane 0 ends up with the result.
for (unsigned Step = NumElts / 2; Step >= 1; Step /= 2) {
unsigned ByteShift = Step * EltBytes;
+
+ // Shift left by Step elements (VSLDB works on bytes, hence v16i8).
+ // Other lanes are don't-care.
SDValue V16i8 = DAG.getBitcast(MVT::v16i8, V);
- SDValue Shifted16i8 = DAG.getNode(
- SystemZISD::SHL_DOUBLE, DL, MVT::v16i8, V16i8, V16i8,
- DAG.getTargetConstant(ByteShift, DL, MVT::i32));
+ SDValue Shifted16i8 =
+ DAG.getNode(SystemZISD::SHL_DOUBLE, DL, MVT::v16i8, V16i8, V16i8,
+ DAG.getTargetConstant(ByteShift, DL, MVT::i32));
SDValue Shifted = DAG.getBitcast(VecVT, Shifted16i8);
+
V = DAG.getNode(EltOp, DL, VecVT, V, Shifted);
}
+ // Return lane 0.
return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, V,
DAG.getConstant(0, DL, MVT::i32));
}
More information about the llvm-commits
mailing list