[llvm] [SystemZ] Custom lowering for vector min/max reductions (PR #229382)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Oct 6 04:38:47 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-systemz
Author: MarkVeerasingam
<details>
<summary>Changes</summary>
Keep `llvm.vector.reduce.{smin,smax,umin,umax}` unexpanded for full-width v16i8, v8i16, and v4i32 vectors, and lower them directly to `VSLDB` + `VMN`/`VMX`.
### Background
Previously, LLVM expanded these reductions into shuffle + min/max trees. On SystemZ, the shuffles were lowered to different instructions such as VMRLG, VPKG etc...
### Example
```llvm
target triple = "s390x-linux-gnu"
define i8 @<!-- -->reduce_smin_v16i8(<16 x i8> %vec) {
entry:
%r = call i8 @<!-- -->llvm.vector.reduce.smin.v16i8(<16 x i8> %vec)
ret i8 %r
}
declare i8 @<!-- -->llvm.vector.reduce.smin.v16i8(<16 x i8>)
```
Output:
```
Before After
vmrlg %v0, %v24, %v24 vsldb %v0, %v24, %v24, 8
vmnb %v0, %v24, %v0 vmnb %v0, %v24, %v0
vpkg %v1, %v0, %v0 vsldb %v1, %v0, %v0, 4
vmnb %v0, %v0, %v1 vmnb %v0, %v0, %v1
vpkf %v1, %v0, %v0 vsldb %v1, %v0, %v0, 2
vmnb %v0, %v0, %v1 vmnb %v0, %v0, %v1
vrepb %v1, %v0, 1 vsldb %v1, %v0, %v0, 1
vmnb %v0, %v0, %v1 vmnb %v0, %v0, %v1
vlgvb %r2, %v0, 0 vlgvb %r2, %v0, 0
```
## Change
Don't expand min/max reductions for full-width vectors with elements smaller than 64 bits. Lower VECREDUCE as a halving tree using VSLDB and the corresponding min/max instruction. Extract the final value from lane 0.
`v2i64`, partial vectors, and targets without vector support are left unchanged.
### Result
The instruction count stays the same, but the generated code is simpler and uses `VSLDB` instead of depending on a shuffle lowering.
### Testing
Added a regression tests covering all three vector sizes and signed/unsigned min/max reductions.
#### Author Note
This is my first time working with the SystemZ vector implementation and lowering, so I'm still getting familiar with the legal types and related details. If there is a more optimal lowering strategy, I'm happy to take a look.
---
Full diff: https://github.com/llvm/llvm-project/pull/229382.diff
4 Files Affected:
- (modified) llvm/lib/Target/SystemZ/SystemZISelLowering.cpp (+53)
- (modified) llvm/lib/Target/SystemZ/SystemZISelLowering.h (+1)
- (modified) llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp (+14-1)
- (added) llvm/test/CodeGen/SystemZ/vec-reduce-minmax.ll (+76)
``````````diff
diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
index a6cc3e94b7e7d..db1d249123242 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
@@ -495,6 +495,14 @@ SystemZTargetLowering::SystemZTargetLowering(const TargetMachine &TM,
// it with VZERO+VSUM
setOperationAction(ISD::VECREDUCE_ADD, VT, Custom);
+ // Custom-lower VECREDUCE_{SMIN,SMAX,UMIN,UMAX} to a halving tree of
+ // native VMN/VMX instructions, avoiding the generic expander's
+ // insert/extract subvector sequence on the illegal v2i32/v1i32 types.
+ setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
+ setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
+ setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom);
+ setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
+
// Map SETCCs onto one of VCE, VCH or VCHL, swapping the operands
// and inverting the result as necessary.
setOperationAction(ISD::SETCC, VT, Custom);
@@ -7270,6 +7278,11 @@ SDValue SystemZTargetLowering::LowerOperation(SDValue Op,
return lowerCTPOP(Op, DAG);
case ISD::VECREDUCE_ADD:
return lowerVECREDUCE_ADD(Op, DAG);
+ case ISD::VECREDUCE_SMIN:
+ case ISD::VECREDUCE_SMAX:
+ case ISD::VECREDUCE_UMIN:
+ case ISD::VECREDUCE_UMAX:
+ return lowerVECREDUCE_MINMAX(Op, DAG);
case ISD::ATOMIC_FENCE:
return lowerATOMIC_FENCE(Op, DAG);
case ISD::ATOMIC_SWAP:
@@ -11465,6 +11478,46 @@ SDValue SystemZTargetLowering::lowerVECREDUCE_ADD(SDValue Op,
DAG.getConstant(OpVT.getVectorNumElements() - 1, DL, MVT::i32));
}
+SDValue SystemZTargetLowering::lowerVECREDUCE_MINMAX(SDValue Op,
+ SelectionDAG &DAG) const {
+ unsigned Opcode = Op.getOpcode();
+ EVT VT = Op.getValueType();
+ SDValue V = Op.getOperand(0);
+ EVT VecVT = V.getValueType();
+ SDLoc DL(Op);
+
+ unsigned EltOp;
+ switch (Opcode) {
+ case ISD::VECREDUCE_SMIN: EltOp = ISD::SMIN; break;
+ case ISD::VECREDUCE_SMAX: EltOp = ISD::SMAX; break;
+ case ISD::VECREDUCE_UMIN: EltOp = ISD::UMIN; break;
+ case ISD::VECREDUCE_UMAX: EltOp = ISD::UMAX; break;
+ default: llvm_unreachable("Unexpected opcode.");
+ }
+
+ unsigned EltBytes = VecVT.getScalarSizeInBits() / 8;
+ unsigned NumElts = VecVT.getVectorNumElements();
+
+ // Halve the number of live lanes each step: combine lane I with lane
+ // I + Step. Lane 0 will end up with the result.
+ for (unsigned Step = NumElts / 2; Step >= 1; Step /= 2) {
+ unsigned ByteShift = Step * EltBytes;
+
+ // Shift left by Step elements (VSLDB works on bytes, hence v16i8).
+ // other lanes are don't-care.
+ SDValue V16i8 = DAG.getBitcast(MVT::v16i8, V);
+ SDValue Shifted16i8 = DAG.getNode(
+ SystemZISD::SHL_DOUBLE, DL, MVT::v16i8, V16i8, V16i8,
+ DAG.getTargetConstant(ByteShift, DL, MVT::i32));
+ SDValue Shifted = DAG.getBitcast(VecVT, Shifted16i8);
+ V = DAG.getNode(EltOp, DL, VecVT, V, Shifted);
+ }
+
+ // Return lane 0.
+ return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, V,
+ DAG.getConstant(0, DL, MVT::i32));
+}
+
static void printFunctionArgExts(const Function *F, raw_fd_ostream &OS) {
FunctionType *FT = F->getFunctionType();
const AttributeList &Attrs = F->getAttributes();
diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.h b/llvm/lib/Target/SystemZ/SystemZISelLowering.h
index 6d99fef8b568b..5a5f8504d6bac 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.h
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.h
@@ -370,6 +370,7 @@ class SystemZTargetLowering : public TargetLowering {
SDValue lowerOR(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerCTPOP(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerVECREDUCE_ADD(SDValue Op, SelectionDAG &DAG) const;
+ SDValue lowerVECREDUCE_MINMAX(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerATOMIC_LOAD(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerATOMIC_STORE(SDValue Op, SelectionDAG &DAG) const;
diff --git a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
index 06f84c78a504d..2153605bbc449 100644
--- a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
@@ -1619,7 +1619,7 @@ bool SystemZTTIImpl::shouldExpandReduction(const IntrinsicInst *II) const {
default:
return true;
// Do not expand vector.reduce.add...
- case Intrinsic::vector_reduce_add:
+ case Intrinsic::vector_reduce_add: {
auto *VType = cast<FixedVectorType>(II->getOperand(0)->getType());
// ...unless the scalar size is i64 or larger,
// or the operand vector is not full, since the
@@ -1627,4 +1627,17 @@ bool SystemZTTIImpl::shouldExpandReduction(const IntrinsicInst *II) const {
return VType->getScalarSizeInBits() >= 64 ||
VType->getPrimitiveSizeInBits() < SystemZ::VectorBits;
}
+ // Do not expand vector.reduce.{smin,smax,umin,umax}, so they reach ISel
+ // and are custom-lowered by lowerVECREDUCE_MINMAX...
+ case Intrinsic::vector_reduce_smin:
+ case Intrinsic::vector_reduce_smax:
+ case Intrinsic::vector_reduce_umin:
+ case Intrinsic::vector_reduce_umax: {
+ auto *VType = cast<FixedVectorType>(II->getOperand(0)->getType());
+ // ...unless the scalar size is i64 (the generic expansion is already
+ // a single vmrlg + vmn/vmx), or the operand vector is not full.
+ return VType->getScalarSizeInBits() >= 64 ||
+ VType->getPrimitiveSizeInBits() < SystemZ::VectorBits;
+ }
+ }
}
diff --git a/llvm/test/CodeGen/SystemZ/vec-reduce-minmax.ll b/llvm/test/CodeGen/SystemZ/vec-reduce-minmax.ll
new file mode 100644
index 0000000000000..b978aa9fbceed
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/vec-reduce-minmax.ll
@@ -0,0 +1,76 @@
+; Test vector min/max reductions on SystemZ.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+define i8 @f_smin_v16i8(<16 x i8> %v) {
+; CHECK-LABEL: f_smin_v16i8:
+; CHECK-NOT: vpk
+; CHECK-NOT: vrep
+; CHECK: vsldb
+; CHECK: vmnb
+; CHECK: vlgvb %r2, {{%v[0-9]+}}, 0
+ %r = call i8 @llvm.vector.reduce.smin.v16i8(<16 x i8> %v)
+ ret i8 %r
+}
+
+define i8 @f_umax_v16i8(<16 x i8> %v) {
+; CHECK-LABEL: f_umax_v16i8:
+; CHECK-NOT: vpk
+; CHECK-NOT: vrep
+; CHECK: vsldb
+; CHECK: vmxlb
+; CHECK: vlgvb %r2, {{%v[0-9]+}}, 0
+ %r = call i8 @llvm.vector.reduce.umax.v16i8(<16 x i8> %v)
+ ret i8 %r
+}
+
+define i16 @f_smax_v8i16(<8 x i16> %v) {
+; CHECK-LABEL: f_smax_v8i16:
+; CHECK-NOT: vpk
+; CHECK-NOT: vrep
+; CHECK: vsldb
+; CHECK: vmxh
+; CHECK: vlgvh %r2, {{%v[0-9]+}}, 0
+ %r = call i16 @llvm.vector.reduce.smax.v8i16(<8 x i16> %v)
+ ret i16 %r
+}
+
+define i16 @f_umin_v8i16(<8 x i16> %v) {
+; CHECK-LABEL: f_umin_v8i16:
+; CHECK-NOT: vpk
+; CHECK-NOT: vrep
+; CHECK: vsldb
+; CHECK: vmnlh
+; CHECK: vlgvh %r2, {{%v[0-9]+}}, 0
+ %r = call i16 @llvm.vector.reduce.umin.v8i16(<8 x i16> %v)
+ ret i16 %r
+}
+
+define i32 @f_smin_v4i32(<4 x i32> %v) {
+; CHECK-LABEL: f_smin_v4i32:
+; CHECK-NOT: vpk
+; CHECK-NOT: vrep
+; CHECK: vsldb
+; CHECK: vmnf
+; CHECK: vlgvf %r2, {{%v[0-9]+}}, 0
+ %r = call i32 @llvm.vector.reduce.smin.v4i32(<4 x i32> %v)
+ ret i32 %r
+}
+
+define i32 @f_umax_v4i32(<4 x i32> %v) {
+; CHECK-LABEL: f_umax_v4i32:
+; CHECK-NOT: vpk
+; CHECK-NOT: vrep
+; CHECK: vsldb
+; CHECK: vmxlf
+; CHECK: vlgvf %r2, {{%v[0-9]+}}, 0
+ %r = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> %v)
+ ret i32 %r
+}
+
+declare i8 @llvm.vector.reduce.smin.v16i8(<16 x i8>)
+declare i8 @llvm.vector.reduce.umax.v16i8(<16 x i8>)
+declare i16 @llvm.vector.reduce.smax.v8i16(<8 x i16>)
+declare i16 @llvm.vector.reduce.umin.v8i16(<8 x i16>)
+declare i32 @llvm.vector.reduce.smin.v4i32(<4 x i32>)
+declare i32 @llvm.vector.reduce.umax.v4i32(<4 x i32>)
``````````
</details>
https://github.com/llvm/llvm-project/pull/229382
More information about the llvm-commits
mailing list