[llvm] [VectorCombine] foldShuffleChainsToReduce - add FADD/FMUL handling (PR #201302)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Jun 3 02:39:59 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-vectorizers
@llvm/pr-subscribers-llvm-transforms
Author: 陈子昂 (Michael-Chen-NJU)
<details>
<summary>Changes</summary>
Extend `foldShuffleChainsToReduce` to fold shuffle-reduction chains of fadd/fmul into the corresponding vector reduction intrinsics (llvm.vector.reduce.fadd / llvm.vector.reduce.fmul).
The transformation requires the `reassoc` fast-math flag on every binop in the chain based on the [langspec](https://llvm.org/docs/LangRef.html#rewrite-based-flags). The output intrinsic receives the intersection of all binops' FMF, and the identity start value is selected via ConstantExpr::getBinOpIdentity (-0.0 for fadd, 1.0 for fmul, respecting nsz for the sign of zero).
Fixes #<!-- -->199030.
---
Full diff: https://github.com/llvm/llvm-project/pull/201302.diff
3 Files Affected:
- (modified) llvm/lib/Transforms/Vectorize/VectorCombine.cpp (+46-4)
- (added) llvm/test/Transforms/VectorCombine/X86/fold-shuffle-chains-to-reduce-fp.ll (+34)
- (modified) llvm/test/Transforms/VectorCombine/fold-shuffle-chains-to-reduce.ll (+134)
``````````diff
diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
index e694dc1fafe5f..74bb710827266 100644
--- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
+++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
@@ -3960,6 +3960,10 @@ bool VectorCombine::foldShuffleChainsToReduce(Instruction &I) {
std::optional<unsigned int> CommonCallOp = std::nullopt;
std::optional<Instruction::BinaryOps> CommonBinOp = std::nullopt;
+ // For floating-point reductions, track FMF intersection across all binops.
+ FastMathFlags CommonFMF;
+ bool IsFloatReduction = false;
+
bool IsFirstCallOrBinInst = true;
bool ShouldBeCallOrBinInst = true;
@@ -4078,7 +4082,9 @@ bool VectorCombine::foldShuffleChainsToReduce(Instruction &I) {
case BinaryOperator::Mul:
case BinaryOperator::Or:
case BinaryOperator::And:
- case BinaryOperator::Xor: {
+ case BinaryOperator::Xor:
+ case BinaryOperator::FAdd:
+ case BinaryOperator::FMul: {
auto *Op0 = BinOp->getOperand(0);
auto *Op1 = BinOp->getOperand(1);
PrevVecV[0] = Op0;
@@ -4088,6 +4094,20 @@ bool VectorCombine::foldShuffleChainsToReduce(Instruction &I) {
default:
return false;
}
+
+ // For FP reductions, require reassoc on every binop and collect FMF.
+ if (*CommonBinOp == Instruction::FAdd ||
+ *CommonBinOp == Instruction::FMul) {
+ if (!BinOp->hasAllowReassoc())
+ return false;
+ if (!IsFloatReduction) {
+ CommonFMF = BinOp->getFastMathFlags();
+ IsFloatReduction = true;
+ } else {
+ CommonFMF &= BinOp->getFastMathFlags();
+ }
+ }
+
ShouldBeCallOrBinInst ^= 1;
OrigCost +=
@@ -4172,6 +4192,13 @@ bool VectorCombine::foldShuffleChainsToReduce(Instruction &I) {
Intrinsic::ID ReducedOp =
(CommonCallOp ? getMinMaxReductionIntrinsicID(*CommonCallOp)
: getReductionForBinop(*CommonBinOp));
+ // getReductionForBinop only covers integer ops; handle FP here.
+ if (ReducedOp == Intrinsic::not_intrinsic && CommonBinOp) {
+ if (*CommonBinOp == Instruction::FAdd)
+ ReducedOp = Intrinsic::vector_reduce_fadd;
+ else if (*CommonBinOp == Instruction::FMul)
+ ReducedOp = Intrinsic::vector_reduce_fmul;
+ }
if (!ReducedOp)
return false;
@@ -4189,7 +4216,12 @@ bool VectorCombine::foldShuffleChainsToReduce(Instruction &I) {
CostKind, 0, ReduceVecTy);
}
- IntrinsicCostAttributes ICA(ReducedOp, ReduceVecTy, {ReduceVecTy});
+ IntrinsicCostAttributes ICA(
+ ReducedOp, ReduceVecTy->getElementType(),
+ IsFloatReduction
+ ? SmallVector<Type *, 2>{ReduceVecTy->getElementType(), ReduceVecTy}
+ : SmallVector<Type *, 2>{ReduceVecTy},
+ IsFloatReduction ? CommonFMF : FastMathFlags());
NewCost += TTI.getIntrinsicInstrCost(ICA, CostKind);
LLVM_DEBUG(dbgs() << "Found reduction shuffle chain: " << I << "\n OldCost : "
@@ -4202,8 +4234,18 @@ bool VectorCombine::foldShuffleChainsToReduce(Instruction &I) {
if (IsPartialReduction)
ReduceInput = Builder.CreateShuffleVector(FinalVecV, ExtractMask);
- auto *ReducedResult = Builder.CreateIntrinsic(
- ReducedOp, {ReduceInput->getType()}, {ReduceInput});
+ CallInst *ReducedResult;
+ if (IsFloatReduction) {
+ Value *Identity = ConstantExpr::getBinOpIdentity(
+ *CommonBinOp, ReduceVecTy->getElementType(), /*AllowRHSConstant=*/false,
+ CommonFMF.noSignedZeros());
+ ReducedResult = Builder.CreateIntrinsic(ReducedOp, {ReduceVecTy},
+ {Identity, ReduceInput});
+ ReducedResult->setFastMathFlags(CommonFMF);
+ } else {
+ ReducedResult =
+ Builder.CreateIntrinsic(ReducedOp, {ReduceVecTy}, {ReduceInput});
+ }
replaceValue(I, *ReducedResult);
return true;
diff --git a/llvm/test/Transforms/VectorCombine/X86/fold-shuffle-chains-to-reduce-fp.ll b/llvm/test/Transforms/VectorCombine/X86/fold-shuffle-chains-to-reduce-fp.ll
new file mode 100644
index 0000000000000..1da7cac67e80f
--- /dev/null
+++ b/llvm/test/Transforms/VectorCombine/X86/fold-shuffle-chains-to-reduce-fp.ll
@@ -0,0 +1,34 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -passes=vector-combine -mtriple=x86_64-unknown-linux-gnu -S | FileCheck %s
+
+; Verify that FP reduction folding works correctly with the X86 cost model.
+
+define float @test_reduce_v8f32_fadd_x86(<8 x float> %a0) {
+; CHECK-LABEL: define float @test_reduce_v8f32_fadd_x86(
+; CHECK-SAME: <8 x float> [[A0:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = call reassoc float @llvm.vector.reduce.fadd.v8f32(float -0.000000e+00, <8 x float> [[A0]])
+; CHECK-NEXT: ret float [[TMP1]]
+;
+ %1 = shufflevector <8 x float> %a0, <8 x float> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
+ %2 = fadd reassoc <8 x float> %a0, %1
+ %3 = shufflevector <8 x float> %2, <8 x float> poison, <8 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+ %4 = fadd reassoc <8 x float> %2, %3
+ %5 = shufflevector <8 x float> %4, <8 x float> poison, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+ %6 = fadd reassoc <8 x float> %4, %5
+ %7 = extractelement <8 x float> %6, i64 0
+ ret float %7
+}
+
+define float @test_reduce_v4f32_fmul_x86(<4 x float> %a0) {
+; CHECK-LABEL: define float @test_reduce_v4f32_fmul_x86(
+; CHECK-SAME: <4 x float> [[A0:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = call reassoc float @llvm.vector.reduce.fmul.v4f32(float 1.000000e+00, <4 x float> [[A0]])
+; CHECK-NEXT: ret float [[TMP1]]
+;
+ %1 = shufflevector <4 x float> %a0, <4 x float> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
+ %2 = fmul reassoc <4 x float> %a0, %1
+ %3 = shufflevector <4 x float> %2, <4 x float> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
+ %4 = fmul reassoc <4 x float> %2, %3
+ %5 = extractelement <4 x float> %4, i64 0
+ ret float %5
+}
diff --git a/llvm/test/Transforms/VectorCombine/fold-shuffle-chains-to-reduce.ll b/llvm/test/Transforms/VectorCombine/fold-shuffle-chains-to-reduce.ll
index 8111309dc5f53..54931f59c2638 100644
--- a/llvm/test/Transforms/VectorCombine/fold-shuffle-chains-to-reduce.ll
+++ b/llvm/test/Transforms/VectorCombine/fold-shuffle-chains-to-reduce.ll
@@ -303,3 +303,137 @@ define i32 @test_no_partial_reduce_v6i32_add(<6 x i32> %a) {
%r = extractelement <6 x i32> %a2, i64 0
ret i32 %r
}
+
+; FADD with reassoc - should fold to vector.reduce.fadd
+define float @test_reduce_v8f32_fadd(<8 x float> %a0) {
+; CHECK-LABEL: define float @test_reduce_v8f32_fadd(
+; CHECK-SAME: <8 x float> [[A0:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = call reassoc float @llvm.vector.reduce.fadd.v8f32(float -0.000000e+00, <8 x float> [[A0]])
+; CHECK-NEXT: ret float [[TMP1]]
+;
+ %1 = shufflevector <8 x float> %a0, <8 x float> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
+ %2 = fadd reassoc <8 x float> %a0, %1
+ %3 = shufflevector <8 x float> %2, <8 x float> poison, <8 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+ %4 = fadd reassoc <8 x float> %2, %3
+ %5 = shufflevector <8 x float> %4, <8 x float> poison, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+ %6 = fadd reassoc <8 x float> %4, %5
+ %7 = extractelement <8 x float> %6, i64 0
+ ret float %7
+}
+
+; FADD with reassoc+nsz+nnan - should fold preserving flags
+define float @test_reduce_v4f32_fadd_flags(<4 x float> %a0) {
+; CHECK-LABEL: define float @test_reduce_v4f32_fadd_flags(
+; CHECK-SAME: <4 x float> [[A0:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = call reassoc nnan nsz float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> [[A0]])
+; CHECK-NEXT: ret float [[TMP1]]
+;
+ %1 = shufflevector <4 x float> %a0, <4 x float> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
+ %2 = fadd reassoc nnan nsz <4 x float> %a0, %1
+ %3 = shufflevector <4 x float> %2, <4 x float> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
+ %4 = fadd reassoc nnan nsz <4 x float> %2, %3
+ %5 = extractelement <4 x float> %4, i64 0
+ ret float %5
+}
+
+; FMUL with reassoc - should fold to vector.reduce.fmul
+define float @test_reduce_v4f32_fmul(<4 x float> %a0) {
+; CHECK-LABEL: define float @test_reduce_v4f32_fmul(
+; CHECK-SAME: <4 x float> [[A0:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = call reassoc float @llvm.vector.reduce.fmul.v4f32(float 1.000000e+00, <4 x float> [[A0]])
+; CHECK-NEXT: ret float [[TMP1]]
+;
+ %1 = shufflevector <4 x float> %a0, <4 x float> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
+ %2 = fmul reassoc <4 x float> %a0, %1
+ %3 = shufflevector <4 x float> %2, <4 x float> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
+ %4 = fmul reassoc <4 x float> %2, %3
+ %5 = extractelement <4 x float> %4, i64 0
+ ret float %5
+}
+
+; Double type FADD with reassoc
+define double @test_reduce_v4f64_fadd(<4 x double> %a0) {
+; CHECK-LABEL: define double @test_reduce_v4f64_fadd(
+; CHECK-SAME: <4 x double> [[A0:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = call reassoc double @llvm.vector.reduce.fadd.v4f64(double -0.000000e+00, <4 x double> [[A0]])
+; CHECK-NEXT: ret double [[TMP1]]
+;
+ %1 = shufflevector <4 x double> %a0, <4 x double> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
+ %2 = fadd reassoc <4 x double> %a0, %1
+ %3 = shufflevector <4 x double> %2, <4 x double> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
+ %4 = fadd reassoc <4 x double> %2, %3
+ %5 = extractelement <4 x double> %4, i64 0
+ ret double %5
+}
+
+; Negative test: FADD without reassoc - should NOT fold
+define float @test_no_reduce_v4f32_fadd_no_reassoc(<4 x float> %a0) {
+; CHECK-LABEL: define float @test_no_reduce_v4f32_fadd_no_reassoc(
+; CHECK-SAME: <4 x float> [[A0:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x float> [[A0]], <4 x float> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP2:%.*]] = fadd <4 x float> [[A0]], [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP4:%.*]] = fadd <4 x float> [[TMP2]], [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP4]], i64 0
+; CHECK-NEXT: ret float [[TMP5]]
+;
+ %1 = shufflevector <4 x float> %a0, <4 x float> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
+ %2 = fadd <4 x float> %a0, %1
+ %3 = shufflevector <4 x float> %2, <4 x float> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
+ %4 = fadd <4 x float> %2, %3
+ %5 = extractelement <4 x float> %4, i64 0
+ ret float %5
+}
+
+; Negative test: FMUL without reassoc - should NOT fold
+define float @test_no_reduce_v4f32_fmul_no_reassoc(<4 x float> %a0) {
+; CHECK-LABEL: define float @test_no_reduce_v4f32_fmul_no_reassoc(
+; CHECK-SAME: <4 x float> [[A0:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x float> [[A0]], <4 x float> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP2:%.*]] = fmul <4 x float> [[A0]], [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP4:%.*]] = fmul <4 x float> [[TMP2]], [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP4]], i64 0
+; CHECK-NEXT: ret float [[TMP5]]
+;
+ %1 = shufflevector <4 x float> %a0, <4 x float> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
+ %2 = fmul <4 x float> %a0, %1
+ %3 = shufflevector <4 x float> %2, <4 x float> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
+ %4 = fmul <4 x float> %2, %3
+ %5 = extractelement <4 x float> %4, i64 0
+ ret float %5
+}
+
+; Negative test: FADD with partial reassoc (only on one binop) - should NOT fold
+define float @test_no_reduce_v4f32_fadd_partial_reassoc(<4 x float> %a0) {
+; CHECK-LABEL: define float @test_no_reduce_v4f32_fadd_partial_reassoc(
+; CHECK-SAME: <4 x float> [[A0:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x float> [[A0]], <4 x float> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP2:%.*]] = fadd <4 x float> [[A0]], [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP4:%.*]] = fadd reassoc <4 x float> [[TMP2]], [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP4]], i64 0
+; CHECK-NEXT: ret float [[TMP5]]
+;
+ %1 = shufflevector <4 x float> %a0, <4 x float> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
+ %2 = fadd <4 x float> %a0, %1
+ %3 = shufflevector <4 x float> %2, <4 x float> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
+ %4 = fadd reassoc <4 x float> %2, %3
+ %5 = extractelement <4 x float> %4, i64 0
+ ret float %5
+}
+
+; FADD with different FMF on each binop - output should have intersection
+define float @test_reduce_v4f32_fadd_fmf_intersect(<4 x float> %a0) {
+; CHECK-LABEL: define float @test_reduce_v4f32_fadd_fmf_intersect(
+; CHECK-SAME: <4 x float> [[A0:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = call reassoc nnan float @llvm.vector.reduce.fadd.v4f32(float -0.000000e+00, <4 x float> [[A0]])
+; CHECK-NEXT: ret float [[TMP1]]
+;
+ %1 = shufflevector <4 x float> %a0, <4 x float> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
+ %2 = fadd reassoc nnan nsz <4 x float> %a0, %1
+ %3 = shufflevector <4 x float> %2, <4 x float> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
+ %4 = fadd reassoc nnan <4 x float> %2, %3
+ %5 = extractelement <4 x float> %4, i64 0
+ ret float %5
+}
``````````
</details>
https://github.com/llvm/llvm-project/pull/201302
More information about the llvm-commits
mailing list