[llvm] 1b10cd9 - [SLP]Do not blacklist ordered-reduction operands on failed root attempt
via llvm-commits
llvm-commits at lists.llvm.org
Tue Jul 14 06:58:08 PDT 2026
Author: Alexey Bataev
Date: 2026-07-14T09:58:03-04:00
New Revision: 1b10cd91d12d783fe8754c100a3afaacfee3bf9f
URL: https://github.com/llvm/llvm-project/commit/1b10cd91d12d783fe8754c100a3afaacfee3bf9f
DIFF: https://github.com/llvm/llvm-project/commit/1b10cd91d12d783fe8754c100a3afaacfee3bf9f.diff
LOG: [SLP]Do not blacklist ordered-reduction operands on failed root attempt
An ordered reduction pulls its leaf operands into ReductionOps via the
fallback in matchAssociativeReduction. When such a reduction fails to
vectorize, marking every reduction op as analyzed also blocks those leaves,
which may still be valid reduction roots on their own.
Fixes https://github.com/llvm/llvm-project/pull/185320#issuecomment-4925949343
Reviewers: hiraditya, bababuck, RKSimon
Pull Request: https://github.com/llvm/llvm-project/pull/208511
Added:
Modified:
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
llvm/test/Transforms/SLPVectorizer/AArch64/non-power-of-2-with-adjusted-gathers.ll
llvm/test/Transforms/SLPVectorizer/X86/control-deps-schedule-data-recalculate.ll
llvm/test/Transforms/SLPVectorizer/X86/reduction-root-multiuse-same-opcode.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 7e22ba3bd149c..4202ea299770a 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -29735,9 +29735,17 @@ class HorizontalReduction {
[](ArrayRef<Value *> RedV) {
return RedV.size() < 2 || !allConstant(RedV) || !isSplat(RedV);
})) {
- for (ReductionOpsType &RdxOps : ReductionOps)
- for (Value *RdxOp : RdxOps)
- V.analyzedReductionRoot(cast<Instruction>(RdxOp));
+ // For ordered reductions the leaves are pulled in via the fallback in
+ // matchAssociativeReduction and may still be valid reduction roots on
+ // their own; only the root is a dead end. For unordered reductions keep
+ // marking every reduction op as analyzed.
+ if (RK == ReductionOrdering::Ordered) {
+ V.analyzedReductionRoot(cast<Instruction>(ReductionRoot));
+ } else {
+ for (ReductionOpsType &RdxOps : ReductionOps)
+ for (Value *RdxOp : RdxOps)
+ V.analyzedReductionRoot(cast<Instruction>(RdxOp));
+ }
return nullptr;
}
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/non-power-of-2-with-adjusted-gathers.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/non-power-of-2-with-adjusted-gathers.ll
index ef92fe5d782ae..2e5259ad9bfa1 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/non-power-of-2-with-adjusted-gathers.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/non-power-of-2-with-adjusted-gathers.ll
@@ -36,23 +36,23 @@ define i1 @test(ptr %arg, ptr %arg1, i1 %arg2, i1 %arg3, i1 %arg4) {
; CHECK-NEXT: [[AND63:%.*]] = and i1 [[ARG2]], [[ICMP62]]
; CHECK-NEXT: [[ICMP65:%.*]] = icmp ult ptr [[GETELEMENTPTR]], null
; CHECK-NEXT: [[AND66:%.*]] = and i1 [[ICMP65]], false
-; CHECK-NEXT: [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP6]])
-; CHECK-NEXT: [[OP_RDX24:%.*]] = or i1 [[TMP8]], [[AND]]
-; CHECK-NEXT: [[OP_RDX25:%.*]] = or i1 false, [[AND12]]
+; CHECK-NEXT: [[TMP15:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP6]])
+; CHECK-NEXT: [[OP_RDX30:%.*]] = or i1 [[TMP15]], [[AND]]
+; CHECK-NEXT: [[OP_RDX31:%.*]] = or i1 false, [[AND12]]
; CHECK-NEXT: [[OP_RDX26:%.*]] = or i1 [[AND15]], false
; CHECK-NEXT: [[OP_RDX27:%.*]] = or i1 false, [[AND47]]
; CHECK-NEXT: [[OP_RDX28:%.*]] = or i1 [[AND50]], [[AND53]]
; CHECK-NEXT: [[OP_RDX29:%.*]] = or i1 [[AND56]], [[AND59]]
-; CHECK-NEXT: [[OP_RDX30:%.*]] = or i1 [[AND63]], [[AND66]]
-; CHECK-NEXT: [[OP_RDX31:%.*]] = or i1 [[ARG3]], [[ARG2]]
+; CHECK-NEXT: [[OP_RDX33:%.*]] = or i1 [[AND63]], [[AND66]]
+; CHECK-NEXT: [[OP_RDX34:%.*]] = or i1 [[ARG3]], [[ARG2]]
; CHECK-NEXT: [[OP_RDX32:%.*]] = or i1 [[ARG4]], false
-; CHECK-NEXT: [[OP_RDX33:%.*]] = or i1 [[OP_RDX24]], [[OP_RDX25]]
-; CHECK-NEXT: [[OP_RDX34:%.*]] = or i1 [[OP_RDX26]], [[OP_RDX27]]
-; CHECK-NEXT: [[OP_RDX35:%.*]] = or i1 [[OP_RDX28]], [[OP_RDX29]]
; CHECK-NEXT: [[OP_RDX36:%.*]] = or i1 [[OP_RDX30]], [[OP_RDX31]]
+; CHECK-NEXT: [[OP_RDX38:%.*]] = or i1 [[OP_RDX26]], [[OP_RDX27]]
+; CHECK-NEXT: [[OP_RDX35:%.*]] = or i1 [[OP_RDX28]], [[OP_RDX29]]
; CHECK-NEXT: [[OP_RDX37:%.*]] = or i1 [[OP_RDX33]], [[OP_RDX34]]
-; CHECK-NEXT: [[OP_RDX38:%.*]] = or i1 [[OP_RDX35]], [[OP_RDX36]]
-; CHECK-NEXT: [[OP_RDX39:%.*]] = or i1 [[OP_RDX37]], [[OP_RDX38]]
+; CHECK-NEXT: [[OP_RDX46:%.*]] = or i1 [[OP_RDX36]], [[OP_RDX38]]
+; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[OP_RDX35]], [[OP_RDX37]]
+; CHECK-NEXT: [[OP_RDX39:%.*]] = or i1 [[OP_RDX46]], [[TMP8]]
; CHECK-NEXT: [[OP_RDX40:%.*]] = or i1 [[OP_RDX39]], [[OP_RDX32]]
; CHECK-NEXT: ret i1 [[OP_RDX40]]
;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/control-deps-schedule-data-recalculate.ll b/llvm/test/Transforms/SLPVectorizer/X86/control-deps-schedule-data-recalculate.ll
index 1ec65da663fee..1d28e37cb2efc 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/control-deps-schedule-data-recalculate.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/control-deps-schedule-data-recalculate.ll
@@ -7,19 +7,18 @@ define i32 @test(i32 %mul) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[H:%.*]] = alloca [4 x i32], align 16
; CHECK-NEXT: [[ADD:%.*]] = add i32 0, 0
-; CHECK-NEXT: [[ADD4:%.*]] = add i32 [[ADD]], 0
-; CHECK-NEXT: [[CALL:%.*]] = tail call i32 @f1(i32 [[ADD4]])
+; CHECK-NEXT: [[CALL:%.*]] = tail call i32 @f1(i32 [[ADD]])
; CHECK-NEXT: [[MUL1:%.*]] = shl i32 0, 1
; CHECK-NEXT: [[ADD5:%.*]] = add i32 [[CALL]], [[MUL1]]
; CHECK-NEXT: store i32 [[ADD5]], ptr [[H]], align 16
; CHECK-NEXT: [[ARRAYINIT_ELEMENT:%.*]] = getelementptr i8, ptr [[H]], i64 4
; CHECK-NEXT: [[ADD6:%.*]] = add i32 0, 0
; CHECK-NEXT: [[ADD7:%.*]] = add i32 [[ADD6]], [[MUL]]
-; CHECK-NEXT: [[ADD9:%.*]] = add i32 [[ADD7]], [[ADD4]]
+; CHECK-NEXT: [[ADD9:%.*]] = add i32 [[ADD7]], [[ADD]]
; CHECK-NEXT: store i32 [[ADD9]], ptr [[ARRAYINIT_ELEMENT]], align 4
; CHECK-NEXT: [[ARRAYINIT_ELEMENT10:%.*]] = getelementptr i8, ptr [[H]], i64 8
-; CHECK-NEXT: [[ADD11:%.*]] = or i32 [[ADD]], 0
-; CHECK-NEXT: [[ADD12:%.*]] = add i32 [[ADD11]], [[ADD4]]
+; CHECK-NEXT: [[ADD11:%.*]] = or i32 0, 0
+; CHECK-NEXT: [[ADD12:%.*]] = add i32 [[ADD11]], [[ADD]]
; CHECK-NEXT: store i32 [[ADD12]], ptr [[ARRAYINIT_ELEMENT10]], align 8
; CHECK-NEXT: [[ARRAYINIT_ELEMENT13:%.*]] = getelementptr i8, ptr [[H]], i64 12
; CHECK-NEXT: store i32 0, ptr [[ARRAYINIT_ELEMENT13]], align 4
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reduction-root-multiuse-same-opcode.ll b/llvm/test/Transforms/SLPVectorizer/X86/reduction-root-multiuse-same-opcode.ll
index b1bda3f9db86a..64c376295bed2 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reduction-root-multiuse-same-opcode.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reduction-root-multiuse-same-opcode.ll
@@ -5,25 +5,9 @@ define i32 @test(ptr %p, i32 %n, ptr %p2) {
; CHECK-LABEL: define i32 @test(
; CHECK-SAME: ptr [[P:%.*]], i32 [[N:%.*]], ptr [[P2:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i16>, ptr [[P]], align 2
-; CHECK-NEXT: [[TMP1:%.*]] = sext <4 x i16> [[TMP0]] to <4 x i32>
-; CHECK-NEXT: [[ARRAYIDX_4:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 8
-; CHECK-NEXT: [[TMP7:%.*]] = load i16, ptr [[ARRAYIDX_4]], align 2
-; CHECK-NEXT: [[CONV_4:%.*]] = sext i16 [[TMP7]] to i32
-; CHECK-NEXT: [[ARRAYIDX_5:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 10
-; CHECK-NEXT: [[TMP3:%.*]] = load i16, ptr [[ARRAYIDX_5]], align 2
-; CHECK-NEXT: [[CONV_5:%.*]] = sext i16 [[TMP3]] to i32
-; CHECK-NEXT: [[ARRAYIDX_6:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 12
-; CHECK-NEXT: [[TMP4:%.*]] = load i16, ptr [[ARRAYIDX_6]], align 2
-; CHECK-NEXT: [[CONV_6:%.*]] = sext i16 [[TMP4]] to i32
-; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP1]])
-; CHECK-NEXT: [[OP_RDX:%.*]] = add i32 [[TMP5]], [[CONV_4]]
-; CHECK-NEXT: [[OP_RDX1:%.*]] = add i32 [[CONV_5]], [[CONV_6]]
-; CHECK-NEXT: [[OP_RDX2:%.*]] = add i32 [[OP_RDX]], [[OP_RDX1]]
-; CHECK-NEXT: [[ARRAYIDX_7:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 14
-; CHECK-NEXT: [[TMP6:%.*]] = load i16, ptr [[ARRAYIDX_7]], align 2
-; CHECK-NEXT: [[CONV_7:%.*]] = sext i16 [[TMP6]] to i32
-; CHECK-NEXT: [[TMP2:%.*]] = add nsw i32 [[OP_RDX2]], [[CONV_7]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <8 x i16>, ptr [[P]], align 2
+; CHECK-NEXT: [[TMP1:%.*]] = sext <8 x i16> [[TMP0]] to <8 x i32>
+; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TMP1]])
; CHECK-NEXT: store i32 [[TMP2]], ptr [[P2]], align 4
; CHECK-NEXT: [[ADD1:%.*]] = add nsw i32 [[TMP2]], [[N]]
; CHECK-NEXT: ret i32 [[ADD1]]
More information about the llvm-commits
mailing list