[llvm] 571819c - [SLP]Recalculate dependencies for all cleared entries
Alexey Bataev via llvm-commits
llvm-commits at lists.llvm.org
Fri Dec 26 11:18:11 PST 2025
Author: Alexey Bataev
Date: 2025-12-26T11:17:14-08:00
New Revision: 571819cb7931f071c09b4129d8ec130ef63fe8dd
URL: https://github.com/llvm/llvm-project/commit/571819cb7931f071c09b4129d8ec130ef63fe8dd
DIFF: https://github.com/llvm/llvm-project/commit/571819cb7931f071c09b4129d8ec130ef63fe8dd.diff
LOG: [SLP]Recalculate dependencies for all cleared entries
Need to recalculate the dependencies for all cleared items to avoid
a crash, if the entry is used in other vector nodes
Fixes #173469
Added:
llvm/test/Transforms/SLPVectorizer/X86/non-schedulable-recalculate-deps.ll
llvm/test/Transforms/SLPVectorizer/X86/non-schedulable-with-copyable-op.ll
Modified:
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index b78cfca8436d8..6fd9759521543 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -5905,7 +5905,8 @@ class slpvectorizer::BoUpSLP {
/// bundles which depend on the original bundle.
void calculateDependencies(ScheduleBundle &Bundle, bool InsertInReadyList,
BoUpSLP *SLP,
- ArrayRef<ScheduleData *> ControlDeps = {});
+ ArrayRef<ScheduleData *> ControlDeps = {},
+ bool NonSchedulable = false);
/// Sets all instruction in the scheduling region to un-scheduled.
void resetSchedule();
@@ -21269,9 +21270,7 @@ BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
if (ScheduleData *OpSD = getScheduleData(Op);
OpSD && OpSD->hasValidDependencies()) {
OpSD->clearDirectDependencies();
- if (RegionHasStackSave ||
- !isGuaranteedToTransferExecutionToSuccessor(OpSD->getInst()))
- ControlDependentMembers.push_back(OpSD);
+ ControlDependentMembers.push_back(OpSD);
}
}
}
@@ -21279,7 +21278,7 @@ BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
if (!ControlDependentMembers.empty()) {
ScheduleBundle Invalid = ScheduleBundle::invalid();
calculateDependencies(Invalid, /*InsertInReadyList=*/true, SLP,
- ControlDependentMembers);
+ ControlDependentMembers, /*NonSchedulable=*/true);
}
return nullptr;
}
@@ -21663,7 +21662,7 @@ void BoUpSLP::BlockScheduling::initScheduleData(Instruction *FromI,
void BoUpSLP::BlockScheduling::calculateDependencies(
ScheduleBundle &Bundle, bool InsertInReadyList, BoUpSLP *SLP,
- ArrayRef<ScheduleData *> ControlDeps) {
+ ArrayRef<ScheduleData *> ControlDeps, bool NonSchedulable) {
SmallVector<ScheduleEntity *> WorkList;
auto ProcessNode = [&](ScheduleEntity *SE) {
if (auto *CD = dyn_cast<ScheduleCopyableData>(SE)) {
@@ -21748,7 +21747,8 @@ void BoUpSLP::BlockScheduling::calculateDependencies(
// The operand is a copyable element - skip.
unsigned &NumOps = UserToNumOps.try_emplace(U, 0).first->getSecond();
++NumOps;
- if (areAllOperandsReplacedByCopyableData(
+ if (!NonSchedulable &&
+ areAllOperandsReplacedByCopyableData(
cast<Instruction>(U), BundleMember->getInst(), *SLP, NumOps))
continue;
BundleMember->incDependencies();
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/non-schedulable-recalculate-deps.ll b/llvm/test/Transforms/SLPVectorizer/X86/non-schedulable-recalculate-deps.ll
new file mode 100644
index 0000000000000..566cf106ea47f
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/non-schedulable-recalculate-deps.ll
@@ -0,0 +1,61 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+
+define void @test(i32 %g, i1 %tobool1.not) {
+; CHECK-LABEL: define void @test(
+; CHECK-SAME: i32 [[G:%.*]], i1 [[TOBOOL1_NOT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[WHILE_BODY:.*]]
+; CHECK: [[WHILE_BODY]]:
+; CHECK-NEXT: [[G_ADDR_0334:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[REASS_SUB35:%.*]], %[[J:.*]] ]
+; CHECK-NEXT: [[I_ADDR_032:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[SUB14:%.*]], %[[J]] ]
+; CHECK-NEXT: br i1 [[TOBOOL1_NOT]], label %[[IF_END:.*]], label %[[J]]
+; CHECK: [[IF_END]]:
+; CHECK-NEXT: [[SUB3:%.*]] = add i32 [[G]], 1
+; CHECK-NEXT: [[REM6:%.*]] = srem i32 1, [[G]]
+; CHECK-NEXT: [[SUB8:%.*]] = add i32 [[REM6]], 1
+; CHECK-NEXT: [[REM10:%.*]] = srem i32 1, [[G]]
+; CHECK-NEXT: [[SUB11:%.*]] = add i32 [[REM10]], 1
+; CHECK-NEXT: [[SUB12:%.*]] = add i32 [[SUB11]], [[G]]
+; CHECK-NEXT: br label %[[J]]
+; CHECK: [[J]]:
+; CHECK-NEXT: [[I_ADDR_1:%.*]] = phi i32 [ [[I_ADDR_032]], %[[WHILE_BODY]] ], [ [[SUB8]], %[[IF_END]] ]
+; CHECK-NEXT: [[G_ADDR_1:%.*]] = phi i32 [ [[G_ADDR_0334]], %[[WHILE_BODY]] ], [ [[SUB12]], %[[IF_END]] ]
+; CHECK-NEXT: [[F_1:%.*]] = phi i32 [ [[G]], %[[WHILE_BODY]] ], [ [[SUB3]], %[[IF_END]] ]
+; CHECK-NEXT: [[I_ADDR_1_FR11:%.*]] = freeze i32 [[I_ADDR_1]]
+; CHECK-NEXT: [[DIV:%.*]] = select i1 [[TOBOOL1_NOT]], i32 [[I_ADDR_1_FR11]], i32 0
+; CHECK-NEXT: [[SUB14]] = or i32 [[DIV]], [[G]]
+; CHECK-NEXT: [[F_1_FR10:%.*]] = freeze i32 [[F_1]]
+; CHECK-NEXT: [[DIV16:%.*]] = select i1 [[TOBOOL1_NOT]], i32 [[F_1_FR10]], i32 0
+; CHECK-NEXT: [[REASS_SUB35]] = or i32 [[DIV16]], [[G_ADDR_1]]
+; CHECK-NEXT: br label %[[WHILE_BODY]]
+;
+entry:
+ br label %while.body
+
+while.body:
+ %g.addr.0334 = phi i32 [ 0, %entry ], [ %reass.sub35, %j ]
+ %i.addr.032 = phi i32 [ 0, %entry ], [ %sub14, %j ]
+ br i1 %tobool1.not, label %if.end, label %j
+
+if.end:
+ %sub3 = add i32 %g, 1
+ %rem6 = srem i32 1, %g
+ %sub8 = add i32 %rem6, 1
+ %rem10 = srem i32 1, %g
+ %sub11 = add i32 %rem10, 1
+ %sub12 = add i32 %sub11, %g
+ br label %j
+
+j:
+ %i.addr.1 = phi i32 [ %i.addr.032, %while.body ], [ %sub8, %if.end ]
+ %g.addr.1 = phi i32 [ %g.addr.0334, %while.body ], [ %sub12, %if.end ]
+ %f.1 = phi i32 [ %g, %while.body ], [ %sub3, %if.end ]
+ %i.addr.1.fr11 = freeze i32 %i.addr.1
+ %div = select i1 %tobool1.not, i32 %i.addr.1.fr11, i32 0
+ %sub14 = or i32 %div, %g
+ %f.1.fr10 = freeze i32 %f.1
+ %div16 = select i1 %tobool1.not, i32 %f.1.fr10, i32 0
+ %reass.sub35 = or i32 %div16, %g.addr.1
+ br label %while.body
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/non-schedulable-with-copyable-op.ll b/llvm/test/Transforms/SLPVectorizer/X86/non-schedulable-with-copyable-op.ll
new file mode 100644
index 0000000000000..688767713b571
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/non-schedulable-with-copyable-op.ll
@@ -0,0 +1,89 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+
+define i32 @test(i32 %xor4.i, i32 %xor18.i, i1 %tobool.not.i) {
+; CHECK-LABEL: define i32 @test(
+; CHECK-SAME: i32 [[XOR4_I:%.*]], i32 [[XOR18_I:%.*]], i1 [[TOBOOL_NOT_I:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[BB:.*]]
+; CHECK: [[BB]]:
+; CHECK-NEXT: [[XOR375678_I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[XOR37_I:%.*]], %[[BB]] ]
+; CHECK-NEXT: [[XOR385777_I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[XOR35_I:%.*]], %[[BB]] ]
+; CHECK-NEXT: [[XOR445876_I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[AND43_I:%.*]], %[[BB]] ]
+; CHECK-NEXT: [[XOR505975_I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[XOR48_I:%.*]], %[[BB]] ]
+; CHECK-NEXT: [[XOR316272_I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[XOR31_I:%.*]], %[[BB]] ]
+; CHECK-NEXT: [[XOR536470_I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[XOR53_I:%.*]], %[[BB]] ]
+; CHECK-NEXT: [[XOR30_I:%.*]] = tail call i32 @llvm.fshl.i32(i32 [[XOR4_I]], i32 1, i32 1)
+; CHECK-NEXT: [[XOR31_I]] = xor i32 [[XOR30_I]], [[XOR4_I]]
+; CHECK-NEXT: [[XOR11_I:%.*]] = tail call i32 @llvm.fshl.i32(i32 [[XOR4_I]], i32 [[XOR375678_I]], i32 1)
+; CHECK-NEXT: [[XOR7_I:%.*]] = tail call i32 @llvm.fshl.i32(i32 [[XOR4_I]], i32 [[XOR316272_I]], i32 1)
+; CHECK-NEXT: [[AND22_I:%.*]] = and i32 [[XOR11_I]], [[XOR7_I]]
+; CHECK-NEXT: [[XOR23_I:%.*]] = xor i32 [[XOR18_I]], [[AND22_I]]
+; CHECK-NEXT: [[XOR32_I:%.*]] = xor i32 [[XOR23_I]], 1
+; CHECK-NEXT: [[XOR35_I]] = tail call i32 @llvm.fshl.i32(i32 [[XOR32_I]], i32 [[XOR18_I]], i32 1)
+; CHECK-NEXT: [[AND36_I:%.*]] = and i32 [[XOR35_I]], 1
+; CHECK-NEXT: [[XOR37_I]] = xor i32 [[AND36_I]], 1
+; CHECK-NEXT: [[XOR14_I:%.*]] = tail call i32 @llvm.fshl.i32(i32 1, i32 [[XOR536470_I]], i32 1)
+; CHECK-NEXT: [[AND_I:%.*]] = and i32 [[XOR14_I]], [[XOR11_I]]
+; CHECK-NEXT: [[TMP0:%.*]] = xor i32 [[XOR4_I]], [[AND_I]]
+; CHECK-NEXT: [[XOR39_I:%.*]] = xor i32 [[TMP0]], 1
+; CHECK-NEXT: [[XOR42_I:%.*]] = tail call i32 @llvm.fshl.i32(i32 [[XOR39_I]], i32 [[XOR23_I]], i32 1)
+; CHECK-NEXT: [[AND43_I]] = and i32 [[XOR42_I]], [[XOR4_I]]
+; CHECK-NEXT: [[XOR15_I:%.*]] = xor i32 [[XOR536470_I]], [[XOR4_I]]
+; CHECK-NEXT: [[XOR18_I3:%.*]] = tail call i32 @llvm.fshl.i32(i32 1, i32 [[XOR15_I]], i32 1)
+; CHECK-NEXT: [[AND25_I:%.*]] = and i32 [[XOR7_I]], [[XOR18_I3]]
+; CHECK-NEXT: [[TMP1:%.*]] = xor i32 [[AND25_I]], [[XOR4_I]]
+; CHECK-NEXT: [[XOR45_I:%.*]] = xor i32 [[TMP1]], 1
+; CHECK-NEXT: [[XOR48_I]] = tail call i32 @llvm.fshl.i32(i32 [[XOR45_I]], i32 1, i32 1)
+; CHECK-NEXT: [[XOR53_I]] = xor i32 [[XOR4_I]], 1
+; CHECK-NEXT: br i1 [[TOBOOL_NOT_I]], label %[[EXIT:.*]], label %[[BB]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[TMP2:%.*]] = xor i32 [[XOR385777_I]], [[XOR445876_I]]
+; CHECK-NEXT: [[TMP3:%.*]] = xor i32 [[TMP2]], [[XOR505975_I]]
+; CHECK-NEXT: [[XOR2_I:%.*]] = xor i32 [[TMP3]], [[XOR375678_I]]
+; CHECK-NEXT: ret i32 [[XOR2_I]]
+;
+entry:
+ br label %bb
+
+bb:
+ %xor375678.i = phi i32 [ 0, %entry ], [ %xor37.i, %bb ]
+ %xor385777.i = phi i32 [ 0, %entry ], [ %xor35.i, %bb ]
+ %xor445876.i = phi i32 [ 0, %entry ], [ %and43.i, %bb ]
+ %xor505975.i = phi i32 [ 0, %entry ], [ %xor48.i, %bb ]
+ %xor316272.i = phi i32 [ 0, %entry ], [ %xor31.i, %bb ]
+ %xor536470.i = phi i32 [ 0, %entry ], [ %xor53.i, %bb ]
+ %xor30.i = tail call i32 @llvm.fshl.i32(i32 %xor4.i, i32 1, i32 1)
+ %xor31.i = xor i32 %xor30.i, %xor4.i
+ %xor11.i = tail call i32 @llvm.fshl.i32(i32 %xor4.i, i32 %xor375678.i, i32 1)
+ %xor7.i = tail call i32 @llvm.fshl.i32(i32 %xor4.i, i32 %xor316272.i, i32 1)
+ %and22.i = and i32 %xor11.i, %xor7.i
+ %xor23.i = xor i32 %xor18.i, %and22.i
+ %xor32.i = xor i32 %xor23.i, 1
+ %xor35.i = tail call i32 @llvm.fshl.i32(i32 %xor32.i, i32 %xor18.i, i32 1)
+ %and36.i = and i32 %xor35.i, 1
+ %xor37.i = xor i32 %and36.i, 1
+ %xor14.i = tail call i32 @llvm.fshl.i32(i32 1, i32 %xor536470.i, i32 1)
+ %and.i = and i32 %xor14.i, %xor11.i
+ %0 = xor i32 %xor4.i, %and.i
+ %xor39.i = xor i32 %0, 1
+ %xor42.i = tail call i32 @llvm.fshl.i32(i32 %xor39.i, i32 %xor23.i, i32 1)
+ %and43.i = and i32 %xor42.i, %xor4.i
+ %xor15.i = xor i32 %xor536470.i, %xor4.i
+ %xor18.i3 = tail call i32 @llvm.fshl.i32(i32 1, i32 %xor15.i, i32 1)
+ %and25.i = and i32 %xor7.i, %xor18.i3
+ %1 = xor i32 %and25.i, %xor4.i
+ %xor45.i = xor i32 %1, 1
+ %xor48.i = tail call i32 @llvm.fshl.i32(i32 %xor45.i, i32 1, i32 1)
+ %xor53.i = xor i32 %xor4.i, 1
+ br i1 %tobool.not.i, label %exit, label %bb
+
+exit:
+ %2 = xor i32 %xor385777.i, %xor445876.i
+ %3 = xor i32 %2, %xor505975.i
+ %xor2.i = xor i32 %3, %xor375678.i
+ ret i32 %xor2.i
+}
+
+declare i32 @llvm.fshl.i32(i32, i32, i32)
+
More information about the llvm-commits
mailing list