[llvm] [SLP]Fix unscheduled-deps assert for copyable elements peeled into reassoc nodes (PR #214262)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Aug 5 08:52:26 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-vectorizers
Author: Alexey Bataev (alexey-bataev)
<details>
<summary>Changes</summary>
Consume the copyable element's self-use count at the absorbed-copyable
release so the reassociated-operand release cannot fire twice, and keep
peeled copyable elements in KeptReassocScalars so erasure and external
uses treat them as surviving scalars.
Fixes #<!-- -->214181
Fixes #<!-- -->214163
---
Full diff: https://github.com/llvm/llvm-project/pull/214262.diff
3 Files Affected:
- (modified) llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp (+17-5)
- (added) llvm/test/Transforms/SLPVectorizer/X86/reassoc-peeled-copyable.ll (+56)
- (modified) llvm/test/Transforms/SLPVectorizer/X86/reassociate-ops.ll (+91)
``````````diff
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index e3a4229e8e8cf..c00b3c2ca035a 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -5324,8 +5324,15 @@ class slpvectorizer::BoUpSLP {
// copyable fmul turned into fmuladd(a, b, -0.0)) does not appear in
// the operand columns of its own node, so the scan above never
// releases the schedule data of the copyable instruction itself.
- // Release it here to keep the unscheduled-deps counters balanced.
+ // Release it here to keep the unscheduled-deps counters balanced,
+ // consuming its self-use count so the reassociated-operand release
+ // below cannot release the same schedule data twice.
if (isa<ScheduleCopyableData>(BundleMember) && !FoundInOpColumns) {
+ auto UseIt = OperandsUses.find(In);
+ if (UseIt != OperandsUses.end() && UseIt->second > 0) {
+ --UseIt->getSecond();
+ --TotalOpCount;
+ }
if (ScheduleData *OpSD = getScheduleData(In))
DecrUnsched(OpSD, /*IsControl=*/false);
}
@@ -8985,13 +8992,18 @@ void BoUpSLP::buildExternalUses(
const ExtraValueToDebugLocsMap &ExternallyUsedValues) {
const size_t NumVectScalars = ScalarToTreeEntries.size() + 1;
DenseMap<Value *, unsigned> ScalarToExtUses;
- // Peeled scalars still claimed by the tree (gathered or listed in some
- // entry's scalars) survive as plain code, along with the peeled scalars
+ // Peeled scalars still claimed by the tree (gathered, listed in some
+ // entry's scalars, or modeled as a copyable element, which is emitted as
+ // a scalar) survive as plain code, along with the peeled scalars
// in their operand chains, which no vector node can rematerialize.
KeptReassocScalars.clear();
SmallVector<const Value *, 8> KeptWorklist;
- for (const Value *V : ReassocScalarToTreeEntries.keys())
- if ((isGathered(V) || !getTreeEntries(V).empty()) &&
+ for (const auto &[V, Owners] : ReassocScalarToTreeEntries)
+ if ((isGathered(V) || !getTreeEntries(V).empty() ||
+ any_of(Owners,
+ [V](const TreeEntry *TE) {
+ return TE->isCopyableElement(const_cast<Value *>(V));
+ })) &&
KeptReassocScalars.insert(V).second)
KeptWorklist.push_back(V);
while (!KeptWorklist.empty()) {
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reassoc-peeled-copyable.ll b/llvm/test/Transforms/SLPVectorizer/X86/reassoc-peeled-copyable.ll
new file mode 100644
index 0000000000000..a310324e2458e
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reassoc-peeled-copyable.ll
@@ -0,0 +1,56 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes=slp-vectorizer -slp-threshold=-100 -S < %s -mtriple=x86_64-unknown-linux -mcpu=corei7-avx | FileCheck %s
+
+ at a = global i32 0, align 4
+ at b = global i64 0, align 8
+ at c = global i32 0, align 4
+
+; The copyable element %add4 is peeled into the flattened node; it must
+; survive as a scalar for its out-of-tree user %rem.
+define void @test_peeled_copyable_kept() {
+; CHECK-LABEL: define void @test_peeled_copyable_kept(
+; CHECK-SAME: ) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr @a, align 4
+; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr @c, align 4
+; CHECK-NEXT: [[TMP2:%.*]] = load i64, ptr @b, align 8
+; CHECK-NEXT: [[CONV1:%.*]] = sext i32 [[TMP1]] to i64
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x i32> poison, i32 [[TMP0]], i64 0
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <2 x i32> [[TMP3]], i32 [[TMP1]], i64 1
+; CHECK-NEXT: [[TMP5:%.*]] = zext <2 x i32> [[TMP4]] to <2 x i64>
+; CHECK-NEXT: [[TMP6:%.*]] = sext <2 x i32> [[TMP4]] to <2 x i64>
+; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <2 x i64> [[TMP5]], <2 x i64> [[TMP6]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT: [[ADD4:%.*]] = sub nsw i64 7, [[CONV1]]
+; CHECK-NEXT: [[TMP8:%.*]] = insertelement <2 x i64> <i64 poison, i64 7>, i64 [[TMP2]], i64 0
+; CHECK-NEXT: [[TMP9:%.*]] = add <2 x i64> <i64 1, i64 0>, [[TMP8]]
+; CHECK-NEXT: [[TMP10:%.*]] = sub <2 x i64> [[TMP9]], [[TMP7]]
+; CHECK-NEXT: [[REM:%.*]] = srem i64 [[CONV1]], [[ADD4]]
+; CHECK-NEXT: [[TMP11:%.*]] = extractelement <2 x i64> [[TMP10]], i64 0
+; CHECK-NEXT: [[AND:%.*]] = and i64 [[TMP11]], [[REM]]
+; CHECK-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i64 [[AND]], 0
+; CHECK-NEXT: br i1 [[TOBOOL_NOT]], label %[[IF_END:.*]], label %[[WHILE_BODY:.*]]
+; CHECK: [[WHILE_BODY]]:
+; CHECK-NEXT: br label %[[WHILE_BODY]]
+; CHECK: [[IF_END]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ %0 = load i32, ptr @a, align 4
+ %1 = load i32, ptr @c, align 4
+ %conv1 = sext i32 %1 to i64
+ %2 = load i64, ptr @b, align 8
+ %conv2 = zext i32 %0 to i64
+ %reass.sub = sub i64 %2, %conv2
+ %add = add i64 %reass.sub, 1
+ %add4 = sub nsw i64 7, %conv1
+ %rem = srem i64 %conv1, %add4
+ %and = and i64 %add, %rem
+ %tobool.not = icmp eq i64 %and, 0
+ br i1 %tobool.not, label %if.end, label %while.body
+
+while.body:
+ br label %while.body
+
+if.end:
+ ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reassociate-ops.ll b/llvm/test/Transforms/SLPVectorizer/X86/reassociate-ops.ll
index 5db2d0b3165fb..46ea5799b5b95 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reassociate-ops.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reassociate-ops.ll
@@ -1358,3 +1358,94 @@ k.exit.7: ; preds = %k.exit, %k.exit.thr
%6 = phi i32 [ %.reass5.4, %k.exit ], [ %4, %k.exit.thread ]
br label %.preheader
}
+
+ at a = global i32 0, align 4
+ at b = global i64 0, align 8
+ at c = global i32 0, align 4
+
+; The copyable element %add4 is peeled into the flattened node; its schedule
+; data must be released exactly once.
+define void @test_reassoc_copyable_chain_link() {
+; CHECK-LABEL: define void @test_reassoc_copyable_chain_link(
+; CHECK-SAME: ) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr @a, align 4
+; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr @c, align 4
+; CHECK-NEXT: [[CONV1:%.*]] = sext i32 [[TMP1]] to i64
+; CHECK-NEXT: [[TMP2:%.*]] = load i64, ptr @b, align 8
+; CHECK-NEXT: [[CONV2:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-NEXT: [[REASS_SUB:%.*]] = sub i64 [[TMP2]], [[CONV2]]
+; CHECK-NEXT: [[ADD:%.*]] = add i64 [[REASS_SUB]], 1
+; CHECK-NEXT: [[ADD4:%.*]] = sub nsw i64 7, [[CONV1]]
+; CHECK-NEXT: [[REM:%.*]] = srem i64 [[CONV1]], [[ADD4]]
+; CHECK-NEXT: [[AND:%.*]] = and i64 [[ADD]], [[REM]]
+; CHECK-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i64 [[AND]], 0
+; CHECK-NEXT: br i1 [[TOBOOL_NOT]], label %[[IF_END:.*]], label %[[WHILE_BODY:.*]]
+; CHECK: [[WHILE_BODY]]:
+; CHECK-NEXT: br label %[[WHILE_BODY]]
+; CHECK: [[IF_END]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ %0 = load i32, ptr @a, align 4
+ %1 = load i32, ptr @c, align 4
+ %conv1 = sext i32 %1 to i64
+ %2 = load i64, ptr @b, align 8
+ %conv2 = zext i32 %0 to i64
+ %reass.sub = sub i64 %2, %conv2
+ %add = add i64 %reass.sub, 1
+ %add4 = sub nsw i64 7, %conv1
+ %rem = srem i64 %conv1, %add4
+ %and = and i64 %add, %rem
+ %tobool.not = icmp eq i64 %and, 0
+ br i1 %tobool.not, label %if.end, label %while.body
+
+while.body:
+ br label %while.body
+
+if.end:
+ ret void
+}
+
+; Same as above, but the copyable element is a sub in an add-family node.
+define i32 @test_reassoc_copyable_sub_chain_link() {
+; CHECK-LABEL: define i32 @test_reassoc_copyable_sub_chain_link(
+; CHECK-SAME: ) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr @b, align 4
+; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP0]], 2
+; CHECK-NEXT: store i32 [[ADD]], ptr @c, align 4
+; CHECK-NEXT: [[TMP1:%.*]] = shl i32 [[ADD]], 30
+; CHECK-NEXT: [[ADD1:%.*]] = sub i32 1073741824, [[TMP1]]
+; CHECK-NEXT: [[DIV:%.*]] = sdiv i32 2, [[ADD]]
+; CHECK-NEXT: [[TMP2:%.*]] = shl i32 [[TMP0]], 1
+; CHECK-NEXT: [[REASS_SUB:%.*]] = sub i32 [[DIV]], [[TMP2]]
+; CHECK-NEXT: [[SUB:%.*]] = add i32 [[REASS_SUB]], -5
+; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[ADD1]], [[SUB]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[IF_THEN:.*]], label %[[IF_END:.*]]
+; CHECK: [[IF_THEN]]:
+; CHECK-NEXT: store i32 0, ptr @a, align 4
+; CHECK-NEXT: br label %[[IF_END]]
+; CHECK: [[IF_END]]:
+; CHECK-NEXT: ret i32 0
+;
+entry:
+ %0 = load i32, ptr @b, align 4
+ %add = add nsw i32 %0, 2
+ store i32 %add, ptr @c, align 4
+ %1 = shl i32 %add, 30
+ %add1 = sub i32 1073741824, %1
+ %div = sdiv i32 2, %add
+ %2 = shl i32 %0, 1
+ %reass.sub = sub i32 %div, %2
+ %sub = add i32 %reass.sub, -5
+ %cmp = icmp slt i32 %add1, %sub
+ br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+ store i32 0, ptr @a, align 4
+ br label %if.end
+
+if.end:
+ ret i32 0
+}
``````````
</details>
https://github.com/llvm/llvm-project/pull/214262
More information about the llvm-commits
mailing list