[llvm] [SLP] Fix dep accounting when cancelling a bundle with copyable elements (PR #223174)

[Alexey Bataev via llvm-commits]

https://github.com/alexey-bataev created https://github.com/llvm/llvm-project/pull/223174

Restore the parent-edge copyable data as a user of the instruction only
if the cancelled element displaced it at creation (chained copyable
self-use); otherwise the extra user dependency is never released and
scheduling asserts.

Fixes #223139


>From 263b34475474081e53e7170a66268d30f7546a92 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev at outlook.com>
Date: Sat, 12 Sep 2026 13:56:16 -0700
Subject: [PATCH] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20initia?=
 =?UTF-8?q?l=20version?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Created using spr 1.3.7
---
 .../Transforms/Vectorize/SLPVectorizer.cpp    | 14 ++++-
 .../X86/cancelled-copyable-self-use-deps.ll   | 58 +++++++++++++++++++
 2 files changed, 69 insertions(+), 3 deletions(-)
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/cancelled-copyable-self-use-deps.ll

diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 488bfb36b6ccc..df9ced3e42d27 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -26173,6 +26173,7 @@ BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
           const auto *It = find(Op, I);
           assert(It != Op.end() && "Lane not set");
           SmallPtrSet<Instruction *, 4> Visited;
+          bool HadSelfUse = false;
           do {
             int Lane = std::distance(Op.begin(), It);
             assert(Lane >= 0 && "Lane not set");
@@ -26186,14 +26187,21 @@ BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
               It = find(make_range(std::next(It), Op.end()), I);
               break;
             }
+            HadSelfUse |= In == I;
             ScheduleCopyableDataMapByInstUser
                 [std::make_pair(std::make_pair(In, EI.EdgeIdx), I)]
                     .pop_back();
             It = find(make_range(std::next(It), Op.end()), I);
           } while (It != Op.end());
-          EdgeInfo UserEI = EI.UserTE->UserTreeIndex;
-          if (ScheduleCopyableData *UserCD = getScheduleCopyableData(UserEI, I))
-            ScheduleCopyableDataMapByUsers[I].insert(UserCD);
+          // Restore the parent-edge copyable data as a user of I only if the
+          // cancelled data displaced it at creation (chained copyable
+          // self-use); otherwise the extra user dependency is never released.
+          if (HadSelfUse) {
+            EdgeInfo UserEI = EI.UserTE->UserTreeIndex;
+            if (ScheduleCopyableData *UserCD =
+                    getScheduleCopyableData(UserEI, I))
+              ScheduleCopyableDataMapByUsers[I].insert(UserCD);
+          }
         }
         if (ScheduleCopyableDataMapByUsers[I].empty())
           ScheduleCopyableDataMapByUsers.erase(I);
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/cancelled-copyable-self-use-deps.ll b/llvm/test/Transforms/SLPVectorizer/X86/cancelled-copyable-self-use-deps.ll
new file mode 100644
index 0000000000000..cbd7cf8dc1779
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/cancelled-copyable-self-use-deps.ll
@@ -0,0 +1,58 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -passes=slp-vectorizer -S < %s -mtriple=x86_64 | FileCheck %s
+
+; The gather node with copyable elements fails to schedule and its bundle is
+; cancelled. The cancellation must not restore the parent-edge copyable data
+; as a user of %conv: it was displaced by another live copyable element, and
+; restoring it charges %conv's schedule data a dependency that is never
+; released, leaving the real bundle [%conv, %add4] unscheduled.
+
+define i16 @test(i64 %d, ptr %a, i32 %conv3, ptr %h) {
+; CHECK-LABEL: define i16 @test(
+; CHECK-SAME: i64 [[D:%.*]], ptr [[A:%.*]], i32 [[CONV3:%.*]], ptr [[H:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = trunc i64 [[D]] to i32
+; CHECK-NEXT:    [[CONV:%.*]] = and i32 [[TMP0]], 1
+; CHECK-NEXT:    [[REASS_ADD:%.*]] = shl i32 [[CONV]], 1
+; CHECK-NEXT:    [[ADD23:%.*]] = add i32 [[REASS_ADD]], 70
+; CHECK-NEXT:    [[XOR4:%.*]] = xor i32 [[ADD23]], 5
+; CHECK-NEXT:    [[ADD7:%.*]] = add i32 [[CONV]], [[XOR4]]
+; CHECK-NEXT:    [[ADD8:%.*]] = add i32 [[ADD7]], [[CONV3]]
+; CHECK-NEXT:    [[TMP12:%.*]] = trunc i32 [[ADD8]] to i16
+; CHECK-NEXT:    store i16 [[TMP12]], ptr [[H]], align 2
+; CHECK-NEXT:    [[ADD4:%.*]] = shl i32 [[CONV3]], 1
+; CHECK-NEXT:    [[ADD13:%.*]] = or i32 [[ADD4]], 1
+; CHECK-NEXT:    [[SUB:%.*]] = add i32 [[ADD13]], [[CONV]]
+; CHECK-NEXT:    [[ADD14:%.*]] = add i32 [[SUB]], [[CONV3]]
+; CHECK-NEXT:    [[TMP13:%.*]] = trunc i32 [[ADD14]] to i16
+; CHECK-NEXT:    store i16 [[TMP13]], ptr [[A]], align 2
+; CHECK-NEXT:    [[TMP14:%.*]] = trunc i32 [[CONV]] to i16
+; CHECK-NEXT:    store i16 [[TMP14]], ptr [[H]], align 2
+; CHECK-NEXT:    [[ADD27:%.*]] = add i32 [[ADD13]], [[REASS_ADD]]
+; CHECK-NEXT:    [[ADD28:%.*]] = add i32 [[ADD27]], [[CONV3]]
+; CHECK-NEXT:    [[TMP15:%.*]] = trunc i32 [[ADD28]] to i16
+; CHECK-NEXT:    ret i16 [[TMP15]]
+;
+entry:
+  %0 = trunc i64 %d to i32
+  %conv = and i32 %0, 1
+  %reass.add = shl i32 %conv, 1
+  %add23 = add i32 %reass.add, 70
+  %xor4 = xor i32 %add23, 5
+  %add7 = add i32 %conv, %xor4
+  %add8 = add i32 %add7, %conv3
+  %conv9 = trunc i32 %add8 to i16
+  store i16 %conv9, ptr %h, align 2
+  %add4 = shl i32 %conv3, 1
+  %add13 = or i32 %add4, 1
+  %sub = add i32 %add13, %conv
+  %add14 = add i32 %sub, %conv3
+  %1 = trunc i32 %add14 to i16
+  store i16 %1, ptr %a, align 2
+  %2 = trunc i32 %conv to i16
+  store i16 %2, ptr %h, align 2
+  %add27 = add i32 %add13, %reass.add
+  %add28 = add i32 %add27, %conv3
+  %3 = trunc i32 %add28 to i16
+  ret i16 %3
+}