[llvm] 8122bb9 - [SLP]Fix a check for non-schedulable instructions

[Alexey Bataev via llvm-commits]

Author: Alexey Bataev
Date: 2025-03-25T04:35:33-07:00
New Revision: 8122bb9dbe39a1dde77eb4aad76bf1c0e70b2d89

URL: https://github.com/llvm/llvm-project/commit/8122bb9dbe39a1dde77eb4aad76bf1c0e70b2d89
DIFF: https://github.com/llvm/llvm-project/commit/8122bb9dbe39a1dde77eb4aad76bf1c0e70b2d89.diff

LOG: [SLP]Fix a check for non-schedulable instructions

Need to fix a check for non-schedulable instructions in
getLastInstructionInBundle function, because this check may not work
correctly during the codegen. Instead, need to check that actually these
instructions were never scheduled, since the scheduling analysis always
performed before the codegen and is stable.

Fixes #132841

Added: 
    llvm/test/Transforms/SLPVectorizer/X86/non-schedulable-instructions-become-schedulable.ll

Modified: 
    llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp

Removed: 
    


################################################################################
diff  --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 4dc398f716b30..7741f96ee897b 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -15071,7 +15071,18 @@ Instruction &BoUpSLP::getLastInstructionInBundle(const TreeEntry *E) {
 
   // Set the insert point to the beginning of the basic block if the entry
   // should not be scheduled.
-  if (doesNotNeedToSchedule(E->Scalars) ||
+  const auto *It = BlocksSchedules.find(BB);
+  auto IsNotScheduledEntry = [&](const TreeEntry *E) {
+    if (E->isGather())
+      return false;
+    // Found previously that the instruction do not need to be scheduled.
+    return It == BlocksSchedules.end() || all_of(E->Scalars, [&](Value *V) {
+             if (!isa<Instruction>(V))
+               return true;
+             return It->second->getScheduleBundles(V).empty();
+           });
+  };
+  if (IsNotScheduledEntry(E) ||
       (!E->isGather() && all_of(E->Scalars, isVectorLikeInstWithConstOps))) {
     if ((E->getOpcode() == Instruction::GetElementPtr &&
          any_of(E->Scalars,
@@ -15098,12 +15109,11 @@ Instruction &BoUpSLP::getLastInstructionInBundle(const TreeEntry *E) {
   // scheduled, and the last instruction is VL.back(). So we start with
   // VL.back() and iterate over schedule data until we reach the end of the
   // bundle. The end of the bundle is marked by null ScheduleData.
-  if (BlocksSchedules.count(BB) && !E->isGather()) {
+  if (It != BlocksSchedules.end() && !E->isGather()) {
     Value *V = E->isOneOf(E->Scalars.back());
     if (doesNotNeedToBeScheduled(V))
       V = *find_if_not(E->Scalars, doesNotNeedToBeScheduled);
-    if (ArrayRef<ScheduleBundle *> Bundles =
-            BlocksSchedules[BB]->getScheduleBundles(V);
+    if (ArrayRef<ScheduleBundle *> Bundles = It->second->getScheduleBundles(V);
         !Bundles.empty()) {
       const auto *It = find_if(
           Bundles, [&](ScheduleBundle *B) { return B->getTreeEntry() == E; });

diff  --git a/llvm/test/Transforms/SLPVectorizer/X86/non-schedulable-instructions-become-schedulable.ll b/llvm/test/Transforms/SLPVectorizer/X86/non-schedulable-instructions-become-schedulable.ll
new file mode 100644
index 0000000000000..382d6ae0e0a6f
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/non-schedulable-instructions-become-schedulable.ll
@@ -0,0 +1,55 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-unknown -mcpu=znver2 < %s | FileCheck %s
+
+define void @test() {
+; CHECK-LABEL: define void @test(
+; CHECK-SAME: ) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br label %[[BB1:.*]]
+; CHECK:       [[IF_THEN_I_I:.*]]:
+; CHECK-NEXT:    br label %[[BB5:.*]]
+; CHECK:       [[BB1]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = zext i1 false to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i64> <i64 poison, i64 0>, i64 [[TMP0]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = add <2 x i64> zeroinitializer, [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <4 x i64> @llvm.vector.insert.v4i64.v2i64(<4 x i64> <i64 0, i64 0, i64 poison, i64 poison>, <2 x i64> [[TMP2]], i64 2)
+; CHECK-NEXT:    [[TMP4:%.*]] = call <4 x i64> @llvm.vector.insert.v4i64.v2i64(<4 x i64> <i64 0, i64 0, i64 poison, i64 poison>, <2 x i64> [[TMP2]], i64 2)
+; CHECK-NEXT:    br i1 false, label %[[BB5]], label %[[BB2:.*]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[TMP6:%.*]] = phi <4 x i64> [ [[TMP3]], %[[BB1]] ], [ poison, %[[IF_THEN_I_I]] ]
+; CHECK-NEXT:    br label %[[BB2]]
+; CHECK:       [[BB2]]:
+; CHECK-NEXT:    [[TMP7:%.*]] = phi <4 x i64> [ [[TMP6]], %[[BB5]] ], [ [[TMP4]], %[[BB1]] ]
+; CHECK-NEXT:    store <4 x i64> [[TMP7]], ptr getelementptr inbounds nuw (i8, ptr null, i64 40), align 8
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %bb1
+
+if.then.i.i:
+  br label %3
+
+bb1:
+  %0 = zext i1 false to i64
+  %1 = add i64 0, %0
+  %2 = add i64 0, 0
+  br i1 false, label %3, label %bb2
+
+3:
+  %pgocount51962 = phi i64 [ 0, %bb1 ], [ 0, %if.then.i.i ]
+  %pgocount62360 = phi i64 [ 0, %bb1 ], [ 0, %if.then.i.i ]
+  %pgocount83056 = phi i64 [ %1, %bb1 ], [ 0, %if.then.i.i ]
+  %pgocount93354 = phi i64 [ %2, %bb1 ], [ 0, %if.then.i.i ]
+  br label %bb2
+
+bb2:
+  %pgocount51961 = phi i64 [ %pgocount51962, %3 ], [ 0, %bb1 ]
+  %pgocount62359 = phi i64 [ %pgocount62360, %3 ], [ 0, %bb1 ]
+  %pgocount83055 = phi i64 [ %pgocount83056, %3 ], [ %1, %bb1 ]
+  %pgocount93353 = phi i64 [ %pgocount93354, %3 ], [ %2, %bb1 ]
+  store i64 %pgocount51961, ptr getelementptr inbounds nuw (i8, ptr null, i64 40), align 8
+  store i64 %pgocount62359, ptr getelementptr inbounds nuw (i8, ptr null, i64 48), align 8
+  store i64 %pgocount83055, ptr getelementptr inbounds nuw (i8, ptr null, i64 56), align 8
+  store i64 %pgocount93353, ptr getelementptr inbounds nuw (i8, ptr null, i64 64), align 8
+  ret void
+}