[llvm] b37a8a7 - [SLP]Fix perfect diamond match for gather nodes with poison in the reuse mask

via llvm-commits llvm-commits at lists.llvm.org
Sun Aug 30 18:19:13 PDT 2026


Author: Alexey Bataev
Date: 2026-08-30T21:19:09-04:00
New Revision: b37a8a70cc98915d157cac2f049e9f46d6da8fe5

URL: https://github.com/llvm/llvm-project/commit/b37a8a70cc98915d157cac2f049e9f46d6da8fe5
DIFF: https://github.com/llvm/llvm-project/commit/b37a8a70cc98915d157cac2f049e9f46d6da8fe5.diff

LOG: [SLP]Fix perfect diamond match for gather nodes with poison in the reuse mask

A poison constant in the gathered bundle leaves a poison lane in the
common mask; expanding the gather through that mask indexed the scalar
list with -1. Expand such lanes to poison instead.

Fixes #219833

Reviewers: 

Pull Request: https://github.com/llvm/llvm-project/pull/219862

Added: 
    llvm/test/Transforms/SLPVectorizer/X86/perfect-diamond-match-poison-reuse.ll

Modified: 
    llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp

Removed: 
    


################################################################################
diff  --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 71ac867d8b4a0..f8b168f0327f7 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -22901,8 +22901,10 @@ ResTy BoUpSLP::processBuildVector(const TreeEntry *E, Type *ScalarTy,
         SmallVector<int> CommonMask = E->getCommonMask();
         SmallVector<Value *> Expanded(E->getVectorFactor());
         for (unsigned I : seq<unsigned>(E->getVectorFactor()))
-          Expanded[I] =
-              CommonMask.empty() ? E->Scalars[I] : E->Scalars[CommonMask[I]];
+          Expanded[I] = CommonMask.empty() ? E->Scalars[I]
+                        : CommonMask[I] == PoisonMaskElem
+                            ? PoisonValue::get(OrigScalarTy)
+                            : E->Scalars[CommonMask[I]];
         return FrontTE->isSame(Expanded);
       };
       if (GatherShuffles.size() == 1 &&

diff  --git a/llvm/test/Transforms/SLPVectorizer/X86/perfect-diamond-match-poison-reuse.ll b/llvm/test/Transforms/SLPVectorizer/X86/perfect-diamond-match-poison-reuse.ll
new file mode 100644
index 0000000000000..2badb1c53eb5d
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/perfect-diamond-match-poison-reuse.ll
@@ -0,0 +1,74 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu | FileCheck %s
+
+define i32 @test(i32 %sel11.sroa.0.0.copyload, i1 %cmp, ptr %g2) {
+; CHECK-LABEL: define i32 @test(
+; CHECK-SAME: i32 [[SEL11_SROA_0_0_COPYLOAD:%.*]], i1 [[CMP:%.*]], ptr [[G2:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LBL_CONT82:.*]]
+; CHECK:       [[LBL_CONT7:.*]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = phi i32 [ [[OR:%.*]], %[[IF_THEN16:.*]] ], [ [[TMP4:%.*]], %[[LBL_FRESH10:.*]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = phi i32 [ [[OR]], %[[IF_THEN16]] ], [ 0, %[[LBL_FRESH10]] ]
+; CHECK-NEXT:    [[BB10_0:%.*]] = phi i32 [ [[SEL11_SROA_0_0_COPYLOAD]], %[[IF_THEN16]] ], [ 0, %[[LBL_FRESH10]] ]
+; CHECK-NEXT:    [[BB13_0:%.*]] = phi i32 [ poison, %[[IF_THEN16]] ], [ [[BB13_1:%.*]], %[[LBL_FRESH10]] ]
+; CHECK-NEXT:    br i1 [[C8_3:%.*]], label %[[LBL_CONT21:.*]], label %[[LBL_FRESH10]]
+; CHECK:       [[LBL_FRESH10]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = phi i32 [ [[TMP0]], %[[LBL_CONT7]] ], [ 0, %[[LBL_CONT82]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = phi i32 [ [[TMP1]], %[[LBL_CONT7]] ], [ 0, %[[LBL_CONT82]] ]
+; CHECK-NEXT:    [[BB10_1:%.*]] = phi i32 [ [[BB10_0]], %[[LBL_CONT7]] ], [ 0, %[[LBL_CONT82]] ]
+; CHECK-NEXT:    [[BB13_1]] = phi i32 [ [[BB13_0]], %[[LBL_CONT7]] ], [ 0, %[[LBL_CONT82]] ]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[LBL_CONT82]], label %[[LBL_CONT7]]
+; CHECK:       [[LBL_CONT21]]:
+; CHECK-NEXT:    br i1 [[CMP]], label %[[IF_END13:.*]], label %[[LBL_CONT82]]
+; CHECK:       [[IF_END13]]:
+; CHECK-NEXT:    ret i32 [[BB10_2:%.*]]
+; CHECK:       [[LBL_CONT82]]:
+; CHECK-NEXT:    [[TMP4]] = phi i32 [ 0, %[[ENTRY]] ], [ [[TMP0]], %[[LBL_CONT21]] ], [ [[TMP2]], %[[LBL_FRESH10]] ]
+; CHECK-NEXT:    [[TMP5:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[TMP0]], %[[LBL_CONT21]] ], [ [[TMP3]], %[[LBL_FRESH10]] ]
+; CHECK-NEXT:    [[REPAIR17_3:%.*]] = phi i1 [ false, %[[ENTRY]] ], [ false, %[[LBL_CONT21]] ], [ true, %[[LBL_FRESH10]] ]
+; CHECK-NEXT:    [[C8_3]] = phi i1 [ false, %[[ENTRY]] ], [ true, %[[LBL_CONT21]] ], [ false, %[[LBL_FRESH10]] ]
+; CHECK-NEXT:    [[BB10_2]] = phi i32 [ 0, %[[ENTRY]] ], [ [[BB10_0]], %[[LBL_CONT21]] ], [ [[BB10_1]], %[[LBL_FRESH10]] ]
+; CHECK-NEXT:    [[BB13_3:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[BB13_0]], %[[LBL_CONT21]] ], [ [[BB13_1]], %[[LBL_FRESH10]] ]
+; CHECK-NEXT:    br i1 [[REPAIR17_3]], label %[[IF_THEN16]], label %[[LBL_FRESH10]]
+; CHECK:       [[IF_THEN16]]:
+; CHECK-NEXT:    [[OR]] = or i32 [[TMP5]], [[BB13_3]]
+; CHECK-NEXT:    store i32 [[OR]], ptr [[G2]], align 4
+; CHECK-NEXT:    br label %[[LBL_CONT7]]
+;
+entry:
+  br label %lbl_cont82
+
+lbl_cont7:
+  %0 = phi i32 [ %or, %if.then16 ], [ %4, %lbl_fresh10 ]
+  %1 = phi i32 [ %or, %if.then16 ], [ 0, %lbl_fresh10 ]
+  %bb10.0 = phi i32 [ %sel11.sroa.0.0.copyload, %if.then16 ], [ 0, %lbl_fresh10 ]
+  %bb13.0 = phi i32 [ poison, %if.then16 ], [ %bb13.1, %lbl_fresh10 ]
+  br i1 %c8.3, label %lbl_cont21, label %lbl_fresh10
+
+lbl_fresh10:
+  %2 = phi i32 [ %0, %lbl_cont7 ], [ 0, %lbl_cont82 ]
+  %3 = phi i32 [ %1, %lbl_cont7 ], [ 0, %lbl_cont82 ]
+  %bb10.1 = phi i32 [ %bb10.0, %lbl_cont7 ], [ 0, %lbl_cont82 ]
+  %bb13.1 = phi i32 [ %bb13.0, %lbl_cont7 ], [ 0, %lbl_cont82 ]
+  br i1 %cmp, label %lbl_cont82, label %lbl_cont7
+
+lbl_cont21:
+  br i1 %cmp, label %if.end13, label %lbl_cont82
+
+if.end13:
+  ret i32 %bb10.2
+
+lbl_cont82:
+  %4 = phi i32 [ 0, %entry ], [ %0, %lbl_cont21 ], [ %2, %lbl_fresh10 ]
+  %5 = phi i32 [ 0, %entry ], [ %0, %lbl_cont21 ], [ %3, %lbl_fresh10 ]
+  %repair17.3 = phi i1 [ false, %entry ], [ false, %lbl_cont21 ], [ true, %lbl_fresh10 ]
+  %c8.3 = phi i1 [ false, %entry ], [ true, %lbl_cont21 ], [ false, %lbl_fresh10 ]
+  %bb10.2 = phi i32 [ 0, %entry ], [ %bb10.0, %lbl_cont21 ], [ %bb10.1, %lbl_fresh10 ]
+  %bb13.3 = phi i32 [ 0, %entry ], [ %bb13.0, %lbl_cont21 ], [ %bb13.1, %lbl_fresh10 ]
+  br i1 %repair17.3, label %if.then16, label %lbl_fresh10
+
+if.then16:
+  %or = or i32 %5, %bb13.3
+  store i32 %or, ptr %g2, align 4
+  br label %lbl_cont7
+}


        


More information about the llvm-commits mailing list