[llvm] [SLP] Relax store chain user limit to 2 (PR #215700)

Akash Dutta via llvm-commits llvm-commits at lists.llvm.org
Tue Aug 18 08:52:35 PDT 2026


https://github.com/akadutta updated https://github.com/llvm/llvm-project/pull/215700

>From 3a9c31d9e36a4bb07683dd679785593b9dd10242 Mon Sep 17 00:00:00 2001
From: akadutta_amdeng <Akash.Dutta at amd.com>
Date: Tue, 11 Aug 2026 18:59:45 -0500
Subject: [PATCH 1/3] relax store chain user limit to 2

---
 .../Transforms/Vectorize/SLPVectorizer.cpp    |  6 +-
 .../AMDGPU/store-chain-two-external-users.ll  | 67 +++++++++++++++++++
 2 files changed, 71 insertions(+), 2 deletions(-)
 create mode 100644 llvm/test/Transforms/SLPVectorizer/AMDGPU/store-chain-two-external-users.ll

diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index a3350cf3f3e52..7eec02574302e 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -28679,13 +28679,15 @@ SLPVectorizerPass::vectorizeStoreChainImpl(ArrayRef<Value *> Chain, BoUpSLP &R,
     // only merges the stores, while the scalars remain live for the other users
     // and all the lanes are gathered back. A single outside use may still be a
     // part of the larger vectorizable graph, same for the values, fed by the
-    // loads, where the vector loads may pay off the gathering.
+    // loads, where the vector loads may pay off the gathering. Likewise two
+    // outside uses (e.g. carry compare and next-limb add) may still be part of
+    // the same tree; require 3+ outside users before early-rejecting here.
     if (S && S.getOpcode() != Instruction::Load &&
         all_of(ValOps.getArrayRef(), [&](Value *V) {
           return none_of(cast<Instruction>(V)->operand_values(),
                          IsaPred<LoadInst>) &&
                  count_if(V->users(),
-                          [&](User *U) { return !Stores.contains(U); }) > 1;
+                          [&](User *U) { return !Stores.contains(U); }) > 2;
         })) {
       Size = 1;
       return false;
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/store-chain-two-external-users.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/store-chain-two-external-users.ll
new file mode 100644
index 0000000000000..8db7283bbdc53
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/store-chain-two-external-users.ll
@@ -0,0 +1,67 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -passes=slp-vectorizer -slp-threshold=-100 -mtriple=amdgpu9.0a-amd-amdhsa < %s | FileCheck %s
+
+; Store-chain operands may have two users outside the store chain and still be
+; worth considering. This mirrors carry-propagating counter updates,
+; where the updated limbs remain live for scalar carry/use
+; chains after the state stores. The old "> 1 outside user" early return rejected
+; this before the cost model; with a "> 2" threshold the chain reaches costing.
+
+declare void @use(i32)
+
+define void @store_chain_two_external_users(ptr %dst, i32 %x0, i32 %x1, i32 %x2, i32 %x3, i32 %y0, i32 %y1, i32 %y2, i32 %y3) {
+; CHECK-LABEL: define void @store_chain_two_external_users(
+; CHECK-SAME: ptr [[DST:%.*]], i32 [[X0:%.*]], i32 [[X1:%.*]], i32 [[X2:%.*]], i32 [[X3:%.*]], i32 [[Y0:%.*]], i32 [[Y1:%.*]], i32 [[Y2:%.*]], i32 [[Y3:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[DST2:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 2
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x i32> poison, i32 [[X0]], i64 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i32> [[TMP0]], i32 [[X1]], i64 1
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x i32> poison, i32 [[Y0]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x i32> [[TMP2]], i32 [[Y1]], i64 1
+; CHECK-NEXT:    [[V0:%.*]] = add <2 x i32> [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <2 x i32> poison, i32 [[X2]], i64 0
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i32> [[TMP5]], i32 [[X3]], i64 1
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <2 x i32> poison, i32 [[Y2]], i64 0
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <2 x i32> [[TMP7]], i32 [[Y3]], i64 1
+; CHECK-NEXT:    [[V1:%.*]] = add <2 x i32> [[TMP6]], [[TMP8]]
+; CHECK-NEXT:    store <2 x i32> [[V0]], ptr [[DST]], align 4
+; CHECK-NEXT:    store <2 x i32> [[V1]], ptr [[DST2]], align 4
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <2 x i32> [[V0]], i64 0
+; CHECK-NEXT:    call void @use(i32 [[TMP10]])
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <2 x i32> [[V0]], i64 1
+; CHECK-NEXT:    call void @use(i32 [[TMP11]])
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <2 x i32> [[V1]], i64 0
+; CHECK-NEXT:    call void @use(i32 [[TMP12]])
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <2 x i32> [[V1]], i64 1
+; CHECK-NEXT:    call void @use(i32 [[TMP13]])
+; CHECK-NEXT:    call void @use(i32 [[TMP10]])
+; CHECK-NEXT:    call void @use(i32 [[TMP11]])
+; CHECK-NEXT:    call void @use(i32 [[TMP12]])
+; CHECK-NEXT:    call void @use(i32 [[TMP13]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %dst1 = getelementptr inbounds i32, ptr %dst, i64 1
+  %dst2 = getelementptr inbounds i32, ptr %dst, i64 2
+  %dst3 = getelementptr inbounds i32, ptr %dst, i64 3
+
+  %a0 = add i32 %x0, %y0
+  %a1 = add i32 %x1, %y1
+  %a2 = add i32 %x2, %y2
+  %a3 = add i32 %x3, %y3
+
+  store i32 %a0, ptr %dst, align 4
+  store i32 %a1, ptr %dst1, align 4
+  store i32 %a2, ptr %dst2, align 4
+  store i32 %a3, ptr %dst3, align 4
+
+  call void @use(i32 %a0)
+  call void @use(i32 %a1)
+  call void @use(i32 %a2)
+  call void @use(i32 %a3)
+  call void @use(i32 %a0)
+  call void @use(i32 %a1)
+  call void @use(i32 %a2)
+  call void @use(i32 %a3)
+  ret void
+}

>From 0c6c56b0f55b5ec84e04ae02f4b891aa16f1ccd0 Mon Sep 17 00:00:00 2001
From: akadutta_amdeng <Akash.Dutta at amd.com>
Date: Tue, 18 Aug 2026 10:34:24 -0500
Subject: [PATCH 2/3] early-reject store chains only for non-local or 3+
 outside users

---
 .../Transforms/Vectorize/SLPVectorizer.cpp    |  27 ++-
 .../AMDGPU/store-chain-two-external-users.ll  | 188 ++++++++++++++----
 2 files changed, 169 insertions(+), 46 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 7eec02574302e..9a69d9deddbbd 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -28679,15 +28679,32 @@ SLPVectorizerPass::vectorizeStoreChainImpl(ArrayRef<Value *> Chain, BoUpSLP &R,
     // only merges the stores, while the scalars remain live for the other users
     // and all the lanes are gathered back. A single outside use may still be a
     // part of the larger vectorizable graph, same for the values, fed by the
-    // loads, where the vector loads may pay off the gathering. Likewise two
-    // outside uses (e.g. carry compare and next-limb add) may still be part of
-    // the same tree; require 3+ outside users before early-rejecting here.
+    // loads, where the vector loads may pay off the gathering. Two outside uses
+    // may still be part of the same tree when they are local carry deps (cmp/add
+    // in the same block); reject 3+ outside users or 2 non-local outside users.
+    auto IsBenignOutsideUser = [](Instruction *V, User *U) {
+      auto *UI = dyn_cast<Instruction>(U);
+      if (!UI || UI->getParent() != V->getParent())
+        return false;
+      return isa<CmpInst>(UI) || isa<BinaryOperator>(UI) || isa<SelectInst>(UI);
+    };
+    auto HasTooManyOutsideUsers = [&](Value *V) {
+      unsigned Outside = 0;
+      bool HasNonBenign = false;
+      for (User *U : V->users()) {
+        if (Stores.contains(U))
+          continue;
+        ++Outside;
+        if (!IsBenignOutsideUser(cast<Instruction>(V), U))
+          HasNonBenign = true;
+      }
+      return Outside > 2 || (Outside == 2 && HasNonBenign);
+    };
     if (S && S.getOpcode() != Instruction::Load &&
         all_of(ValOps.getArrayRef(), [&](Value *V) {
           return none_of(cast<Instruction>(V)->operand_values(),
                          IsaPred<LoadInst>) &&
-                 count_if(V->users(),
-                          [&](User *U) { return !Stores.contains(U); }) > 2;
+                 HasTooManyOutsideUsers(V);
         })) {
       Size = 1;
       return false;
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/store-chain-two-external-users.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/store-chain-two-external-users.ll
index 8db7283bbdc53..ab5820ae87e5a 100644
--- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/store-chain-two-external-users.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/store-chain-two-external-users.ll
@@ -1,67 +1,173 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
 ; RUN: opt -S -passes=slp-vectorizer -slp-threshold=-100 -mtriple=amdgpu9.0a-amd-amdhsa < %s | FileCheck %s
 
-; Store-chain operands may have two users outside the store chain and still be
-; worth considering. This mirrors carry-propagating counter updates,
-; where the updated limbs remain live for scalar carry/use
-; chains after the state stores. The old "> 1 outside user" early return rejected
-; this before the cost model; with a "> 2" threshold the chain reaches costing.
+; Use a negative threshold to test whether the store chain reaches costing,
+; independently of the target cost model.
+;
+; A store chain is rejected when every stored value has either more than two
+; users outside the chain, or exactly two outside users with at least one that
+; is not a same-block cmp, binop, or select.
 
 declare void @use(i32)
 
-define void @store_chain_two_external_users(ptr %dst, i32 %x0, i32 %x1, i32 %x2, i32 %x3, i32 %y0, i32 %y1, i32 %y2, i32 %y3) {
-; CHECK-LABEL: define void @store_chain_two_external_users(
-; CHECK-SAME: ptr [[DST:%.*]], i32 [[X0:%.*]], i32 [[X1:%.*]], i32 [[X2:%.*]], i32 [[X3:%.*]], i32 [[Y0:%.*]], i32 [[Y1:%.*]], i32 [[Y2:%.*]], i32 [[Y3:%.*]]) {
+; Two benign outside users let the store chain reach costing. This distinguishes
+; the refined heuristic from the previous >1 limit and covers values used by
+; both a compare and a dependent add.
+define void @store_chain_two_benign_users(ptr %dst, i32 %x0, i32 %x1, i32 %y0, i32 %y1) {
+; CHECK-LABEL: define void @store_chain_two_benign_users(
+; CHECK-SAME: ptr [[DST:%.*]], i32 [[X0:%.*]], i32 [[X1:%.*]], i32 [[Y0:%.*]], i32 [[Y1:%.*]]) {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    [[DST2:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 2
 ; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x i32> poison, i32 [[X0]], i64 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i32> [[TMP0]], i32 [[X1]], i64 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x i32> poison, i32 [[Y0]], i64 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x i32> [[TMP2]], i32 [[Y1]], i64 1
-; CHECK-NEXT:    [[V0:%.*]] = add <2 x i32> [[TMP1]], [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <2 x i32> poison, i32 [[X2]], i64 0
-; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i32> [[TMP5]], i32 [[X3]], i64 1
-; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <2 x i32> poison, i32 [[Y2]], i64 0
-; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <2 x i32> [[TMP7]], i32 [[Y3]], i64 1
-; CHECK-NEXT:    [[V1:%.*]] = add <2 x i32> [[TMP6]], [[TMP8]]
-; CHECK-NEXT:    store <2 x i32> [[V0]], ptr [[DST]], align 4
-; CHECK-NEXT:    store <2 x i32> [[V1]], ptr [[DST2]], align 4
-; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <2 x i32> [[V0]], i64 0
-; CHECK-NEXT:    call void @use(i32 [[TMP10]])
-; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <2 x i32> [[V0]], i64 1
-; CHECK-NEXT:    call void @use(i32 [[TMP11]])
-; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <2 x i32> [[V1]], i64 0
-; CHECK-NEXT:    call void @use(i32 [[TMP12]])
-; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <2 x i32> [[V1]], i64 1
-; CHECK-NEXT:    call void @use(i32 [[TMP13]])
-; CHECK-NEXT:    call void @use(i32 [[TMP10]])
-; CHECK-NEXT:    call void @use(i32 [[TMP11]])
-; CHECK-NEXT:    call void @use(i32 [[TMP12]])
-; CHECK-NEXT:    call void @use(i32 [[TMP13]])
+; CHECK-NEXT:    [[TMP4:%.*]] = add <2 x i32> [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ult <2 x i32> [[TMP4]], [[TMP3]]
+; CHECK-NEXT:    [[TMP6:%.*]] = zext <2 x i1> [[TMP5]] to <2 x i32>
+; CHECK-NEXT:    [[TMP7:%.*]] = add <2 x i32> [[TMP4]], [[TMP6]]
+; CHECK-NEXT:    store <2 x i32> [[TMP4]], ptr [[DST]], align 4
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x i32> [[TMP7]], i64 0
+; CHECK-NEXT:    call void @use(i32 [[TMP8]])
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <2 x i32> [[TMP7]], i64 1
+; CHECK-NEXT:    call void @use(i32 [[TMP9]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %dst1 = getelementptr inbounds i32, ptr %dst, i64 1
-  %dst2 = getelementptr inbounds i32, ptr %dst, i64 2
-  %dst3 = getelementptr inbounds i32, ptr %dst, i64 3
-
   %a0 = add i32 %x0, %y0
   %a1 = add i32 %x1, %y1
-  %a2 = add i32 %x2, %y2
-  %a3 = add i32 %x3, %y3
-
+  %cmp0 = icmp ult i32 %a0, %y0
+  %cmp1 = icmp ult i32 %a1, %y1
+  %z0 = zext i1 %cmp0 to i32
+  %z1 = zext i1 %cmp1 to i32
+  %b0 = add i32 %a0, %z0
+  %b1 = add i32 %a1, %z1
   store i32 %a0, ptr %dst, align 4
   store i32 %a1, ptr %dst1, align 4
-  store i32 %a2, ptr %dst2, align 4
-  store i32 %a3, ptr %dst3, align 4
+  call void @use(i32 %b0)
+  call void @use(i32 %b1)
+  ret void
+}
 
+; Two non-benign call users keep the store chain scalar.
+; Independent arithmetic may still be vectorized before the scalar
+; stores.
+define void @store_chain_two_call_users(ptr %dst, i32 %x0, i32 %x1, i32 %y0, i32 %y1) {
+; CHECK-LABEL: define void @store_chain_two_call_users(
+; CHECK-SAME: ptr [[DST:%.*]], i32 [[X0:%.*]], i32 [[X1:%.*]], i32 [[Y0:%.*]], i32 [[Y1:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[DST1:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 1
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x i32> poison, i32 [[X0]], i64 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i32> [[TMP0]], i32 [[X1]], i64 1
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x i32> poison, i32 [[Y0]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x i32> [[TMP2]], i32 [[Y1]], i64 1
+; CHECK-NEXT:    [[TMP4:%.*]] = add <2 x i32> [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i32> [[TMP4]], i64 1
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i32> [[TMP4]], i64 0
+; CHECK-NEXT:    store i32 [[TMP6]], ptr [[DST]], align 4
+; CHECK-NEXT:    store i32 [[TMP5]], ptr [[DST1]], align 4
+; CHECK-NEXT:    call void @use(i32 [[TMP6]])
+; CHECK-NEXT:    call void @use(i32 [[TMP6]])
+; CHECK-NEXT:    call void @use(i32 [[TMP5]])
+; CHECK-NEXT:    call void @use(i32 [[TMP5]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %dst1 = getelementptr inbounds i32, ptr %dst, i64 1
+  %a0 = add i32 %x0, %y0
+  %a1 = add i32 %x1, %y1
+  store i32 %a0, ptr %dst, align 4
+  store i32 %a1, ptr %dst1, align 4
+  call void @use(i32 %a0)
   call void @use(i32 %a0)
   call void @use(i32 %a1)
-  call void @use(i32 %a2)
-  call void @use(i32 %a3)
+  call void @use(i32 %a1)
+  ret void
+}
+
+; Three benign outside users keep the store chain scalar.
+define void @store_chain_three_benign_users(ptr %dst, i32 %x0, i32 %x1, i32 %y0, i32 %y1, i1 %cond) {
+; CHECK-LABEL: define void @store_chain_three_benign_users(
+; CHECK-SAME: ptr [[DST:%.*]], i32 [[X0:%.*]], i32 [[X1:%.*]], i32 [[Y0:%.*]], i32 [[Y1:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[DST1:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 1
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x i32> poison, i32 [[X0]], i64 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i32> [[TMP0]], i32 [[X1]], i64 1
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x i32> poison, i32 [[Y0]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x i32> [[TMP2]], i32 [[Y1]], i64 1
+; CHECK-NEXT:    [[TMP4:%.*]] = add <2 x i32> [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ult <2 x i32> [[TMP4]], [[TMP3]]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i1> [[TMP5]], i64 0
+; CHECK-NEXT:    [[Z0:%.*]] = zext i1 [[TMP6]] to i32
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x i1> [[TMP5]], i64 1
+; CHECK-NEXT:    [[Z1:%.*]] = zext i1 [[TMP7]] to i32
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x i32> [[TMP4]], i64 0
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <2 x i32> [[TMP4]], i64 1
+; CHECK-NEXT:    [[TMP10:%.*]] = add <2 x i32> [[TMP4]], [[TMP1]]
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <2 x i1> poison, i1 [[COND]], i64 0
+; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <2 x i1> [[TMP11]], <2 x i1> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = select <2 x i1> [[TMP12]], <2 x i32> [[TMP4]], <2 x i32> [[TMP3]]
+; CHECK-NEXT:    store i32 [[TMP8]], ptr [[DST]], align 4
+; CHECK-NEXT:    store i32 [[TMP9]], ptr [[DST1]], align 4
+; CHECK-NEXT:    call void @use(i32 [[Z0]])
+; CHECK-NEXT:    call void @use(i32 [[Z1]])
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x i32> [[TMP10]], i64 0
+; CHECK-NEXT:    call void @use(i32 [[TMP14]])
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <2 x i32> [[TMP10]], i64 1
+; CHECK-NEXT:    call void @use(i32 [[TMP15]])
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <2 x i32> [[TMP13]], i64 0
+; CHECK-NEXT:    call void @use(i32 [[TMP16]])
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <2 x i32> [[TMP13]], i64 1
+; CHECK-NEXT:    call void @use(i32 [[TMP17]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %dst1 = getelementptr inbounds i32, ptr %dst, i64 1
+  %a0 = add i32 %x0, %y0
+  %a1 = add i32 %x1, %y1
+  %cmp0 = icmp ult i32 %a0, %y0
+  %cmp1 = icmp ult i32 %a1, %y1
+  %z0 = zext i1 %cmp0 to i32
+  %z1 = zext i1 %cmp1 to i32
+  %b0 = add i32 %a0, %x0
+  %b1 = add i32 %a1, %x1
+  %sel0 = select i1 %cond, i32 %a0, i32 %y0
+  %sel1 = select i1 %cond, i32 %a1, i32 %y1
+  store i32 %a0, ptr %dst, align 4
+  store i32 %a1, ptr %dst1, align 4
+  call void @use(i32 %z0)
+  call void @use(i32 %z1)
+  call void @use(i32 %b0)
+  call void @use(i32 %b1)
+  call void @use(i32 %sel0)
+  call void @use(i32 %sel1)
+  ret void
+}
+
+; One non-benign outside user is still allowed, matching the original >1 heuristic.
+define void @store_chain_one_call_user(ptr %dst, i32 %x0, i32 %x1, i32 %y0, i32 %y1) {
+; CHECK-LABEL: define void @store_chain_one_call_user(
+; CHECK-SAME: ptr [[DST:%.*]], i32 [[X0:%.*]], i32 [[X1:%.*]], i32 [[Y0:%.*]], i32 [[Y1:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x i32> poison, i32 [[X0]], i64 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i32> [[TMP0]], i32 [[X1]], i64 1
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x i32> poison, i32 [[Y0]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x i32> [[TMP2]], i32 [[Y1]], i64 1
+; CHECK-NEXT:    [[TMP4:%.*]] = add <2 x i32> [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    store <2 x i32> [[TMP4]], ptr [[DST]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i32> [[TMP4]], i64 0
+; CHECK-NEXT:    call void @use(i32 [[TMP5]])
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i32> [[TMP4]], i64 1
+; CHECK-NEXT:    call void @use(i32 [[TMP6]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %dst1 = getelementptr inbounds i32, ptr %dst, i64 1
+  %a0 = add i32 %x0, %y0
+  %a1 = add i32 %x1, %y1
+  store i32 %a0, ptr %dst, align 4
+  store i32 %a1, ptr %dst1, align 4
   call void @use(i32 %a0)
   call void @use(i32 %a1)
-  call void @use(i32 %a2)
-  call void @use(i32 %a3)
   ret void
 }

>From 76d2e87345f36f2b06f48d4928c71c7967425a36 Mon Sep 17 00:00:00 2001
From: akadutta_amdeng <Akash.Dutta at amd.com>
Date: Tue, 18 Aug 2026 10:52:20 -0500
Subject: [PATCH 3/3] fix format issues

---
 llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 9a69d9deddbbd..9ac8bb26fcbfb 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -28680,8 +28680,9 @@ SLPVectorizerPass::vectorizeStoreChainImpl(ArrayRef<Value *> Chain, BoUpSLP &R,
     // and all the lanes are gathered back. A single outside use may still be a
     // part of the larger vectorizable graph, same for the values, fed by the
     // loads, where the vector loads may pay off the gathering. Two outside uses
-    // may still be part of the same tree when they are local carry deps (cmp/add
-    // in the same block); reject 3+ outside users or 2 non-local outside users.
+    // may still be part of the same tree when they are local carry deps
+    // (cmp/add in the same block); reject 3+ outside users or 2 non-local
+    // outside users.
     auto IsBenignOutsideUser = [](Instruction *V, User *U) {
       auto *UI = dyn_cast<Instruction>(U);
       if (!UI || UI->getParent() != V->getParent())



More information about the llvm-commits mailing list