[llvm] [SLP] Relax store chain user limit to 2 (PR #215700)
Akash Dutta via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 18 08:52:35 PDT 2026
https://github.com/akadutta updated https://github.com/llvm/llvm-project/pull/215700
>From 3a9c31d9e36a4bb07683dd679785593b9dd10242 Mon Sep 17 00:00:00 2001
From: akadutta_amdeng <Akash.Dutta at amd.com>
Date: Tue, 11 Aug 2026 18:59:45 -0500
Subject: [PATCH 1/3] relax store chain user limit to 2
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 6 +-
.../AMDGPU/store-chain-two-external-users.ll | 67 +++++++++++++++++++
2 files changed, 71 insertions(+), 2 deletions(-)
create mode 100644 llvm/test/Transforms/SLPVectorizer/AMDGPU/store-chain-two-external-users.ll
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index a3350cf3f3e52..7eec02574302e 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -28679,13 +28679,15 @@ SLPVectorizerPass::vectorizeStoreChainImpl(ArrayRef<Value *> Chain, BoUpSLP &R,
// only merges the stores, while the scalars remain live for the other users
// and all the lanes are gathered back. A single outside use may still be a
// part of the larger vectorizable graph, same for the values, fed by the
- // loads, where the vector loads may pay off the gathering.
+ // loads, where the vector loads may pay off the gathering. Likewise two
+ // outside uses (e.g. carry compare and next-limb add) may still be part of
+ // the same tree; require 3+ outside users before early-rejecting here.
if (S && S.getOpcode() != Instruction::Load &&
all_of(ValOps.getArrayRef(), [&](Value *V) {
return none_of(cast<Instruction>(V)->operand_values(),
IsaPred<LoadInst>) &&
count_if(V->users(),
- [&](User *U) { return !Stores.contains(U); }) > 1;
+ [&](User *U) { return !Stores.contains(U); }) > 2;
})) {
Size = 1;
return false;
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/store-chain-two-external-users.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/store-chain-two-external-users.ll
new file mode 100644
index 0000000000000..8db7283bbdc53
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/store-chain-two-external-users.ll
@@ -0,0 +1,67 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -passes=slp-vectorizer -slp-threshold=-100 -mtriple=amdgpu9.0a-amd-amdhsa < %s | FileCheck %s
+
+; Store-chain operands may have two users outside the store chain and still be
+; worth considering. This mirrors carry-propagating counter updates,
+; where the updated limbs remain live for scalar carry/use
+; chains after the state stores. The old "> 1 outside user" early return rejected
+; this before the cost model; with a "> 2" threshold the chain reaches costing.
+
+declare void @use(i32)
+
+define void @store_chain_two_external_users(ptr %dst, i32 %x0, i32 %x1, i32 %x2, i32 %x3, i32 %y0, i32 %y1, i32 %y2, i32 %y3) {
+; CHECK-LABEL: define void @store_chain_two_external_users(
+; CHECK-SAME: ptr [[DST:%.*]], i32 [[X0:%.*]], i32 [[X1:%.*]], i32 [[X2:%.*]], i32 [[X3:%.*]], i32 [[Y0:%.*]], i32 [[Y1:%.*]], i32 [[Y2:%.*]], i32 [[Y3:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[DST2:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 2
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x i32> poison, i32 [[X0]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i32> [[TMP0]], i32 [[X1]], i64 1
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x i32> poison, i32 [[Y0]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x i32> [[TMP2]], i32 [[Y1]], i64 1
+; CHECK-NEXT: [[V0:%.*]] = add <2 x i32> [[TMP1]], [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x i32> poison, i32 [[X2]], i64 0
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x i32> [[TMP5]], i32 [[X3]], i64 1
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x i32> poison, i32 [[Y2]], i64 0
+; CHECK-NEXT: [[TMP8:%.*]] = insertelement <2 x i32> [[TMP7]], i32 [[Y3]], i64 1
+; CHECK-NEXT: [[V1:%.*]] = add <2 x i32> [[TMP6]], [[TMP8]]
+; CHECK-NEXT: store <2 x i32> [[V0]], ptr [[DST]], align 4
+; CHECK-NEXT: store <2 x i32> [[V1]], ptr [[DST2]], align 4
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x i32> [[V0]], i64 0
+; CHECK-NEXT: call void @use(i32 [[TMP10]])
+; CHECK-NEXT: [[TMP11:%.*]] = extractelement <2 x i32> [[V0]], i64 1
+; CHECK-NEXT: call void @use(i32 [[TMP11]])
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x i32> [[V1]], i64 0
+; CHECK-NEXT: call void @use(i32 [[TMP12]])
+; CHECK-NEXT: [[TMP13:%.*]] = extractelement <2 x i32> [[V1]], i64 1
+; CHECK-NEXT: call void @use(i32 [[TMP13]])
+; CHECK-NEXT: call void @use(i32 [[TMP10]])
+; CHECK-NEXT: call void @use(i32 [[TMP11]])
+; CHECK-NEXT: call void @use(i32 [[TMP12]])
+; CHECK-NEXT: call void @use(i32 [[TMP13]])
+; CHECK-NEXT: ret void
+;
+entry:
+ %dst1 = getelementptr inbounds i32, ptr %dst, i64 1
+ %dst2 = getelementptr inbounds i32, ptr %dst, i64 2
+ %dst3 = getelementptr inbounds i32, ptr %dst, i64 3
+
+ %a0 = add i32 %x0, %y0
+ %a1 = add i32 %x1, %y1
+ %a2 = add i32 %x2, %y2
+ %a3 = add i32 %x3, %y3
+
+ store i32 %a0, ptr %dst, align 4
+ store i32 %a1, ptr %dst1, align 4
+ store i32 %a2, ptr %dst2, align 4
+ store i32 %a3, ptr %dst3, align 4
+
+ call void @use(i32 %a0)
+ call void @use(i32 %a1)
+ call void @use(i32 %a2)
+ call void @use(i32 %a3)
+ call void @use(i32 %a0)
+ call void @use(i32 %a1)
+ call void @use(i32 %a2)
+ call void @use(i32 %a3)
+ ret void
+}
>From 0c6c56b0f55b5ec84e04ae02f4b891aa16f1ccd0 Mon Sep 17 00:00:00 2001
From: akadutta_amdeng <Akash.Dutta at amd.com>
Date: Tue, 18 Aug 2026 10:34:24 -0500
Subject: [PATCH 2/3] early-reject store chains only for non-local or 3+
outside users
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 27 ++-
.../AMDGPU/store-chain-two-external-users.ll | 188 ++++++++++++++----
2 files changed, 169 insertions(+), 46 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 7eec02574302e..9a69d9deddbbd 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -28679,15 +28679,32 @@ SLPVectorizerPass::vectorizeStoreChainImpl(ArrayRef<Value *> Chain, BoUpSLP &R,
// only merges the stores, while the scalars remain live for the other users
// and all the lanes are gathered back. A single outside use may still be a
// part of the larger vectorizable graph, same for the values, fed by the
- // loads, where the vector loads may pay off the gathering. Likewise two
- // outside uses (e.g. carry compare and next-limb add) may still be part of
- // the same tree; require 3+ outside users before early-rejecting here.
+ // loads, where the vector loads may pay off the gathering. Two outside uses
+ // may still be part of the same tree when they are local carry deps (cmp/add
+ // in the same block); reject 3+ outside users or 2 non-local outside users.
+ auto IsBenignOutsideUser = [](Instruction *V, User *U) {
+ auto *UI = dyn_cast<Instruction>(U);
+ if (!UI || UI->getParent() != V->getParent())
+ return false;
+ return isa<CmpInst>(UI) || isa<BinaryOperator>(UI) || isa<SelectInst>(UI);
+ };
+ auto HasTooManyOutsideUsers = [&](Value *V) {
+ unsigned Outside = 0;
+ bool HasNonBenign = false;
+ for (User *U : V->users()) {
+ if (Stores.contains(U))
+ continue;
+ ++Outside;
+ if (!IsBenignOutsideUser(cast<Instruction>(V), U))
+ HasNonBenign = true;
+ }
+ return Outside > 2 || (Outside == 2 && HasNonBenign);
+ };
if (S && S.getOpcode() != Instruction::Load &&
all_of(ValOps.getArrayRef(), [&](Value *V) {
return none_of(cast<Instruction>(V)->operand_values(),
IsaPred<LoadInst>) &&
- count_if(V->users(),
- [&](User *U) { return !Stores.contains(U); }) > 2;
+ HasTooManyOutsideUsers(V);
})) {
Size = 1;
return false;
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/store-chain-two-external-users.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/store-chain-two-external-users.ll
index 8db7283bbdc53..ab5820ae87e5a 100644
--- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/store-chain-two-external-users.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/store-chain-two-external-users.ll
@@ -1,67 +1,173 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -S -passes=slp-vectorizer -slp-threshold=-100 -mtriple=amdgpu9.0a-amd-amdhsa < %s | FileCheck %s
-; Store-chain operands may have two users outside the store chain and still be
-; worth considering. This mirrors carry-propagating counter updates,
-; where the updated limbs remain live for scalar carry/use
-; chains after the state stores. The old "> 1 outside user" early return rejected
-; this before the cost model; with a "> 2" threshold the chain reaches costing.
+; Use a negative threshold to test whether the store chain reaches costing,
+; independently of the target cost model.
+;
+; A store chain is rejected when every stored value has either more than two
+; users outside the chain, or exactly two outside users with at least one that
+; is not a same-block cmp, binop, or select.
declare void @use(i32)
-define void @store_chain_two_external_users(ptr %dst, i32 %x0, i32 %x1, i32 %x2, i32 %x3, i32 %y0, i32 %y1, i32 %y2, i32 %y3) {
-; CHECK-LABEL: define void @store_chain_two_external_users(
-; CHECK-SAME: ptr [[DST:%.*]], i32 [[X0:%.*]], i32 [[X1:%.*]], i32 [[X2:%.*]], i32 [[X3:%.*]], i32 [[Y0:%.*]], i32 [[Y1:%.*]], i32 [[Y2:%.*]], i32 [[Y3:%.*]]) {
+; Two benign outside users let the store chain reach costing. This distinguishes
+; the refined heuristic from the previous >1 limit and covers values used by
+; both a compare and a dependent add.
+define void @store_chain_two_benign_users(ptr %dst, i32 %x0, i32 %x1, i32 %y0, i32 %y1) {
+; CHECK-LABEL: define void @store_chain_two_benign_users(
+; CHECK-SAME: ptr [[DST:%.*]], i32 [[X0:%.*]], i32 [[X1:%.*]], i32 [[Y0:%.*]], i32 [[Y1:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[DST2:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 2
; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x i32> poison, i32 [[X0]], i64 0
; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i32> [[TMP0]], i32 [[X1]], i64 1
; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x i32> poison, i32 [[Y0]], i64 0
; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x i32> [[TMP2]], i32 [[Y1]], i64 1
-; CHECK-NEXT: [[V0:%.*]] = add <2 x i32> [[TMP1]], [[TMP3]]
-; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x i32> poison, i32 [[X2]], i64 0
-; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x i32> [[TMP5]], i32 [[X3]], i64 1
-; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x i32> poison, i32 [[Y2]], i64 0
-; CHECK-NEXT: [[TMP8:%.*]] = insertelement <2 x i32> [[TMP7]], i32 [[Y3]], i64 1
-; CHECK-NEXT: [[V1:%.*]] = add <2 x i32> [[TMP6]], [[TMP8]]
-; CHECK-NEXT: store <2 x i32> [[V0]], ptr [[DST]], align 4
-; CHECK-NEXT: store <2 x i32> [[V1]], ptr [[DST2]], align 4
-; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x i32> [[V0]], i64 0
-; CHECK-NEXT: call void @use(i32 [[TMP10]])
-; CHECK-NEXT: [[TMP11:%.*]] = extractelement <2 x i32> [[V0]], i64 1
-; CHECK-NEXT: call void @use(i32 [[TMP11]])
-; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x i32> [[V1]], i64 0
-; CHECK-NEXT: call void @use(i32 [[TMP12]])
-; CHECK-NEXT: [[TMP13:%.*]] = extractelement <2 x i32> [[V1]], i64 1
-; CHECK-NEXT: call void @use(i32 [[TMP13]])
-; CHECK-NEXT: call void @use(i32 [[TMP10]])
-; CHECK-NEXT: call void @use(i32 [[TMP11]])
-; CHECK-NEXT: call void @use(i32 [[TMP12]])
-; CHECK-NEXT: call void @use(i32 [[TMP13]])
+; CHECK-NEXT: [[TMP4:%.*]] = add <2 x i32> [[TMP1]], [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = icmp ult <2 x i32> [[TMP4]], [[TMP3]]
+; CHECK-NEXT: [[TMP6:%.*]] = zext <2 x i1> [[TMP5]] to <2 x i32>
+; CHECK-NEXT: [[TMP7:%.*]] = add <2 x i32> [[TMP4]], [[TMP6]]
+; CHECK-NEXT: store <2 x i32> [[TMP4]], ptr [[DST]], align 4
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x i32> [[TMP7]], i64 0
+; CHECK-NEXT: call void @use(i32 [[TMP8]])
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x i32> [[TMP7]], i64 1
+; CHECK-NEXT: call void @use(i32 [[TMP9]])
; CHECK-NEXT: ret void
;
entry:
%dst1 = getelementptr inbounds i32, ptr %dst, i64 1
- %dst2 = getelementptr inbounds i32, ptr %dst, i64 2
- %dst3 = getelementptr inbounds i32, ptr %dst, i64 3
-
%a0 = add i32 %x0, %y0
%a1 = add i32 %x1, %y1
- %a2 = add i32 %x2, %y2
- %a3 = add i32 %x3, %y3
-
+ %cmp0 = icmp ult i32 %a0, %y0
+ %cmp1 = icmp ult i32 %a1, %y1
+ %z0 = zext i1 %cmp0 to i32
+ %z1 = zext i1 %cmp1 to i32
+ %b0 = add i32 %a0, %z0
+ %b1 = add i32 %a1, %z1
store i32 %a0, ptr %dst, align 4
store i32 %a1, ptr %dst1, align 4
- store i32 %a2, ptr %dst2, align 4
- store i32 %a3, ptr %dst3, align 4
+ call void @use(i32 %b0)
+ call void @use(i32 %b1)
+ ret void
+}
+; Two non-benign call users keep the store chain scalar.
+; Independent arithmetic may still be vectorized before the scalar
+; stores.
+define void @store_chain_two_call_users(ptr %dst, i32 %x0, i32 %x1, i32 %y0, i32 %y1) {
+; CHECK-LABEL: define void @store_chain_two_call_users(
+; CHECK-SAME: ptr [[DST:%.*]], i32 [[X0:%.*]], i32 [[X1:%.*]], i32 [[Y0:%.*]], i32 [[Y1:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[DST1:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 1
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x i32> poison, i32 [[X0]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i32> [[TMP0]], i32 [[X1]], i64 1
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x i32> poison, i32 [[Y0]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x i32> [[TMP2]], i32 [[Y1]], i64 1
+; CHECK-NEXT: [[TMP4:%.*]] = add <2 x i32> [[TMP1]], [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x i32> [[TMP4]], i64 1
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x i32> [[TMP4]], i64 0
+; CHECK-NEXT: store i32 [[TMP6]], ptr [[DST]], align 4
+; CHECK-NEXT: store i32 [[TMP5]], ptr [[DST1]], align 4
+; CHECK-NEXT: call void @use(i32 [[TMP6]])
+; CHECK-NEXT: call void @use(i32 [[TMP6]])
+; CHECK-NEXT: call void @use(i32 [[TMP5]])
+; CHECK-NEXT: call void @use(i32 [[TMP5]])
+; CHECK-NEXT: ret void
+;
+entry:
+ %dst1 = getelementptr inbounds i32, ptr %dst, i64 1
+ %a0 = add i32 %x0, %y0
+ %a1 = add i32 %x1, %y1
+ store i32 %a0, ptr %dst, align 4
+ store i32 %a1, ptr %dst1, align 4
+ call void @use(i32 %a0)
call void @use(i32 %a0)
call void @use(i32 %a1)
- call void @use(i32 %a2)
- call void @use(i32 %a3)
+ call void @use(i32 %a1)
+ ret void
+}
+
+; Three benign outside users keep the store chain scalar.
+define void @store_chain_three_benign_users(ptr %dst, i32 %x0, i32 %x1, i32 %y0, i32 %y1, i1 %cond) {
+; CHECK-LABEL: define void @store_chain_three_benign_users(
+; CHECK-SAME: ptr [[DST:%.*]], i32 [[X0:%.*]], i32 [[X1:%.*]], i32 [[Y0:%.*]], i32 [[Y1:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[DST1:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 1
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x i32> poison, i32 [[X0]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i32> [[TMP0]], i32 [[X1]], i64 1
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x i32> poison, i32 [[Y0]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x i32> [[TMP2]], i32 [[Y1]], i64 1
+; CHECK-NEXT: [[TMP4:%.*]] = add <2 x i32> [[TMP1]], [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = icmp ult <2 x i32> [[TMP4]], [[TMP3]]
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x i1> [[TMP5]], i64 0
+; CHECK-NEXT: [[Z0:%.*]] = zext i1 [[TMP6]] to i32
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x i1> [[TMP5]], i64 1
+; CHECK-NEXT: [[Z1:%.*]] = zext i1 [[TMP7]] to i32
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x i32> [[TMP4]], i64 0
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x i32> [[TMP4]], i64 1
+; CHECK-NEXT: [[TMP10:%.*]] = add <2 x i32> [[TMP4]], [[TMP1]]
+; CHECK-NEXT: [[TMP11:%.*]] = insertelement <2 x i1> poison, i1 [[COND]], i64 0
+; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <2 x i1> [[TMP11]], <2 x i1> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP13:%.*]] = select <2 x i1> [[TMP12]], <2 x i32> [[TMP4]], <2 x i32> [[TMP3]]
+; CHECK-NEXT: store i32 [[TMP8]], ptr [[DST]], align 4
+; CHECK-NEXT: store i32 [[TMP9]], ptr [[DST1]], align 4
+; CHECK-NEXT: call void @use(i32 [[Z0]])
+; CHECK-NEXT: call void @use(i32 [[Z1]])
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <2 x i32> [[TMP10]], i64 0
+; CHECK-NEXT: call void @use(i32 [[TMP14]])
+; CHECK-NEXT: [[TMP15:%.*]] = extractelement <2 x i32> [[TMP10]], i64 1
+; CHECK-NEXT: call void @use(i32 [[TMP15]])
+; CHECK-NEXT: [[TMP16:%.*]] = extractelement <2 x i32> [[TMP13]], i64 0
+; CHECK-NEXT: call void @use(i32 [[TMP16]])
+; CHECK-NEXT: [[TMP17:%.*]] = extractelement <2 x i32> [[TMP13]], i64 1
+; CHECK-NEXT: call void @use(i32 [[TMP17]])
+; CHECK-NEXT: ret void
+;
+entry:
+ %dst1 = getelementptr inbounds i32, ptr %dst, i64 1
+ %a0 = add i32 %x0, %y0
+ %a1 = add i32 %x1, %y1
+ %cmp0 = icmp ult i32 %a0, %y0
+ %cmp1 = icmp ult i32 %a1, %y1
+ %z0 = zext i1 %cmp0 to i32
+ %z1 = zext i1 %cmp1 to i32
+ %b0 = add i32 %a0, %x0
+ %b1 = add i32 %a1, %x1
+ %sel0 = select i1 %cond, i32 %a0, i32 %y0
+ %sel1 = select i1 %cond, i32 %a1, i32 %y1
+ store i32 %a0, ptr %dst, align 4
+ store i32 %a1, ptr %dst1, align 4
+ call void @use(i32 %z0)
+ call void @use(i32 %z1)
+ call void @use(i32 %b0)
+ call void @use(i32 %b1)
+ call void @use(i32 %sel0)
+ call void @use(i32 %sel1)
+ ret void
+}
+
+; One non-benign outside user is still allowed, matching the original >1 heuristic.
+define void @store_chain_one_call_user(ptr %dst, i32 %x0, i32 %x1, i32 %y0, i32 %y1) {
+; CHECK-LABEL: define void @store_chain_one_call_user(
+; CHECK-SAME: ptr [[DST:%.*]], i32 [[X0:%.*]], i32 [[X1:%.*]], i32 [[Y0:%.*]], i32 [[Y1:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x i32> poison, i32 [[X0]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i32> [[TMP0]], i32 [[X1]], i64 1
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x i32> poison, i32 [[Y0]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x i32> [[TMP2]], i32 [[Y1]], i64 1
+; CHECK-NEXT: [[TMP4:%.*]] = add <2 x i32> [[TMP1]], [[TMP3]]
+; CHECK-NEXT: store <2 x i32> [[TMP4]], ptr [[DST]], align 4
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x i32> [[TMP4]], i64 0
+; CHECK-NEXT: call void @use(i32 [[TMP5]])
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x i32> [[TMP4]], i64 1
+; CHECK-NEXT: call void @use(i32 [[TMP6]])
+; CHECK-NEXT: ret void
+;
+entry:
+ %dst1 = getelementptr inbounds i32, ptr %dst, i64 1
+ %a0 = add i32 %x0, %y0
+ %a1 = add i32 %x1, %y1
+ store i32 %a0, ptr %dst, align 4
+ store i32 %a1, ptr %dst1, align 4
call void @use(i32 %a0)
call void @use(i32 %a1)
- call void @use(i32 %a2)
- call void @use(i32 %a3)
ret void
}
>From 76d2e87345f36f2b06f48d4928c71c7967425a36 Mon Sep 17 00:00:00 2001
From: akadutta_amdeng <Akash.Dutta at amd.com>
Date: Tue, 18 Aug 2026 10:52:20 -0500
Subject: [PATCH 3/3] fix format issues
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 9a69d9deddbbd..9ac8bb26fcbfb 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -28680,8 +28680,9 @@ SLPVectorizerPass::vectorizeStoreChainImpl(ArrayRef<Value *> Chain, BoUpSLP &R,
// and all the lanes are gathered back. A single outside use may still be a
// part of the larger vectorizable graph, same for the values, fed by the
// loads, where the vector loads may pay off the gathering. Two outside uses
- // may still be part of the same tree when they are local carry deps (cmp/add
- // in the same block); reject 3+ outside users or 2 non-local outside users.
+ // may still be part of the same tree when they are local carry deps
+ // (cmp/add in the same block); reject 3+ outside users or 2 non-local
+ // outside users.
auto IsBenignOutsideUser = [](Instruction *V, User *U) {
auto *UI = dyn_cast<Instruction>(U);
if (!UI || UI->getParent() != V->getParent())
More information about the llvm-commits
mailing list