[llvm] [SLP] Relax store chain user limit to 2 (PR #215700)
Akash Dutta via llvm-commits
llvm-commits at lists.llvm.org
Fri Aug 21 15:51:17 PDT 2026
https://github.com/akadutta updated https://github.com/llvm/llvm-project/pull/215700
>From 3a9c31d9e36a4bb07683dd679785593b9dd10242 Mon Sep 17 00:00:00 2001
From: akadutta_amdeng <Akash.Dutta at amd.com>
Date: Tue, 11 Aug 2026 18:59:45 -0500
Subject: [PATCH 1/5] relax store chain user limit to 2
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 6 +-
.../AMDGPU/store-chain-two-external-users.ll | 67 +++++++++++++++++++
2 files changed, 71 insertions(+), 2 deletions(-)
create mode 100644 llvm/test/Transforms/SLPVectorizer/AMDGPU/store-chain-two-external-users.ll
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index a3350cf3f3e52..7eec02574302e 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -28679,13 +28679,15 @@ SLPVectorizerPass::vectorizeStoreChainImpl(ArrayRef<Value *> Chain, BoUpSLP &R,
// only merges the stores, while the scalars remain live for the other users
// and all the lanes are gathered back. A single outside use may still be a
// part of the larger vectorizable graph, same for the values, fed by the
- // loads, where the vector loads may pay off the gathering.
+ // loads, where the vector loads may pay off the gathering. Likewise two
+ // outside uses (e.g. carry compare and next-limb add) may still be part of
+ // the same tree; require 3+ outside users before early-rejecting here.
if (S && S.getOpcode() != Instruction::Load &&
all_of(ValOps.getArrayRef(), [&](Value *V) {
return none_of(cast<Instruction>(V)->operand_values(),
IsaPred<LoadInst>) &&
count_if(V->users(),
- [&](User *U) { return !Stores.contains(U); }) > 1;
+ [&](User *U) { return !Stores.contains(U); }) > 2;
})) {
Size = 1;
return false;
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/store-chain-two-external-users.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/store-chain-two-external-users.ll
new file mode 100644
index 0000000000000..8db7283bbdc53
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/store-chain-two-external-users.ll
@@ -0,0 +1,67 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -passes=slp-vectorizer -slp-threshold=-100 -mtriple=amdgpu9.0a-amd-amdhsa < %s | FileCheck %s
+
+; Store-chain operands may have two users outside the store chain and still be
+; worth considering. This mirrors carry-propagating counter updates,
+; where the updated limbs remain live for scalar carry/use
+; chains after the state stores. The old "> 1 outside user" early return rejected
+; this before the cost model; with a "> 2" threshold the chain reaches costing.
+
+declare void @use(i32)
+
+define void @store_chain_two_external_users(ptr %dst, i32 %x0, i32 %x1, i32 %x2, i32 %x3, i32 %y0, i32 %y1, i32 %y2, i32 %y3) {
+; CHECK-LABEL: define void @store_chain_two_external_users(
+; CHECK-SAME: ptr [[DST:%.*]], i32 [[X0:%.*]], i32 [[X1:%.*]], i32 [[X2:%.*]], i32 [[X3:%.*]], i32 [[Y0:%.*]], i32 [[Y1:%.*]], i32 [[Y2:%.*]], i32 [[Y3:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[DST2:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 2
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x i32> poison, i32 [[X0]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i32> [[TMP0]], i32 [[X1]], i64 1
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x i32> poison, i32 [[Y0]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x i32> [[TMP2]], i32 [[Y1]], i64 1
+; CHECK-NEXT: [[V0:%.*]] = add <2 x i32> [[TMP1]], [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x i32> poison, i32 [[X2]], i64 0
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x i32> [[TMP5]], i32 [[X3]], i64 1
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x i32> poison, i32 [[Y2]], i64 0
+; CHECK-NEXT: [[TMP8:%.*]] = insertelement <2 x i32> [[TMP7]], i32 [[Y3]], i64 1
+; CHECK-NEXT: [[V1:%.*]] = add <2 x i32> [[TMP6]], [[TMP8]]
+; CHECK-NEXT: store <2 x i32> [[V0]], ptr [[DST]], align 4
+; CHECK-NEXT: store <2 x i32> [[V1]], ptr [[DST2]], align 4
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x i32> [[V0]], i64 0
+; CHECK-NEXT: call void @use(i32 [[TMP10]])
+; CHECK-NEXT: [[TMP11:%.*]] = extractelement <2 x i32> [[V0]], i64 1
+; CHECK-NEXT: call void @use(i32 [[TMP11]])
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x i32> [[V1]], i64 0
+; CHECK-NEXT: call void @use(i32 [[TMP12]])
+; CHECK-NEXT: [[TMP13:%.*]] = extractelement <2 x i32> [[V1]], i64 1
+; CHECK-NEXT: call void @use(i32 [[TMP13]])
+; CHECK-NEXT: call void @use(i32 [[TMP10]])
+; CHECK-NEXT: call void @use(i32 [[TMP11]])
+; CHECK-NEXT: call void @use(i32 [[TMP12]])
+; CHECK-NEXT: call void @use(i32 [[TMP13]])
+; CHECK-NEXT: ret void
+;
+entry:
+ %dst1 = getelementptr inbounds i32, ptr %dst, i64 1
+ %dst2 = getelementptr inbounds i32, ptr %dst, i64 2
+ %dst3 = getelementptr inbounds i32, ptr %dst, i64 3
+
+ %a0 = add i32 %x0, %y0
+ %a1 = add i32 %x1, %y1
+ %a2 = add i32 %x2, %y2
+ %a3 = add i32 %x3, %y3
+
+ store i32 %a0, ptr %dst, align 4
+ store i32 %a1, ptr %dst1, align 4
+ store i32 %a2, ptr %dst2, align 4
+ store i32 %a3, ptr %dst3, align 4
+
+ call void @use(i32 %a0)
+ call void @use(i32 %a1)
+ call void @use(i32 %a2)
+ call void @use(i32 %a3)
+ call void @use(i32 %a0)
+ call void @use(i32 %a1)
+ call void @use(i32 %a2)
+ call void @use(i32 %a3)
+ ret void
+}
>From 0c6c56b0f55b5ec84e04ae02f4b891aa16f1ccd0 Mon Sep 17 00:00:00 2001
From: akadutta_amdeng <Akash.Dutta at amd.com>
Date: Tue, 18 Aug 2026 10:34:24 -0500
Subject: [PATCH 2/5] early-reject store chains only for non-local or 3+
outside users
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 27 ++-
.../AMDGPU/store-chain-two-external-users.ll | 188 ++++++++++++++----
2 files changed, 169 insertions(+), 46 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 7eec02574302e..9a69d9deddbbd 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -28679,15 +28679,32 @@ SLPVectorizerPass::vectorizeStoreChainImpl(ArrayRef<Value *> Chain, BoUpSLP &R,
// only merges the stores, while the scalars remain live for the other users
// and all the lanes are gathered back. A single outside use may still be a
// part of the larger vectorizable graph, same for the values, fed by the
- // loads, where the vector loads may pay off the gathering. Likewise two
- // outside uses (e.g. carry compare and next-limb add) may still be part of
- // the same tree; require 3+ outside users before early-rejecting here.
+ // loads, where the vector loads may pay off the gathering. Two outside uses
+ // may still be part of the same tree when they are local carry deps (cmp/add
+ // in the same block); reject 3+ outside users or 2 non-local outside users.
+ auto IsBenignOutsideUser = [](Instruction *V, User *U) {
+ auto *UI = dyn_cast<Instruction>(U);
+ if (!UI || UI->getParent() != V->getParent())
+ return false;
+ return isa<CmpInst>(UI) || isa<BinaryOperator>(UI) || isa<SelectInst>(UI);
+ };
+ auto HasTooManyOutsideUsers = [&](Value *V) {
+ unsigned Outside = 0;
+ bool HasNonBenign = false;
+ for (User *U : V->users()) {
+ if (Stores.contains(U))
+ continue;
+ ++Outside;
+ if (!IsBenignOutsideUser(cast<Instruction>(V), U))
+ HasNonBenign = true;
+ }
+ return Outside > 2 || (Outside == 2 && HasNonBenign);
+ };
if (S && S.getOpcode() != Instruction::Load &&
all_of(ValOps.getArrayRef(), [&](Value *V) {
return none_of(cast<Instruction>(V)->operand_values(),
IsaPred<LoadInst>) &&
- count_if(V->users(),
- [&](User *U) { return !Stores.contains(U); }) > 2;
+ HasTooManyOutsideUsers(V);
})) {
Size = 1;
return false;
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/store-chain-two-external-users.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/store-chain-two-external-users.ll
index 8db7283bbdc53..ab5820ae87e5a 100644
--- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/store-chain-two-external-users.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/store-chain-two-external-users.ll
@@ -1,67 +1,173 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -S -passes=slp-vectorizer -slp-threshold=-100 -mtriple=amdgpu9.0a-amd-amdhsa < %s | FileCheck %s
-; Store-chain operands may have two users outside the store chain and still be
-; worth considering. This mirrors carry-propagating counter updates,
-; where the updated limbs remain live for scalar carry/use
-; chains after the state stores. The old "> 1 outside user" early return rejected
-; this before the cost model; with a "> 2" threshold the chain reaches costing.
+; Use a negative threshold to test whether the store chain reaches costing,
+; independently of the target cost model.
+;
+; A store chain is rejected when every stored value has either more than two
+; users outside the chain, or exactly two outside users with at least one that
+; is not a same-block cmp, binop, or select.
declare void @use(i32)
-define void @store_chain_two_external_users(ptr %dst, i32 %x0, i32 %x1, i32 %x2, i32 %x3, i32 %y0, i32 %y1, i32 %y2, i32 %y3) {
-; CHECK-LABEL: define void @store_chain_two_external_users(
-; CHECK-SAME: ptr [[DST:%.*]], i32 [[X0:%.*]], i32 [[X1:%.*]], i32 [[X2:%.*]], i32 [[X3:%.*]], i32 [[Y0:%.*]], i32 [[Y1:%.*]], i32 [[Y2:%.*]], i32 [[Y3:%.*]]) {
+; Two benign outside users let the store chain reach costing. This distinguishes
+; the refined heuristic from the previous >1 limit and covers values used by
+; both a compare and a dependent add.
+define void @store_chain_two_benign_users(ptr %dst, i32 %x0, i32 %x1, i32 %y0, i32 %y1) {
+; CHECK-LABEL: define void @store_chain_two_benign_users(
+; CHECK-SAME: ptr [[DST:%.*]], i32 [[X0:%.*]], i32 [[X1:%.*]], i32 [[Y0:%.*]], i32 [[Y1:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[DST2:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 2
; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x i32> poison, i32 [[X0]], i64 0
; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i32> [[TMP0]], i32 [[X1]], i64 1
; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x i32> poison, i32 [[Y0]], i64 0
; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x i32> [[TMP2]], i32 [[Y1]], i64 1
-; CHECK-NEXT: [[V0:%.*]] = add <2 x i32> [[TMP1]], [[TMP3]]
-; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x i32> poison, i32 [[X2]], i64 0
-; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x i32> [[TMP5]], i32 [[X3]], i64 1
-; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x i32> poison, i32 [[Y2]], i64 0
-; CHECK-NEXT: [[TMP8:%.*]] = insertelement <2 x i32> [[TMP7]], i32 [[Y3]], i64 1
-; CHECK-NEXT: [[V1:%.*]] = add <2 x i32> [[TMP6]], [[TMP8]]
-; CHECK-NEXT: store <2 x i32> [[V0]], ptr [[DST]], align 4
-; CHECK-NEXT: store <2 x i32> [[V1]], ptr [[DST2]], align 4
-; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x i32> [[V0]], i64 0
-; CHECK-NEXT: call void @use(i32 [[TMP10]])
-; CHECK-NEXT: [[TMP11:%.*]] = extractelement <2 x i32> [[V0]], i64 1
-; CHECK-NEXT: call void @use(i32 [[TMP11]])
-; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x i32> [[V1]], i64 0
-; CHECK-NEXT: call void @use(i32 [[TMP12]])
-; CHECK-NEXT: [[TMP13:%.*]] = extractelement <2 x i32> [[V1]], i64 1
-; CHECK-NEXT: call void @use(i32 [[TMP13]])
-; CHECK-NEXT: call void @use(i32 [[TMP10]])
-; CHECK-NEXT: call void @use(i32 [[TMP11]])
-; CHECK-NEXT: call void @use(i32 [[TMP12]])
-; CHECK-NEXT: call void @use(i32 [[TMP13]])
+; CHECK-NEXT: [[TMP4:%.*]] = add <2 x i32> [[TMP1]], [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = icmp ult <2 x i32> [[TMP4]], [[TMP3]]
+; CHECK-NEXT: [[TMP6:%.*]] = zext <2 x i1> [[TMP5]] to <2 x i32>
+; CHECK-NEXT: [[TMP7:%.*]] = add <2 x i32> [[TMP4]], [[TMP6]]
+; CHECK-NEXT: store <2 x i32> [[TMP4]], ptr [[DST]], align 4
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x i32> [[TMP7]], i64 0
+; CHECK-NEXT: call void @use(i32 [[TMP8]])
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x i32> [[TMP7]], i64 1
+; CHECK-NEXT: call void @use(i32 [[TMP9]])
; CHECK-NEXT: ret void
;
entry:
%dst1 = getelementptr inbounds i32, ptr %dst, i64 1
- %dst2 = getelementptr inbounds i32, ptr %dst, i64 2
- %dst3 = getelementptr inbounds i32, ptr %dst, i64 3
-
%a0 = add i32 %x0, %y0
%a1 = add i32 %x1, %y1
- %a2 = add i32 %x2, %y2
- %a3 = add i32 %x3, %y3
-
+ %cmp0 = icmp ult i32 %a0, %y0
+ %cmp1 = icmp ult i32 %a1, %y1
+ %z0 = zext i1 %cmp0 to i32
+ %z1 = zext i1 %cmp1 to i32
+ %b0 = add i32 %a0, %z0
+ %b1 = add i32 %a1, %z1
store i32 %a0, ptr %dst, align 4
store i32 %a1, ptr %dst1, align 4
- store i32 %a2, ptr %dst2, align 4
- store i32 %a3, ptr %dst3, align 4
+ call void @use(i32 %b0)
+ call void @use(i32 %b1)
+ ret void
+}
+; Two non-benign call users keep the store chain scalar.
+; Independent arithmetic may still be vectorized before the scalar
+; stores.
+define void @store_chain_two_call_users(ptr %dst, i32 %x0, i32 %x1, i32 %y0, i32 %y1) {
+; CHECK-LABEL: define void @store_chain_two_call_users(
+; CHECK-SAME: ptr [[DST:%.*]], i32 [[X0:%.*]], i32 [[X1:%.*]], i32 [[Y0:%.*]], i32 [[Y1:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[DST1:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 1
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x i32> poison, i32 [[X0]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i32> [[TMP0]], i32 [[X1]], i64 1
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x i32> poison, i32 [[Y0]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x i32> [[TMP2]], i32 [[Y1]], i64 1
+; CHECK-NEXT: [[TMP4:%.*]] = add <2 x i32> [[TMP1]], [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x i32> [[TMP4]], i64 1
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x i32> [[TMP4]], i64 0
+; CHECK-NEXT: store i32 [[TMP6]], ptr [[DST]], align 4
+; CHECK-NEXT: store i32 [[TMP5]], ptr [[DST1]], align 4
+; CHECK-NEXT: call void @use(i32 [[TMP6]])
+; CHECK-NEXT: call void @use(i32 [[TMP6]])
+; CHECK-NEXT: call void @use(i32 [[TMP5]])
+; CHECK-NEXT: call void @use(i32 [[TMP5]])
+; CHECK-NEXT: ret void
+;
+entry:
+ %dst1 = getelementptr inbounds i32, ptr %dst, i64 1
+ %a0 = add i32 %x0, %y0
+ %a1 = add i32 %x1, %y1
+ store i32 %a0, ptr %dst, align 4
+ store i32 %a1, ptr %dst1, align 4
+ call void @use(i32 %a0)
call void @use(i32 %a0)
call void @use(i32 %a1)
- call void @use(i32 %a2)
- call void @use(i32 %a3)
+ call void @use(i32 %a1)
+ ret void
+}
+
+; Three benign outside users keep the store chain scalar.
+define void @store_chain_three_benign_users(ptr %dst, i32 %x0, i32 %x1, i32 %y0, i32 %y1, i1 %cond) {
+; CHECK-LABEL: define void @store_chain_three_benign_users(
+; CHECK-SAME: ptr [[DST:%.*]], i32 [[X0:%.*]], i32 [[X1:%.*]], i32 [[Y0:%.*]], i32 [[Y1:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[DST1:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 1
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x i32> poison, i32 [[X0]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i32> [[TMP0]], i32 [[X1]], i64 1
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x i32> poison, i32 [[Y0]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x i32> [[TMP2]], i32 [[Y1]], i64 1
+; CHECK-NEXT: [[TMP4:%.*]] = add <2 x i32> [[TMP1]], [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = icmp ult <2 x i32> [[TMP4]], [[TMP3]]
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x i1> [[TMP5]], i64 0
+; CHECK-NEXT: [[Z0:%.*]] = zext i1 [[TMP6]] to i32
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x i1> [[TMP5]], i64 1
+; CHECK-NEXT: [[Z1:%.*]] = zext i1 [[TMP7]] to i32
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x i32> [[TMP4]], i64 0
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x i32> [[TMP4]], i64 1
+; CHECK-NEXT: [[TMP10:%.*]] = add <2 x i32> [[TMP4]], [[TMP1]]
+; CHECK-NEXT: [[TMP11:%.*]] = insertelement <2 x i1> poison, i1 [[COND]], i64 0
+; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <2 x i1> [[TMP11]], <2 x i1> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP13:%.*]] = select <2 x i1> [[TMP12]], <2 x i32> [[TMP4]], <2 x i32> [[TMP3]]
+; CHECK-NEXT: store i32 [[TMP8]], ptr [[DST]], align 4
+; CHECK-NEXT: store i32 [[TMP9]], ptr [[DST1]], align 4
+; CHECK-NEXT: call void @use(i32 [[Z0]])
+; CHECK-NEXT: call void @use(i32 [[Z1]])
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <2 x i32> [[TMP10]], i64 0
+; CHECK-NEXT: call void @use(i32 [[TMP14]])
+; CHECK-NEXT: [[TMP15:%.*]] = extractelement <2 x i32> [[TMP10]], i64 1
+; CHECK-NEXT: call void @use(i32 [[TMP15]])
+; CHECK-NEXT: [[TMP16:%.*]] = extractelement <2 x i32> [[TMP13]], i64 0
+; CHECK-NEXT: call void @use(i32 [[TMP16]])
+; CHECK-NEXT: [[TMP17:%.*]] = extractelement <2 x i32> [[TMP13]], i64 1
+; CHECK-NEXT: call void @use(i32 [[TMP17]])
+; CHECK-NEXT: ret void
+;
+entry:
+ %dst1 = getelementptr inbounds i32, ptr %dst, i64 1
+ %a0 = add i32 %x0, %y0
+ %a1 = add i32 %x1, %y1
+ %cmp0 = icmp ult i32 %a0, %y0
+ %cmp1 = icmp ult i32 %a1, %y1
+ %z0 = zext i1 %cmp0 to i32
+ %z1 = zext i1 %cmp1 to i32
+ %b0 = add i32 %a0, %x0
+ %b1 = add i32 %a1, %x1
+ %sel0 = select i1 %cond, i32 %a0, i32 %y0
+ %sel1 = select i1 %cond, i32 %a1, i32 %y1
+ store i32 %a0, ptr %dst, align 4
+ store i32 %a1, ptr %dst1, align 4
+ call void @use(i32 %z0)
+ call void @use(i32 %z1)
+ call void @use(i32 %b0)
+ call void @use(i32 %b1)
+ call void @use(i32 %sel0)
+ call void @use(i32 %sel1)
+ ret void
+}
+
+; One non-benign outside user is still allowed, matching the original >1 heuristic.
+define void @store_chain_one_call_user(ptr %dst, i32 %x0, i32 %x1, i32 %y0, i32 %y1) {
+; CHECK-LABEL: define void @store_chain_one_call_user(
+; CHECK-SAME: ptr [[DST:%.*]], i32 [[X0:%.*]], i32 [[X1:%.*]], i32 [[Y0:%.*]], i32 [[Y1:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x i32> poison, i32 [[X0]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i32> [[TMP0]], i32 [[X1]], i64 1
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x i32> poison, i32 [[Y0]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x i32> [[TMP2]], i32 [[Y1]], i64 1
+; CHECK-NEXT: [[TMP4:%.*]] = add <2 x i32> [[TMP1]], [[TMP3]]
+; CHECK-NEXT: store <2 x i32> [[TMP4]], ptr [[DST]], align 4
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x i32> [[TMP4]], i64 0
+; CHECK-NEXT: call void @use(i32 [[TMP5]])
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x i32> [[TMP4]], i64 1
+; CHECK-NEXT: call void @use(i32 [[TMP6]])
+; CHECK-NEXT: ret void
+;
+entry:
+ %dst1 = getelementptr inbounds i32, ptr %dst, i64 1
+ %a0 = add i32 %x0, %y0
+ %a1 = add i32 %x1, %y1
+ store i32 %a0, ptr %dst, align 4
+ store i32 %a1, ptr %dst1, align 4
call void @use(i32 %a0)
call void @use(i32 %a1)
- call void @use(i32 %a2)
- call void @use(i32 %a3)
ret void
}
>From 76d2e87345f36f2b06f48d4928c71c7967425a36 Mon Sep 17 00:00:00 2001
From: akadutta_amdeng <Akash.Dutta at amd.com>
Date: Tue, 18 Aug 2026 10:52:20 -0500
Subject: [PATCH 3/5] fix format issues
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 9a69d9deddbbd..9ac8bb26fcbfb 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -28680,8 +28680,9 @@ SLPVectorizerPass::vectorizeStoreChainImpl(ArrayRef<Value *> Chain, BoUpSLP &R,
// and all the lanes are gathered back. A single outside use may still be a
// part of the larger vectorizable graph, same for the values, fed by the
// loads, where the vector loads may pay off the gathering. Two outside uses
- // may still be part of the same tree when they are local carry deps (cmp/add
- // in the same block); reject 3+ outside users or 2 non-local outside users.
+ // may still be part of the same tree when they are local carry deps
+ // (cmp/add in the same block); reject 3+ outside users or 2 non-local
+ // outside users.
auto IsBenignOutsideUser = [](Instruction *V, User *U) {
auto *UI = dyn_cast<Instruction>(U);
if (!UI || UI->getParent() != V->getParent())
>From eebf9f1ee3bc16d6e7f08de76067d94595ee5aaa Mon Sep 17 00:00:00 2001
From: akadutta_amdeng <Akash.Dutta at amd.com>
Date: Tue, 18 Aug 2026 13:12:51 -0500
Subject: [PATCH 4/5] cap no. of uses, pass inst into lambda
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 26 ++++++++++++-------
1 file changed, 16 insertions(+), 10 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 9ac8bb26fcbfb..a02068e9c27af 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -28683,29 +28683,35 @@ SLPVectorizerPass::vectorizeStoreChainImpl(ArrayRef<Value *> Chain, BoUpSLP &R,
// may still be part of the same tree when they are local carry deps
// (cmp/add in the same block); reject 3+ outside users or 2 non-local
// outside users.
- auto IsBenignOutsideUser = [](Instruction *V, User *U) {
+ auto IsBenignOutsideUser = [](Instruction *I, User *U) {
auto *UI = dyn_cast<Instruction>(U);
- if (!UI || UI->getParent() != V->getParent())
+ if (!UI || UI->getParent() != I->getParent())
return false;
- return isa<CmpInst>(UI) || isa<BinaryOperator>(UI) || isa<SelectInst>(UI);
+ return isa<CmpInst, BinaryOperator, SelectInst>(UI);
};
- auto HasTooManyOutsideUsers = [&](Value *V) {
+ auto HasTooManyOutsideUsers = [&](Instruction *I) {
+ // Cap the walk: more than one store use + two outside users is enough to
+ // reject, so skip values with clearly too many uses.
+ if (I->hasNUsesOrMore(Stores.size() + 3))
+ return true;
unsigned Outside = 0;
bool HasNonBenign = false;
- for (User *U : V->users()) {
+ for (User *U : I->users()) {
if (Stores.contains(U))
continue;
++Outside;
- if (!IsBenignOutsideUser(cast<Instruction>(V), U))
+ if (Outside > 2)
+ return true;
+ if (!IsBenignOutsideUser(I, U))
HasNonBenign = true;
}
- return Outside > 2 || (Outside == 2 && HasNonBenign);
+ return Outside == 2 && HasNonBenign;
};
if (S && S.getOpcode() != Instruction::Load &&
all_of(ValOps.getArrayRef(), [&](Value *V) {
- return none_of(cast<Instruction>(V)->operand_values(),
- IsaPred<LoadInst>) &&
- HasTooManyOutsideUsers(V);
+ auto *I = cast<Instruction>(V);
+ return none_of(I->operand_values(), IsaPred<LoadInst>) &&
+ HasTooManyOutsideUsers(I);
})) {
Size = 1;
return false;
>From ac4c20da53ff613d100de76ddd3148a5c6f4f664 Mon Sep 17 00:00:00 2001
From: akadutta_amdeng <Akash.Dutta at amd.com>
Date: Fri, 21 Aug 2026 11:45:06 -0500
Subject: [PATCH 5/5] update benign outside user criteria
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 23 +-
.../AMDGPU/store-chain-two-external-users.ll | 324 +++++++++++++++++-
2 files changed, 336 insertions(+), 11 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index a02068e9c27af..42612e857cf0d 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -28680,19 +28680,22 @@ SLPVectorizerPass::vectorizeStoreChainImpl(ArrayRef<Value *> Chain, BoUpSLP &R,
// and all the lanes are gathered back. A single outside use may still be a
// part of the larger vectorizable graph, same for the values, fed by the
// loads, where the vector loads may pay off the gathering. Two outside uses
- // may still be part of the same tree when they are local carry deps
- // (cmp/add in the same block); reject 3+ outside users or 2 non-local
- // outside users.
- auto IsBenignOutsideUser = [](Instruction *I, User *U) {
+ // may still be part of the same profitable tree when they are carry deps
+ // (cmp/add/select, or a two-incoming phi); reject 3+ outside users
+ // or 2 non-benign outside users.
+ auto IsBenignOutsideUser = [](User *U) {
auto *UI = dyn_cast<Instruction>(U);
- if (!UI || UI->getParent() != I->getParent())
+ if (!UI)
return false;
- return isa<CmpInst, BinaryOperator, SelectInst>(UI);
+ if (auto *PN = dyn_cast<PHINode>(UI))
+ return PN->getNumIncomingValues() == 2;
+ if (auto *BO = dyn_cast<BinaryOperator>(UI))
+ return !BO->isIntDivRem() && !BO->isFPDivRem();
+ return isa<CmpInst, SelectInst>(UI);
};
auto HasTooManyOutsideUsers = [&](Instruction *I) {
- // Cap the walk: more than one store use + two outside users is enough to
- // reject, so skip values with clearly too many uses.
- if (I->hasNUsesOrMore(Stores.size() + 3))
+ // To save compilation time, bail out if the use list is huge.
+ if (I->hasNUsesOrMore(UsesLimit))
return true;
unsigned Outside = 0;
bool HasNonBenign = false;
@@ -28702,7 +28705,7 @@ SLPVectorizerPass::vectorizeStoreChainImpl(ArrayRef<Value *> Chain, BoUpSLP &R,
++Outside;
if (Outside > 2)
return true;
- if (!IsBenignOutsideUser(I, U))
+ if (!IsBenignOutsideUser(U))
HasNonBenign = true;
}
return Outside == 2 && HasNonBenign;
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/store-chain-two-external-users.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/store-chain-two-external-users.ll
index ab5820ae87e5a..0ba42e2b62bec 100644
--- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/store-chain-two-external-users.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/store-chain-two-external-users.ll
@@ -6,7 +6,8 @@
;
; A store chain is rejected when every stored value has either more than two
; users outside the chain, or exactly two outside users with at least one that
-; is not a same-block cmp, binop, or select.
+; is not a benign carry dependence. A benign outside user is a cmp, binop, or
+; select (in any block), or a phi with exactly two incoming values.
declare void @use(i32)
@@ -171,3 +172,324 @@ entry:
call void @use(i32 %a1)
ret void
}
+
+; Two outside users that are cmp/binop/select but live in a *different* block are
+; still benign: the heuristic is block-agnostic. Each stored value has exactly
+; two cross-block add users, so the chain reaches costing.
+define void @store_chain_two_cross_block_benign(ptr %dst, i32 %x0, i32 %x1, i32 %y0, i32 %y1, i1 %cond) {
+; CHECK-LABEL: define void @store_chain_two_cross_block_benign(
+; CHECK-SAME: ptr [[DST:%.*]], i32 [[X0:%.*]], i32 [[X1:%.*]], i32 [[Y0:%.*]], i32 [[Y1:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x i32> poison, i32 [[X0]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i32> [[TMP0]], i32 [[X1]], i64 1
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x i32> poison, i32 [[Y0]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x i32> [[TMP2]], i32 [[Y1]], i64 1
+; CHECK-NEXT: [[TMP4:%.*]] = add <2 x i32> [[TMP1]], [[TMP3]]
+; CHECK-NEXT: store <2 x i32> [[TMP4]], ptr [[DST]], align 4
+; CHECK-NEXT: br i1 [[COND]], label %[[USE:.*]], label %[[EXIT:.*]]
+; CHECK: [[USE]]:
+; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x i32> [[TMP2]], i32 [[X0]], i64 1
+; CHECK-NEXT: [[TMP7:%.*]] = add <2 x i32> [[TMP5]], [[TMP6]]
+; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> poison, <2 x i32> <i32 1, i32 poison>
+; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <2 x i32> [[TMP8]], <2 x i32> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP10:%.*]] = insertelement <2 x i32> poison, i32 [[Y1]], i64 0
+; CHECK-NEXT: [[TMP11:%.*]] = insertelement <2 x i32> [[TMP10]], i32 [[X1]], i64 1
+; CHECK-NEXT: [[TMP12:%.*]] = add <2 x i32> [[TMP9]], [[TMP11]]
+; CHECK-NEXT: [[TMP13:%.*]] = extractelement <2 x i32> [[TMP7]], i64 0
+; CHECK-NEXT: call void @use(i32 [[TMP13]])
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <2 x i32> [[TMP7]], i64 1
+; CHECK-NEXT: call void @use(i32 [[TMP14]])
+; CHECK-NEXT: [[TMP15:%.*]] = extractelement <2 x i32> [[TMP12]], i64 0
+; CHECK-NEXT: call void @use(i32 [[TMP15]])
+; CHECK-NEXT: [[TMP16:%.*]] = extractelement <2 x i32> [[TMP12]], i64 1
+; CHECK-NEXT: call void @use(i32 [[TMP16]])
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ %dst1 = getelementptr inbounds i32, ptr %dst, i64 1
+ %a0 = add i32 %x0, %y0
+ %a1 = add i32 %x1, %y1
+ store i32 %a0, ptr %dst, align 4
+ store i32 %a1, ptr %dst1, align 4
+ br i1 %cond, label %use, label %exit
+use:
+ %b0 = add i32 %a0, %y0
+ %c0 = add i32 %a0, %x0
+ %b1 = add i32 %a1, %y1
+ %c1 = add i32 %a1, %x1
+ call void @use(i32 %b0)
+ call void @use(i32 %c0)
+ call void @use(i32 %b1)
+ call void @use(i32 %c1)
+ br label %exit
+exit:
+ ret void
+}
+
+; A two-incoming phi is a benign outside user. Each incremented limb feeds one
+; two-incoming phi plus one add, so the store chain reaches costing.
+define void @store_chain_two_incoming_phi(ptr %state, i32 %init0, i32 %init1, i32 %n) {
+; CHECK-LABEL: define void @store_chain_two_incoming_phi(
+; CHECK-SAME: ptr [[STATE:%.*]], i32 [[INIT0:%.*]], i32 [[INIT1:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x i32> poison, i32 [[INIT0]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i32> [[TMP0]], i32 [[INIT1]], i64 1
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = phi <2 x i32> [ [[TMP1]], %[[ENTRY]] ], [ [[TMP3:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[TMP3]] = add <2 x i32> [[TMP2]], splat (i32 1)
+; CHECK-NEXT: store <2 x i32> [[TMP3]], ptr [[STATE]], align 4
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <2 x i32> poison, i32 [[IV]], i64 0
+; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = add <2 x i32> [[TMP3]], [[TMP5]]
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x i32> [[TMP6]], i64 0
+; CHECK-NEXT: call void @use(i32 [[TMP7]])
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x i32> [[TMP6]], i64 1
+; CHECK-NEXT: call void @use(i32 [[TMP8]])
+; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT: [[DONE:%.*]] = icmp eq i32 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ %s1 = getelementptr inbounds i32, ptr %state, i64 1
+ br label %loop
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+ %c0 = phi i32 [ %init0, %entry ], [ %new0, %loop ]
+ %c1 = phi i32 [ %init1, %entry ], [ %new1, %loop ]
+ %new0 = add i32 %c0, 1
+ %new1 = add i32 %c1, 1
+ store i32 %new0, ptr %state, align 4
+ store i32 %new1, ptr %s1, align 4
+ %u0 = add i32 %new0, %iv
+ %u1 = add i32 %new1, %iv
+ call void @use(i32 %u0)
+ call void @use(i32 %u1)
+ %iv.next = add i32 %iv, 1
+ %done = icmp eq i32 %iv.next, %n
+ br i1 %done, label %exit, label %loop
+exit:
+ ret void
+}
+
+; Two non-benign outside users keep the store chain scalar even when they live in
+; a different block: block placement does not make a call benign.
+define void @store_chain_two_cross_block_calls(ptr %dst, i32 %x0, i32 %x1, i32 %y0, i32 %y1, i1 %cond) {
+; CHECK-LABEL: define void @store_chain_two_cross_block_calls(
+; CHECK-SAME: ptr [[DST:%.*]], i32 [[X0:%.*]], i32 [[X1:%.*]], i32 [[Y0:%.*]], i32 [[Y1:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[DST1:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 1
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x i32> poison, i32 [[X0]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i32> [[TMP0]], i32 [[X1]], i64 1
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x i32> poison, i32 [[Y0]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x i32> [[TMP2]], i32 [[Y1]], i64 1
+; CHECK-NEXT: [[TMP4:%.*]] = add <2 x i32> [[TMP1]], [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x i32> [[TMP4]], i64 1
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x i32> [[TMP4]], i64 0
+; CHECK-NEXT: store i32 [[TMP6]], ptr [[DST]], align 4
+; CHECK-NEXT: store i32 [[TMP5]], ptr [[DST1]], align 4
+; CHECK-NEXT: br i1 [[COND]], label %[[USE:.*]], label %[[EXIT:.*]]
+; CHECK: [[USE]]:
+; CHECK-NEXT: call void @use(i32 [[TMP6]])
+; CHECK-NEXT: call void @use(i32 [[TMP6]])
+; CHECK-NEXT: call void @use(i32 [[TMP5]])
+; CHECK-NEXT: call void @use(i32 [[TMP5]])
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ %dst1 = getelementptr inbounds i32, ptr %dst, i64 1
+ %a0 = add i32 %x0, %y0
+ %a1 = add i32 %x1, %y1
+ store i32 %a0, ptr %dst, align 4
+ store i32 %a1, ptr %dst1, align 4
+ br i1 %cond, label %use, label %exit
+use:
+ call void @use(i32 %a0)
+ call void @use(i32 %a0)
+ call void @use(i32 %a1)
+ call void @use(i32 %a1)
+ br label %exit
+exit:
+ ret void
+}
+
+; Exactly two outside users where one is benign (add) and one is non-benign
+; (call) keeps the store chain scalar: a single non-benign user among the two is
+; enough to reject.
+define void @store_chain_two_mixed_users(ptr %dst, i32 %x0, i32 %x1, i32 %y0, i32 %y1) {
+; CHECK-LABEL: define void @store_chain_two_mixed_users(
+; CHECK-SAME: ptr [[DST:%.*]], i32 [[X0:%.*]], i32 [[X1:%.*]], i32 [[Y0:%.*]], i32 [[Y1:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[DST1:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 1
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x i32> poison, i32 [[X0]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i32> [[TMP0]], i32 [[X1]], i64 1
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x i32> poison, i32 [[Y0]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x i32> [[TMP2]], i32 [[Y1]], i64 1
+; CHECK-NEXT: [[TMP4:%.*]] = add <2 x i32> [[TMP1]], [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x i32> [[TMP4]], i64 1
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x i32> [[TMP4]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = add <2 x i32> [[TMP4]], [[TMP3]]
+; CHECK-NEXT: store i32 [[TMP6]], ptr [[DST]], align 4
+; CHECK-NEXT: store i32 [[TMP5]], ptr [[DST1]], align 4
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x i32> [[TMP7]], i64 0
+; CHECK-NEXT: call void @use(i32 [[TMP8]])
+; CHECK-NEXT: call void @use(i32 [[TMP6]])
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x i32> [[TMP7]], i64 1
+; CHECK-NEXT: call void @use(i32 [[TMP9]])
+; CHECK-NEXT: call void @use(i32 [[TMP5]])
+; CHECK-NEXT: ret void
+;
+entry:
+ %dst1 = getelementptr inbounds i32, ptr %dst, i64 1
+ %a0 = add i32 %x0, %y0
+ %a1 = add i32 %x1, %y1
+ %b0 = add i32 %a0, %y0
+ %b1 = add i32 %a1, %y1
+ store i32 %a0, ptr %dst, align 4
+ store i32 %a1, ptr %dst1, align 4
+ call void @use(i32 %b0)
+ call void @use(i32 %a0)
+ call void @use(i32 %b1)
+ call void @use(i32 %a1)
+ ret void
+}
+
+; Two two-incoming join phis are benign. Each stored value has exactly two such
+; outside users, so the store chain reaches costing.
+define void @store_chain_two_join_phis(ptr %dst, i32 %x0, i32 %x1, i32 %y0, i32 %y1, i1 %cond) {
+; CHECK-LABEL: define void @store_chain_two_join_phis(
+; CHECK-SAME: ptr [[DST:%.*]], i32 [[X0:%.*]], i32 [[X1:%.*]], i32 [[Y0:%.*]], i32 [[Y1:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x i32> poison, i32 [[X0]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i32> [[TMP0]], i32 [[X1]], i64 1
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x i32> poison, i32 [[Y0]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x i32> [[TMP2]], i32 [[Y1]], i64 1
+; CHECK-NEXT: [[TMP4:%.*]] = add <2 x i32> [[TMP1]], [[TMP3]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LEFT:.*]], label %[[RIGHT:.*]]
+; CHECK: [[LEFT]]:
+; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> poison, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT: br label %[[MERGE:.*]]
+; CHECK: [[RIGHT]]:
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x i32> [[TMP2]], i32 [[X0]], i64 1
+; CHECK-NEXT: [[TMP8:%.*]] = insertelement <2 x i32> poison, i32 [[Y1]], i64 0
+; CHECK-NEXT: [[TMP9:%.*]] = insertelement <2 x i32> [[TMP8]], i32 [[X1]], i64 1
+; CHECK-NEXT: br label %[[MERGE]]
+; CHECK: [[MERGE]]:
+; CHECK-NEXT: [[TMP10:%.*]] = phi <2 x i32> [ [[TMP5]], %[[LEFT]] ], [ [[TMP7]], %[[RIGHT]] ]
+; CHECK-NEXT: [[TMP11:%.*]] = phi <2 x i32> [ [[TMP6]], %[[LEFT]] ], [ [[TMP9]], %[[RIGHT]] ]
+; CHECK-NEXT: store <2 x i32> [[TMP4]], ptr [[DST]], align 4
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x i32> [[TMP10]], i64 0
+; CHECK-NEXT: call void @use(i32 [[TMP12]])
+; CHECK-NEXT: [[TMP13:%.*]] = extractelement <2 x i32> [[TMP10]], i64 1
+; CHECK-NEXT: call void @use(i32 [[TMP13]])
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <2 x i32> [[TMP11]], i64 0
+; CHECK-NEXT: call void @use(i32 [[TMP14]])
+; CHECK-NEXT: [[TMP15:%.*]] = extractelement <2 x i32> [[TMP11]], i64 1
+; CHECK-NEXT: call void @use(i32 [[TMP15]])
+; CHECK-NEXT: ret void
+;
+entry:
+ %dst1 = getelementptr inbounds i32, ptr %dst, i64 1
+ %a0 = add i32 %x0, %y0
+ %a1 = add i32 %x1, %y1
+ br i1 %cond, label %left, label %right
+left:
+ br label %merge
+right:
+ br label %merge
+merge:
+ %p0 = phi i32 [ %a0, %left ], [ %y0, %right ]
+ %q0 = phi i32 [ %a0, %left ], [ %x0, %right ]
+ %p1 = phi i32 [ %a1, %left ], [ %y1, %right ]
+ %q1 = phi i32 [ %a1, %left ], [ %x1, %right ]
+ store i32 %a0, ptr %dst, align 4
+ store i32 %a1, ptr %dst1, align 4
+ call void @use(i32 %p0)
+ call void @use(i32 %q0)
+ call void @use(i32 %p1)
+ call void @use(i32 %q1)
+ ret void
+}
+
+; Three-incoming phis are not benign. Exactly two such outside users keep the
+; store chain scalar.
+define void @store_chain_two_three_incoming_phis(ptr %dst, i32 %x0, i32 %x1, i32 %y0, i32 %y1, i32 %sel) {
+; CHECK-LABEL: define void @store_chain_two_three_incoming_phis(
+; CHECK-SAME: ptr [[DST:%.*]], i32 [[X0:%.*]], i32 [[X1:%.*]], i32 [[Y0:%.*]], i32 [[Y1:%.*]], i32 [[SEL:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[DST1:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 1
+; CHECK-NEXT: [[A0:%.*]] = add i32 [[X0]], [[Y0]]
+; CHECK-NEXT: [[A1:%.*]] = add i32 [[X1]], [[Y1]]
+; CHECK-NEXT: switch i32 [[SEL]], label %[[BB2:.*]] [
+; CHECK-NEXT: i32 0, label %[[BB0:.*]]
+; CHECK-NEXT: i32 1, label %[[BB1:.*]]
+; CHECK-NEXT: ]
+; CHECK: [[BB0]]:
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x i32> poison, i32 [[A0]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <2 x i32> [[TMP0]], <2 x i32> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x i32> poison, i32 [[A1]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[MERGE:.*]]
+; CHECK: [[BB1]]:
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <2 x i32> poison, i32 [[Y0]], i64 0
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x i32> [[TMP4]], i32 [[X0]], i64 1
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x i32> poison, i32 [[Y1]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x i32> [[TMP6]], i32 [[X1]], i64 1
+; CHECK-NEXT: br label %[[MERGE]]
+; CHECK: [[BB2]]:
+; CHECK-NEXT: [[TMP8:%.*]] = insertelement <2 x i32> poison, i32 [[X0]], i64 0
+; CHECK-NEXT: [[TMP9:%.*]] = insertelement <2 x i32> [[TMP8]], i32 [[Y0]], i64 1
+; CHECK-NEXT: [[TMP10:%.*]] = insertelement <2 x i32> poison, i32 [[X1]], i64 0
+; CHECK-NEXT: [[TMP11:%.*]] = insertelement <2 x i32> [[TMP10]], i32 [[Y1]], i64 1
+; CHECK-NEXT: br label %[[MERGE]]
+; CHECK: [[MERGE]]:
+; CHECK-NEXT: [[TMP12:%.*]] = phi <2 x i32> [ [[TMP1]], %[[BB0]] ], [ [[TMP5]], %[[BB1]] ], [ [[TMP9]], %[[BB2]] ]
+; CHECK-NEXT: [[TMP13:%.*]] = phi <2 x i32> [ [[TMP3]], %[[BB0]] ], [ [[TMP7]], %[[BB1]] ], [ [[TMP11]], %[[BB2]] ]
+; CHECK-NEXT: store i32 [[A0]], ptr [[DST]], align 4
+; CHECK-NEXT: store i32 [[A1]], ptr [[DST1]], align 4
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <2 x i32> [[TMP12]], i64 0
+; CHECK-NEXT: call void @use(i32 [[TMP14]])
+; CHECK-NEXT: [[TMP15:%.*]] = extractelement <2 x i32> [[TMP12]], i64 1
+; CHECK-NEXT: call void @use(i32 [[TMP15]])
+; CHECK-NEXT: [[TMP16:%.*]] = extractelement <2 x i32> [[TMP13]], i64 0
+; CHECK-NEXT: call void @use(i32 [[TMP16]])
+; CHECK-NEXT: [[TMP17:%.*]] = extractelement <2 x i32> [[TMP13]], i64 1
+; CHECK-NEXT: call void @use(i32 [[TMP17]])
+; CHECK-NEXT: ret void
+;
+entry:
+ %dst1 = getelementptr inbounds i32, ptr %dst, i64 1
+ %a0 = add i32 %x0, %y0
+ %a1 = add i32 %x1, %y1
+ switch i32 %sel, label %else [
+ i32 0, label %then0
+ i32 1, label %then1
+ ]
+then0:
+ br label %merge
+then1:
+ br label %merge
+else:
+ br label %merge
+merge:
+ %p0 = phi i32 [ %a0, %then0 ], [ %y0, %then1 ], [ %x0, %else ]
+ %q0 = phi i32 [ %a0, %then0 ], [ %x0, %then1 ], [ %y0, %else ]
+ %p1 = phi i32 [ %a1, %then0 ], [ %y1, %then1 ], [ %x1, %else ]
+ %q1 = phi i32 [ %a1, %then0 ], [ %x1, %then1 ], [ %y1, %else ]
+ store i32 %a0, ptr %dst, align 4
+ store i32 %a1, ptr %dst1, align 4
+ call void @use(i32 %p0)
+ call void @use(i32 %q0)
+ call void @use(i32 %p1)
+ call void @use(i32 %q1)
+ ret void
+}
More information about the llvm-commits
mailing list