[llvm] [SimplifyCFG] Avoid sinking loads/stores that impact vectorization (PR #222587)
Ashutosh Nema via llvm-commits
llvm-commits at lists.llvm.org
Wed Sep 23 02:51:45 PDT 2026
https://github.com/nema-ashutosh updated https://github.com/llvm/llvm-project/pull/222587
>From 80b0eaba405d38f798c006e0a361b96271f0a077 Mon Sep 17 00:00:00 2001
From: Ashutosh Nema <ashu1212 at gmail.com>
Date: Thu, 10 Sep 2026 15:48:41 +0530
Subject: [PATCH 1/3] [SimplifyCFG] Avoid sinking loads/stores that impact
vectorization
Sinking load/store memory ops through a pointer PHI forces later
vectorization to use suboptimal code generation. Keep them separate
when independent masked loads or stores are cheaper, matching the
existing store-sinking profitability guard and applying it to loads
as well.
Fixes #222516
---
llvm/lib/Transforms/Utils/SimplifyCFG.cpp | 121 ++++++++++-
.../sink-common-load-different-pointers.ll | 148 ++++++++++++++
.../X86/sink-common-load-gather-cost.ll | 189 +++++++++++++++++
.../sink-common-store-different-pointers.ll | 152 ++++++++++++++
.../X86/sink-common-store-scatter-cost.ll | 190 ++++++++++++++++++
5 files changed, 797 insertions(+), 3 deletions(-)
create mode 100644 llvm/test/Transforms/SimplifyCFG/X86/sink-common-load-different-pointers.ll
create mode 100644 llvm/test/Transforms/SimplifyCFG/X86/sink-common-load-gather-cost.ll
create mode 100644 llvm/test/Transforms/SimplifyCFG/X86/sink-common-store-different-pointers.ll
create mode 100644 llvm/test/Transforms/SimplifyCFG/X86/sink-common-store-scatter-cost.ll
diff --git a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
index 6a2601487b178b..9eb0d7e4bd5ee8 100644
--- a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
@@ -2420,10 +2420,106 @@ static void sinkLastInstruction(ArrayRef<BasicBlock*> Blocks) {
}
}
+/// Estimate whether commoning \p NumMemOps loads or stores like \p I, which
+/// access distinct addresses, penalizes a later vectorizer. Commoning needs a
+/// pointer PHI, so the result can only be widened as a gather or scatter, while
+/// separate blocks allow one independent masked load or store each.
+static bool sinkingMemOpsPenalizesVectorization(const TargetTransformInfo &TTI,
+ Instruction *I,
+ unsigned NumMemOps) {
+ bool IsLoad = isa<LoadInst>(I);
+ assert((IsLoad || isa<StoreInst>(I)) && "Expected a load or store");
+ Type *ScalarTy = getLoadStoreType(I);
+ Align Alignment = getLoadStoreAlignment(I);
+ unsigned AS = getLoadStoreAddressSpace(I);
+ Intrinsic::ID MaskedID =
+ IsLoad ? Intrinsic::masked_load : Intrinsic::masked_store;
+ Intrinsic::ID GatherScatterID =
+ IsLoad ? Intrinsic::masked_gather : Intrinsic::masked_scatter;
+
+ // There is nothing to preserve unless the operations can be widened in place.
+ if (!VectorType::isValidElementType(ScalarTy) ||
+ !(IsLoad ? TTI.isLegalMaskedLoad(ScalarTy, Alignment, AS)
+ : TTI.isLegalMaskedStore(ScalarTy, Alignment, AS)))
+ return false;
+
+ // Without a gather or scatter the commoned operation cannot be widened.
+ if (!(IsLoad ? TTI.isLegalMaskedGather(ScalarTy, Alignment)
+ : TTI.isLegalMaskedScatter(ScalarTy, Alignment)))
+ return true;
+
+ TypeSize EltWidth = I->getDataLayout().getTypeSizeInBits(ScalarTy);
+ if (EltWidth.isScalable() || EltWidth.isZero())
+ return true;
+
+ auto PenalizesForVF = [&](ElementCount VF) {
+ auto *VecTy = VectorType::get(ScalarTy, VF);
+ Value *Ptr = getLoadStorePointerOperand(I);
+ auto *PtrVecTy = VectorType::get(Ptr->getType(), VF);
+ constexpr TargetTransformInfo::TargetCostKind CostKind =
+ TargetTransformInfo::TCK_RecipThroughput;
+
+ InstructionCost GatherScatterCost =
+ TTI.getAddressComputationCost(PtrVecTy, nullptr, nullptr, CostKind) +
+ TTI.getMemIntrinsicInstrCost(
+ MemIntrinsicCostAttributes(GatherScatterID, VecTy, Ptr,
+ /*VariableMask=*/true, Alignment, I),
+ CostKind);
+ InstructionCost MaskedCost =
+ TTI.getMemIntrinsicInstrCost(
+ MemIntrinsicCostAttributes(MaskedID, VecTy, Alignment, AS),
+ CostKind) *
+ NumMemOps;
+
+ if (!GatherScatterCost.isValid())
+ return true;
+ if (!MaskedCost.isValid())
+ return false;
+
+ LLVM_DEBUG(dbgs() << "SINK: " << (VF.isScalable() ? "scalable" : "fixed")
+ << " VF " << VF.getKnownMinValue() << ": "
+ << (IsLoad ? "gather" : "scatter") << " cost "
+ << GatherScatterCost << " vs " << NumMemOps << " masked "
+ << (IsLoad ? "loads " : "stores ") << MaskedCost << "\n");
+ return GatherScatterCost >= MaskedCost;
+ };
+
+ // One vector register's worth of elements, at vscale=1 for scalable vectors.
+ auto VFFrom = [&](TargetTransformInfo::RegisterKind RK,
+ bool Scalable) -> std::optional<ElementCount> {
+ TypeSize RegWidth = TTI.getRegisterBitWidth(RK);
+ if (RegWidth.isScalable() != Scalable || RegWidth.isZero())
+ return std::nullopt;
+ unsigned NumElts = RegWidth.getKnownMinValue() / EltWidth.getFixedValue();
+ if (NumElts < (Scalable ? 1u : 2u))
+ return std::nullopt;
+ return ElementCount::get(NumElts, Scalable);
+ };
+
+ std::optional<ElementCount> FixedVF =
+ VFFrom(TargetTransformInfo::RGK_FixedWidthVector, /*Scalable=*/false);
+ std::optional<ElementCount> ScalableVF =
+ VFFrom(TargetTransformInfo::RGK_ScalableVector, /*Scalable=*/true);
+
+ // Both forms are legal but no vector factor can be formed, so the target
+ // vectorizes in a way this cannot reason about. Keep the operations separate.
+ if (!FixedVF && !ScalableVF)
+ return true;
+
+ // A vectorizer may pick either kind of vector, so keep the operations
+ // separate if commoning them does not pay off for one of them.
+ bool Penalizes = false;
+ if (FixedVF)
+ Penalizes |= PenalizesForVF(*FixedVF);
+ if (ScalableVF)
+ Penalizes |= PenalizesForVF(*ScalableVF);
+ return Penalizes;
+}
+
/// Check whether BB's predecessors end with unconditional branches. If it is
/// true, sink any common code from the predecessors to BB.
-static bool sinkCommonCodeFromPredecessors(BasicBlock *BB,
- DomTreeUpdater *DTU) {
+static bool sinkCommonCodeFromPredecessors(BasicBlock *BB, DomTreeUpdater *DTU,
+ const TargetTransformInfo &TTI) {
// We support two situations:
// (1) all incoming arcs are unconditional
// (2) there are non-unconditional incoming arcs
@@ -2523,10 +2619,29 @@ static bool sinkCommonCodeFromPredecessors(BasicBlock *BB,
return false;
};
+ // Check whether the memory operations in \p Insts all access the same
+ // address, so that commoning them needs no PHI for the pointer operand.
+ auto HaveSameMemAddress = [](ArrayRef<Instruction *> Insts) {
+ Value *Ptr = getLoadStorePointerOperand(Insts.front());
+ auto *PtrI = dyn_cast<Instruction>(Ptr);
+ return all_of(drop_begin(Insts), [&](Instruction *I) {
+ Value *OtherPtr = getLoadStorePointerOperand(I);
+ if (OtherPtr == Ptr)
+ return true;
+ auto *OtherPtrI = dyn_cast<Instruction>(OtherPtr);
+ return PtrI && OtherPtrI && PtrI->isIdenticalTo(OtherPtrI);
+ });
+ };
+
// Okay, we *could* sink last ScanIdx instructions. But how many can we
// actually sink before encountering instruction that is unprofitable to
// sink?
auto ProfitableToSinkInstruction = [&](LockstepReverseIterator<true> &LRI) {
+ ArrayRef<Instruction *> Insts = *LRI;
+ if (isa<LoadInst, StoreInst>(Insts[0]) && !HaveSameMemAddress(Insts) &&
+ sinkingMemOpsPenalizesVectorization(TTI, Insts[0], Insts.size()))
+ return false;
+
unsigned NumPHIInsts = 0;
for (Use &U : (*LRI)[0]->operands()) {
auto It = PHIOperands.find(&U);
@@ -9319,7 +9434,7 @@ bool SimplifyCFGOpt::simplifyOnce(BasicBlock *BB) {
return true;
if (SinkCommon && Options.SinkCommonInsts) {
- if (sinkCommonCodeFromPredecessors(BB, DTU) ||
+ if (sinkCommonCodeFromPredecessors(BB, DTU, TTI) ||
mergeCompatibleInvokes(BB, DTU)) {
// sinkCommonCodeFromPredecessors() does not automatically CSE PHI's,
// so we may now how duplicate PHI's.
diff --git a/llvm/test/Transforms/SimplifyCFG/X86/sink-common-load-different-pointers.ll b/llvm/test/Transforms/SimplifyCFG/X86/sink-common-load-different-pointers.ll
new file mode 100644
index 00000000000000..5e2eb7f64c0afa
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/X86/sink-common-load-different-pointers.ll
@@ -0,0 +1,148 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt < %s -mtriple=x86_64-- -mattr=+avx512f -passes='simplifycfg<sink-common-insts>' -S | FileCheck %s --check-prefixes=MASKED
+; RUN: opt < %s -mtriple=x86_64-- -mattr=+avx2 -passes='simplifycfg<sink-common-insts>' -S | FileCheck %s --check-prefixes=NOGATHER
+; RUN: opt < %s -mtriple=x86_64-- -mattr=+sse2 -passes='simplifycfg<sink-common-insts>' -S | FileCheck %s --check-prefixes=SCALAR
+
+; Loads from different address expressions can only be commoned through a
+; pointer PHI. On a target with masked loads, keep them in their original
+; blocks so that a vectorizer can widen each one on its own. AVX2 reaches the
+; same result without a gather, which cannot be widened at all.
+define i32 @different_load_pointers(i1 %cond, ptr %a, ptr %b, i64 %index) {
+; MASKED-LABEL: define i32 @different_load_pointers(
+; MASKED-SAME: i1 [[COND:%.*]], ptr [[A:%.*]], ptr [[B:%.*]], i64 [[INDEX:%.*]]) #[[ATTR0:[0-9]+]] {
+; MASKED-NEXT: [[ENTRY:.*:]]
+; MASKED-NEXT: br i1 [[COND]], label %[[IF_THEN:.*]], label %[[IF_ELSE:.*]]
+; MASKED: [[IF_THEN]]:
+; MASKED-NEXT: [[A_GEP:%.*]] = getelementptr i32, ptr [[A]], i64 [[INDEX]]
+; MASKED-NEXT: [[X:%.*]] = load i32, ptr [[A_GEP]], align 4
+; MASKED-NEXT: br label %[[EXIT:.*]]
+; MASKED: [[IF_ELSE]]:
+; MASKED-NEXT: [[B_GEP:%.*]] = getelementptr i32, ptr [[B]], i64 [[INDEX]]
+; MASKED-NEXT: [[Y:%.*]] = load i32, ptr [[B_GEP]], align 4
+; MASKED-NEXT: br label %[[EXIT]]
+; MASKED: [[EXIT]]:
+; MASKED-NEXT: [[R:%.*]] = phi i32 [ [[X]], %[[IF_THEN]] ], [ [[Y]], %[[IF_ELSE]] ]
+; MASKED-NEXT: ret i32 [[R]]
+;
+; NOGATHER-LABEL: define i32 @different_load_pointers(
+; NOGATHER-SAME: i1 [[COND:%.*]], ptr [[A:%.*]], ptr [[B:%.*]], i64 [[INDEX:%.*]]) #[[ATTR0:[0-9]+]] {
+; NOGATHER-NEXT: [[ENTRY:.*:]]
+; NOGATHER-NEXT: br i1 [[COND]], label %[[IF_THEN:.*]], label %[[IF_ELSE:.*]]
+; NOGATHER: [[IF_THEN]]:
+; NOGATHER-NEXT: [[A_GEP:%.*]] = getelementptr i32, ptr [[A]], i64 [[INDEX]]
+; NOGATHER-NEXT: [[X:%.*]] = load i32, ptr [[A_GEP]], align 4
+; NOGATHER-NEXT: br label %[[EXIT:.*]]
+; NOGATHER: [[IF_ELSE]]:
+; NOGATHER-NEXT: [[B_GEP:%.*]] = getelementptr i32, ptr [[B]], i64 [[INDEX]]
+; NOGATHER-NEXT: [[Y:%.*]] = load i32, ptr [[B_GEP]], align 4
+; NOGATHER-NEXT: br label %[[EXIT]]
+; NOGATHER: [[EXIT]]:
+; NOGATHER-NEXT: [[R:%.*]] = phi i32 [ [[X]], %[[IF_THEN]] ], [ [[Y]], %[[IF_ELSE]] ]
+; NOGATHER-NEXT: ret i32 [[R]]
+;
+; SCALAR-LABEL: define i32 @different_load_pointers(
+; SCALAR-SAME: i1 [[COND:%.*]], ptr [[A:%.*]], ptr [[B:%.*]], i64 [[INDEX:%.*]]) #[[ATTR0:[0-9]+]] {
+; SCALAR-NEXT: [[ENTRY:.*:]]
+; SCALAR-NEXT: [[A_B:%.*]] = select i1 [[COND]], ptr [[A]], ptr [[B]]
+; SCALAR-NEXT: [[B_GEP:%.*]] = getelementptr i32, ptr [[A_B]], i64 [[INDEX]]
+; SCALAR-NEXT: [[Y:%.*]] = load i32, ptr [[B_GEP]], align 4
+; SCALAR-NEXT: ret i32 [[Y]]
+;
+entry:
+ br i1 %cond, label %if.then, label %if.else
+
+if.then:
+ %a.gep = getelementptr i32, ptr %a, i64 %index
+ %x = load i32, ptr %a.gep, align 4
+ br label %exit
+
+if.else:
+ %b.gep = getelementptr i32, ptr %b, i64 %index
+ %y = load i32, ptr %b.gep, align 4
+ br label %exit
+
+exit:
+ %r = phi i32 [ %x, %if.then ], [ %y, %if.else ]
+ ret i32 %r
+}
+
+; A pointer PHI is not needed when both loads use the same address, so these
+; are commoned on any target.
+define i32 @same_load_pointer(i1 %cond, ptr %a) {
+; MASKED-LABEL: define i32 @same_load_pointer(
+; MASKED-SAME: i1 [[COND:%.*]], ptr [[A:%.*]]) #[[ATTR0]] {
+; MASKED-NEXT: [[ENTRY:.*:]]
+; MASKED-NEXT: [[X:%.*]] = load i32, ptr [[A]], align 4
+; MASKED-NEXT: ret i32 [[X]]
+;
+; NOGATHER-LABEL: define i32 @same_load_pointer(
+; NOGATHER-SAME: i1 [[COND:%.*]], ptr [[A:%.*]]) #[[ATTR0]] {
+; NOGATHER-NEXT: [[ENTRY:.*:]]
+; NOGATHER-NEXT: [[X:%.*]] = load i32, ptr [[A]], align 4
+; NOGATHER-NEXT: ret i32 [[X]]
+;
+; SCALAR-LABEL: define i32 @same_load_pointer(
+; SCALAR-SAME: i1 [[COND:%.*]], ptr [[A:%.*]]) #[[ATTR0]] {
+; SCALAR-NEXT: [[ENTRY:.*:]]
+; SCALAR-NEXT: [[X:%.*]] = load i32, ptr [[A]], align 4
+; SCALAR-NEXT: ret i32 [[X]]
+;
+entry:
+ br i1 %cond, label %if.then, label %if.else
+
+if.then:
+ %x = load i32, ptr %a, align 4
+ br label %exit
+
+if.else:
+ %y = load i32, ptr %a, align 4
+ br label %exit
+
+exit:
+ %r = phi i32 [ %x, %if.then ], [ %y, %if.else ]
+ ret i32 %r
+}
+
+; Types that the target cannot load under a mask are still commoned.
+define i8 @different_load_pointers_unsupported_type(i1 %cond, ptr %a, ptr %b, i64 %index) {
+; MASKED-LABEL: define i8 @different_load_pointers_unsupported_type(
+; MASKED-SAME: i1 [[COND:%.*]], ptr [[A:%.*]], ptr [[B:%.*]], i64 [[INDEX:%.*]]) #[[ATTR0]] {
+; MASKED-NEXT: [[ENTRY:.*:]]
+; MASKED-NEXT: [[A_B:%.*]] = select i1 [[COND]], ptr [[A]], ptr [[B]]
+; MASKED-NEXT: [[B_GEP:%.*]] = getelementptr i8, ptr [[A_B]], i64 [[INDEX]]
+; MASKED-NEXT: [[Y:%.*]] = load i8, ptr [[B_GEP]], align 1
+; MASKED-NEXT: ret i8 [[Y]]
+;
+; NOGATHER-LABEL: define i8 @different_load_pointers_unsupported_type(
+; NOGATHER-SAME: i1 [[COND:%.*]], ptr [[A:%.*]], ptr [[B:%.*]], i64 [[INDEX:%.*]]) #[[ATTR0]] {
+; NOGATHER-NEXT: [[ENTRY:.*:]]
+; NOGATHER-NEXT: [[A_B:%.*]] = select i1 [[COND]], ptr [[A]], ptr [[B]]
+; NOGATHER-NEXT: [[B_GEP:%.*]] = getelementptr i8, ptr [[A_B]], i64 [[INDEX]]
+; NOGATHER-NEXT: [[Y:%.*]] = load i8, ptr [[B_GEP]], align 1
+; NOGATHER-NEXT: ret i8 [[Y]]
+;
+; SCALAR-LABEL: define i8 @different_load_pointers_unsupported_type(
+; SCALAR-SAME: i1 [[COND:%.*]], ptr [[A:%.*]], ptr [[B:%.*]], i64 [[INDEX:%.*]]) #[[ATTR0]] {
+; SCALAR-NEXT: [[ENTRY:.*:]]
+; SCALAR-NEXT: [[A_B:%.*]] = select i1 [[COND]], ptr [[A]], ptr [[B]]
+; SCALAR-NEXT: [[B_GEP:%.*]] = getelementptr i8, ptr [[A_B]], i64 [[INDEX]]
+; SCALAR-NEXT: [[Y:%.*]] = load i8, ptr [[B_GEP]], align 1
+; SCALAR-NEXT: ret i8 [[Y]]
+;
+entry:
+ br i1 %cond, label %if.then, label %if.else
+
+if.then:
+ %a.gep = getelementptr i8, ptr %a, i64 %index
+ %x = load i8, ptr %a.gep, align 1
+ br label %exit
+
+if.else:
+ %b.gep = getelementptr i8, ptr %b, i64 %index
+ %y = load i8, ptr %b.gep, align 1
+ br label %exit
+
+exit:
+ %r = phi i8 [ %x, %if.then ], [ %y, %if.else ]
+ ret i8 %r
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/X86/sink-common-load-gather-cost.ll b/llvm/test/Transforms/SimplifyCFG/X86/sink-common-load-gather-cost.ll
new file mode 100644
index 00000000000000..e73ea1bf031627
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/X86/sink-common-load-gather-cost.ll
@@ -0,0 +1,189 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt < %s -mtriple=x86_64-- -mattr=+avx512f -passes='simplifycfg<sink-common-insts>' -S | FileCheck %s --check-prefixes=VF16
+; RUN: opt < %s -mtriple=x86_64-- -mattr=+avx512f,+prefer-256-bit -passes='simplifycfg<sink-common-insts>' -S | FileCheck %s --check-prefixes=VF8
+
+; Whether commoning these loads is worthwhile is decided by cost, not by
+; legality. Masked loads and gathers are both available, but a gather only pays
+; off once it replaces enough separate masked loads. The wider the vector, the
+; more loads a single gather has to absorb to break even.
+define i32 @many_load_pointers(i32 %sel, i64 %index, ptr %p0, ptr %p1, ptr %p2, ptr %p3, ptr %p4, ptr %p5, ptr %p6, ptr %p7, ptr %p8, ptr %p9, ptr %p10) {
+; VF16-LABEL: define i32 @many_load_pointers(
+; VF16-SAME: i32 [[SEL:%.*]], i64 [[INDEX:%.*]], ptr [[P0:%.*]], ptr [[P1:%.*]], ptr [[P2:%.*]], ptr [[P3:%.*]], ptr [[P4:%.*]], ptr [[P5:%.*]], ptr [[P6:%.*]], ptr [[P7:%.*]], ptr [[P8:%.*]], ptr [[P9:%.*]], ptr [[P10:%.*]]) #[[ATTR0:[0-9]+]] {
+; VF16-NEXT: [[ENTRY:.*:]]
+; VF16-NEXT: switch i32 [[SEL]], label %[[CASE0:.*]] [
+; VF16-NEXT: i32 1, label %[[CASE1:.*]]
+; VF16-NEXT: i32 2, label %[[CASE2:.*]]
+; VF16-NEXT: i32 3, label %[[CASE3:.*]]
+; VF16-NEXT: i32 4, label %[[CASE4:.*]]
+; VF16-NEXT: i32 5, label %[[CASE5:.*]]
+; VF16-NEXT: i32 6, label %[[CASE6:.*]]
+; VF16-NEXT: i32 7, label %[[CASE7:.*]]
+; VF16-NEXT: i32 8, label %[[CASE8:.*]]
+; VF16-NEXT: i32 9, label %[[CASE9:.*]]
+; VF16-NEXT: i32 10, label %[[CASE10:.*]]
+; VF16-NEXT: ]
+; VF16: [[CASE0]]:
+; VF16-NEXT: [[GEP0:%.*]] = getelementptr i32, ptr [[P0]], i64 [[INDEX]]
+; VF16-NEXT: [[V0:%.*]] = load i32, ptr [[GEP0]], align 4
+; VF16-NEXT: br label %[[EXIT:.*]]
+; VF16: [[CASE1]]:
+; VF16-NEXT: [[GEP1:%.*]] = getelementptr i32, ptr [[P1]], i64 [[INDEX]]
+; VF16-NEXT: [[V1:%.*]] = load i32, ptr [[GEP1]], align 4
+; VF16-NEXT: br label %[[EXIT]]
+; VF16: [[CASE2]]:
+; VF16-NEXT: [[GEP2:%.*]] = getelementptr i32, ptr [[P2]], i64 [[INDEX]]
+; VF16-NEXT: [[V2:%.*]] = load i32, ptr [[GEP2]], align 4
+; VF16-NEXT: br label %[[EXIT]]
+; VF16: [[CASE3]]:
+; VF16-NEXT: [[GEP3:%.*]] = getelementptr i32, ptr [[P3]], i64 [[INDEX]]
+; VF16-NEXT: [[V3:%.*]] = load i32, ptr [[GEP3]], align 4
+; VF16-NEXT: br label %[[EXIT]]
+; VF16: [[CASE4]]:
+; VF16-NEXT: [[GEP4:%.*]] = getelementptr i32, ptr [[P4]], i64 [[INDEX]]
+; VF16-NEXT: [[V4:%.*]] = load i32, ptr [[GEP4]], align 4
+; VF16-NEXT: br label %[[EXIT]]
+; VF16: [[CASE5]]:
+; VF16-NEXT: [[GEP5:%.*]] = getelementptr i32, ptr [[P5]], i64 [[INDEX]]
+; VF16-NEXT: [[V5:%.*]] = load i32, ptr [[GEP5]], align 4
+; VF16-NEXT: br label %[[EXIT]]
+; VF16: [[CASE6]]:
+; VF16-NEXT: [[GEP6:%.*]] = getelementptr i32, ptr [[P6]], i64 [[INDEX]]
+; VF16-NEXT: [[V6:%.*]] = load i32, ptr [[GEP6]], align 4
+; VF16-NEXT: br label %[[EXIT]]
+; VF16: [[CASE7]]:
+; VF16-NEXT: [[GEP7:%.*]] = getelementptr i32, ptr [[P7]], i64 [[INDEX]]
+; VF16-NEXT: [[V7:%.*]] = load i32, ptr [[GEP7]], align 4
+; VF16-NEXT: br label %[[EXIT]]
+; VF16: [[CASE8]]:
+; VF16-NEXT: [[GEP8:%.*]] = getelementptr i32, ptr [[P8]], i64 [[INDEX]]
+; VF16-NEXT: [[V8:%.*]] = load i32, ptr [[GEP8]], align 4
+; VF16-NEXT: br label %[[EXIT]]
+; VF16: [[CASE9]]:
+; VF16-NEXT: [[GEP9:%.*]] = getelementptr i32, ptr [[P9]], i64 [[INDEX]]
+; VF16-NEXT: [[V9:%.*]] = load i32, ptr [[GEP9]], align 4
+; VF16-NEXT: br label %[[EXIT]]
+; VF16: [[CASE10]]:
+; VF16-NEXT: [[GEP10:%.*]] = getelementptr i32, ptr [[P10]], i64 [[INDEX]]
+; VF16-NEXT: [[V10:%.*]] = load i32, ptr [[GEP10]], align 4
+; VF16-NEXT: br label %[[EXIT]]
+; VF16: [[EXIT]]:
+; VF16-NEXT: [[RESULT:%.*]] = phi i32 [ [[V0]], %[[CASE0]] ], [ [[V1]], %[[CASE1]] ], [ [[V2]], %[[CASE2]] ], [ [[V3]], %[[CASE3]] ], [ [[V4]], %[[CASE4]] ], [ [[V5]], %[[CASE5]] ], [ [[V6]], %[[CASE6]] ], [ [[V7]], %[[CASE7]] ], [ [[V8]], %[[CASE8]] ], [ [[V9]], %[[CASE9]] ], [ [[V10]], %[[CASE10]] ]
+; VF16-NEXT: ret i32 [[RESULT]]
+;
+; VF8-LABEL: define i32 @many_load_pointers(
+; VF8-SAME: i32 [[SEL:%.*]], i64 [[INDEX:%.*]], ptr [[P0:%.*]], ptr [[P1:%.*]], ptr [[P2:%.*]], ptr [[P3:%.*]], ptr [[P4:%.*]], ptr [[P5:%.*]], ptr [[P6:%.*]], ptr [[P7:%.*]], ptr [[P8:%.*]], ptr [[P9:%.*]], ptr [[P10:%.*]]) #[[ATTR0:[0-9]+]] {
+; VF8-NEXT: [[ENTRY:.*]]:
+; VF8-NEXT: switch i32 [[SEL]], label %[[EXIT:.*]] [
+; VF8-NEXT: i32 1, label %[[CASE1:.*]]
+; VF8-NEXT: i32 2, label %[[CASE2:.*]]
+; VF8-NEXT: i32 3, label %[[CASE3:.*]]
+; VF8-NEXT: i32 4, label %[[CASE4:.*]]
+; VF8-NEXT: i32 5, label %[[CASE5:.*]]
+; VF8-NEXT: i32 6, label %[[CASE6:.*]]
+; VF8-NEXT: i32 7, label %[[CASE7:.*]]
+; VF8-NEXT: i32 8, label %[[CASE8:.*]]
+; VF8-NEXT: i32 9, label %[[CASE9:.*]]
+; VF8-NEXT: i32 10, label %[[CASE10:.*]]
+; VF8-NEXT: ]
+; VF8: [[CASE1]]:
+; VF8-NEXT: br label %[[EXIT]]
+; VF8: [[CASE2]]:
+; VF8-NEXT: br label %[[EXIT]]
+; VF8: [[CASE3]]:
+; VF8-NEXT: br label %[[EXIT]]
+; VF8: [[CASE4]]:
+; VF8-NEXT: br label %[[EXIT]]
+; VF8: [[CASE5]]:
+; VF8-NEXT: br label %[[EXIT]]
+; VF8: [[CASE6]]:
+; VF8-NEXT: br label %[[EXIT]]
+; VF8: [[CASE7]]:
+; VF8-NEXT: br label %[[EXIT]]
+; VF8: [[CASE8]]:
+; VF8-NEXT: br label %[[EXIT]]
+; VF8: [[CASE9]]:
+; VF8-NEXT: br label %[[EXIT]]
+; VF8: [[CASE10]]:
+; VF8-NEXT: br label %[[EXIT]]
+; VF8: [[EXIT]]:
+; VF8-NEXT: [[P10_SINK:%.*]] = phi ptr [ [[P10]], %[[CASE10]] ], [ [[P9]], %[[CASE9]] ], [ [[P8]], %[[CASE8]] ], [ [[P7]], %[[CASE7]] ], [ [[P6]], %[[CASE6]] ], [ [[P5]], %[[CASE5]] ], [ [[P4]], %[[CASE4]] ], [ [[P3]], %[[CASE3]] ], [ [[P2]], %[[CASE2]] ], [ [[P1]], %[[CASE1]] ], [ [[P0]], %[[ENTRY]] ]
+; VF8-NEXT: [[GEP10:%.*]] = getelementptr i32, ptr [[P10_SINK]], i64 [[INDEX]]
+; VF8-NEXT: [[V10:%.*]] = load i32, ptr [[GEP10]], align 4
+; VF8-NEXT: ret i32 [[V10]]
+;
+entry:
+ switch i32 %sel, label %case0 [
+ i32 1, label %case1
+ i32 2, label %case2
+ i32 3, label %case3
+ i32 4, label %case4
+ i32 5, label %case5
+ i32 6, label %case6
+ i32 7, label %case7
+ i32 8, label %case8
+ i32 9, label %case9
+ i32 10, label %case10
+ ]
+
+case0:
+ %gep0 = getelementptr i32, ptr %p0, i64 %index
+ %v0 = load i32, ptr %gep0, align 4
+ br label %exit
+
+case1:
+ %gep1 = getelementptr i32, ptr %p1, i64 %index
+ %v1 = load i32, ptr %gep1, align 4
+ br label %exit
+
+case2:
+ %gep2 = getelementptr i32, ptr %p2, i64 %index
+ %v2 = load i32, ptr %gep2, align 4
+ br label %exit
+
+case3:
+ %gep3 = getelementptr i32, ptr %p3, i64 %index
+ %v3 = load i32, ptr %gep3, align 4
+ br label %exit
+
+case4:
+ %gep4 = getelementptr i32, ptr %p4, i64 %index
+ %v4 = load i32, ptr %gep4, align 4
+ br label %exit
+
+case5:
+ %gep5 = getelementptr i32, ptr %p5, i64 %index
+ %v5 = load i32, ptr %gep5, align 4
+ br label %exit
+
+case6:
+ %gep6 = getelementptr i32, ptr %p6, i64 %index
+ %v6 = load i32, ptr %gep6, align 4
+ br label %exit
+
+case7:
+ %gep7 = getelementptr i32, ptr %p7, i64 %index
+ %v7 = load i32, ptr %gep7, align 4
+ br label %exit
+
+case8:
+ %gep8 = getelementptr i32, ptr %p8, i64 %index
+ %v8 = load i32, ptr %gep8, align 4
+ br label %exit
+
+case9:
+ %gep9 = getelementptr i32, ptr %p9, i64 %index
+ %v9 = load i32, ptr %gep9, align 4
+ br label %exit
+
+case10:
+ %gep10 = getelementptr i32, ptr %p10, i64 %index
+ %v10 = load i32, ptr %gep10, align 4
+ br label %exit
+
+exit:
+ %result = phi i32 [ %v0, %case0 ], [ %v1, %case1 ], [ %v2, %case2 ],
+ [ %v3, %case3 ], [ %v4, %case4 ], [ %v5, %case5 ],
+ [ %v6, %case6 ], [ %v7, %case7 ], [ %v8, %case8 ],
+ [ %v9, %case9 ], [ %v10, %case10 ]
+ ret i32 %result
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/X86/sink-common-store-different-pointers.ll b/llvm/test/Transforms/SimplifyCFG/X86/sink-common-store-different-pointers.ll
new file mode 100644
index 00000000000000..14ff3d351ea40f
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/X86/sink-common-store-different-pointers.ll
@@ -0,0 +1,152 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt < %s -mtriple=x86_64-- -mattr=+avx512f -passes='simplifycfg<sink-common-insts>' -S | FileCheck %s --check-prefixes=MASKED
+; RUN: opt < %s -mtriple=x86_64-- -mattr=+sse2 -passes='simplifycfg<sink-common-insts>' -S | FileCheck %s --check-prefixes=SCALAR
+
+; Stores to different address expressions can only be commoned through a
+; pointer PHI. On a target with masked stores or scatters, keep them in their
+; original blocks so that a vectorizer can widen each one on its own.
+define void @different_store_pointers(i1 %cond, ptr %a, ptr %b, i64 %index, i32 %x, i32 %y) {
+; MASKED-LABEL: define void @different_store_pointers(
+; MASKED-SAME: i1 [[COND:%.*]], ptr [[A:%.*]], ptr [[B:%.*]], i64 [[INDEX:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]]) #[[ATTR0:[0-9]+]] {
+; MASKED-NEXT: [[ENTRY:.*:]]
+; MASKED-NEXT: br i1 [[COND]], label %[[IF_THEN:.*]], label %[[IF_ELSE:.*]]
+; MASKED: [[IF_THEN]]:
+; MASKED-NEXT: [[A_GEP:%.*]] = getelementptr i32, ptr [[A]], i64 [[INDEX]]
+; MASKED-NEXT: store i32 [[X]], ptr [[A_GEP]], align 4
+; MASKED-NEXT: br label %[[EXIT:.*]]
+; MASKED: [[IF_ELSE]]:
+; MASKED-NEXT: [[B_GEP:%.*]] = getelementptr i32, ptr [[B]], i64 [[INDEX]]
+; MASKED-NEXT: store i32 [[Y]], ptr [[B_GEP]], align 4
+; MASKED-NEXT: br label %[[EXIT]]
+; MASKED: [[EXIT]]:
+; MASKED-NEXT: ret void
+;
+; SCALAR-LABEL: define void @different_store_pointers(
+; SCALAR-SAME: i1 [[COND:%.*]], ptr [[A:%.*]], ptr [[B:%.*]], i64 [[INDEX:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]]) #[[ATTR0:[0-9]+]] {
+; SCALAR-NEXT: [[ENTRY:.*:]]
+; SCALAR-NEXT: [[A_B:%.*]] = select i1 [[COND]], ptr [[A]], ptr [[B]]
+; SCALAR-NEXT: [[X_Y:%.*]] = select i1 [[COND]], i32 [[X]], i32 [[Y]]
+; SCALAR-NEXT: [[B_GEP:%.*]] = getelementptr i32, ptr [[A_B]], i64 [[INDEX]]
+; SCALAR-NEXT: store i32 [[X_Y]], ptr [[B_GEP]], align 4
+; SCALAR-NEXT: ret void
+;
+entry:
+ br i1 %cond, label %if.then, label %if.else
+
+if.then:
+ %a.gep = getelementptr i32, ptr %a, i64 %index
+ store i32 %x, ptr %a.gep
+ br label %exit
+
+if.else:
+ %b.gep = getelementptr i32, ptr %b, i64 %index
+ store i32 %y, ptr %b.gep
+ br label %exit
+
+exit:
+ ret void
+}
+
+; A pointer PHI is not needed when both stores write to the same address, so
+; these are commoned on any target.
+define void @same_store_pointer(i1 %cond, ptr %a, i32 %x, i32 %y) {
+; MASKED-LABEL: define void @same_store_pointer(
+; MASKED-SAME: i1 [[COND:%.*]], ptr [[A:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]]) #[[ATTR0]] {
+; MASKED-NEXT: [[ENTRY:.*:]]
+; MASKED-NEXT: [[X_Y:%.*]] = select i1 [[COND]], i32 [[X]], i32 [[Y]]
+; MASKED-NEXT: store i32 [[X_Y]], ptr [[A]], align 4
+; MASKED-NEXT: ret void
+;
+; SCALAR-LABEL: define void @same_store_pointer(
+; SCALAR-SAME: i1 [[COND:%.*]], ptr [[A:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]]) #[[ATTR0]] {
+; SCALAR-NEXT: [[ENTRY:.*:]]
+; SCALAR-NEXT: [[X_Y:%.*]] = select i1 [[COND]], i32 [[X]], i32 [[Y]]
+; SCALAR-NEXT: store i32 [[X_Y]], ptr [[A]], align 4
+; SCALAR-NEXT: ret void
+;
+entry:
+ br i1 %cond, label %if.then, label %if.else
+
+if.then:
+ store i32 %x, ptr %a
+ br label %exit
+
+if.else:
+ store i32 %y, ptr %a
+ br label %exit
+
+exit:
+ ret void
+}
+
+; Same, for distinct instructions computing the same address.
+define void @same_store_address_expression(i1 %cond, ptr %a, i64 %index, i32 %x, i32 %y) {
+; MASKED-LABEL: define void @same_store_address_expression(
+; MASKED-SAME: i1 [[COND:%.*]], ptr [[A:%.*]], i64 [[INDEX:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]]) #[[ATTR0]] {
+; MASKED-NEXT: [[ENTRY:.*:]]
+; MASKED-NEXT: [[X_Y:%.*]] = select i1 [[COND]], i32 [[X]], i32 [[Y]]
+; MASKED-NEXT: [[B_GEP:%.*]] = getelementptr i32, ptr [[A]], i64 [[INDEX]]
+; MASKED-NEXT: store i32 [[X_Y]], ptr [[B_GEP]], align 4
+; MASKED-NEXT: ret void
+;
+; SCALAR-LABEL: define void @same_store_address_expression(
+; SCALAR-SAME: i1 [[COND:%.*]], ptr [[A:%.*]], i64 [[INDEX:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]]) #[[ATTR0]] {
+; SCALAR-NEXT: [[ENTRY:.*:]]
+; SCALAR-NEXT: [[X_Y:%.*]] = select i1 [[COND]], i32 [[X]], i32 [[Y]]
+; SCALAR-NEXT: [[B_GEP:%.*]] = getelementptr i32, ptr [[A]], i64 [[INDEX]]
+; SCALAR-NEXT: store i32 [[X_Y]], ptr [[B_GEP]], align 4
+; SCALAR-NEXT: ret void
+;
+entry:
+ br i1 %cond, label %if.then, label %if.else
+
+if.then:
+ %a.gep = getelementptr i32, ptr %a, i64 %index
+ store i32 %x, ptr %a.gep
+ br label %exit
+
+if.else:
+ %b.gep = getelementptr i32, ptr %a, i64 %index
+ store i32 %y, ptr %b.gep
+ br label %exit
+
+exit:
+ ret void
+}
+
+; Types that the target cannot mask are still commoned.
+define void @different_store_pointers_unsupported_type(i1 %cond, ptr %a, ptr %b, i64 %index, i8 %x, i8 %y) {
+; MASKED-LABEL: define void @different_store_pointers_unsupported_type(
+; MASKED-SAME: i1 [[COND:%.*]], ptr [[A:%.*]], ptr [[B:%.*]], i64 [[INDEX:%.*]], i8 [[X:%.*]], i8 [[Y:%.*]]) #[[ATTR0]] {
+; MASKED-NEXT: [[ENTRY:.*:]]
+; MASKED-NEXT: [[A_B:%.*]] = select i1 [[COND]], ptr [[A]], ptr [[B]]
+; MASKED-NEXT: [[X_Y:%.*]] = select i1 [[COND]], i8 [[X]], i8 [[Y]]
+; MASKED-NEXT: [[B_GEP:%.*]] = getelementptr i8, ptr [[A_B]], i64 [[INDEX]]
+; MASKED-NEXT: store i8 [[X_Y]], ptr [[B_GEP]], align 1
+; MASKED-NEXT: ret void
+;
+; SCALAR-LABEL: define void @different_store_pointers_unsupported_type(
+; SCALAR-SAME: i1 [[COND:%.*]], ptr [[A:%.*]], ptr [[B:%.*]], i64 [[INDEX:%.*]], i8 [[X:%.*]], i8 [[Y:%.*]]) #[[ATTR0]] {
+; SCALAR-NEXT: [[ENTRY:.*:]]
+; SCALAR-NEXT: [[A_B:%.*]] = select i1 [[COND]], ptr [[A]], ptr [[B]]
+; SCALAR-NEXT: [[X_Y:%.*]] = select i1 [[COND]], i8 [[X]], i8 [[Y]]
+; SCALAR-NEXT: [[B_GEP:%.*]] = getelementptr i8, ptr [[A_B]], i64 [[INDEX]]
+; SCALAR-NEXT: store i8 [[X_Y]], ptr [[B_GEP]], align 1
+; SCALAR-NEXT: ret void
+;
+entry:
+ br i1 %cond, label %if.then, label %if.else
+
+if.then:
+ %a.gep = getelementptr i8, ptr %a, i64 %index
+ store i8 %x, ptr %a.gep
+ br label %exit
+
+if.else:
+ %b.gep = getelementptr i8, ptr %b, i64 %index
+ store i8 %y, ptr %b.gep
+ br label %exit
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/X86/sink-common-store-scatter-cost.ll b/llvm/test/Transforms/SimplifyCFG/X86/sink-common-store-scatter-cost.ll
new file mode 100644
index 00000000000000..b9dbe6a93ec59d
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/X86/sink-common-store-scatter-cost.ll
@@ -0,0 +1,190 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt < %s -mtriple=x86_64-- -mattr=+avx512f -passes='simplifycfg<sink-common-insts>' -S | FileCheck %s --check-prefixes=VF16
+; RUN: opt < %s -mtriple=x86_64-- -mattr=+avx512f,+prefer-256-bit -passes='simplifycfg<sink-common-insts>' -S | FileCheck %s --check-prefixes=VF8
+
+; Whether commoning these stores is worthwhile is decided by cost, not by
+; legality: masked stores and scatters are both available here, but a scatter
+; only pays off once it replaces enough separate masked stores. The wider the
+; vector, the more stores a single scatter has to absorb to break even.
+
+define void @many_store_pointers(i32 %sel, i64 %index, i32 %v, ptr %p0, ptr %p1, ptr %p2, ptr %p3, ptr %p4, ptr %p5, ptr %p6, ptr %p7, ptr %p8, ptr %p9, ptr %p10) {
+; VF16-LABEL: define void @many_store_pointers(
+; VF16-SAME: i32 [[SEL:%.*]], i64 [[INDEX:%.*]], i32 [[V:%.*]], ptr [[P0:%.*]], ptr [[P1:%.*]], ptr [[P2:%.*]], ptr [[P3:%.*]], ptr [[P4:%.*]], ptr [[P5:%.*]], ptr [[P6:%.*]], ptr [[P7:%.*]], ptr [[P8:%.*]], ptr [[P9:%.*]], ptr [[P10:%.*]]) #[[ATTR0:[0-9]+]] {
+; VF16-NEXT: [[ENTRY:.*:]]
+; VF16-NEXT: switch i32 [[SEL]], label %[[EXIT:.*]] [
+; VF16-NEXT: i32 0, label %[[CASE0:.*]]
+; VF16-NEXT: i32 1, label %[[CASE1:.*]]
+; VF16-NEXT: i32 2, label %[[CASE2:.*]]
+; VF16-NEXT: i32 3, label %[[CASE3:.*]]
+; VF16-NEXT: i32 4, label %[[CASE4:.*]]
+; VF16-NEXT: i32 5, label %[[CASE5:.*]]
+; VF16-NEXT: i32 6, label %[[CASE6:.*]]
+; VF16-NEXT: i32 7, label %[[CASE7:.*]]
+; VF16-NEXT: i32 8, label %[[CASE8:.*]]
+; VF16-NEXT: i32 9, label %[[CASE9:.*]]
+; VF16-NEXT: i32 10, label %[[CASE10:.*]]
+; VF16-NEXT: ]
+; VF16: [[CASE0]]:
+; VF16-NEXT: [[GEP0:%.*]] = getelementptr i32, ptr [[P0]], i64 [[INDEX]]
+; VF16-NEXT: store i32 [[V]], ptr [[GEP0]], align 4
+; VF16-NEXT: br label %[[EXIT]]
+; VF16: [[CASE1]]:
+; VF16-NEXT: [[GEP1:%.*]] = getelementptr i32, ptr [[P1]], i64 [[INDEX]]
+; VF16-NEXT: store i32 [[V]], ptr [[GEP1]], align 4
+; VF16-NEXT: br label %[[EXIT]]
+; VF16: [[CASE2]]:
+; VF16-NEXT: [[GEP2:%.*]] = getelementptr i32, ptr [[P2]], i64 [[INDEX]]
+; VF16-NEXT: store i32 [[V]], ptr [[GEP2]], align 4
+; VF16-NEXT: br label %[[EXIT]]
+; VF16: [[CASE3]]:
+; VF16-NEXT: [[GEP3:%.*]] = getelementptr i32, ptr [[P3]], i64 [[INDEX]]
+; VF16-NEXT: store i32 [[V]], ptr [[GEP3]], align 4
+; VF16-NEXT: br label %[[EXIT]]
+; VF16: [[CASE4]]:
+; VF16-NEXT: [[GEP4:%.*]] = getelementptr i32, ptr [[P4]], i64 [[INDEX]]
+; VF16-NEXT: store i32 [[V]], ptr [[GEP4]], align 4
+; VF16-NEXT: br label %[[EXIT]]
+; VF16: [[CASE5]]:
+; VF16-NEXT: [[GEP5:%.*]] = getelementptr i32, ptr [[P5]], i64 [[INDEX]]
+; VF16-NEXT: store i32 [[V]], ptr [[GEP5]], align 4
+; VF16-NEXT: br label %[[EXIT]]
+; VF16: [[CASE6]]:
+; VF16-NEXT: [[GEP6:%.*]] = getelementptr i32, ptr [[P6]], i64 [[INDEX]]
+; VF16-NEXT: store i32 [[V]], ptr [[GEP6]], align 4
+; VF16-NEXT: br label %[[EXIT]]
+; VF16: [[CASE7]]:
+; VF16-NEXT: [[GEP7:%.*]] = getelementptr i32, ptr [[P7]], i64 [[INDEX]]
+; VF16-NEXT: store i32 [[V]], ptr [[GEP7]], align 4
+; VF16-NEXT: br label %[[EXIT]]
+; VF16: [[CASE8]]:
+; VF16-NEXT: [[GEP8:%.*]] = getelementptr i32, ptr [[P8]], i64 [[INDEX]]
+; VF16-NEXT: store i32 [[V]], ptr [[GEP8]], align 4
+; VF16-NEXT: br label %[[EXIT]]
+; VF16: [[CASE9]]:
+; VF16-NEXT: [[GEP9:%.*]] = getelementptr i32, ptr [[P9]], i64 [[INDEX]]
+; VF16-NEXT: store i32 [[V]], ptr [[GEP9]], align 4
+; VF16-NEXT: br label %[[EXIT]]
+; VF16: [[CASE10]]:
+; VF16-NEXT: [[GEP10:%.*]] = getelementptr i32, ptr [[P10]], i64 [[INDEX]]
+; VF16-NEXT: store i32 [[V]], ptr [[GEP10]], align 4
+; VF16-NEXT: br label %[[EXIT]]
+; VF16: [[EXIT]]:
+; VF16-NEXT: ret void
+;
+; VF8-LABEL: define void @many_store_pointers(
+; VF8-SAME: i32 [[SEL:%.*]], i64 [[INDEX:%.*]], i32 [[V:%.*]], ptr [[P0:%.*]], ptr [[P1:%.*]], ptr [[P2:%.*]], ptr [[P3:%.*]], ptr [[P4:%.*]], ptr [[P5:%.*]], ptr [[P6:%.*]], ptr [[P7:%.*]], ptr [[P8:%.*]], ptr [[P9:%.*]], ptr [[P10:%.*]]) #[[ATTR0:[0-9]+]] {
+; VF8-NEXT: [[ENTRY:.*]]:
+; VF8-NEXT: switch i32 [[SEL]], label %[[EXIT:.*]] [
+; VF8-NEXT: i32 0, label %[[EXIT_SINK_SPLIT:.*]]
+; VF8-NEXT: i32 1, label %[[CASE1:.*]]
+; VF8-NEXT: i32 2, label %[[CASE2:.*]]
+; VF8-NEXT: i32 3, label %[[CASE3:.*]]
+; VF8-NEXT: i32 4, label %[[CASE4:.*]]
+; VF8-NEXT: i32 5, label %[[CASE5:.*]]
+; VF8-NEXT: i32 6, label %[[CASE6:.*]]
+; VF8-NEXT: i32 7, label %[[CASE7:.*]]
+; VF8-NEXT: i32 8, label %[[CASE8:.*]]
+; VF8-NEXT: i32 9, label %[[CASE9:.*]]
+; VF8-NEXT: i32 10, label %[[CASE10:.*]]
+; VF8-NEXT: ]
+; VF8: [[CASE1]]:
+; VF8-NEXT: br label %[[EXIT_SINK_SPLIT]]
+; VF8: [[CASE2]]:
+; VF8-NEXT: br label %[[EXIT_SINK_SPLIT]]
+; VF8: [[CASE3]]:
+; VF8-NEXT: br label %[[EXIT_SINK_SPLIT]]
+; VF8: [[CASE4]]:
+; VF8-NEXT: br label %[[EXIT_SINK_SPLIT]]
+; VF8: [[CASE5]]:
+; VF8-NEXT: br label %[[EXIT_SINK_SPLIT]]
+; VF8: [[CASE6]]:
+; VF8-NEXT: br label %[[EXIT_SINK_SPLIT]]
+; VF8: [[CASE7]]:
+; VF8-NEXT: br label %[[EXIT_SINK_SPLIT]]
+; VF8: [[CASE8]]:
+; VF8-NEXT: br label %[[EXIT_SINK_SPLIT]]
+; VF8: [[CASE9]]:
+; VF8-NEXT: br label %[[EXIT_SINK_SPLIT]]
+; VF8: [[CASE10]]:
+; VF8-NEXT: br label %[[EXIT_SINK_SPLIT]]
+; VF8: [[EXIT_SINK_SPLIT]]:
+; VF8-NEXT: [[P10_SINK:%.*]] = phi ptr [ [[P10]], %[[CASE10]] ], [ [[P9]], %[[CASE9]] ], [ [[P8]], %[[CASE8]] ], [ [[P7]], %[[CASE7]] ], [ [[P6]], %[[CASE6]] ], [ [[P5]], %[[CASE5]] ], [ [[P4]], %[[CASE4]] ], [ [[P3]], %[[CASE3]] ], [ [[P2]], %[[CASE2]] ], [ [[P1]], %[[CASE1]] ], [ [[P0]], %[[ENTRY]] ]
+; VF8-NEXT: [[GEP10:%.*]] = getelementptr i32, ptr [[P10_SINK]], i64 [[INDEX]]
+; VF8-NEXT: store i32 [[V]], ptr [[GEP10]], align 4
+; VF8-NEXT: br label %[[EXIT]]
+; VF8: [[EXIT]]:
+; VF8-NEXT: ret void
+;
+entry:
+ switch i32 %sel, label %exit [
+ i32 0, label %case0
+ i32 1, label %case1
+ i32 2, label %case2
+ i32 3, label %case3
+ i32 4, label %case4
+ i32 5, label %case5
+ i32 6, label %case6
+ i32 7, label %case7
+ i32 8, label %case8
+ i32 9, label %case9
+ i32 10, label %case10
+ ]
+
+case0:
+ %gep0 = getelementptr i32, ptr %p0, i64 %index
+ store i32 %v, ptr %gep0
+ br label %exit
+
+case1:
+ %gep1 = getelementptr i32, ptr %p1, i64 %index
+ store i32 %v, ptr %gep1
+ br label %exit
+
+case2:
+ %gep2 = getelementptr i32, ptr %p2, i64 %index
+ store i32 %v, ptr %gep2
+ br label %exit
+
+case3:
+ %gep3 = getelementptr i32, ptr %p3, i64 %index
+ store i32 %v, ptr %gep3
+ br label %exit
+
+case4:
+ %gep4 = getelementptr i32, ptr %p4, i64 %index
+ store i32 %v, ptr %gep4
+ br label %exit
+
+case5:
+ %gep5 = getelementptr i32, ptr %p5, i64 %index
+ store i32 %v, ptr %gep5
+ br label %exit
+
+case6:
+ %gep6 = getelementptr i32, ptr %p6, i64 %index
+ store i32 %v, ptr %gep6
+ br label %exit
+
+case7:
+ %gep7 = getelementptr i32, ptr %p7, i64 %index
+ store i32 %v, ptr %gep7
+ br label %exit
+
+case8:
+ %gep8 = getelementptr i32, ptr %p8, i64 %index
+ store i32 %v, ptr %gep8
+ br label %exit
+
+case9:
+ %gep9 = getelementptr i32, ptr %p9, i64 %index
+ store i32 %v, ptr %gep9
+ br label %exit
+
+case10:
+ %gep10 = getelementptr i32, ptr %p10, i64 %index
+ store i32 %v, ptr %gep10
+ br label %exit
+
+exit:
+ ret void
+}
>From 6a97913f52bdbc589c82f2988c33725209ffc814 Mon Sep 17 00:00:00 2001
From: Ashutosh Nema <ashu1212 at gmail.com>
Date: Tue, 22 Sep 2026 19:52:43 +0530
Subject: [PATCH 2/3] [SimplifyCFG] Rename sink cost guard and drop redundant
InstructionCost
---
llvm/lib/Transforms/Utils/SimplifyCFG.cpp | 13 +-
.../X86/sink-common-loads-vectorize.ll | 130 ++++++++++++++++++
2 files changed, 134 insertions(+), 9 deletions(-)
create mode 100644 llvm/test/Transforms/PhaseOrdering/X86/sink-common-loads-vectorize.ll
diff --git a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
index 9eb0d7e4bd5ee8..cbef2f5b826972 100644
--- a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
@@ -2424,9 +2424,9 @@ static void sinkLastInstruction(ArrayRef<BasicBlock*> Blocks) {
/// access distinct addresses, penalizes a later vectorizer. Commoning needs a
/// pointer PHI, so the result can only be widened as a gather or scatter, while
/// separate blocks allow one independent masked load or store each.
-static bool sinkingMemOpsPenalizesVectorization(const TargetTransformInfo &TTI,
- Instruction *I,
- unsigned NumMemOps) {
+static bool
+sinkingMemOpCouldPenalizeVectorization(const TargetTransformInfo &TTI,
+ Instruction *I, unsigned NumMemOps) {
bool IsLoad = isa<LoadInst>(I);
assert((IsLoad || isa<StoreInst>(I)) && "Expected a load or store");
Type *ScalarTy = getLoadStoreType(I);
@@ -2471,11 +2471,6 @@ static bool sinkingMemOpsPenalizesVectorization(const TargetTransformInfo &TTI,
CostKind) *
NumMemOps;
- if (!GatherScatterCost.isValid())
- return true;
- if (!MaskedCost.isValid())
- return false;
-
LLVM_DEBUG(dbgs() << "SINK: " << (VF.isScalable() ? "scalable" : "fixed")
<< " VF " << VF.getKnownMinValue() << ": "
<< (IsLoad ? "gather" : "scatter") << " cost "
@@ -2639,7 +2634,7 @@ static bool sinkCommonCodeFromPredecessors(BasicBlock *BB, DomTreeUpdater *DTU,
auto ProfitableToSinkInstruction = [&](LockstepReverseIterator<true> &LRI) {
ArrayRef<Instruction *> Insts = *LRI;
if (isa<LoadInst, StoreInst>(Insts[0]) && !HaveSameMemAddress(Insts) &&
- sinkingMemOpsPenalizesVectorization(TTI, Insts[0], Insts.size()))
+ sinkingMemOpCouldPenalizeVectorization(TTI, Insts[0], Insts.size()))
return false;
unsigned NumPHIInsts = 0;
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/sink-common-loads-vectorize.ll b/llvm/test/Transforms/PhaseOrdering/X86/sink-common-loads-vectorize.ll
new file mode 100644
index 00000000000000..9b51fd20f20812
--- /dev/null
+++ b/llvm/test/Transforms/PhaseOrdering/X86/sink-common-loads-vectorize.ll
@@ -0,0 +1,130 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt -S -passes="default<O2>" -mattr=+avx512f -force-vector-interleave=1 < %s | FileCheck %s
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Conditional loads from two pairs of arrays. Sinking them through pointer
+; PHIs would force gathers; keeping them separate lets LV emit masked loads.
+define i32 @cond_load_pairs(ptr noalias %B, ptr noalias %C, ptr noalias %Y, ptr noalias %Z, ptr noalias %P) {
+; CHECK-LABEL: define i32 @cond_load_pairs(
+; CHECK-SAME: ptr noalias nofree readonly captures(none) [[B:%.*]], ptr noalias nofree readonly captures(none) [[C:%.*]], ptr noalias nofree readonly captures(none) [[Y:%.*]], ptr noalias nofree readonly captures(none) [[Z:%.*]], ptr noalias nofree readonly captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[VECTOR_PH:.*]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI4:%.*]] = phi <16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP40:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[P]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD7:%.*]] = load <16 x i32>, ptr [[TMP1]], align 4
+; CHECK-NEXT: [[TMP8:%.*]] = icmp sgt <16 x i32> [[WIDE_LOAD7]], zeroinitializer
+; CHECK-NEXT: [[TMP12:%.*]] = xor <16 x i1> [[TMP8]], splat (i1 true)
+; CHECK-NEXT: [[TMP13:%.*]] = getelementptr [4 x i8], ptr [[Y]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD10:%.*]] = tail call <16 x i32> @llvm.masked.load.v16i32.p0(ptr align 4 [[TMP13]], <16 x i1> [[TMP12]], <16 x i32> poison)
+; CHECK-NEXT: [[TMP17:%.*]] = getelementptr [4 x i8], ptr [[Z]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD14:%.*]] = tail call <16 x i32> @llvm.masked.load.v16i32.p0(ptr align 4 [[TMP17]], <16 x i1> [[TMP12]], <16 x i32> poison)
+; CHECK-NEXT: [[TMP24:%.*]] = add nsw <16 x i32> [[WIDE_MASKED_LOAD14]], [[WIDE_MASKED_LOAD10]]
+; CHECK-NEXT: [[TMP25:%.*]] = getelementptr [4 x i8], ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD18:%.*]] = tail call <16 x i32> @llvm.masked.load.v16i32.p0(ptr align 4 [[TMP25]], <16 x i1> [[TMP8]], <16 x i32> poison)
+; CHECK-NEXT: [[TMP29:%.*]] = getelementptr [4 x i8], ptr [[C]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD22:%.*]] = tail call <16 x i32> @llvm.masked.load.v16i32.p0(ptr align 4 [[TMP29]], <16 x i1> [[TMP8]], <16 x i32> poison)
+; CHECK-NEXT: [[TMP36:%.*]] = add nsw <16 x i32> [[WIDE_MASKED_LOAD22]], [[WIDE_MASKED_LOAD18]]
+; CHECK-NEXT: [[PREDPHI25:%.*]] = select <16 x i1> [[TMP8]], <16 x i32> [[TMP36]], <16 x i32> [[TMP24]]
+; CHECK-NEXT: [[TMP40]] = add <16 x i32> [[PREDPHI25]], [[VEC_PHI4]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP11:%.*]] = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP40]])
+; CHECK-NEXT: ret i32 [[TMP11]]
+;
+entry:
+ br label %for.body
+
+for.body:
+ %i = phi i64 [ %inc, %for.inc ], [ 0, %entry ]
+ %sum = phi i32 [ %sum.next, %for.inc ], [ 0, %entry ]
+ %p.gep = getelementptr inbounds i32, ptr %P, i64 %i
+ %p.val = load i32, ptr %p.gep, align 4
+ %cond = icmp sgt i32 %p.val, 0
+ br i1 %cond, label %if.then, label %if.else
+
+if.then:
+ %b.gep = getelementptr inbounds i32, ptr %B, i64 %i
+ %b.val = load i32, ptr %b.gep, align 4
+ %c.gep = getelementptr inbounds i32, ptr %C, i64 %i
+ %c.val = load i32, ptr %c.gep, align 4
+ %then.add = add nsw i32 %b.val, %c.val
+ br label %for.inc
+
+if.else:
+ %y.gep = getelementptr inbounds i32, ptr %Y, i64 %i
+ %y.val = load i32, ptr %y.gep, align 4
+ %z.gep = getelementptr inbounds i32, ptr %Z, i64 %i
+ %z.val = load i32, ptr %z.gep, align 4
+ %else.add = add nsw i32 %y.val, %z.val
+ br label %for.inc
+
+for.inc:
+ %add = phi i32 [ %then.add, %if.then ], [ %else.add, %if.else ]
+ %sum.next = add nsw i32 %sum, %add
+ %inc = add nuw nsw i64 %i, 1
+ %exitcond = icmp eq i64 %inc, 1024
+ br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+ ret i32 %sum.next
+}
+
+; Conditional stores to two arrays. Sinking them through a pointer PHI
+; would force a scatter; keeping them separate lets LV emit masked stores.
+define void @cond_store_pair(ptr noalias %B, ptr noalias %Y, ptr noalias %P, i32 %x) {
+; CHECK-LABEL: define void @cond_store_pair(
+; CHECK-SAME: ptr noalias nofree writeonly captures(none) [[B:%.*]], ptr noalias nofree writeonly captures(none) [[Y:%.*]], ptr noalias nofree readonly captures(none) [[P:%.*]], i32 [[X:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
+; CHECK-NEXT: [[VECTOR_PH:.*]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i32> poison, i32 [[X]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x i32> [[BROADCAST_SPLATINSERT]], <16 x i32> poison, <16 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT10:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[P]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD4:%.*]] = load <16 x i32>, ptr [[TMP1]], align 4
+; CHECK-NEXT: [[TMP8:%.*]] = icmp sgt <16 x i32> [[WIDE_LOAD4]], zeroinitializer
+; CHECK-NEXT: [[TMP12:%.*]] = xor <16 x i1> [[TMP8]], splat (i1 true)
+; CHECK-NEXT: [[TMP13:%.*]] = getelementptr [4 x i8], ptr [[Y]], i64 [[INDEX]]
+; CHECK-NEXT: tail call void @llvm.masked.store.v16i32.p0(<16 x i32> [[BROADCAST_SPLAT]], ptr align 4 [[TMP13]], <16 x i1> [[TMP12]])
+; CHECK-NEXT: [[TMP17:%.*]] = getelementptr [4 x i8], ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: tail call void @llvm.masked.store.v16i32.p0(<16 x i32> [[BROADCAST_SPLAT]], ptr align 4 [[TMP17]], <16 x i1> [[TMP8]])
+; CHECK-NEXT: [[INDEX_NEXT10]] = add nuw i64 [[INDEX]], 16
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT10]], 1024
+; CHECK-NEXT: br i1 [[TMP5]], label %[[FOR_END:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[FOR_END]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %i = phi i64 [ %inc, %for.inc ], [ 0, %entry ]
+ %p.gep = getelementptr inbounds i32, ptr %P, i64 %i
+ %p.val = load i32, ptr %p.gep, align 4
+ %cond = icmp sgt i32 %p.val, 0
+ br i1 %cond, label %if.then, label %if.else
+
+if.then:
+ %b.gep = getelementptr inbounds i32, ptr %B, i64 %i
+ store i32 %x, ptr %b.gep, align 4
+ br label %for.inc
+
+if.else:
+ %y.gep = getelementptr inbounds i32, ptr %Y, i64 %i
+ store i32 %x, ptr %y.gep, align 4
+ br label %for.inc
+
+for.inc:
+ %inc = add nuw nsw i64 %i, 1
+ %exitcond = icmp eq i64 %inc, 1024
+ br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+ ret void
+}
>From 0b16d75d0ecce41f3d2b08f0b74a01a35c7cabec Mon Sep 17 00:00:00 2001
From: Ashutosh Nema <ashu1212 at gmail.com>
Date: Wed, 23 Sep 2026 15:19:47 +0530
Subject: [PATCH 3/3] [SimplifyCFG] During sink use unit stride and masked
legality instead of cost
---
llvm/include/llvm/Analysis/VectorUtils.h | 10 +
.../Transforms/Utils/SimplifyCFGOptions.h | 6 +
llvm/lib/Analysis/VectorUtils.cpp | 15 +
.../lib/Transforms/Scalar/SimplifyCFGPass.cpp | 10 +
llvm/lib/Transforms/Utils/SimplifyCFG.cpp | 131 +++----
.../Vectorize/LoopVectorizationPlanner.cpp | 8 +-
.../sink-common-load-different-pointers.ll | 338 +++++++++++++-----
.../X86/sink-common-load-gather-cost.ll | 189 ----------
.../sink-common-store-different-pointers.ll | 275 ++++++++++----
.../X86/sink-common-store-scatter-cost.ll | 190 ----------
10 files changed, 549 insertions(+), 623 deletions(-)
delete mode 100644 llvm/test/Transforms/SimplifyCFG/X86/sink-common-load-gather-cost.ll
delete mode 100644 llvm/test/Transforms/SimplifyCFG/X86/sink-common-store-scatter-cost.ll
diff --git a/llvm/include/llvm/Analysis/VectorUtils.h b/llvm/include/llvm/Analysis/VectorUtils.h
index b177d9eec21896..78ea52d1708478 100644
--- a/llvm/include/llvm/Analysis/VectorUtils.h
+++ b/llvm/include/llvm/Analysis/VectorUtils.h
@@ -132,6 +132,16 @@ namespace Intrinsic {
typedef unsigned ID;
}
+/// Return true if the target supports a masked load or store of \p ScalarTy.
+LLVM_ABI bool isLegalMaskedLoadOrStore(const TargetTransformInfo &TTI,
+ bool IsLoad, Type *ScalarTy,
+ Align Alignment, unsigned AddressSpace);
+
+/// Return true if the target supports a gather or scatter of \p ScalarTy.
+LLVM_ABI bool isLegalGatherOrScatter(const TargetTransformInfo &TTI,
+ bool IsLoad, Type *ScalarTy,
+ Align Alignment, ElementCount VF);
+
/// Identify if the intrinsic is trivially vectorizable.
/// This method returns true if the intrinsic's argument types are all scalars
/// for the scalar form of the intrinsic and all vectors (or scalars handled by
diff --git a/llvm/include/llvm/Transforms/Utils/SimplifyCFGOptions.h b/llvm/include/llvm/Transforms/Utils/SimplifyCFGOptions.h
index 2d0f95741077f5..7024e5a9f2434d 100644
--- a/llvm/include/llvm/Transforms/Utils/SimplifyCFGOptions.h
+++ b/llvm/include/llvm/Transforms/Utils/SimplifyCFGOptions.h
@@ -19,6 +19,8 @@
namespace llvm {
class AssumptionCache;
+class LoopInfo;
+class ScalarEvolution;
struct SimplifyCFGOptions {
int BonusInstThreshold = 1;
@@ -35,6 +37,10 @@ struct SimplifyCFGOptions {
bool SpeculateUnpredictables = false;
AssumptionCache *AC = nullptr;
+ // Only used to identify unit-stride memory accesses when sinking. Both may be
+ // null, in which case no access is treated as unit-stride.
+ LoopInfo *LI = nullptr;
+ ScalarEvolution *SE = nullptr;
// Support 'builder' pattern to set members by name at construction time.
SimplifyCFGOptions &bonusInstThreshold(int I) {
diff --git a/llvm/lib/Analysis/VectorUtils.cpp b/llvm/lib/Analysis/VectorUtils.cpp
index f28fc4afc70ef2..5434c3e5dffe5c 100644
--- a/llvm/lib/Analysis/VectorUtils.cpp
+++ b/llvm/lib/Analysis/VectorUtils.cpp
@@ -39,6 +39,21 @@ static cl::opt<unsigned> MaxInterleaveGroupFactor(
cl::desc("Maximum factor for an interleaved access group (default = 8)"),
cl::init(8));
+bool llvm::isLegalMaskedLoadOrStore(const TargetTransformInfo &TTI, bool IsLoad,
+ Type *ScalarTy, Align Alignment,
+ unsigned AddressSpace) {
+ return IsLoad ? TTI.isLegalMaskedLoad(ScalarTy, Alignment, AddressSpace)
+ : TTI.isLegalMaskedStore(ScalarTy, Alignment, AddressSpace);
+}
+
+bool llvm::isLegalGatherOrScatter(const TargetTransformInfo &TTI, bool IsLoad,
+ Type *ScalarTy, Align Alignment,
+ ElementCount VF) {
+ Type *VectorTy = toVectorTy(ScalarTy, VF);
+ return IsLoad ? TTI.isLegalMaskedGather(VectorTy, Alignment)
+ : TTI.isLegalMaskedScatter(VectorTy, Alignment);
+}
+
/// Return true if all of the intrinsic's arguments and return type are scalars
/// for the scalar form of the intrinsic, and vectors for the vector form of the
/// intrinsic (except operands that are marked as always being scalar by
diff --git a/llvm/lib/Transforms/Scalar/SimplifyCFGPass.cpp b/llvm/lib/Transforms/Scalar/SimplifyCFGPass.cpp
index 2ef04eb6a337fe..7d365900caca00 100644
--- a/llvm/lib/Transforms/Scalar/SimplifyCFGPass.cpp
+++ b/llvm/lib/Transforms/Scalar/SimplifyCFGPass.cpp
@@ -28,6 +28,8 @@
#include "llvm/Analysis/CFG.h"
#include "llvm/Analysis/DomTreeUpdater.h"
#include "llvm/Analysis/GlobalsModRef.h"
+#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/CFG.h"
@@ -379,6 +381,10 @@ PreservedAnalyses SimplifyCFGPass::run(Function &F,
FunctionAnalysisManager &AM) {
auto &TTI = AM.getResult<TargetIRAnalysis>(F);
Options.AC = &AM.getResult<AssumptionAnalysis>(F);
+ // Use loop info and SCEV only if they have already been computed, to keep
+ // sinking decisions from requiring extra analyses.
+ Options.LI = AM.getCachedResult<LoopAnalysis>(F);
+ Options.SE = AM.getCachedResult<ScalarEvolutionAnalysis>(F);
DominatorTree *DT = nullptr;
if (RequireAndPreserveDomTree)
DT = &AM.getResult<DominatorTreeAnalysis>(F);
@@ -415,6 +421,10 @@ struct CFGSimplifyPass : public FunctionPass {
return false;
Options.AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
+ if (auto *LIWP = getAnalysisIfAvailable<LoopInfoWrapperPass>())
+ Options.LI = &LIWP->getLoopInfo();
+ if (auto *SEWP = getAnalysisIfAvailable<ScalarEvolutionWrapperPass>())
+ Options.SE = &SEWP->getSE();
DominatorTree *DT = nullptr;
if (RequireAndPreserveDomTree)
DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
diff --git a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
index cbef2f5b826972..d3bce7dbe998db 100644
--- a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
@@ -29,10 +29,15 @@
#include "llvm/Analysis/GuardUtils.h"
#include "llvm/Analysis/InstructionSimplify.h"
#include "llvm/Analysis/Loads.h"
+#include "llvm/Analysis/LoopAccessAnalysis.h"
+#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/MemorySSA.h"
#include "llvm/Analysis/MemorySSAUpdater.h"
+#include "llvm/Analysis/ScalarEvolution.h"
+#include "llvm/Analysis/ScalarEvolutionExpressions.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Analysis/ValueTracking.h"
+#include "llvm/Analysis/VectorUtils.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/CFG.h"
@@ -2420,101 +2425,55 @@ static void sinkLastInstruction(ArrayRef<BasicBlock*> Blocks) {
}
}
-/// Estimate whether commoning \p NumMemOps loads or stores like \p I, which
-/// access distinct addresses, penalizes a later vectorizer. Commoning needs a
-/// pointer PHI, so the result can only be widened as a gather or scatter, while
-/// separate blocks allow one independent masked load or store each.
+/// Return true if \p I is a memory access with a stride of one element per
+/// iteration of the loop it belongs to.
+static bool isUnitStrideMemOp(Instruction *I, const SimplifyCFGOptions &Opts) {
+ if (!Opts.LI || !Opts.SE)
+ return false;
+ Loop *L = Opts.LI->getLoopFor(I->getParent());
+ if (!L)
+ return false;
+ Type *AccessTy = getLoadStoreType(I);
+ const DataLayout &DL = I->getDataLayout();
+ if (DL.getTypeAllocSizeInBits(AccessTy) != DL.getTypeSizeInBits(AccessTy))
+ return false;
+ Value *Ptr = getLoadStorePointerOperand(I);
+ PredicatedScalarEvolution PSE(*Opts.SE, *L);
+ auto *AR = dyn_cast<SCEVAddRecExpr>(PSE.getSCEV(Ptr));
+ if (!AR)
+ return false;
+ std::optional<int64_t> Stride =
+ getStrideFromAddRec(AR, L, AccessTy, Ptr, PSE);
+ return Stride == 1 || Stride == -1;
+}
+
+/// Estimate whether commoning the loads or stores \p Insts, which access
+/// distinct addresses, penalizes a later vectorizer. Commoning needs a pointer
+/// PHI, so consecutive accesses that each could be widened into a masked load
+/// or store are widened as a gather or scatter instead.
static bool
sinkingMemOpCouldPenalizeVectorization(const TargetTransformInfo &TTI,
- Instruction *I, unsigned NumMemOps) {
+ ArrayRef<Instruction *> Insts,
+ const SimplifyCFGOptions &Opts) {
+ Instruction *I = Insts.front();
bool IsLoad = isa<LoadInst>(I);
assert((IsLoad || isa<StoreInst>(I)) && "Expected a load or store");
- Type *ScalarTy = getLoadStoreType(I);
- Align Alignment = getLoadStoreAlignment(I);
- unsigned AS = getLoadStoreAddressSpace(I);
- Intrinsic::ID MaskedID =
- IsLoad ? Intrinsic::masked_load : Intrinsic::masked_store;
- Intrinsic::ID GatherScatterID =
- IsLoad ? Intrinsic::masked_gather : Intrinsic::masked_scatter;
-
- // There is nothing to preserve unless the operations can be widened in place.
- if (!VectorType::isValidElementType(ScalarTy) ||
- !(IsLoad ? TTI.isLegalMaskedLoad(ScalarTy, Alignment, AS)
- : TTI.isLegalMaskedStore(ScalarTy, Alignment, AS)))
+ if (!all_of(Insts,
+ [&](Instruction *I) { return isUnitStrideMemOp(I, Opts); }))
return false;
- // Without a gather or scatter the commoned operation cannot be widened.
- if (!(IsLoad ? TTI.isLegalMaskedGather(ScalarTy, Alignment)
- : TTI.isLegalMaskedScatter(ScalarTy, Alignment)))
- return true;
-
- TypeSize EltWidth = I->getDataLayout().getTypeSizeInBits(ScalarTy);
- if (EltWidth.isScalable() || EltWidth.isZero())
- return true;
-
- auto PenalizesForVF = [&](ElementCount VF) {
- auto *VecTy = VectorType::get(ScalarTy, VF);
- Value *Ptr = getLoadStorePointerOperand(I);
- auto *PtrVecTy = VectorType::get(Ptr->getType(), VF);
- constexpr TargetTransformInfo::TargetCostKind CostKind =
- TargetTransformInfo::TCK_RecipThroughput;
-
- InstructionCost GatherScatterCost =
- TTI.getAddressComputationCost(PtrVecTy, nullptr, nullptr, CostKind) +
- TTI.getMemIntrinsicInstrCost(
- MemIntrinsicCostAttributes(GatherScatterID, VecTy, Ptr,
- /*VariableMask=*/true, Alignment, I),
- CostKind);
- InstructionCost MaskedCost =
- TTI.getMemIntrinsicInstrCost(
- MemIntrinsicCostAttributes(MaskedID, VecTy, Alignment, AS),
- CostKind) *
- NumMemOps;
-
- LLVM_DEBUG(dbgs() << "SINK: " << (VF.isScalable() ? "scalable" : "fixed")
- << " VF " << VF.getKnownMinValue() << ": "
- << (IsLoad ? "gather" : "scatter") << " cost "
- << GatherScatterCost << " vs " << NumMemOps << " masked "
- << (IsLoad ? "loads " : "stores ") << MaskedCost << "\n");
- return GatherScatterCost >= MaskedCost;
- };
-
- // One vector register's worth of elements, at vscale=1 for scalable vectors.
- auto VFFrom = [&](TargetTransformInfo::RegisterKind RK,
- bool Scalable) -> std::optional<ElementCount> {
- TypeSize RegWidth = TTI.getRegisterBitWidth(RK);
- if (RegWidth.isScalable() != Scalable || RegWidth.isZero())
- return std::nullopt;
- unsigned NumElts = RegWidth.getKnownMinValue() / EltWidth.getFixedValue();
- if (NumElts < (Scalable ? 1u : 2u))
- return std::nullopt;
- return ElementCount::get(NumElts, Scalable);
- };
-
- std::optional<ElementCount> FixedVF =
- VFFrom(TargetTransformInfo::RGK_FixedWidthVector, /*Scalable=*/false);
- std::optional<ElementCount> ScalableVF =
- VFFrom(TargetTransformInfo::RGK_ScalableVector, /*Scalable=*/true);
-
- // Both forms are legal but no vector factor can be formed, so the target
- // vectorizes in a way this cannot reason about. Keep the operations separate.
- if (!FixedVF && !ScalableVF)
- return true;
-
- // A vectorizer may pick either kind of vector, so keep the operations
- // separate if commoning them does not pay off for one of them.
- bool Penalizes = false;
- if (FixedVF)
- Penalizes |= PenalizesForVF(*FixedVF);
- if (ScalableVF)
- Penalizes |= PenalizesForVF(*ScalableVF);
- return Penalizes;
+ Type *ScalarTy = getLoadStoreType(I);
+ return VectorType::isValidElementType(ScalarTy) &&
+ isLegalMaskedLoadOrStore(TTI, IsLoad, ScalarTy,
+ getLoadStoreAlignment(I),
+ getLoadStoreAddressSpace(I));
}
/// Check whether BB's predecessors end with unconditional branches. If it is
/// true, sink any common code from the predecessors to BB.
static bool sinkCommonCodeFromPredecessors(BasicBlock *BB, DomTreeUpdater *DTU,
- const TargetTransformInfo &TTI) {
+ const TargetTransformInfo &TTI,
+ const SimplifyCFGOptions &Options) {
// We support two situations:
// (1) all incoming arcs are unconditional
// (2) there are non-unconditional incoming arcs
@@ -2634,7 +2593,7 @@ static bool sinkCommonCodeFromPredecessors(BasicBlock *BB, DomTreeUpdater *DTU,
auto ProfitableToSinkInstruction = [&](LockstepReverseIterator<true> &LRI) {
ArrayRef<Instruction *> Insts = *LRI;
if (isa<LoadInst, StoreInst>(Insts[0]) && !HaveSameMemAddress(Insts) &&
- sinkingMemOpCouldPenalizeVectorization(TTI, Insts[0], Insts.size()))
+ sinkingMemOpCouldPenalizeVectorization(TTI, Insts, Options))
return false;
unsigned NumPHIInsts = 0;
@@ -9429,7 +9388,7 @@ bool SimplifyCFGOpt::simplifyOnce(BasicBlock *BB) {
return true;
if (SinkCommon && Options.SinkCommonInsts) {
- if (sinkCommonCodeFromPredecessors(BB, DTU, TTI) ||
+ if (sinkCommonCodeFromPredecessors(BB, DTU, TTI, Options) ||
mergeCompatibleInvokes(BB, DTU)) {
// sinkCommonCodeFromPredecessors() does not automatically CSE PHI's,
// so we may now how duplicate PHI's.
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
index 7ecedd6e189e92..be4b0227806e23 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
@@ -143,17 +143,15 @@ bool VFSelectionContext::isLegalMaskedLoadOrStore(bool IsLoad, Type *ScalarTy,
Align Alignment,
unsigned AddressSpace) const {
return ForceTargetSupportsMaskedMemoryOps ||
- (IsLoad ? TTI.isLegalMaskedLoad(ScalarTy, Alignment, AddressSpace)
- : TTI.isLegalMaskedStore(ScalarTy, Alignment, AddressSpace));
+ llvm::isLegalMaskedLoadOrStore(TTI, IsLoad, ScalarTy, Alignment,
+ AddressSpace);
}
bool VFSelectionContext::isLegalGatherOrScatter(bool IsLoad, Type *ScalarTy,
Align Alignment,
ElementCount VF) const {
- Type *VectorTy = toVectorTy(ScalarTy, VF);
return ForceTargetSupportsGatherScatterOps ||
- (IsLoad ? TTI.isLegalMaskedGather(VectorTy, Alignment)
- : TTI.isLegalMaskedScatter(VectorTy, Alignment));
+ llvm::isLegalGatherOrScatter(TTI, IsLoad, ScalarTy, Alignment, VF);
}
bool VFSelectionContext::supportsScalableVectors() const {
diff --git a/llvm/test/Transforms/SimplifyCFG/X86/sink-common-load-different-pointers.ll b/llvm/test/Transforms/SimplifyCFG/X86/sink-common-load-different-pointers.ll
index 5e2eb7f64c0afa..98993e25e2d35b 100644
--- a/llvm/test/Transforms/SimplifyCFG/X86/sink-common-load-different-pointers.ll
+++ b/llvm/test/Transforms/SimplifyCFG/X86/sink-common-load-different-pointers.ll
@@ -1,46 +1,18 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
-; RUN: opt < %s -mtriple=x86_64-- -mattr=+avx512f -passes='simplifycfg<sink-common-insts>' -S | FileCheck %s --check-prefixes=MASKED
-; RUN: opt < %s -mtriple=x86_64-- -mattr=+avx2 -passes='simplifycfg<sink-common-insts>' -S | FileCheck %s --check-prefixes=NOGATHER
-; RUN: opt < %s -mtriple=x86_64-- -mattr=+sse2 -passes='simplifycfg<sink-common-insts>' -S | FileCheck %s --check-prefixes=SCALAR
-
-; Loads from different address expressions can only be commoned through a
-; pointer PHI. On a target with masked loads, keep them in their original
-; blocks so that a vectorizer can widen each one on its own. AVX2 reaches the
-; same result without a gather, which cannot be widened at all.
-define i32 @different_load_pointers(i1 %cond, ptr %a, ptr %b, i64 %index) {
-; MASKED-LABEL: define i32 @different_load_pointers(
+; RUN: opt < %s -mtriple=x86_64-- -mattr=+avx512f -passes='require<scalar-evolution>,simplifycfg<sink-common-insts>' -S | FileCheck %s --check-prefixes=MASKED
+; RUN: opt < %s -mtriple=x86_64-- -mattr=+sse2 -passes='require<scalar-evolution>,simplifycfg<sink-common-insts>' -S | FileCheck %s --check-prefixes=SCALAR
+
+; Loads outside a loop are commoned because they are not consecutive accesses.
+define i32 @non_loop_load_pointers(i1 %cond, ptr %a, ptr %b, i64 %index) {
+; MASKED-LABEL: define i32 @non_loop_load_pointers(
; MASKED-SAME: i1 [[COND:%.*]], ptr [[A:%.*]], ptr [[B:%.*]], i64 [[INDEX:%.*]]) #[[ATTR0:[0-9]+]] {
; MASKED-NEXT: [[ENTRY:.*:]]
-; MASKED-NEXT: br i1 [[COND]], label %[[IF_THEN:.*]], label %[[IF_ELSE:.*]]
-; MASKED: [[IF_THEN]]:
-; MASKED-NEXT: [[A_GEP:%.*]] = getelementptr i32, ptr [[A]], i64 [[INDEX]]
-; MASKED-NEXT: [[X:%.*]] = load i32, ptr [[A_GEP]], align 4
-; MASKED-NEXT: br label %[[EXIT:.*]]
-; MASKED: [[IF_ELSE]]:
-; MASKED-NEXT: [[B_GEP:%.*]] = getelementptr i32, ptr [[B]], i64 [[INDEX]]
+; MASKED-NEXT: [[A_B:%.*]] = select i1 [[COND]], ptr [[A]], ptr [[B]]
+; MASKED-NEXT: [[B_GEP:%.*]] = getelementptr i32, ptr [[A_B]], i64 [[INDEX]]
; MASKED-NEXT: [[Y:%.*]] = load i32, ptr [[B_GEP]], align 4
-; MASKED-NEXT: br label %[[EXIT]]
-; MASKED: [[EXIT]]:
-; MASKED-NEXT: [[R:%.*]] = phi i32 [ [[X]], %[[IF_THEN]] ], [ [[Y]], %[[IF_ELSE]] ]
-; MASKED-NEXT: ret i32 [[R]]
-;
-; NOGATHER-LABEL: define i32 @different_load_pointers(
-; NOGATHER-SAME: i1 [[COND:%.*]], ptr [[A:%.*]], ptr [[B:%.*]], i64 [[INDEX:%.*]]) #[[ATTR0:[0-9]+]] {
-; NOGATHER-NEXT: [[ENTRY:.*:]]
-; NOGATHER-NEXT: br i1 [[COND]], label %[[IF_THEN:.*]], label %[[IF_ELSE:.*]]
-; NOGATHER: [[IF_THEN]]:
-; NOGATHER-NEXT: [[A_GEP:%.*]] = getelementptr i32, ptr [[A]], i64 [[INDEX]]
-; NOGATHER-NEXT: [[X:%.*]] = load i32, ptr [[A_GEP]], align 4
-; NOGATHER-NEXT: br label %[[EXIT:.*]]
-; NOGATHER: [[IF_ELSE]]:
-; NOGATHER-NEXT: [[B_GEP:%.*]] = getelementptr i32, ptr [[B]], i64 [[INDEX]]
-; NOGATHER-NEXT: [[Y:%.*]] = load i32, ptr [[B_GEP]], align 4
-; NOGATHER-NEXT: br label %[[EXIT]]
-; NOGATHER: [[EXIT]]:
-; NOGATHER-NEXT: [[R:%.*]] = phi i32 [ [[X]], %[[IF_THEN]] ], [ [[Y]], %[[IF_ELSE]] ]
-; NOGATHER-NEXT: ret i32 [[R]]
+; MASKED-NEXT: ret i32 [[Y]]
;
-; SCALAR-LABEL: define i32 @different_load_pointers(
+; SCALAR-LABEL: define i32 @non_loop_load_pointers(
; SCALAR-SAME: i1 [[COND:%.*]], ptr [[A:%.*]], ptr [[B:%.*]], i64 [[INDEX:%.*]]) #[[ATTR0:[0-9]+]] {
; SCALAR-NEXT: [[ENTRY:.*:]]
; SCALAR-NEXT: [[A_B:%.*]] = select i1 [[COND]], ptr [[A]], ptr [[B]]
@@ -66,83 +38,279 @@ exit:
ret i32 %r
}
-; A pointer PHI is not needed when both loads use the same address, so these
-; are commoned on any target.
-define i32 @same_load_pointer(i1 %cond, ptr %a) {
-; MASKED-LABEL: define i32 @same_load_pointer(
-; MASKED-SAME: i1 [[COND:%.*]], ptr [[A:%.*]]) #[[ATTR0]] {
-; MASKED-NEXT: [[ENTRY:.*:]]
-; MASKED-NEXT: [[X:%.*]] = load i32, ptr [[A]], align 4
-; MASKED-NEXT: ret i32 [[X]]
-;
-; NOGATHER-LABEL: define i32 @same_load_pointer(
-; NOGATHER-SAME: i1 [[COND:%.*]], ptr [[A:%.*]]) #[[ATTR0]] {
-; NOGATHER-NEXT: [[ENTRY:.*:]]
-; NOGATHER-NEXT: [[X:%.*]] = load i32, ptr [[A]], align 4
-; NOGATHER-NEXT: ret i32 [[X]]
+; Keep consecutive loads separate when masked loads are legal.
+define i32 @consecutive_load_pointers(i1 %cond, ptr %a, ptr %b) {
+; MASKED-LABEL: define i32 @consecutive_load_pointers(
+; MASKED-SAME: i1 [[COND:%.*]], ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; MASKED-NEXT: [[ENTRY:.*]]:
+; MASKED-NEXT: br label %[[LOOP:.*]]
+; MASKED: [[LOOP]]:
+; MASKED-NEXT: [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[LATCH:.*]] ]
+; MASKED-NEXT: [[SUM:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[SUM_NEXT:%.*]], %[[LATCH]] ]
+; MASKED-NEXT: br i1 [[COND]], label %[[IF_THEN:.*]], label %[[IF_ELSE:.*]]
+; MASKED: [[IF_THEN]]:
+; MASKED-NEXT: [[A_GEP:%.*]] = getelementptr i32, ptr [[A]], i64 [[I]]
+; MASKED-NEXT: [[X:%.*]] = load i32, ptr [[A_GEP]], align 4
+; MASKED-NEXT: br label %[[LATCH]]
+; MASKED: [[IF_ELSE]]:
+; MASKED-NEXT: [[B_GEP:%.*]] = getelementptr i32, ptr [[B]], i64 [[I]]
+; MASKED-NEXT: [[Y:%.*]] = load i32, ptr [[B_GEP]], align 4
+; MASKED-NEXT: br label %[[LATCH]]
+; MASKED: [[LATCH]]:
+; MASKED-NEXT: [[R:%.*]] = phi i32 [ [[X]], %[[IF_THEN]] ], [ [[Y]], %[[IF_ELSE]] ]
+; MASKED-NEXT: [[SUM_NEXT]] = add i32 [[SUM]], [[R]]
+; MASKED-NEXT: [[INC]] = add nuw nsw i64 [[I]], 1
+; MASKED-NEXT: [[DONE:%.*]] = icmp eq i64 [[INC]], 1024
+; MASKED-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; MASKED: [[EXIT]]:
+; MASKED-NEXT: ret i32 [[SUM_NEXT]]
;
-; SCALAR-LABEL: define i32 @same_load_pointer(
-; SCALAR-SAME: i1 [[COND:%.*]], ptr [[A:%.*]]) #[[ATTR0]] {
-; SCALAR-NEXT: [[ENTRY:.*:]]
-; SCALAR-NEXT: [[X:%.*]] = load i32, ptr [[A]], align 4
-; SCALAR-NEXT: ret i32 [[X]]
+; SCALAR-LABEL: define i32 @consecutive_load_pointers(
+; SCALAR-SAME: i1 [[COND:%.*]], ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; SCALAR-NEXT: [[ENTRY:.*]]:
+; SCALAR-NEXT: br label %[[LOOP:.*]]
+; SCALAR: [[LOOP]]:
+; SCALAR-NEXT: [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[LOOP]] ]
+; SCALAR-NEXT: [[SUM:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[SUM_NEXT:%.*]], %[[LOOP]] ]
+; SCALAR-NEXT: [[A_B:%.*]] = select i1 [[COND]], ptr [[A]], ptr [[B]]
+; SCALAR-NEXT: [[B_GEP:%.*]] = getelementptr i32, ptr [[A_B]], i64 [[I]]
+; SCALAR-NEXT: [[Y:%.*]] = load i32, ptr [[B_GEP]], align 4
+; SCALAR-NEXT: [[SUM_NEXT]] = add i32 [[SUM]], [[Y]]
+; SCALAR-NEXT: [[INC]] = add nuw nsw i64 [[I]], 1
+; SCALAR-NEXT: [[DONE:%.*]] = icmp eq i64 [[INC]], 1024
+; SCALAR-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; SCALAR: [[EXIT]]:
+; SCALAR-NEXT: ret i32 [[SUM_NEXT]]
;
entry:
+ br label %loop
+
+loop:
+ %i = phi i64 [ 0, %entry ], [ %inc, %latch ]
+ %sum = phi i32 [ 0, %entry ], [ %sum.next, %latch ]
br i1 %cond, label %if.then, label %if.else
if.then:
- %x = load i32, ptr %a, align 4
- br label %exit
+ %a.gep = getelementptr i32, ptr %a, i64 %i
+ %x = load i32, ptr %a.gep, align 4
+ br label %latch
if.else:
- %y = load i32, ptr %a, align 4
- br label %exit
+ %b.gep = getelementptr i32, ptr %b, i64 %i
+ %y = load i32, ptr %b.gep, align 4
+ br label %latch
+
+latch:
+ %r = phi i32 [ %x, %if.then ], [ %y, %if.else ]
+ %sum.next = add i32 %sum, %r
+ %inc = add nuw nsw i64 %i, 1
+ %done = icmp eq i64 %inc, 1024
+ br i1 %done, label %exit, label %loop
exit:
+ ret i32 %sum.next
+}
+
+; Loads with a non-unit stride are commoned.
+define i32 @strided_load_pointers(i1 %cond, ptr %a, ptr %b) {
+; MASKED-LABEL: define i32 @strided_load_pointers(
+; MASKED-SAME: i1 [[COND:%.*]], ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; MASKED-NEXT: [[ENTRY:.*]]:
+; MASKED-NEXT: br label %[[LOOP:.*]]
+; MASKED: [[LOOP]]:
+; MASKED-NEXT: [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[LOOP]] ]
+; MASKED-NEXT: [[SUM:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[SUM_NEXT:%.*]], %[[LOOP]] ]
+; MASKED-NEXT: [[INDEX:%.*]] = shl nuw nsw i64 [[I]], 1
+; MASKED-NEXT: [[A_B:%.*]] = select i1 [[COND]], ptr [[A]], ptr [[B]]
+; MASKED-NEXT: [[B_GEP:%.*]] = getelementptr i32, ptr [[A_B]], i64 [[INDEX]]
+; MASKED-NEXT: [[Y:%.*]] = load i32, ptr [[B_GEP]], align 4
+; MASKED-NEXT: [[SUM_NEXT]] = add i32 [[SUM]], [[Y]]
+; MASKED-NEXT: [[INC]] = add nuw nsw i64 [[I]], 1
+; MASKED-NEXT: [[DONE:%.*]] = icmp eq i64 [[INC]], 1024
+; MASKED-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; MASKED: [[EXIT]]:
+; MASKED-NEXT: ret i32 [[SUM_NEXT]]
+;
+; SCALAR-LABEL: define i32 @strided_load_pointers(
+; SCALAR-SAME: i1 [[COND:%.*]], ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; SCALAR-NEXT: [[ENTRY:.*]]:
+; SCALAR-NEXT: br label %[[LOOP:.*]]
+; SCALAR: [[LOOP]]:
+; SCALAR-NEXT: [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[LOOP]] ]
+; SCALAR-NEXT: [[SUM:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[SUM_NEXT:%.*]], %[[LOOP]] ]
+; SCALAR-NEXT: [[INDEX:%.*]] = shl nuw nsw i64 [[I]], 1
+; SCALAR-NEXT: [[A_B:%.*]] = select i1 [[COND]], ptr [[A]], ptr [[B]]
+; SCALAR-NEXT: [[B_GEP:%.*]] = getelementptr i32, ptr [[A_B]], i64 [[INDEX]]
+; SCALAR-NEXT: [[Y:%.*]] = load i32, ptr [[B_GEP]], align 4
+; SCALAR-NEXT: [[SUM_NEXT]] = add i32 [[SUM]], [[Y]]
+; SCALAR-NEXT: [[INC]] = add nuw nsw i64 [[I]], 1
+; SCALAR-NEXT: [[DONE:%.*]] = icmp eq i64 [[INC]], 1024
+; SCALAR-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; SCALAR: [[EXIT]]:
+; SCALAR-NEXT: ret i32 [[SUM_NEXT]]
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i64 [ 0, %entry ], [ %inc, %latch ]
+ %sum = phi i32 [ 0, %entry ], [ %sum.next, %latch ]
+ %index = shl nuw nsw i64 %i, 1
+ br i1 %cond, label %if.then, label %if.else
+
+if.then:
+ %a.gep = getelementptr i32, ptr %a, i64 %index
+ %x = load i32, ptr %a.gep, align 4
+ br label %latch
+
+if.else:
+ %b.gep = getelementptr i32, ptr %b, i64 %index
+ %y = load i32, ptr %b.gep, align 4
+ br label %latch
+
+latch:
%r = phi i32 [ %x, %if.then ], [ %y, %if.else ]
- ret i32 %r
+ %sum.next = add i32 %sum, %r
+ %inc = add nuw nsw i64 %i, 1
+ %done = icmp eq i64 %inc, 1024
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret i32 %sum.next
}
-; Types that the target cannot load under a mask are still commoned.
-define i8 @different_load_pointers_unsupported_type(i1 %cond, ptr %a, ptr %b, i64 %index) {
-; MASKED-LABEL: define i8 @different_load_pointers_unsupported_type(
-; MASKED-SAME: i1 [[COND:%.*]], ptr [[A:%.*]], ptr [[B:%.*]], i64 [[INDEX:%.*]]) #[[ATTR0]] {
-; MASKED-NEXT: [[ENTRY:.*:]]
+; Consecutive types without masked-load support are commoned.
+define i8 @consecutive_load_pointers_unsupported_type(i1 %cond, ptr %a,
+; MASKED-LABEL: define i8 @consecutive_load_pointers_unsupported_type(
+; MASKED-SAME: i1 [[COND:%.*]], ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; MASKED-NEXT: [[ENTRY:.*]]:
+; MASKED-NEXT: br label %[[LOOP:.*]]
+; MASKED: [[LOOP]]:
+; MASKED-NEXT: [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[LOOP]] ]
; MASKED-NEXT: [[A_B:%.*]] = select i1 [[COND]], ptr [[A]], ptr [[B]]
-; MASKED-NEXT: [[B_GEP:%.*]] = getelementptr i8, ptr [[A_B]], i64 [[INDEX]]
+; MASKED-NEXT: [[B_GEP:%.*]] = getelementptr i8, ptr [[A_B]], i64 [[I]]
; MASKED-NEXT: [[Y:%.*]] = load i8, ptr [[B_GEP]], align 1
+; MASKED-NEXT: [[INC]] = add nuw nsw i64 [[I]], 1
+; MASKED-NEXT: [[DONE:%.*]] = icmp eq i64 [[INC]], 1024
+; MASKED-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; MASKED: [[EXIT]]:
; MASKED-NEXT: ret i8 [[Y]]
;
-; NOGATHER-LABEL: define i8 @different_load_pointers_unsupported_type(
-; NOGATHER-SAME: i1 [[COND:%.*]], ptr [[A:%.*]], ptr [[B:%.*]], i64 [[INDEX:%.*]]) #[[ATTR0]] {
-; NOGATHER-NEXT: [[ENTRY:.*:]]
-; NOGATHER-NEXT: [[A_B:%.*]] = select i1 [[COND]], ptr [[A]], ptr [[B]]
-; NOGATHER-NEXT: [[B_GEP:%.*]] = getelementptr i8, ptr [[A_B]], i64 [[INDEX]]
-; NOGATHER-NEXT: [[Y:%.*]] = load i8, ptr [[B_GEP]], align 1
-; NOGATHER-NEXT: ret i8 [[Y]]
-;
-; SCALAR-LABEL: define i8 @different_load_pointers_unsupported_type(
-; SCALAR-SAME: i1 [[COND:%.*]], ptr [[A:%.*]], ptr [[B:%.*]], i64 [[INDEX:%.*]]) #[[ATTR0]] {
-; SCALAR-NEXT: [[ENTRY:.*:]]
+; SCALAR-LABEL: define i8 @consecutive_load_pointers_unsupported_type(
+; SCALAR-SAME: i1 [[COND:%.*]], ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; SCALAR-NEXT: [[ENTRY:.*]]:
+; SCALAR-NEXT: br label %[[LOOP:.*]]
+; SCALAR: [[LOOP]]:
+; SCALAR-NEXT: [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[LOOP]] ]
; SCALAR-NEXT: [[A_B:%.*]] = select i1 [[COND]], ptr [[A]], ptr [[B]]
-; SCALAR-NEXT: [[B_GEP:%.*]] = getelementptr i8, ptr [[A_B]], i64 [[INDEX]]
+; SCALAR-NEXT: [[B_GEP:%.*]] = getelementptr i8, ptr [[A_B]], i64 [[I]]
; SCALAR-NEXT: [[Y:%.*]] = load i8, ptr [[B_GEP]], align 1
+; SCALAR-NEXT: [[INC]] = add nuw nsw i64 [[I]], 1
+; SCALAR-NEXT: [[DONE:%.*]] = icmp eq i64 [[INC]], 1024
+; SCALAR-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; SCALAR: [[EXIT]]:
; SCALAR-NEXT: ret i8 [[Y]]
;
+ ptr %b) {
entry:
+ br label %loop
+
+loop:
+ %i = phi i64 [ 0, %entry ], [ %inc, %latch ]
br i1 %cond, label %if.then, label %if.else
if.then:
- %a.gep = getelementptr i8, ptr %a, i64 %index
+ %a.gep = getelementptr i8, ptr %a, i64 %i
%x = load i8, ptr %a.gep, align 1
- br label %exit
+ br label %latch
if.else:
- %b.gep = getelementptr i8, ptr %b, i64 %index
+ %b.gep = getelementptr i8, ptr %b, i64 %i
%y = load i8, ptr %b.gep, align 1
- br label %exit
+ br label %latch
-exit:
+latch:
%r = phi i8 [ %x, %if.then ], [ %y, %if.else ]
+ %inc = add nuw nsw i64 %i, 1
+ %done = icmp eq i64 %inc, 1024
+ br i1 %done, label %exit, label %loop
+
+exit:
ret i8 %r
}
+
+; Reverse-consecutive loads are also kept separate.
+define i32 @reverse_consecutive_load_pointers(i1 %cond, ptr %a, ptr %b) {
+; MASKED-LABEL: define i32 @reverse_consecutive_load_pointers(
+; MASKED-SAME: i1 [[COND:%.*]], ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; MASKED-NEXT: [[ENTRY:.*]]:
+; MASKED-NEXT: br label %[[LOOP:.*]]
+; MASKED: [[LOOP]]:
+; MASKED-NEXT: [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[LATCH:.*]] ]
+; MASKED-NEXT: [[SUM:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[SUM_NEXT:%.*]], %[[LATCH]] ]
+; MASKED-NEXT: [[INDEX:%.*]] = sub nuw nsw i64 1023, [[I]]
+; MASKED-NEXT: br i1 [[COND]], label %[[IF_THEN:.*]], label %[[IF_ELSE:.*]]
+; MASKED: [[IF_THEN]]:
+; MASKED-NEXT: [[A_GEP:%.*]] = getelementptr i32, ptr [[A]], i64 [[INDEX]]
+; MASKED-NEXT: [[X:%.*]] = load i32, ptr [[A_GEP]], align 4
+; MASKED-NEXT: br label %[[LATCH]]
+; MASKED: [[IF_ELSE]]:
+; MASKED-NEXT: [[B_GEP:%.*]] = getelementptr i32, ptr [[B]], i64 [[INDEX]]
+; MASKED-NEXT: [[Y:%.*]] = load i32, ptr [[B_GEP]], align 4
+; MASKED-NEXT: br label %[[LATCH]]
+; MASKED: [[LATCH]]:
+; MASKED-NEXT: [[R:%.*]] = phi i32 [ [[X]], %[[IF_THEN]] ], [ [[Y]], %[[IF_ELSE]] ]
+; MASKED-NEXT: [[SUM_NEXT]] = add i32 [[SUM]], [[R]]
+; MASKED-NEXT: [[INC]] = add nuw nsw i64 [[I]], 1
+; MASKED-NEXT: [[DONE:%.*]] = icmp eq i64 [[INC]], 1024
+; MASKED-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; MASKED: [[EXIT]]:
+; MASKED-NEXT: ret i32 [[SUM_NEXT]]
+;
+; SCALAR-LABEL: define i32 @reverse_consecutive_load_pointers(
+; SCALAR-SAME: i1 [[COND:%.*]], ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; SCALAR-NEXT: [[ENTRY:.*]]:
+; SCALAR-NEXT: br label %[[LOOP:.*]]
+; SCALAR: [[LOOP]]:
+; SCALAR-NEXT: [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[LOOP]] ]
+; SCALAR-NEXT: [[SUM:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[SUM_NEXT:%.*]], %[[LOOP]] ]
+; SCALAR-NEXT: [[INDEX:%.*]] = sub nuw nsw i64 1023, [[I]]
+; SCALAR-NEXT: [[A_B:%.*]] = select i1 [[COND]], ptr [[A]], ptr [[B]]
+; SCALAR-NEXT: [[B_GEP:%.*]] = getelementptr i32, ptr [[A_B]], i64 [[INDEX]]
+; SCALAR-NEXT: [[Y:%.*]] = load i32, ptr [[B_GEP]], align 4
+; SCALAR-NEXT: [[SUM_NEXT]] = add i32 [[SUM]], [[Y]]
+; SCALAR-NEXT: [[INC]] = add nuw nsw i64 [[I]], 1
+; SCALAR-NEXT: [[DONE:%.*]] = icmp eq i64 [[INC]], 1024
+; SCALAR-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; SCALAR: [[EXIT]]:
+; SCALAR-NEXT: ret i32 [[SUM_NEXT]]
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i64 [ 0, %entry ], [ %inc, %latch ]
+ %sum = phi i32 [ 0, %entry ], [ %sum.next, %latch ]
+ %index = sub nuw nsw i64 1023, %i
+ br i1 %cond, label %if.then, label %if.else
+
+if.then:
+ %a.gep = getelementptr i32, ptr %a, i64 %index
+ %x = load i32, ptr %a.gep, align 4
+ br label %latch
+
+if.else:
+ %b.gep = getelementptr i32, ptr %b, i64 %index
+ %y = load i32, ptr %b.gep, align 4
+ br label %latch
+
+latch:
+ %r = phi i32 [ %x, %if.then ], [ %y, %if.else ]
+ %sum.next = add i32 %sum, %r
+ %inc = add nuw nsw i64 %i, 1
+ %done = icmp eq i64 %inc, 1024
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret i32 %sum.next
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/X86/sink-common-load-gather-cost.ll b/llvm/test/Transforms/SimplifyCFG/X86/sink-common-load-gather-cost.ll
deleted file mode 100644
index e73ea1bf031627..00000000000000
--- a/llvm/test/Transforms/SimplifyCFG/X86/sink-common-load-gather-cost.ll
+++ /dev/null
@@ -1,189 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
-; RUN: opt < %s -mtriple=x86_64-- -mattr=+avx512f -passes='simplifycfg<sink-common-insts>' -S | FileCheck %s --check-prefixes=VF16
-; RUN: opt < %s -mtriple=x86_64-- -mattr=+avx512f,+prefer-256-bit -passes='simplifycfg<sink-common-insts>' -S | FileCheck %s --check-prefixes=VF8
-
-; Whether commoning these loads is worthwhile is decided by cost, not by
-; legality. Masked loads and gathers are both available, but a gather only pays
-; off once it replaces enough separate masked loads. The wider the vector, the
-; more loads a single gather has to absorb to break even.
-define i32 @many_load_pointers(i32 %sel, i64 %index, ptr %p0, ptr %p1, ptr %p2, ptr %p3, ptr %p4, ptr %p5, ptr %p6, ptr %p7, ptr %p8, ptr %p9, ptr %p10) {
-; VF16-LABEL: define i32 @many_load_pointers(
-; VF16-SAME: i32 [[SEL:%.*]], i64 [[INDEX:%.*]], ptr [[P0:%.*]], ptr [[P1:%.*]], ptr [[P2:%.*]], ptr [[P3:%.*]], ptr [[P4:%.*]], ptr [[P5:%.*]], ptr [[P6:%.*]], ptr [[P7:%.*]], ptr [[P8:%.*]], ptr [[P9:%.*]], ptr [[P10:%.*]]) #[[ATTR0:[0-9]+]] {
-; VF16-NEXT: [[ENTRY:.*:]]
-; VF16-NEXT: switch i32 [[SEL]], label %[[CASE0:.*]] [
-; VF16-NEXT: i32 1, label %[[CASE1:.*]]
-; VF16-NEXT: i32 2, label %[[CASE2:.*]]
-; VF16-NEXT: i32 3, label %[[CASE3:.*]]
-; VF16-NEXT: i32 4, label %[[CASE4:.*]]
-; VF16-NEXT: i32 5, label %[[CASE5:.*]]
-; VF16-NEXT: i32 6, label %[[CASE6:.*]]
-; VF16-NEXT: i32 7, label %[[CASE7:.*]]
-; VF16-NEXT: i32 8, label %[[CASE8:.*]]
-; VF16-NEXT: i32 9, label %[[CASE9:.*]]
-; VF16-NEXT: i32 10, label %[[CASE10:.*]]
-; VF16-NEXT: ]
-; VF16: [[CASE0]]:
-; VF16-NEXT: [[GEP0:%.*]] = getelementptr i32, ptr [[P0]], i64 [[INDEX]]
-; VF16-NEXT: [[V0:%.*]] = load i32, ptr [[GEP0]], align 4
-; VF16-NEXT: br label %[[EXIT:.*]]
-; VF16: [[CASE1]]:
-; VF16-NEXT: [[GEP1:%.*]] = getelementptr i32, ptr [[P1]], i64 [[INDEX]]
-; VF16-NEXT: [[V1:%.*]] = load i32, ptr [[GEP1]], align 4
-; VF16-NEXT: br label %[[EXIT]]
-; VF16: [[CASE2]]:
-; VF16-NEXT: [[GEP2:%.*]] = getelementptr i32, ptr [[P2]], i64 [[INDEX]]
-; VF16-NEXT: [[V2:%.*]] = load i32, ptr [[GEP2]], align 4
-; VF16-NEXT: br label %[[EXIT]]
-; VF16: [[CASE3]]:
-; VF16-NEXT: [[GEP3:%.*]] = getelementptr i32, ptr [[P3]], i64 [[INDEX]]
-; VF16-NEXT: [[V3:%.*]] = load i32, ptr [[GEP3]], align 4
-; VF16-NEXT: br label %[[EXIT]]
-; VF16: [[CASE4]]:
-; VF16-NEXT: [[GEP4:%.*]] = getelementptr i32, ptr [[P4]], i64 [[INDEX]]
-; VF16-NEXT: [[V4:%.*]] = load i32, ptr [[GEP4]], align 4
-; VF16-NEXT: br label %[[EXIT]]
-; VF16: [[CASE5]]:
-; VF16-NEXT: [[GEP5:%.*]] = getelementptr i32, ptr [[P5]], i64 [[INDEX]]
-; VF16-NEXT: [[V5:%.*]] = load i32, ptr [[GEP5]], align 4
-; VF16-NEXT: br label %[[EXIT]]
-; VF16: [[CASE6]]:
-; VF16-NEXT: [[GEP6:%.*]] = getelementptr i32, ptr [[P6]], i64 [[INDEX]]
-; VF16-NEXT: [[V6:%.*]] = load i32, ptr [[GEP6]], align 4
-; VF16-NEXT: br label %[[EXIT]]
-; VF16: [[CASE7]]:
-; VF16-NEXT: [[GEP7:%.*]] = getelementptr i32, ptr [[P7]], i64 [[INDEX]]
-; VF16-NEXT: [[V7:%.*]] = load i32, ptr [[GEP7]], align 4
-; VF16-NEXT: br label %[[EXIT]]
-; VF16: [[CASE8]]:
-; VF16-NEXT: [[GEP8:%.*]] = getelementptr i32, ptr [[P8]], i64 [[INDEX]]
-; VF16-NEXT: [[V8:%.*]] = load i32, ptr [[GEP8]], align 4
-; VF16-NEXT: br label %[[EXIT]]
-; VF16: [[CASE9]]:
-; VF16-NEXT: [[GEP9:%.*]] = getelementptr i32, ptr [[P9]], i64 [[INDEX]]
-; VF16-NEXT: [[V9:%.*]] = load i32, ptr [[GEP9]], align 4
-; VF16-NEXT: br label %[[EXIT]]
-; VF16: [[CASE10]]:
-; VF16-NEXT: [[GEP10:%.*]] = getelementptr i32, ptr [[P10]], i64 [[INDEX]]
-; VF16-NEXT: [[V10:%.*]] = load i32, ptr [[GEP10]], align 4
-; VF16-NEXT: br label %[[EXIT]]
-; VF16: [[EXIT]]:
-; VF16-NEXT: [[RESULT:%.*]] = phi i32 [ [[V0]], %[[CASE0]] ], [ [[V1]], %[[CASE1]] ], [ [[V2]], %[[CASE2]] ], [ [[V3]], %[[CASE3]] ], [ [[V4]], %[[CASE4]] ], [ [[V5]], %[[CASE5]] ], [ [[V6]], %[[CASE6]] ], [ [[V7]], %[[CASE7]] ], [ [[V8]], %[[CASE8]] ], [ [[V9]], %[[CASE9]] ], [ [[V10]], %[[CASE10]] ]
-; VF16-NEXT: ret i32 [[RESULT]]
-;
-; VF8-LABEL: define i32 @many_load_pointers(
-; VF8-SAME: i32 [[SEL:%.*]], i64 [[INDEX:%.*]], ptr [[P0:%.*]], ptr [[P1:%.*]], ptr [[P2:%.*]], ptr [[P3:%.*]], ptr [[P4:%.*]], ptr [[P5:%.*]], ptr [[P6:%.*]], ptr [[P7:%.*]], ptr [[P8:%.*]], ptr [[P9:%.*]], ptr [[P10:%.*]]) #[[ATTR0:[0-9]+]] {
-; VF8-NEXT: [[ENTRY:.*]]:
-; VF8-NEXT: switch i32 [[SEL]], label %[[EXIT:.*]] [
-; VF8-NEXT: i32 1, label %[[CASE1:.*]]
-; VF8-NEXT: i32 2, label %[[CASE2:.*]]
-; VF8-NEXT: i32 3, label %[[CASE3:.*]]
-; VF8-NEXT: i32 4, label %[[CASE4:.*]]
-; VF8-NEXT: i32 5, label %[[CASE5:.*]]
-; VF8-NEXT: i32 6, label %[[CASE6:.*]]
-; VF8-NEXT: i32 7, label %[[CASE7:.*]]
-; VF8-NEXT: i32 8, label %[[CASE8:.*]]
-; VF8-NEXT: i32 9, label %[[CASE9:.*]]
-; VF8-NEXT: i32 10, label %[[CASE10:.*]]
-; VF8-NEXT: ]
-; VF8: [[CASE1]]:
-; VF8-NEXT: br label %[[EXIT]]
-; VF8: [[CASE2]]:
-; VF8-NEXT: br label %[[EXIT]]
-; VF8: [[CASE3]]:
-; VF8-NEXT: br label %[[EXIT]]
-; VF8: [[CASE4]]:
-; VF8-NEXT: br label %[[EXIT]]
-; VF8: [[CASE5]]:
-; VF8-NEXT: br label %[[EXIT]]
-; VF8: [[CASE6]]:
-; VF8-NEXT: br label %[[EXIT]]
-; VF8: [[CASE7]]:
-; VF8-NEXT: br label %[[EXIT]]
-; VF8: [[CASE8]]:
-; VF8-NEXT: br label %[[EXIT]]
-; VF8: [[CASE9]]:
-; VF8-NEXT: br label %[[EXIT]]
-; VF8: [[CASE10]]:
-; VF8-NEXT: br label %[[EXIT]]
-; VF8: [[EXIT]]:
-; VF8-NEXT: [[P10_SINK:%.*]] = phi ptr [ [[P10]], %[[CASE10]] ], [ [[P9]], %[[CASE9]] ], [ [[P8]], %[[CASE8]] ], [ [[P7]], %[[CASE7]] ], [ [[P6]], %[[CASE6]] ], [ [[P5]], %[[CASE5]] ], [ [[P4]], %[[CASE4]] ], [ [[P3]], %[[CASE3]] ], [ [[P2]], %[[CASE2]] ], [ [[P1]], %[[CASE1]] ], [ [[P0]], %[[ENTRY]] ]
-; VF8-NEXT: [[GEP10:%.*]] = getelementptr i32, ptr [[P10_SINK]], i64 [[INDEX]]
-; VF8-NEXT: [[V10:%.*]] = load i32, ptr [[GEP10]], align 4
-; VF8-NEXT: ret i32 [[V10]]
-;
-entry:
- switch i32 %sel, label %case0 [
- i32 1, label %case1
- i32 2, label %case2
- i32 3, label %case3
- i32 4, label %case4
- i32 5, label %case5
- i32 6, label %case6
- i32 7, label %case7
- i32 8, label %case8
- i32 9, label %case9
- i32 10, label %case10
- ]
-
-case0:
- %gep0 = getelementptr i32, ptr %p0, i64 %index
- %v0 = load i32, ptr %gep0, align 4
- br label %exit
-
-case1:
- %gep1 = getelementptr i32, ptr %p1, i64 %index
- %v1 = load i32, ptr %gep1, align 4
- br label %exit
-
-case2:
- %gep2 = getelementptr i32, ptr %p2, i64 %index
- %v2 = load i32, ptr %gep2, align 4
- br label %exit
-
-case3:
- %gep3 = getelementptr i32, ptr %p3, i64 %index
- %v3 = load i32, ptr %gep3, align 4
- br label %exit
-
-case4:
- %gep4 = getelementptr i32, ptr %p4, i64 %index
- %v4 = load i32, ptr %gep4, align 4
- br label %exit
-
-case5:
- %gep5 = getelementptr i32, ptr %p5, i64 %index
- %v5 = load i32, ptr %gep5, align 4
- br label %exit
-
-case6:
- %gep6 = getelementptr i32, ptr %p6, i64 %index
- %v6 = load i32, ptr %gep6, align 4
- br label %exit
-
-case7:
- %gep7 = getelementptr i32, ptr %p7, i64 %index
- %v7 = load i32, ptr %gep7, align 4
- br label %exit
-
-case8:
- %gep8 = getelementptr i32, ptr %p8, i64 %index
- %v8 = load i32, ptr %gep8, align 4
- br label %exit
-
-case9:
- %gep9 = getelementptr i32, ptr %p9, i64 %index
- %v9 = load i32, ptr %gep9, align 4
- br label %exit
-
-case10:
- %gep10 = getelementptr i32, ptr %p10, i64 %index
- %v10 = load i32, ptr %gep10, align 4
- br label %exit
-
-exit:
- %result = phi i32 [ %v0, %case0 ], [ %v1, %case1 ], [ %v2, %case2 ],
- [ %v3, %case3 ], [ %v4, %case4 ], [ %v5, %case5 ],
- [ %v6, %case6 ], [ %v7, %case7 ], [ %v8, %case8 ],
- [ %v9, %case9 ], [ %v10, %case10 ]
- ret i32 %result
-}
diff --git a/llvm/test/Transforms/SimplifyCFG/X86/sink-common-store-different-pointers.ll b/llvm/test/Transforms/SimplifyCFG/X86/sink-common-store-different-pointers.ll
index 14ff3d351ea40f..fb60c86ec8c1e5 100644
--- a/llvm/test/Transforms/SimplifyCFG/X86/sink-common-store-different-pointers.ll
+++ b/llvm/test/Transforms/SimplifyCFG/X86/sink-common-store-different-pointers.ll
@@ -1,27 +1,19 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
-; RUN: opt < %s -mtriple=x86_64-- -mattr=+avx512f -passes='simplifycfg<sink-common-insts>' -S | FileCheck %s --check-prefixes=MASKED
-; RUN: opt < %s -mtriple=x86_64-- -mattr=+sse2 -passes='simplifycfg<sink-common-insts>' -S | FileCheck %s --check-prefixes=SCALAR
-
-; Stores to different address expressions can only be commoned through a
-; pointer PHI. On a target with masked stores or scatters, keep them in their
-; original blocks so that a vectorizer can widen each one on its own.
-define void @different_store_pointers(i1 %cond, ptr %a, ptr %b, i64 %index, i32 %x, i32 %y) {
-; MASKED-LABEL: define void @different_store_pointers(
+; RUN: opt < %s -mtriple=x86_64-- -mattr=+avx512f -passes='require<scalar-evolution>,simplifycfg<sink-common-insts>' -S | FileCheck %s --check-prefixes=MASKED
+; RUN: opt < %s -mtriple=x86_64-- -mattr=+sse2 -passes='require<scalar-evolution>,simplifycfg<sink-common-insts>' -S | FileCheck %s --check-prefixes=SCALAR
+
+; Stores outside a loop are commoned because they are not consecutive accesses.
+define void @non_loop_store_pointers(i1 %cond, ptr %a, ptr %b, i64 %index,
+; MASKED-LABEL: define void @non_loop_store_pointers(
; MASKED-SAME: i1 [[COND:%.*]], ptr [[A:%.*]], ptr [[B:%.*]], i64 [[INDEX:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]]) #[[ATTR0:[0-9]+]] {
; MASKED-NEXT: [[ENTRY:.*:]]
-; MASKED-NEXT: br i1 [[COND]], label %[[IF_THEN:.*]], label %[[IF_ELSE:.*]]
-; MASKED: [[IF_THEN]]:
-; MASKED-NEXT: [[A_GEP:%.*]] = getelementptr i32, ptr [[A]], i64 [[INDEX]]
-; MASKED-NEXT: store i32 [[X]], ptr [[A_GEP]], align 4
-; MASKED-NEXT: br label %[[EXIT:.*]]
-; MASKED: [[IF_ELSE]]:
-; MASKED-NEXT: [[B_GEP:%.*]] = getelementptr i32, ptr [[B]], i64 [[INDEX]]
-; MASKED-NEXT: store i32 [[Y]], ptr [[B_GEP]], align 4
-; MASKED-NEXT: br label %[[EXIT]]
-; MASKED: [[EXIT]]:
+; MASKED-NEXT: [[A_B:%.*]] = select i1 [[COND]], ptr [[A]], ptr [[B]]
+; MASKED-NEXT: [[X_Y:%.*]] = select i1 [[COND]], i32 [[X]], i32 [[Y]]
+; MASKED-NEXT: [[B_GEP:%.*]] = getelementptr i32, ptr [[A_B]], i64 [[INDEX]]
+; MASKED-NEXT: store i32 [[X_Y]], ptr [[B_GEP]], align 4
; MASKED-NEXT: ret void
;
-; SCALAR-LABEL: define void @different_store_pointers(
+; SCALAR-LABEL: define void @non_loop_store_pointers(
; SCALAR-SAME: i1 [[COND:%.*]], ptr [[A:%.*]], ptr [[B:%.*]], i64 [[INDEX:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]]) #[[ATTR0:[0-9]+]] {
; SCALAR-NEXT: [[ENTRY:.*:]]
; SCALAR-NEXT: [[A_B:%.*]] = select i1 [[COND]], ptr [[A]], ptr [[B]]
@@ -30,122 +22,269 @@ define void @different_store_pointers(i1 %cond, ptr %a, ptr %b, i64 %index, i32
; SCALAR-NEXT: store i32 [[X_Y]], ptr [[B_GEP]], align 4
; SCALAR-NEXT: ret void
;
+ i32 %x, i32 %y) {
entry:
br i1 %cond, label %if.then, label %if.else
if.then:
%a.gep = getelementptr i32, ptr %a, i64 %index
- store i32 %x, ptr %a.gep
+ store i32 %x, ptr %a.gep, align 4
br label %exit
if.else:
%b.gep = getelementptr i32, ptr %b, i64 %index
- store i32 %y, ptr %b.gep
+ store i32 %y, ptr %b.gep, align 4
br label %exit
exit:
ret void
}
-; A pointer PHI is not needed when both stores write to the same address, so
-; these are commoned on any target.
-define void @same_store_pointer(i1 %cond, ptr %a, i32 %x, i32 %y) {
-; MASKED-LABEL: define void @same_store_pointer(
-; MASKED-SAME: i1 [[COND:%.*]], ptr [[A:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]]) #[[ATTR0]] {
-; MASKED-NEXT: [[ENTRY:.*:]]
-; MASKED-NEXT: [[X_Y:%.*]] = select i1 [[COND]], i32 [[X]], i32 [[Y]]
-; MASKED-NEXT: store i32 [[X_Y]], ptr [[A]], align 4
+; Keep consecutive stores separate when masked stores are legal.
+define void @consecutive_store_pointers(i1 %cond, ptr %a, ptr %b, i32 %x,
+; MASKED-LABEL: define void @consecutive_store_pointers(
+; MASKED-SAME: i1 [[COND:%.*]], ptr [[A:%.*]], ptr [[B:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]]) #[[ATTR0]] {
+; MASKED-NEXT: [[ENTRY:.*]]:
+; MASKED-NEXT: br label %[[LOOP:.*]]
+; MASKED: [[LOOP]]:
+; MASKED-NEXT: [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[LATCH:.*]] ]
+; MASKED-NEXT: br i1 [[COND]], label %[[IF_THEN:.*]], label %[[IF_ELSE:.*]]
+; MASKED: [[IF_THEN]]:
+; MASKED-NEXT: [[A_GEP:%.*]] = getelementptr i32, ptr [[A]], i64 [[I]]
+; MASKED-NEXT: store i32 [[X]], ptr [[A_GEP]], align 4
+; MASKED-NEXT: br label %[[LATCH]]
+; MASKED: [[IF_ELSE]]:
+; MASKED-NEXT: [[B_GEP:%.*]] = getelementptr i32, ptr [[B]], i64 [[I]]
+; MASKED-NEXT: store i32 [[Y]], ptr [[B_GEP]], align 4
+; MASKED-NEXT: br label %[[LATCH]]
+; MASKED: [[LATCH]]:
+; MASKED-NEXT: [[INC]] = add nuw nsw i64 [[I]], 1
+; MASKED-NEXT: [[DONE:%.*]] = icmp eq i64 [[INC]], 1024
+; MASKED-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; MASKED: [[EXIT]]:
; MASKED-NEXT: ret void
;
-; SCALAR-LABEL: define void @same_store_pointer(
-; SCALAR-SAME: i1 [[COND:%.*]], ptr [[A:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]]) #[[ATTR0]] {
-; SCALAR-NEXT: [[ENTRY:.*:]]
+; SCALAR-LABEL: define void @consecutive_store_pointers(
+; SCALAR-SAME: i1 [[COND:%.*]], ptr [[A:%.*]], ptr [[B:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]]) #[[ATTR0]] {
+; SCALAR-NEXT: [[ENTRY:.*]]:
+; SCALAR-NEXT: br label %[[LOOP:.*]]
+; SCALAR: [[LOOP]]:
+; SCALAR-NEXT: [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[LOOP]] ]
+; SCALAR-NEXT: [[A_B:%.*]] = select i1 [[COND]], ptr [[A]], ptr [[B]]
; SCALAR-NEXT: [[X_Y:%.*]] = select i1 [[COND]], i32 [[X]], i32 [[Y]]
-; SCALAR-NEXT: store i32 [[X_Y]], ptr [[A]], align 4
+; SCALAR-NEXT: [[B_GEP:%.*]] = getelementptr i32, ptr [[A_B]], i64 [[I]]
+; SCALAR-NEXT: store i32 [[X_Y]], ptr [[B_GEP]], align 4
+; SCALAR-NEXT: [[INC]] = add nuw nsw i64 [[I]], 1
+; SCALAR-NEXT: [[DONE:%.*]] = icmp eq i64 [[INC]], 1024
+; SCALAR-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; SCALAR: [[EXIT]]:
; SCALAR-NEXT: ret void
;
+ i32 %y) {
entry:
+ br label %loop
+
+loop:
+ %i = phi i64 [ 0, %entry ], [ %inc, %latch ]
br i1 %cond, label %if.then, label %if.else
if.then:
- store i32 %x, ptr %a
- br label %exit
+ %a.gep = getelementptr i32, ptr %a, i64 %i
+ store i32 %x, ptr %a.gep, align 4
+ br label %latch
if.else:
- store i32 %y, ptr %a
- br label %exit
+ %b.gep = getelementptr i32, ptr %b, i64 %i
+ store i32 %y, ptr %b.gep, align 4
+ br label %latch
+
+latch:
+ %inc = add nuw nsw i64 %i, 1
+ %done = icmp eq i64 %inc, 1024
+ br i1 %done, label %exit, label %loop
exit:
ret void
}
-; Same, for distinct instructions computing the same address.
-define void @same_store_address_expression(i1 %cond, ptr %a, i64 %index, i32 %x, i32 %y) {
-; MASKED-LABEL: define void @same_store_address_expression(
-; MASKED-SAME: i1 [[COND:%.*]], ptr [[A:%.*]], i64 [[INDEX:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]]) #[[ATTR0]] {
-; MASKED-NEXT: [[ENTRY:.*:]]
+; Stores with a non-unit stride are commoned.
+define void @strided_store_pointers(i1 %cond, ptr %a, ptr %b, i32 %x,
+; MASKED-LABEL: define void @strided_store_pointers(
+; MASKED-SAME: i1 [[COND:%.*]], ptr [[A:%.*]], ptr [[B:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]]) #[[ATTR0]] {
+; MASKED-NEXT: [[ENTRY:.*]]:
+; MASKED-NEXT: br label %[[LOOP:.*]]
+; MASKED: [[LOOP]]:
+; MASKED-NEXT: [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[LOOP]] ]
+; MASKED-NEXT: [[INDEX:%.*]] = shl nuw nsw i64 [[I]], 1
+; MASKED-NEXT: [[A_B:%.*]] = select i1 [[COND]], ptr [[A]], ptr [[B]]
; MASKED-NEXT: [[X_Y:%.*]] = select i1 [[COND]], i32 [[X]], i32 [[Y]]
-; MASKED-NEXT: [[B_GEP:%.*]] = getelementptr i32, ptr [[A]], i64 [[INDEX]]
+; MASKED-NEXT: [[B_GEP:%.*]] = getelementptr i32, ptr [[A_B]], i64 [[INDEX]]
; MASKED-NEXT: store i32 [[X_Y]], ptr [[B_GEP]], align 4
+; MASKED-NEXT: [[INC]] = add nuw nsw i64 [[I]], 1
+; MASKED-NEXT: [[DONE:%.*]] = icmp eq i64 [[INC]], 1024
+; MASKED-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; MASKED: [[EXIT]]:
; MASKED-NEXT: ret void
;
-; SCALAR-LABEL: define void @same_store_address_expression(
-; SCALAR-SAME: i1 [[COND:%.*]], ptr [[A:%.*]], i64 [[INDEX:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]]) #[[ATTR0]] {
-; SCALAR-NEXT: [[ENTRY:.*:]]
+; SCALAR-LABEL: define void @strided_store_pointers(
+; SCALAR-SAME: i1 [[COND:%.*]], ptr [[A:%.*]], ptr [[B:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]]) #[[ATTR0]] {
+; SCALAR-NEXT: [[ENTRY:.*]]:
+; SCALAR-NEXT: br label %[[LOOP:.*]]
+; SCALAR: [[LOOP]]:
+; SCALAR-NEXT: [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[LOOP]] ]
+; SCALAR-NEXT: [[INDEX:%.*]] = shl nuw nsw i64 [[I]], 1
+; SCALAR-NEXT: [[A_B:%.*]] = select i1 [[COND]], ptr [[A]], ptr [[B]]
; SCALAR-NEXT: [[X_Y:%.*]] = select i1 [[COND]], i32 [[X]], i32 [[Y]]
-; SCALAR-NEXT: [[B_GEP:%.*]] = getelementptr i32, ptr [[A]], i64 [[INDEX]]
+; SCALAR-NEXT: [[B_GEP:%.*]] = getelementptr i32, ptr [[A_B]], i64 [[INDEX]]
; SCALAR-NEXT: store i32 [[X_Y]], ptr [[B_GEP]], align 4
+; SCALAR-NEXT: [[INC]] = add nuw nsw i64 [[I]], 1
+; SCALAR-NEXT: [[DONE:%.*]] = icmp eq i64 [[INC]], 1024
+; SCALAR-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; SCALAR: [[EXIT]]:
; SCALAR-NEXT: ret void
;
+ i32 %y) {
entry:
+ br label %loop
+
+loop:
+ %i = phi i64 [ 0, %entry ], [ %inc, %latch ]
+ %index = shl nuw nsw i64 %i, 1
br i1 %cond, label %if.then, label %if.else
if.then:
%a.gep = getelementptr i32, ptr %a, i64 %index
- store i32 %x, ptr %a.gep
- br label %exit
+ store i32 %x, ptr %a.gep, align 4
+ br label %latch
if.else:
- %b.gep = getelementptr i32, ptr %a, i64 %index
- store i32 %y, ptr %b.gep
- br label %exit
+ %b.gep = getelementptr i32, ptr %b, i64 %index
+ store i32 %y, ptr %b.gep, align 4
+ br label %latch
+
+latch:
+ %inc = add nuw nsw i64 %i, 1
+ %done = icmp eq i64 %inc, 1024
+ br i1 %done, label %exit, label %loop
exit:
ret void
}
-; Types that the target cannot mask are still commoned.
-define void @different_store_pointers_unsupported_type(i1 %cond, ptr %a, ptr %b, i64 %index, i8 %x, i8 %y) {
-; MASKED-LABEL: define void @different_store_pointers_unsupported_type(
-; MASKED-SAME: i1 [[COND:%.*]], ptr [[A:%.*]], ptr [[B:%.*]], i64 [[INDEX:%.*]], i8 [[X:%.*]], i8 [[Y:%.*]]) #[[ATTR0]] {
-; MASKED-NEXT: [[ENTRY:.*:]]
+; Stores to identical addresses are commoned.
+define void @same_store_address(i1 %cond, ptr %a, i32 %x, i32 %y) {
+; MASKED-LABEL: define void @same_store_address(
+; MASKED-SAME: i1 [[COND:%.*]], ptr [[A:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]]) #[[ATTR0]] {
+; MASKED-NEXT: [[ENTRY:.*]]:
+; MASKED-NEXT: br label %[[LOOP:.*]]
+; MASKED: [[LOOP]]:
+; MASKED-NEXT: [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[LOOP]] ]
+; MASKED-NEXT: [[X_Y:%.*]] = select i1 [[COND]], i32 [[X]], i32 [[Y]]
+; MASKED-NEXT: [[B_GEP:%.*]] = getelementptr i32, ptr [[A]], i64 [[I]]
+; MASKED-NEXT: store i32 [[X_Y]], ptr [[B_GEP]], align 4
+; MASKED-NEXT: [[INC]] = add nuw nsw i64 [[I]], 1
+; MASKED-NEXT: [[DONE:%.*]] = icmp eq i64 [[INC]], 1024
+; MASKED-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; MASKED: [[EXIT]]:
+; MASKED-NEXT: ret void
+;
+; SCALAR-LABEL: define void @same_store_address(
+; SCALAR-SAME: i1 [[COND:%.*]], ptr [[A:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]]) #[[ATTR0]] {
+; SCALAR-NEXT: [[ENTRY:.*]]:
+; SCALAR-NEXT: br label %[[LOOP:.*]]
+; SCALAR: [[LOOP]]:
+; SCALAR-NEXT: [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[LOOP]] ]
+; SCALAR-NEXT: [[X_Y:%.*]] = select i1 [[COND]], i32 [[X]], i32 [[Y]]
+; SCALAR-NEXT: [[B_GEP:%.*]] = getelementptr i32, ptr [[A]], i64 [[I]]
+; SCALAR-NEXT: store i32 [[X_Y]], ptr [[B_GEP]], align 4
+; SCALAR-NEXT: [[INC]] = add nuw nsw i64 [[I]], 1
+; SCALAR-NEXT: [[DONE:%.*]] = icmp eq i64 [[INC]], 1024
+; SCALAR-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; SCALAR: [[EXIT]]:
+; SCALAR-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i64 [ 0, %entry ], [ %inc, %latch ]
+ br i1 %cond, label %if.then, label %if.else
+
+if.then:
+ %a.gep = getelementptr i32, ptr %a, i64 %i
+ store i32 %x, ptr %a.gep, align 4
+ br label %latch
+
+if.else:
+ %b.gep = getelementptr i32, ptr %a, i64 %i
+ store i32 %y, ptr %b.gep, align 4
+ br label %latch
+
+latch:
+ %inc = add nuw nsw i64 %i, 1
+ %done = icmp eq i64 %inc, 1024
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Consecutive types without masked-store support are commoned.
+define void @consecutive_store_pointers_unsupported_type(
+; MASKED-LABEL: define void @consecutive_store_pointers_unsupported_type(
+; MASKED-SAME: i1 [[COND:%.*]], ptr [[A:%.*]], ptr [[B:%.*]], i8 [[X:%.*]], i8 [[Y:%.*]]) #[[ATTR0]] {
+; MASKED-NEXT: [[ENTRY:.*]]:
+; MASKED-NEXT: br label %[[LOOP:.*]]
+; MASKED: [[LOOP]]:
+; MASKED-NEXT: [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[LOOP]] ]
; MASKED-NEXT: [[A_B:%.*]] = select i1 [[COND]], ptr [[A]], ptr [[B]]
; MASKED-NEXT: [[X_Y:%.*]] = select i1 [[COND]], i8 [[X]], i8 [[Y]]
-; MASKED-NEXT: [[B_GEP:%.*]] = getelementptr i8, ptr [[A_B]], i64 [[INDEX]]
+; MASKED-NEXT: [[B_GEP:%.*]] = getelementptr i8, ptr [[A_B]], i64 [[I]]
; MASKED-NEXT: store i8 [[X_Y]], ptr [[B_GEP]], align 1
+; MASKED-NEXT: [[INC]] = add nuw nsw i64 [[I]], 1
+; MASKED-NEXT: [[DONE:%.*]] = icmp eq i64 [[INC]], 1024
+; MASKED-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; MASKED: [[EXIT]]:
; MASKED-NEXT: ret void
;
-; SCALAR-LABEL: define void @different_store_pointers_unsupported_type(
-; SCALAR-SAME: i1 [[COND:%.*]], ptr [[A:%.*]], ptr [[B:%.*]], i64 [[INDEX:%.*]], i8 [[X:%.*]], i8 [[Y:%.*]]) #[[ATTR0]] {
-; SCALAR-NEXT: [[ENTRY:.*:]]
+; SCALAR-LABEL: define void @consecutive_store_pointers_unsupported_type(
+; SCALAR-SAME: i1 [[COND:%.*]], ptr [[A:%.*]], ptr [[B:%.*]], i8 [[X:%.*]], i8 [[Y:%.*]]) #[[ATTR0]] {
+; SCALAR-NEXT: [[ENTRY:.*]]:
+; SCALAR-NEXT: br label %[[LOOP:.*]]
+; SCALAR: [[LOOP]]:
+; SCALAR-NEXT: [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[LOOP]] ]
; SCALAR-NEXT: [[A_B:%.*]] = select i1 [[COND]], ptr [[A]], ptr [[B]]
; SCALAR-NEXT: [[X_Y:%.*]] = select i1 [[COND]], i8 [[X]], i8 [[Y]]
-; SCALAR-NEXT: [[B_GEP:%.*]] = getelementptr i8, ptr [[A_B]], i64 [[INDEX]]
+; SCALAR-NEXT: [[B_GEP:%.*]] = getelementptr i8, ptr [[A_B]], i64 [[I]]
; SCALAR-NEXT: store i8 [[X_Y]], ptr [[B_GEP]], align 1
+; SCALAR-NEXT: [[INC]] = add nuw nsw i64 [[I]], 1
+; SCALAR-NEXT: [[DONE:%.*]] = icmp eq i64 [[INC]], 1024
+; SCALAR-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; SCALAR: [[EXIT]]:
; SCALAR-NEXT: ret void
;
+ i1 %cond, ptr %a, ptr %b, i8 %x, i8 %y) {
entry:
+ br label %loop
+
+loop:
+ %i = phi i64 [ 0, %entry ], [ %inc, %latch ]
br i1 %cond, label %if.then, label %if.else
if.then:
- %a.gep = getelementptr i8, ptr %a, i64 %index
- store i8 %x, ptr %a.gep
- br label %exit
+ %a.gep = getelementptr i8, ptr %a, i64 %i
+ store i8 %x, ptr %a.gep, align 1
+ br label %latch
if.else:
- %b.gep = getelementptr i8, ptr %b, i64 %index
- store i8 %y, ptr %b.gep
- br label %exit
+ %b.gep = getelementptr i8, ptr %b, i64 %i
+ store i8 %y, ptr %b.gep, align 1
+ br label %latch
+
+latch:
+ %inc = add nuw nsw i64 %i, 1
+ %done = icmp eq i64 %inc, 1024
+ br i1 %done, label %exit, label %loop
exit:
ret void
diff --git a/llvm/test/Transforms/SimplifyCFG/X86/sink-common-store-scatter-cost.ll b/llvm/test/Transforms/SimplifyCFG/X86/sink-common-store-scatter-cost.ll
deleted file mode 100644
index b9dbe6a93ec59d..00000000000000
--- a/llvm/test/Transforms/SimplifyCFG/X86/sink-common-store-scatter-cost.ll
+++ /dev/null
@@ -1,190 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
-; RUN: opt < %s -mtriple=x86_64-- -mattr=+avx512f -passes='simplifycfg<sink-common-insts>' -S | FileCheck %s --check-prefixes=VF16
-; RUN: opt < %s -mtriple=x86_64-- -mattr=+avx512f,+prefer-256-bit -passes='simplifycfg<sink-common-insts>' -S | FileCheck %s --check-prefixes=VF8
-
-; Whether commoning these stores is worthwhile is decided by cost, not by
-; legality: masked stores and scatters are both available here, but a scatter
-; only pays off once it replaces enough separate masked stores. The wider the
-; vector, the more stores a single scatter has to absorb to break even.
-
-define void @many_store_pointers(i32 %sel, i64 %index, i32 %v, ptr %p0, ptr %p1, ptr %p2, ptr %p3, ptr %p4, ptr %p5, ptr %p6, ptr %p7, ptr %p8, ptr %p9, ptr %p10) {
-; VF16-LABEL: define void @many_store_pointers(
-; VF16-SAME: i32 [[SEL:%.*]], i64 [[INDEX:%.*]], i32 [[V:%.*]], ptr [[P0:%.*]], ptr [[P1:%.*]], ptr [[P2:%.*]], ptr [[P3:%.*]], ptr [[P4:%.*]], ptr [[P5:%.*]], ptr [[P6:%.*]], ptr [[P7:%.*]], ptr [[P8:%.*]], ptr [[P9:%.*]], ptr [[P10:%.*]]) #[[ATTR0:[0-9]+]] {
-; VF16-NEXT: [[ENTRY:.*:]]
-; VF16-NEXT: switch i32 [[SEL]], label %[[EXIT:.*]] [
-; VF16-NEXT: i32 0, label %[[CASE0:.*]]
-; VF16-NEXT: i32 1, label %[[CASE1:.*]]
-; VF16-NEXT: i32 2, label %[[CASE2:.*]]
-; VF16-NEXT: i32 3, label %[[CASE3:.*]]
-; VF16-NEXT: i32 4, label %[[CASE4:.*]]
-; VF16-NEXT: i32 5, label %[[CASE5:.*]]
-; VF16-NEXT: i32 6, label %[[CASE6:.*]]
-; VF16-NEXT: i32 7, label %[[CASE7:.*]]
-; VF16-NEXT: i32 8, label %[[CASE8:.*]]
-; VF16-NEXT: i32 9, label %[[CASE9:.*]]
-; VF16-NEXT: i32 10, label %[[CASE10:.*]]
-; VF16-NEXT: ]
-; VF16: [[CASE0]]:
-; VF16-NEXT: [[GEP0:%.*]] = getelementptr i32, ptr [[P0]], i64 [[INDEX]]
-; VF16-NEXT: store i32 [[V]], ptr [[GEP0]], align 4
-; VF16-NEXT: br label %[[EXIT]]
-; VF16: [[CASE1]]:
-; VF16-NEXT: [[GEP1:%.*]] = getelementptr i32, ptr [[P1]], i64 [[INDEX]]
-; VF16-NEXT: store i32 [[V]], ptr [[GEP1]], align 4
-; VF16-NEXT: br label %[[EXIT]]
-; VF16: [[CASE2]]:
-; VF16-NEXT: [[GEP2:%.*]] = getelementptr i32, ptr [[P2]], i64 [[INDEX]]
-; VF16-NEXT: store i32 [[V]], ptr [[GEP2]], align 4
-; VF16-NEXT: br label %[[EXIT]]
-; VF16: [[CASE3]]:
-; VF16-NEXT: [[GEP3:%.*]] = getelementptr i32, ptr [[P3]], i64 [[INDEX]]
-; VF16-NEXT: store i32 [[V]], ptr [[GEP3]], align 4
-; VF16-NEXT: br label %[[EXIT]]
-; VF16: [[CASE4]]:
-; VF16-NEXT: [[GEP4:%.*]] = getelementptr i32, ptr [[P4]], i64 [[INDEX]]
-; VF16-NEXT: store i32 [[V]], ptr [[GEP4]], align 4
-; VF16-NEXT: br label %[[EXIT]]
-; VF16: [[CASE5]]:
-; VF16-NEXT: [[GEP5:%.*]] = getelementptr i32, ptr [[P5]], i64 [[INDEX]]
-; VF16-NEXT: store i32 [[V]], ptr [[GEP5]], align 4
-; VF16-NEXT: br label %[[EXIT]]
-; VF16: [[CASE6]]:
-; VF16-NEXT: [[GEP6:%.*]] = getelementptr i32, ptr [[P6]], i64 [[INDEX]]
-; VF16-NEXT: store i32 [[V]], ptr [[GEP6]], align 4
-; VF16-NEXT: br label %[[EXIT]]
-; VF16: [[CASE7]]:
-; VF16-NEXT: [[GEP7:%.*]] = getelementptr i32, ptr [[P7]], i64 [[INDEX]]
-; VF16-NEXT: store i32 [[V]], ptr [[GEP7]], align 4
-; VF16-NEXT: br label %[[EXIT]]
-; VF16: [[CASE8]]:
-; VF16-NEXT: [[GEP8:%.*]] = getelementptr i32, ptr [[P8]], i64 [[INDEX]]
-; VF16-NEXT: store i32 [[V]], ptr [[GEP8]], align 4
-; VF16-NEXT: br label %[[EXIT]]
-; VF16: [[CASE9]]:
-; VF16-NEXT: [[GEP9:%.*]] = getelementptr i32, ptr [[P9]], i64 [[INDEX]]
-; VF16-NEXT: store i32 [[V]], ptr [[GEP9]], align 4
-; VF16-NEXT: br label %[[EXIT]]
-; VF16: [[CASE10]]:
-; VF16-NEXT: [[GEP10:%.*]] = getelementptr i32, ptr [[P10]], i64 [[INDEX]]
-; VF16-NEXT: store i32 [[V]], ptr [[GEP10]], align 4
-; VF16-NEXT: br label %[[EXIT]]
-; VF16: [[EXIT]]:
-; VF16-NEXT: ret void
-;
-; VF8-LABEL: define void @many_store_pointers(
-; VF8-SAME: i32 [[SEL:%.*]], i64 [[INDEX:%.*]], i32 [[V:%.*]], ptr [[P0:%.*]], ptr [[P1:%.*]], ptr [[P2:%.*]], ptr [[P3:%.*]], ptr [[P4:%.*]], ptr [[P5:%.*]], ptr [[P6:%.*]], ptr [[P7:%.*]], ptr [[P8:%.*]], ptr [[P9:%.*]], ptr [[P10:%.*]]) #[[ATTR0:[0-9]+]] {
-; VF8-NEXT: [[ENTRY:.*]]:
-; VF8-NEXT: switch i32 [[SEL]], label %[[EXIT:.*]] [
-; VF8-NEXT: i32 0, label %[[EXIT_SINK_SPLIT:.*]]
-; VF8-NEXT: i32 1, label %[[CASE1:.*]]
-; VF8-NEXT: i32 2, label %[[CASE2:.*]]
-; VF8-NEXT: i32 3, label %[[CASE3:.*]]
-; VF8-NEXT: i32 4, label %[[CASE4:.*]]
-; VF8-NEXT: i32 5, label %[[CASE5:.*]]
-; VF8-NEXT: i32 6, label %[[CASE6:.*]]
-; VF8-NEXT: i32 7, label %[[CASE7:.*]]
-; VF8-NEXT: i32 8, label %[[CASE8:.*]]
-; VF8-NEXT: i32 9, label %[[CASE9:.*]]
-; VF8-NEXT: i32 10, label %[[CASE10:.*]]
-; VF8-NEXT: ]
-; VF8: [[CASE1]]:
-; VF8-NEXT: br label %[[EXIT_SINK_SPLIT]]
-; VF8: [[CASE2]]:
-; VF8-NEXT: br label %[[EXIT_SINK_SPLIT]]
-; VF8: [[CASE3]]:
-; VF8-NEXT: br label %[[EXIT_SINK_SPLIT]]
-; VF8: [[CASE4]]:
-; VF8-NEXT: br label %[[EXIT_SINK_SPLIT]]
-; VF8: [[CASE5]]:
-; VF8-NEXT: br label %[[EXIT_SINK_SPLIT]]
-; VF8: [[CASE6]]:
-; VF8-NEXT: br label %[[EXIT_SINK_SPLIT]]
-; VF8: [[CASE7]]:
-; VF8-NEXT: br label %[[EXIT_SINK_SPLIT]]
-; VF8: [[CASE8]]:
-; VF8-NEXT: br label %[[EXIT_SINK_SPLIT]]
-; VF8: [[CASE9]]:
-; VF8-NEXT: br label %[[EXIT_SINK_SPLIT]]
-; VF8: [[CASE10]]:
-; VF8-NEXT: br label %[[EXIT_SINK_SPLIT]]
-; VF8: [[EXIT_SINK_SPLIT]]:
-; VF8-NEXT: [[P10_SINK:%.*]] = phi ptr [ [[P10]], %[[CASE10]] ], [ [[P9]], %[[CASE9]] ], [ [[P8]], %[[CASE8]] ], [ [[P7]], %[[CASE7]] ], [ [[P6]], %[[CASE6]] ], [ [[P5]], %[[CASE5]] ], [ [[P4]], %[[CASE4]] ], [ [[P3]], %[[CASE3]] ], [ [[P2]], %[[CASE2]] ], [ [[P1]], %[[CASE1]] ], [ [[P0]], %[[ENTRY]] ]
-; VF8-NEXT: [[GEP10:%.*]] = getelementptr i32, ptr [[P10_SINK]], i64 [[INDEX]]
-; VF8-NEXT: store i32 [[V]], ptr [[GEP10]], align 4
-; VF8-NEXT: br label %[[EXIT]]
-; VF8: [[EXIT]]:
-; VF8-NEXT: ret void
-;
-entry:
- switch i32 %sel, label %exit [
- i32 0, label %case0
- i32 1, label %case1
- i32 2, label %case2
- i32 3, label %case3
- i32 4, label %case4
- i32 5, label %case5
- i32 6, label %case6
- i32 7, label %case7
- i32 8, label %case8
- i32 9, label %case9
- i32 10, label %case10
- ]
-
-case0:
- %gep0 = getelementptr i32, ptr %p0, i64 %index
- store i32 %v, ptr %gep0
- br label %exit
-
-case1:
- %gep1 = getelementptr i32, ptr %p1, i64 %index
- store i32 %v, ptr %gep1
- br label %exit
-
-case2:
- %gep2 = getelementptr i32, ptr %p2, i64 %index
- store i32 %v, ptr %gep2
- br label %exit
-
-case3:
- %gep3 = getelementptr i32, ptr %p3, i64 %index
- store i32 %v, ptr %gep3
- br label %exit
-
-case4:
- %gep4 = getelementptr i32, ptr %p4, i64 %index
- store i32 %v, ptr %gep4
- br label %exit
-
-case5:
- %gep5 = getelementptr i32, ptr %p5, i64 %index
- store i32 %v, ptr %gep5
- br label %exit
-
-case6:
- %gep6 = getelementptr i32, ptr %p6, i64 %index
- store i32 %v, ptr %gep6
- br label %exit
-
-case7:
- %gep7 = getelementptr i32, ptr %p7, i64 %index
- store i32 %v, ptr %gep7
- br label %exit
-
-case8:
- %gep8 = getelementptr i32, ptr %p8, i64 %index
- store i32 %v, ptr %gep8
- br label %exit
-
-case9:
- %gep9 = getelementptr i32, ptr %p9, i64 %index
- store i32 %v, ptr %gep9
- br label %exit
-
-case10:
- %gep10 = getelementptr i32, ptr %p10, i64 %index
- store i32 %v, ptr %gep10
- br label %exit
-
-exit:
- ret void
-}
More information about the llvm-commits
mailing list