[llvm] [VPlan] Generalize noalias-licm-check to replicate regions (NFC) (PR #187017)
Ramkumar Ramachandra via llvm-commits
llvm-commits at lists.llvm.org
Fri Mar 27 01:23:24 PDT 2026
https://github.com/artagnon updated https://github.com/llvm/llvm-project/pull/187017
>From 455c83d81021dfc1086042f826dfbb5d5510b68f Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Tue, 17 Mar 2026 12:05:38 +0000
Subject: [PATCH 1/5] [VPlan] Generalize noalias-licm-check to replicate
regions (NFC)
In order to use the cannotHoistOrSinkWithNoAlias check in use-sites
after replicate regions are created, generalize it to work to with
replicate regions, using a variant of VPBlockUtils::blocksOnly.
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 46 +++++++++----------
llvm/lib/Transforms/Vectorize/VPlanUtils.h | 15 ++++++
2 files changed, 37 insertions(+), 24 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 7e8496a568643..f84a0e2df837a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -188,24 +188,24 @@ class SinkStoreInfo {
};
/// Check if a memory operation doesn't alias with memory operations in blocks
-/// between \p FirstBB and \p LastBB using scoped noalias metadata. If
-/// \p SinkInfo is std::nullopt, only recipes that may write to memory are
-/// checked (for load hoisting). Otherwise recipes that both read and write
-/// memory are checked, and SCEV is used to prove no-alias between the group
-/// leader and other replicate recipes (for store sinking).
+/// between \p FirstBB and \p LastBB, which is expected to be a valid range in
+/// a shallow-traversal of the vector loop region in \p Plan. We check aliasing
+/// with using scoped noalias metadata. If \p SinkInfo is std::nullopt, only
+/// recipes that may write to memory are checked (for load hoisting). Otherwise
+/// recipes that both read and write memory are checked, and SCEV is used to
+/// prove no-alias between the group leader and other replicate recipes (for
+/// store sinking).
static bool
-canHoistOrSinkWithNoAliasCheck(const MemoryLocation &MemLoc,
+canHoistOrSinkWithNoAliasCheck(const MemoryLocation &MemLoc, VPlan &Plan,
VPBasicBlock *FirstBB, VPBasicBlock *LastBB,
std::optional<SinkStoreInfo> SinkInfo = {}) {
bool CheckReads = SinkInfo.has_value();
if (!MemLoc.AATags.Scope)
return false;
- for (VPBlockBase *Block = FirstBB; Block;
- Block = Block->getSingleSuccessor()) {
- assert(Block->getNumSuccessors() <= 1 &&
- "Expected at most one successor in block chain");
- auto *VPBB = cast<VPBasicBlock>(Block);
+ for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
+ vp_depth_first_shallow(Plan.getVectorLoopRegion()->getEntry()),
+ FirstBB, LastBB)) {
for (VPRecipeBase &R : *VPBB) {
if (SinkInfo && SinkInfo->shouldSkip(R))
continue;
@@ -223,14 +223,12 @@ canHoistOrSinkWithNoAliasCheck(const MemoryLocation &MemLoc,
if (ScopedNoAliasAAResult::alias(*Loc, MemLoc) != AliasResult::NoAlias)
return false;
}
-
- if (Block == LastBB)
- break;
}
return true;
}
-/// Collect either replicated Loads or Stores grouped by their address SCEV.
+/// Collect either replicated Loads or Stores grouped by their address SCEV, in
+/// a shallow-traversal of the vector loop region in \p Plan.
template <unsigned Opcode>
static SmallVector<SmallVector<VPReplicateRecipe *, 4>>
collectGroupedReplicateMemOps(
@@ -241,9 +239,8 @@ collectGroupedReplicateMemOps(
constexpr bool IsLoad = (Opcode == Instruction::Load);
SmallDenseMap<const SCEV *, SmallVector<VPReplicateRecipe *, 4>>
RecipesByAddress;
- for (VPBlockBase *Block :
- vp_depth_first_shallow(Plan.getVectorLoopRegion()->getEntry())) {
- auto *VPBB = cast<VPBasicBlock>(Block);
+ for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
+ vp_depth_first_shallow(Plan.getVectorLoopRegion()->getEntry()))) {
for (VPRecipeBase &R : *VPBB) {
auto *RepR = dyn_cast<VPReplicateRecipe>(&R);
if (!RepR || RepR->getOpcode() != Opcode || !FilterFn(RepR))
@@ -4841,7 +4838,8 @@ void VPlanTransforms::hoistPredicatedLoads(VPlan &Plan,
// Check that the load doesn't alias with stores between first and last.
auto LoadLoc = vputils::getMemoryLocation(*EarliestLoad);
- if (!LoadLoc || !canHoistOrSinkWithNoAliasCheck(*LoadLoc, FirstBB, LastBB))
+ if (!LoadLoc ||
+ !canHoistOrSinkWithNoAliasCheck(*LoadLoc, Plan, FirstBB, LastBB))
continue;
// Collect common metadata from all loads in the group.
@@ -4876,7 +4874,7 @@ void VPlanTransforms::hoistPredicatedLoads(VPlan &Plan,
static bool
canSinkStoreWithNoAliasCheck(ArrayRef<VPReplicateRecipe *> StoresToSink,
PredicatedScalarEvolution &PSE, const Loop &L,
- VPTypeAnalysis &TypeInfo) {
+ VPlan &Plan) {
auto StoreLoc = vputils::getMemoryLocation(*StoresToSink.front());
if (!StoreLoc || !StoreLoc->AATags.Scope)
return false;
@@ -4888,8 +4886,10 @@ canSinkStoreWithNoAliasCheck(ArrayRef<VPReplicateRecipe *> StoresToSink,
VPBasicBlock *FirstBB = StoresToSink.front()->getParent();
VPBasicBlock *LastBB = StoresToSink.back()->getParent();
+ VPTypeAnalysis TypeInfo(Plan);
SinkStoreInfo SinkInfo(StoresToSinkSet, *StoresToSink[0], PSE, L, TypeInfo);
- return canHoistOrSinkWithNoAliasCheck(*StoreLoc, FirstBB, LastBB, SinkInfo);
+ return canHoistOrSinkWithNoAliasCheck(*StoreLoc, Plan, FirstBB, LastBB,
+ SinkInfo);
}
void VPlanTransforms::sinkPredicatedStores(VPlan &Plan,
@@ -4900,10 +4900,8 @@ void VPlanTransforms::sinkPredicatedStores(VPlan &Plan,
if (Groups.empty())
return;
- VPTypeAnalysis TypeInfo(Plan);
-
for (auto &Group : Groups) {
- if (!canSinkStoreWithNoAliasCheck(Group, PSE, *L, TypeInfo))
+ if (!canSinkStoreWithNoAliasCheck(Group, PSE, *L, Plan))
continue;
// Use the last (most dominated) store's location for the unconditional
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.h b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
index f4a0f7feb43a8..50a7a1ac75b74 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
@@ -286,6 +286,21 @@ class VPBlockUtils {
});
}
+ /// A variant of blocksOnly that only returns blocks between \p FirstBB and \p
+ /// LastBB.
+ template <typename BlockTy, typename T>
+ static SmallVector<VPBasicBlock *>
+ blocksOnly(const T &Range, VPBasicBlock *FirstBB, VPBasicBlock *LastBB) {
+ auto Blocks = to_vector(blocksOnly<BlockTy, T>(Range));
+ auto *FirstIt = find(Blocks, FirstBB);
+ auto *LastIt = find(Blocks, LastBB);
+ assert(FirstIt != Blocks.end() && LastIt != Blocks.end() &&
+ "FirstBB and LastBB don't correspond to Range");
+ Blocks.erase(Blocks.begin(), FirstIt);
+ Blocks.erase(LastIt, Blocks.end());
+ return Blocks;
+ }
+
/// Inserts \p BlockPtr on the edge between \p From and \p To. That is, update
/// \p From's successor to \p To to point to \p BlockPtr and \p To's
/// predecessor from \p From to \p BlockPtr. \p From and \p To are added to \p
>From fbfe91c26b568cb6e7408d073bf112d60578af49 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Thu, 19 Mar 2026 21:46:42 +0000
Subject: [PATCH 2/5] [VPlan] Improve API to disallow misuse, deep traversal
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 12 ++++-------
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 21 +++++++++++++++++++
llvm/lib/Transforms/Vectorize/VPlanUtils.h | 18 ++++------------
3 files changed, 29 insertions(+), 22 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index f84a0e2df837a..39ff14e83dc6d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -196,16 +196,14 @@ class SinkStoreInfo {
/// prove no-alias between the group leader and other replicate recipes (for
/// store sinking).
static bool
-canHoistOrSinkWithNoAliasCheck(const MemoryLocation &MemLoc, VPlan &Plan,
+canHoistOrSinkWithNoAliasCheck(const MemoryLocation &MemLoc,
VPBasicBlock *FirstBB, VPBasicBlock *LastBB,
std::optional<SinkStoreInfo> SinkInfo = {}) {
bool CheckReads = SinkInfo.has_value();
if (!MemLoc.AATags.Scope)
return false;
- for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
- vp_depth_first_shallow(Plan.getVectorLoopRegion()->getEntry()),
- FirstBB, LastBB)) {
+ for (VPBasicBlock *VPBB : VPBlockUtils::blocksChainBetween(FirstBB, LastBB)) {
for (VPRecipeBase &R : *VPBB) {
if (SinkInfo && SinkInfo->shouldSkip(R))
continue;
@@ -4838,8 +4836,7 @@ void VPlanTransforms::hoistPredicatedLoads(VPlan &Plan,
// Check that the load doesn't alias with stores between first and last.
auto LoadLoc = vputils::getMemoryLocation(*EarliestLoad);
- if (!LoadLoc ||
- !canHoistOrSinkWithNoAliasCheck(*LoadLoc, Plan, FirstBB, LastBB))
+ if (!LoadLoc || !canHoistOrSinkWithNoAliasCheck(*LoadLoc, FirstBB, LastBB))
continue;
// Collect common metadata from all loads in the group.
@@ -4888,8 +4885,7 @@ canSinkStoreWithNoAliasCheck(ArrayRef<VPReplicateRecipe *> StoresToSink,
VPBasicBlock *LastBB = StoresToSink.back()->getParent();
VPTypeAnalysis TypeInfo(Plan);
SinkStoreInfo SinkInfo(StoresToSinkSet, *StoresToSink[0], PSE, L, TypeInfo);
- return canHoistOrSinkWithNoAliasCheck(*StoreLoc, Plan, FirstBB, LastBB,
- SinkInfo);
+ return canHoistOrSinkWithNoAliasCheck(*StoreLoc, FirstBB, LastBB, SinkInfo);
}
void VPlanTransforms::sinkPredicatedStores(VPlan &Plan,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 7921a6fa0411e..b47b1fdee3e12 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -609,6 +609,27 @@ VPSingleDefRecipe *vputils::findHeaderMask(VPlan &Plan) {
return HeaderMask;
}
+SmallVector<VPBasicBlock *>
+VPBlockUtils::blocksChainBetween(VPBasicBlock *FirstBB, VPBasicBlock *LastBB) {
+ assert(FirstBB->getPlan() == LastBB->getPlan() &&
+ "FirstBB and LastBB from different VPlans");
+#ifndef NDEBUG
+ VPDominatorTree VPDT(*FirstBB->getPlan());
+#endif
+ assert(VPDT.properlyDominates(FirstBB, LastBB) &&
+ "Expected FirstBB to dominate LastBB");
+ VPlan &Plan = *FirstBB->getPlan();
+ auto Blocks = to_vector(VPBlockUtils::blocksOnly<VPBasicBlock>(
+ vp_depth_first_deep(Plan.getVectorLoopRegion()->getEntry())));
+ auto *FirstIt = find(Blocks, FirstBB);
+ auto *LastIt = find(Blocks, LastBB);
+ assert(FirstIt != Blocks.end() && LastIt != Blocks.end() &&
+ "FirstBB and LastBB don't correspond to Range");
+ Blocks.erase(Blocks.begin(), FirstIt);
+ Blocks.erase(LastIt, Blocks.end());
+ return Blocks;
+}
+
bool VPBlockUtils::isHeader(const VPBlockBase *VPB,
const VPDominatorTree &VPDT) {
auto *VPBB = dyn_cast<VPBasicBlock>(VPB);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.h b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
index 50a7a1ac75b74..7a25a9b65f215 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
@@ -286,20 +286,10 @@ class VPBlockUtils {
});
}
- /// A variant of blocksOnly that only returns blocks between \p FirstBB and \p
- /// LastBB.
- template <typename BlockTy, typename T>
- static SmallVector<VPBasicBlock *>
- blocksOnly(const T &Range, VPBasicBlock *FirstBB, VPBasicBlock *LastBB) {
- auto Blocks = to_vector(blocksOnly<BlockTy, T>(Range));
- auto *FirstIt = find(Blocks, FirstBB);
- auto *LastIt = find(Blocks, LastBB);
- assert(FirstIt != Blocks.end() && LastIt != Blocks.end() &&
- "FirstBB and LastBB don't correspond to Range");
- Blocks.erase(Blocks.begin(), FirstIt);
- Blocks.erase(LastIt, Blocks.end());
- return Blocks;
- }
+ /// Returns the blocks between \p FirstBB and \p LastBB, where FirstBB
+ /// to LastBB forms a chain within the vector loop region.
+ static SmallVector<VPBasicBlock *> blocksChainBetween(VPBasicBlock *FirstBB,
+ VPBasicBlock *LastBB);
/// Inserts \p BlockPtr on the edge between \p From and \p To. That is, update
/// \p From's successor to \p To to point to \p BlockPtr and \p To's
>From 25a0f1a942a65a040031a9fadb65bce4056c88f0 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Thu, 19 Mar 2026 22:21:12 +0000
Subject: [PATCH 3/5] [VPlan] Fix thinkos, include nice test update!
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 24 ++++++-------
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 11 +++---
llvm/lib/Transforms/Vectorize/VPlanUtils.h | 2 +-
...predicated-loads-with-predicated-stores.ll | 36 ++++++-------------
4 files changed, 28 insertions(+), 45 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 39ff14e83dc6d..85d6f2b7357d2 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -188,13 +188,12 @@ class SinkStoreInfo {
};
/// Check if a memory operation doesn't alias with memory operations in blocks
-/// between \p FirstBB and \p LastBB, which is expected to be a valid range in
-/// a shallow-traversal of the vector loop region in \p Plan. We check aliasing
-/// with using scoped noalias metadata. If \p SinkInfo is std::nullopt, only
-/// recipes that may write to memory are checked (for load hoisting). Otherwise
-/// recipes that both read and write memory are checked, and SCEV is used to
-/// prove no-alias between the group leader and other replicate recipes (for
-/// store sinking).
+/// between \p FirstBB and \p LastBB, which is expected to be a chain in \p
+/// Plan. We check aliasing with using scoped noalias metadata. If \p SinkInfo
+/// is std::nullopt, only recipes that may write to memory are checked (for load
+/// hoisting). Otherwise recipes that both read and write memory are checked,
+/// and SCEV is used to prove no-alias between the group leader and other
+/// replicate recipes (for store sinking).
static bool
canHoistOrSinkWithNoAliasCheck(const MemoryLocation &MemLoc,
VPBasicBlock *FirstBB, VPBasicBlock *LastBB,
@@ -226,7 +225,7 @@ canHoistOrSinkWithNoAliasCheck(const MemoryLocation &MemLoc,
}
/// Collect either replicated Loads or Stores grouped by their address SCEV, in
-/// a shallow-traversal of the vector loop region in \p Plan.
+/// a deep-traversal of the vector loop region in \p Plan.
template <unsigned Opcode>
static SmallVector<SmallVector<VPReplicateRecipe *, 4>>
collectGroupedReplicateMemOps(
@@ -238,7 +237,7 @@ collectGroupedReplicateMemOps(
SmallDenseMap<const SCEV *, SmallVector<VPReplicateRecipe *, 4>>
RecipesByAddress;
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
- vp_depth_first_shallow(Plan.getVectorLoopRegion()->getEntry()))) {
+ vp_depth_first_deep(Plan.getVectorLoopRegion()->getEntry()))) {
for (VPRecipeBase &R : *VPBB) {
auto *RepR = dyn_cast<VPReplicateRecipe>(&R);
if (!RepR || RepR->getOpcode() != Opcode || !FilterFn(RepR))
@@ -4871,7 +4870,7 @@ void VPlanTransforms::hoistPredicatedLoads(VPlan &Plan,
static bool
canSinkStoreWithNoAliasCheck(ArrayRef<VPReplicateRecipe *> StoresToSink,
PredicatedScalarEvolution &PSE, const Loop &L,
- VPlan &Plan) {
+ VPTypeAnalysis &TypeInfo) {
auto StoreLoc = vputils::getMemoryLocation(*StoresToSink.front());
if (!StoreLoc || !StoreLoc->AATags.Scope)
return false;
@@ -4883,7 +4882,6 @@ canSinkStoreWithNoAliasCheck(ArrayRef<VPReplicateRecipe *> StoresToSink,
VPBasicBlock *FirstBB = StoresToSink.front()->getParent();
VPBasicBlock *LastBB = StoresToSink.back()->getParent();
- VPTypeAnalysis TypeInfo(Plan);
SinkStoreInfo SinkInfo(StoresToSinkSet, *StoresToSink[0], PSE, L, TypeInfo);
return canHoistOrSinkWithNoAliasCheck(*StoreLoc, FirstBB, LastBB, SinkInfo);
}
@@ -4896,8 +4894,10 @@ void VPlanTransforms::sinkPredicatedStores(VPlan &Plan,
if (Groups.empty())
return;
+ VPTypeAnalysis TypeInfo(Plan);
+
for (auto &Group : Groups) {
- if (!canSinkStoreWithNoAliasCheck(Group, PSE, *L, Plan))
+ if (!canSinkStoreWithNoAliasCheck(Group, PSE, *L, TypeInfo))
continue;
// Use the last (most dominated) store's location for the unconditional
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index b47b1fdee3e12..831dd2e2bc823 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -618,14 +618,11 @@ VPBlockUtils::blocksChainBetween(VPBasicBlock *FirstBB, VPBasicBlock *LastBB) {
#endif
assert(VPDT.properlyDominates(FirstBB, LastBB) &&
"Expected FirstBB to dominate LastBB");
- VPlan &Plan = *FirstBB->getPlan();
- auto Blocks = to_vector(VPBlockUtils::blocksOnly<VPBasicBlock>(
- vp_depth_first_deep(Plan.getVectorLoopRegion()->getEntry())));
- auto *FirstIt = find(Blocks, FirstBB);
+ auto Blocks = to_vector(
+ VPBlockUtils::blocksOnly<VPBasicBlock>(vp_depth_first_deep(FirstBB)));
auto *LastIt = find(Blocks, LastBB);
- assert(FirstIt != Blocks.end() && LastIt != Blocks.end() &&
- "FirstBB and LastBB don't correspond to Range");
- Blocks.erase(Blocks.begin(), FirstIt);
+ assert(LastIt != Blocks.end() &&
+ "LastBB unreachable from FirstBB in depth-first traversal");
Blocks.erase(LastIt, Blocks.end());
return Blocks;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.h b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
index 7a25a9b65f215..1f84fea786013 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
@@ -287,7 +287,7 @@ class VPBlockUtils {
}
/// Returns the blocks between \p FirstBB and \p LastBB, where FirstBB
- /// to LastBB forms a chain within the vector loop region.
+ /// to LastBB forms a chain.
static SmallVector<VPBasicBlock *> blocksChainBetween(VPBasicBlock *FirstBB,
VPBasicBlock *LastBB);
diff --git a/llvm/test/Transforms/LoopVectorize/hoist-predicated-loads-with-predicated-stores.ll b/llvm/test/Transforms/LoopVectorize/hoist-predicated-loads-with-predicated-stores.ll
index 38527baffd2e1..cdbed2411916b 100644
--- a/llvm/test/Transforms/LoopVectorize/hoist-predicated-loads-with-predicated-stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/hoist-predicated-loads-with-predicated-stores.ll
@@ -851,8 +851,6 @@ define void @sink_multiple_store_groups_alias_via_scev(ptr %dst, ptr %src) {
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE7:.*]] ]
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 16
; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 16
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <2 x i64> poison, i64 [[OFFSET_IDX]], i32 0
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x i64> [[TMP4]], i64 [[TMP1]], i32 1
; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr double, ptr [[SRC]], i64 [[OFFSET_IDX]]
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr double, ptr [[SRC]], i64 [[TMP1]]
; CHECK-NEXT: [[GEP_FLAG:%.*]] = getelementptr i8, ptr [[GEP_SRC]], i64 152
@@ -866,40 +864,28 @@ define void @sink_multiple_store_groups_alias_via_scev(ptr %dst, ptr %src) {
; CHECK-NEXT: [[TMP14:%.*]] = load double, ptr [[TMP3]], align 8, !alias.scope [[META85]]
; CHECK-NEXT: [[TMP15:%.*]] = insertelement <2 x double> poison, double [[TMP13]], i32 0
; CHECK-NEXT: [[TMP16:%.*]] = insertelement <2 x double> [[TMP15]], double [[TMP14]], i32 1
-; CHECK-NEXT: [[TMP17:%.*]] = xor <2 x i1> [[TMP12]], splat (i1 true)
; CHECK-NEXT: [[TMP18:%.*]] = fadd <2 x double> [[TMP16]], splat (double 8.000000e+00)
-; CHECK-NEXT: [[TMP36:%.*]] = extractelement <2 x i1> [[TMP17]], i32 0
-; CHECK-NEXT: br i1 [[TMP36]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
-; CHECK: [[PRED_STORE_IF]]:
; CHECK-NEXT: [[TMP20:%.*]] = getelementptr double, ptr [[DST]], i64 [[OFFSET_IDX]]
-; CHECK-NEXT: [[TMP19:%.*]] = extractelement <2 x double> [[TMP18]], i32 0
-; CHECK-NEXT: store double [[TMP19]], ptr [[TMP20]], align 8, !alias.scope [[META88:![0-9]+]], !noalias [[META85]]
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; CHECK: [[PRED_STORE_CONTINUE]]:
-; CHECK-NEXT: [[TMP39:%.*]] = extractelement <2 x i1> [[TMP17]], i32 1
-; CHECK-NEXT: br i1 [[TMP39]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]]
-; CHECK: [[PRED_STORE_IF2]]:
; CHECK-NEXT: [[TMP21:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP1]]
-; CHECK-NEXT: [[TMP22:%.*]] = extractelement <2 x double> [[TMP18]], i32 1
-; CHECK-NEXT: store double [[TMP22]], ptr [[TMP21]], align 8, !alias.scope [[META88]], !noalias [[META85]]
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE3]]
-; CHECK: [[PRED_STORE_CONTINUE3]]:
+; CHECK-NEXT: [[TMP17:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP20]], i32 0
+; CHECK-NEXT: [[TMP22:%.*]] = insertelement <2 x ptr> [[TMP17]], ptr [[TMP21]], i32 1
+; CHECK-NEXT: [[TMP19:%.*]] = select <2 x i1> [[TMP12]], <2 x double> [[TMP16]], <2 x double> [[TMP18]]
+; CHECK-NEXT: [[TMP24:%.*]] = extractelement <2 x double> [[TMP19]], i32 0
+; CHECK-NEXT: store double [[TMP24]], ptr [[TMP20]], align 8, !alias.scope [[META88:![0-9]+]], !noalias [[META85]]
+; CHECK-NEXT: [[TMP26:%.*]] = extractelement <2 x double> [[TMP19]], i32 1
+; CHECK-NEXT: store double [[TMP26]], ptr [[TMP21]], align 8, !alias.scope [[META88]], !noalias [[META85]]
; CHECK-NEXT: [[TMP42:%.*]] = extractelement <2 x i1> [[TMP12]], i32 0
; CHECK-NEXT: br i1 [[TMP42]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]]
; CHECK: [[PRED_STORE_IF4]]:
-; CHECK-NEXT: [[TMP43:%.*]] = getelementptr double, ptr [[DST]], i64 [[OFFSET_IDX]]
-; CHECK-NEXT: store double [[TMP13]], ptr [[TMP43]], align 8, !alias.scope [[META88]], !noalias [[META85]]
-; CHECK-NEXT: [[TMP44:%.*]] = getelementptr i8, ptr [[TMP43]], i64 8
-; CHECK-NEXT: store double 1.000000e+01, ptr [[TMP44]], align 8, !alias.scope [[META88]], !noalias [[META85]]
+; CHECK-NEXT: [[TMP23:%.*]] = getelementptr i8, ptr [[TMP20]], i64 8
+; CHECK-NEXT: store double 1.000000e+01, ptr [[TMP23]], align 8, !alias.scope [[META88]], !noalias [[META85]]
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE5]]
; CHECK: [[PRED_STORE_CONTINUE5]]:
; CHECK-NEXT: [[TMP45:%.*]] = extractelement <2 x i1> [[TMP12]], i32 1
; CHECK-NEXT: br i1 [[TMP45]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7]]
; CHECK: [[PRED_STORE_IF6]]:
-; CHECK-NEXT: [[TMP46:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP1]]
-; CHECK-NEXT: store double [[TMP14]], ptr [[TMP46]], align 8, !alias.scope [[META88]], !noalias [[META85]]
-; CHECK-NEXT: [[TMP47:%.*]] = getelementptr i8, ptr [[TMP46]], i64 8
-; CHECK-NEXT: store double 1.000000e+01, ptr [[TMP47]], align 8, !alias.scope [[META88]], !noalias [[META85]]
+; CHECK-NEXT: [[TMP25:%.*]] = getelementptr i8, ptr [[TMP21]], i64 8
+; CHECK-NEXT: store double 1.000000e+01, ptr [[TMP25]], align 8, !alias.scope [[META88]], !noalias [[META85]]
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE7]]
; CHECK: [[PRED_STORE_CONTINUE7]]:
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
>From 722660bcbddd720907bdda11c6c38a2ca986de11 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Thu, 19 Mar 2026 22:49:39 +0000
Subject: [PATCH 4/5] [VPlan] Fix off-by-one error; bad test update
---
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 2 +-
...predicated-loads-with-predicated-stores.ll | 36 +++++++++++++------
2 files changed, 26 insertions(+), 12 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 831dd2e2bc823..c7ba50e43d12a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -623,7 +623,7 @@ VPBlockUtils::blocksChainBetween(VPBasicBlock *FirstBB, VPBasicBlock *LastBB) {
auto *LastIt = find(Blocks, LastBB);
assert(LastIt != Blocks.end() &&
"LastBB unreachable from FirstBB in depth-first traversal");
- Blocks.erase(LastIt, Blocks.end());
+ Blocks.erase(std::next(LastIt), Blocks.end());
return Blocks;
}
diff --git a/llvm/test/Transforms/LoopVectorize/hoist-predicated-loads-with-predicated-stores.ll b/llvm/test/Transforms/LoopVectorize/hoist-predicated-loads-with-predicated-stores.ll
index cdbed2411916b..38527baffd2e1 100644
--- a/llvm/test/Transforms/LoopVectorize/hoist-predicated-loads-with-predicated-stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/hoist-predicated-loads-with-predicated-stores.ll
@@ -851,6 +851,8 @@ define void @sink_multiple_store_groups_alias_via_scev(ptr %dst, ptr %src) {
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE7:.*]] ]
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 16
; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 16
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <2 x i64> poison, i64 [[OFFSET_IDX]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x i64> [[TMP4]], i64 [[TMP1]], i32 1
; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr double, ptr [[SRC]], i64 [[OFFSET_IDX]]
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr double, ptr [[SRC]], i64 [[TMP1]]
; CHECK-NEXT: [[GEP_FLAG:%.*]] = getelementptr i8, ptr [[GEP_SRC]], i64 152
@@ -864,28 +866,40 @@ define void @sink_multiple_store_groups_alias_via_scev(ptr %dst, ptr %src) {
; CHECK-NEXT: [[TMP14:%.*]] = load double, ptr [[TMP3]], align 8, !alias.scope [[META85]]
; CHECK-NEXT: [[TMP15:%.*]] = insertelement <2 x double> poison, double [[TMP13]], i32 0
; CHECK-NEXT: [[TMP16:%.*]] = insertelement <2 x double> [[TMP15]], double [[TMP14]], i32 1
+; CHECK-NEXT: [[TMP17:%.*]] = xor <2 x i1> [[TMP12]], splat (i1 true)
; CHECK-NEXT: [[TMP18:%.*]] = fadd <2 x double> [[TMP16]], splat (double 8.000000e+00)
+; CHECK-NEXT: [[TMP36:%.*]] = extractelement <2 x i1> [[TMP17]], i32 0
+; CHECK-NEXT: br i1 [[TMP36]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; CHECK: [[PRED_STORE_IF]]:
; CHECK-NEXT: [[TMP20:%.*]] = getelementptr double, ptr [[DST]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT: [[TMP19:%.*]] = extractelement <2 x double> [[TMP18]], i32 0
+; CHECK-NEXT: store double [[TMP19]], ptr [[TMP20]], align 8, !alias.scope [[META88:![0-9]+]], !noalias [[META85]]
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; CHECK: [[PRED_STORE_CONTINUE]]:
+; CHECK-NEXT: [[TMP39:%.*]] = extractelement <2 x i1> [[TMP17]], i32 1
+; CHECK-NEXT: br i1 [[TMP39]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]]
+; CHECK: [[PRED_STORE_IF2]]:
; CHECK-NEXT: [[TMP21:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP1]]
-; CHECK-NEXT: [[TMP17:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP20]], i32 0
-; CHECK-NEXT: [[TMP22:%.*]] = insertelement <2 x ptr> [[TMP17]], ptr [[TMP21]], i32 1
-; CHECK-NEXT: [[TMP19:%.*]] = select <2 x i1> [[TMP12]], <2 x double> [[TMP16]], <2 x double> [[TMP18]]
-; CHECK-NEXT: [[TMP24:%.*]] = extractelement <2 x double> [[TMP19]], i32 0
-; CHECK-NEXT: store double [[TMP24]], ptr [[TMP20]], align 8, !alias.scope [[META88:![0-9]+]], !noalias [[META85]]
-; CHECK-NEXT: [[TMP26:%.*]] = extractelement <2 x double> [[TMP19]], i32 1
-; CHECK-NEXT: store double [[TMP26]], ptr [[TMP21]], align 8, !alias.scope [[META88]], !noalias [[META85]]
+; CHECK-NEXT: [[TMP22:%.*]] = extractelement <2 x double> [[TMP18]], i32 1
+; CHECK-NEXT: store double [[TMP22]], ptr [[TMP21]], align 8, !alias.scope [[META88]], !noalias [[META85]]
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE3]]
+; CHECK: [[PRED_STORE_CONTINUE3]]:
; CHECK-NEXT: [[TMP42:%.*]] = extractelement <2 x i1> [[TMP12]], i32 0
; CHECK-NEXT: br i1 [[TMP42]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]]
; CHECK: [[PRED_STORE_IF4]]:
-; CHECK-NEXT: [[TMP23:%.*]] = getelementptr i8, ptr [[TMP20]], i64 8
-; CHECK-NEXT: store double 1.000000e+01, ptr [[TMP23]], align 8, !alias.scope [[META88]], !noalias [[META85]]
+; CHECK-NEXT: [[TMP43:%.*]] = getelementptr double, ptr [[DST]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT: store double [[TMP13]], ptr [[TMP43]], align 8, !alias.scope [[META88]], !noalias [[META85]]
+; CHECK-NEXT: [[TMP44:%.*]] = getelementptr i8, ptr [[TMP43]], i64 8
+; CHECK-NEXT: store double 1.000000e+01, ptr [[TMP44]], align 8, !alias.scope [[META88]], !noalias [[META85]]
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE5]]
; CHECK: [[PRED_STORE_CONTINUE5]]:
; CHECK-NEXT: [[TMP45:%.*]] = extractelement <2 x i1> [[TMP12]], i32 1
; CHECK-NEXT: br i1 [[TMP45]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7]]
; CHECK: [[PRED_STORE_IF6]]:
-; CHECK-NEXT: [[TMP25:%.*]] = getelementptr i8, ptr [[TMP21]], i64 8
-; CHECK-NEXT: store double 1.000000e+01, ptr [[TMP25]], align 8, !alias.scope [[META88]], !noalias [[META85]]
+; CHECK-NEXT: [[TMP46:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP1]]
+; CHECK-NEXT: store double [[TMP14]], ptr [[TMP46]], align 8, !alias.scope [[META88]], !noalias [[META85]]
+; CHECK-NEXT: [[TMP47:%.*]] = getelementptr i8, ptr [[TMP46]], i64 8
+; CHECK-NEXT: store double 1.000000e+01, ptr [[TMP47]], align 8, !alias.scope [[META88]], !noalias [[META85]]
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE7]]
; CHECK: [[PRED_STORE_CONTINUE7]]:
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
>From bf535bd365685426b9b482ac9e4672d110ecce5d Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Fri, 27 Mar 2026 08:21:36 +0000
Subject: [PATCH 5/5] [VPlan] Tighten assert to single-succ chain
---
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 13 ++++++++-----
1 file changed, 8 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index c7ba50e43d12a..29140ff871880 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -611,13 +611,16 @@ VPSingleDefRecipe *vputils::findHeaderMask(VPlan &Plan) {
SmallVector<VPBasicBlock *>
VPBlockUtils::blocksChainBetween(VPBasicBlock *FirstBB, VPBasicBlock *LastBB) {
- assert(FirstBB->getPlan() == LastBB->getPlan() &&
- "FirstBB and LastBB from different VPlans");
+ assert(FirstBB->getEnclosingLoopRegion() ==
+ LastBB->getEnclosingLoopRegion() &&
+ "FirstBB and LastBB from different regions");
#ifndef NDEBUG
- VPDominatorTree VPDT(*FirstBB->getPlan());
+ bool InSingleSuccChain = false;
+ for (VPBlockBase *Succ = FirstBB; Succ; Succ = Succ->getSingleSuccessor())
+ InSingleSuccChain |= (Succ == LastBB);
+ assert(InSingleSuccChain &&
+ "LastBB not reachable from FirstBB in single-successor chain");
#endif
- assert(VPDT.properlyDominates(FirstBB, LastBB) &&
- "Expected FirstBB to dominate LastBB");
auto Blocks = to_vector(
VPBlockUtils::blocksOnly<VPBasicBlock>(vp_depth_first_deep(FirstBB)));
auto *LastIt = find(Blocks, LastBB);
More information about the llvm-commits
mailing list