[llvm] [LV] Respect access type and alignment when speculating loads (PR #229379)
Dávid Bolvanský via llvm-commits
llvm-commits at lists.llvm.org
Thu Oct 8 04:33:37 PDT 2026
https://github.com/davidbolvansky updated https://github.com/llvm/llvm-project/pull/229379
>From cd71c149cab00f9335065f80d9125ebb512cd634 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?D=C3=A1vid=20Bolvansk=C3=BD?= <david.bolvansky at gmail.com>
Date: Tue, 6 Oct 2026 18:10:19 +0200
Subject: [PATCH] [LV] Respect access type and alignment when speculating loads
---
.../Vectorize/LoopVectorizationLegality.h | 11 ++-
.../Vectorize/LoopVectorizationLegality.cpp | 46 ++++++++-----
.../predicated-load-access-safety.ll | 69 +++++++++++++++++--
3 files changed, 97 insertions(+), 29 deletions(-)
diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
index 6dd44f8a71538..afa4b59fcb544 100644
--- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
+++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
@@ -26,8 +26,10 @@
#ifndef LLVM_TRANSFORMS_VECTORIZE_LOOPVECTORIZATIONLEGALITY_H
#define LLVM_TRANSFORMS_VECTORIZE_LOOPVECTORIZATIONLEGALITY_H
+#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/MapVector.h"
#include "llvm/Analysis/LoopAccessAnalysis.h"
+#include "llvm/Support/Alignment.h"
#include "llvm/Support/TypeSize.h"
#include "llvm/Transforms/Utils/LoopUtils.h"
@@ -621,15 +623,18 @@ class LoopVectorizationLegality {
/// for it.
bool canUncountableExitConditionLoadBeMoved(BasicBlock *ExitingBlock);
+ /// Maps a pointer and access type to its known safe alignment.
+ using SafeAccessesTy = SmallDenseMap<std::pair<Value *, Type *>, Align, 8>;
+
/// Return true if all of the instructions in the block can be speculatively
/// executed, and record the loads/stores that require masking.
- /// \p SafePtrs is a list of addresses that are known to be legal and we know
- /// that we can read from them without segfault.
+ /// \p SafeAccesses records pointers that can be accessed unconditionally,
+ /// together with the access type and alignment justified for each pointer.
/// \p MaskedOp is a list of instructions that have to be transformed into
/// calls to the appropriate masked intrinsic when the loop is vectorized
/// or dropped if the instruction is a conditional assume intrinsic.
bool
- blockCanBePredicated(BasicBlock *BB, SmallPtrSetImpl<Value *> &SafePtrs,
+ blockCanBePredicated(BasicBlock *BB, SafeAccessesTy &SafeAccesses,
SmallPtrSetImpl<const Instruction *> &MaskedOp) const;
/// Updates the vectorization state by adding \p Phi to the inductions list.
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 2dfeef4488098..6caccd871f107 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -1362,7 +1362,7 @@ bool LoopVectorizationLegality::blockNeedsPredication(
}
bool LoopVectorizationLegality::blockCanBePredicated(
- BasicBlock *BB, SmallPtrSetImpl<Value *> &SafePtrs,
+ BasicBlock *BB, SafeAccessesTy &SafeAccesses,
SmallPtrSetImpl<const Instruction *> &MaskedOp) const {
for (Instruction &I : *BB) {
// We can predicate blocks with calls to assume, as long as we drop them in
@@ -1390,7 +1390,8 @@ bool LoopVectorizationLegality::blockCanBePredicated(
// Loads are handled via masking (or speculated if safe to do so.)
if (auto *LI = dyn_cast<LoadInst>(&I)) {
- if (!SafePtrs.count(LI->getPointerOperand()))
+ auto It = SafeAccesses.find({LI->getPointerOperand(), LI->getType()});
+ if (It == SafeAccesses.end() || It->second < LI->getAlign())
MaskedOp.insert(LI);
continue;
}
@@ -1421,19 +1422,26 @@ bool LoopVectorizationLegality::canVectorizeWithIfConvert() {
assert(TheLoop->getNumBlocks() > 1 && "Single block loops are vectorizable");
- // A list of pointers which are known to be dereferenceable within scope of
- // the loop body for each iteration of the loop which executes. That is,
- // the memory pointed to can be dereferenced (with the access size implied by
- // the value's type) unconditionally within the loop header without
- // introducing a new fault.
- SmallPtrSet<Value *, 8> SafePointers;
+ // Record the access type and maximum alignment known to be safe for each
+ // pointer in every executed iteration. Loads with these properties can be
+ // executed unconditionally without introducing a new fault.
+ SafeAccessesTy SafeAccesses;
+ auto RecordSafeAccess = [&SafeAccesses](Instruction &I) {
+ Value *Ptr = getLoadStorePointerOperand(&I);
+ if (!Ptr)
+ return;
+ Align Alignment = getLoadStoreAlignment(&I);
+ auto [It, Inserted] =
+ SafeAccesses.try_emplace({Ptr, getLoadStoreType(&I)}, Alignment);
+ if (!Inserted)
+ It->second = std::max(It->second, Alignment);
+ };
- // Collect safe addresses.
+ // Collect safe accesses.
for (BasicBlock *BB : TheLoop->blocks()) {
if (!blockNeedsPredication(BB)) {
for (Instruction &I : *BB)
- if (auto *Ptr = getLoadStorePointerOperand(&I))
- SafePointers.insert(Ptr);
+ RecordSafeAccess(I);
continue;
}
@@ -1495,7 +1503,7 @@ bool LoopVectorizationLegality::canVectorizeWithIfConvert() {
CanSpeculatePointerOp(LI->getPointerOperand()) &&
isDereferenceableAndAlignedInLoop(LI, TheLoop, SE, *DT, AC,
&Predicates))
- SafePointers.insert(LI->getPointerOperand());
+ RecordSafeAccess(*LI);
Predicates.clear();
}
}
@@ -1520,7 +1528,7 @@ bool LoopVectorizationLegality::canVectorizeWithIfConvert() {
// We must be able to predicate all blocks that need to be predicated.
if (blockNeedsPredication(BB) &&
- !blockCanBePredicated(BB, SafePointers, ConditionallyExecutedOps)) {
+ !blockCanBePredicated(BB, SafeAccesses, ConditionallyExecutedOps)) {
reportVectorizationFailure(
"Control flow cannot be substituted for a select", "NoCFGForSelect",
ORE, TheLoop, BB->getTerminator());
@@ -2066,14 +2074,14 @@ bool LoopVectorizationLegality::canFoldTailByMasking() const {
LLVM_DEBUG(dbgs() << "LV: checking if tail can be folded by masking.\n");
- // The list of pointers that we can safely read and write to remains empty.
- SmallPtrSet<Value *, 8> SafePointers;
+ // No accesses are known to be safe in inactive lanes of the folded tail.
+ SafeAccessesTy SafeAccesses;
// Check all blocks for predication, including those that ordinarily do not
// need predication such as the header block.
SmallPtrSet<const Instruction *, 8> TmpMaskedOp;
for (BasicBlock *BB : TheLoop->blocks()) {
- if (!blockCanBePredicated(BB, SafePointers, TmpMaskedOp)) {
+ if (!blockCanBePredicated(BB, SafeAccesses, TmpMaskedOp)) {
LLVM_DEBUG(dbgs() << "LV: Cannot fold tail by masking.\n");
return false;
}
@@ -2085,15 +2093,15 @@ bool LoopVectorizationLegality::canFoldTailByMasking() const {
}
void LoopVectorizationLegality::prepareToFoldTailByMasking() {
- // The list of pointers that we can safely read and write to remains empty.
- SmallPtrSet<Value *, 8> SafePointers;
+ // No accesses are known to be safe in inactive lanes of the folded tail.
+ SafeAccessesTy SafeAccesses;
// Mark all blocks for predication, including those that ordinarily do not
// need predication such as the header block, and collect instructions needing
// predication in TailFoldedMaskedOp.
for (BasicBlock *BB : TheLoop->blocks()) {
[[maybe_unused]] bool R =
- blockCanBePredicated(BB, SafePointers, TailFoldedMaskedOp);
+ blockCanBePredicated(BB, SafeAccesses, TailFoldedMaskedOp);
assert(R && "Must be able to predicate block when tail-folding.");
}
}
diff --git a/llvm/test/Transforms/LoopVectorize/predicated-load-access-safety.ll b/llvm/test/Transforms/LoopVectorize/predicated-load-access-safety.ll
index 64d3427039dbb..26f5d314ad4b2 100644
--- a/llvm/test/Transforms/LoopVectorize/predicated-load-access-safety.ll
+++ b/llvm/test/Transforms/LoopVectorize/predicated-load-access-safety.ll
@@ -13,7 +13,7 @@ define void @wider_predicated_load(ptr noalias %src, ptr noalias %dst, i64 %n) {
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE6:.*]] ]
; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1
; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 3
@@ -30,7 +30,38 @@ define void @wider_predicated_load(ptr noalias %src, ptr noalias %dst, i64 %n) {
; CHECK-NEXT: [[TMP14:%.*]] = insertelement <4 x i8> [[TMP13]], i8 [[TMP10]], i64 2
; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x i8> [[TMP14]], i8 [[TMP11]], i64 3
; CHECK-NEXT: [[TMP16:%.*]] = icmp ne <4 x i8> [[TMP15]], zeroinitializer
-; CHECK-NEXT: [[TMP32:%.*]] = load <4 x i64>, ptr [[TMP4]], align 1
+; CHECK-NEXT: [[TMP17:%.*]] = extractelement <4 x i1> [[TMP16]], i64 0
+; CHECK-NEXT: br i1 [[TMP17]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
+; CHECK: [[PRED_LOAD_IF]]:
+; CHECK-NEXT: [[TMP18:%.*]] = load i64, ptr [[TMP4]], align 1
+; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x i64> poison, i64 [[TMP18]], i64 0
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE]]
+; CHECK: [[PRED_LOAD_CONTINUE]]:
+; CHECK-NEXT: [[TMP20:%.*]] = phi <4 x i64> [ poison, %[[VECTOR_BODY]] ], [ [[TMP19]], %[[PRED_LOAD_IF]] ]
+; CHECK-NEXT: [[TMP21:%.*]] = extractelement <4 x i1> [[TMP16]], i64 1
+; CHECK-NEXT: br i1 [[TMP21]], label %[[PRED_LOAD_IF1:.*]], label %[[PRED_LOAD_CONTINUE2:.*]]
+; CHECK: [[PRED_LOAD_IF1]]:
+; CHECK-NEXT: [[TMP22:%.*]] = load i64, ptr [[TMP5]], align 1
+; CHECK-NEXT: [[TMP23:%.*]] = insertelement <4 x i64> [[TMP20]], i64 [[TMP22]], i64 1
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE2]]
+; CHECK: [[PRED_LOAD_CONTINUE2]]:
+; CHECK-NEXT: [[TMP24:%.*]] = phi <4 x i64> [ [[TMP20]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP23]], %[[PRED_LOAD_IF1]] ]
+; CHECK-NEXT: [[TMP25:%.*]] = extractelement <4 x i1> [[TMP16]], i64 2
+; CHECK-NEXT: br i1 [[TMP25]], label %[[PRED_LOAD_IF3:.*]], label %[[PRED_LOAD_CONTINUE4:.*]]
+; CHECK: [[PRED_LOAD_IF3]]:
+; CHECK-NEXT: [[TMP26:%.*]] = load i64, ptr [[TMP6]], align 1
+; CHECK-NEXT: [[TMP27:%.*]] = insertelement <4 x i64> [[TMP24]], i64 [[TMP26]], i64 2
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE4]]
+; CHECK: [[PRED_LOAD_CONTINUE4]]:
+; CHECK-NEXT: [[TMP28:%.*]] = phi <4 x i64> [ [[TMP24]], %[[PRED_LOAD_CONTINUE2]] ], [ [[TMP27]], %[[PRED_LOAD_IF3]] ]
+; CHECK-NEXT: [[TMP29:%.*]] = extractelement <4 x i1> [[TMP16]], i64 3
+; CHECK-NEXT: br i1 [[TMP29]], label %[[PRED_LOAD_IF5:.*]], label %[[PRED_LOAD_CONTINUE6]]
+; CHECK: [[PRED_LOAD_IF5]]:
+; CHECK-NEXT: [[TMP30:%.*]] = load i64, ptr [[TMP7]], align 1
+; CHECK-NEXT: [[TMP31:%.*]] = insertelement <4 x i64> [[TMP28]], i64 [[TMP30]], i64 3
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE6]]
+; CHECK: [[PRED_LOAD_CONTINUE6]]:
+; CHECK-NEXT: [[TMP32:%.*]] = phi <4 x i64> [ [[TMP28]], %[[PRED_LOAD_CONTINUE4]] ], [ [[TMP31]], %[[PRED_LOAD_IF5]] ]
; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP16]], <4 x i64> [[TMP32]], <4 x i64> zeroinitializer
; CHECK-NEXT: [[TMP33:%.*]] = getelementptr i64, ptr [[DST]], i64 [[INDEX]]
; CHECK-NEXT: store <4 x i64> [[PREDPHI]], ptr [[TMP33]], align 1
@@ -80,7 +111,7 @@ define void @stronger_predicated_alignment(ptr noalias %src, ptr noalias %dst, i
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE6:.*]] ]
; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1
; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 3
@@ -97,14 +128,38 @@ define void @stronger_predicated_alignment(ptr noalias %src, ptr noalias %dst, i
; CHECK-NEXT: [[TMP14:%.*]] = insertelement <4 x i32> [[TMP13]], i32 [[TMP10]], i64 2
; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x i32> [[TMP14]], i32 [[TMP11]], i64 3
; CHECK-NEXT: [[TMP16:%.*]] = icmp ne <4 x i32> [[TMP15]], zeroinitializer
+; CHECK-NEXT: [[TMP17:%.*]] = extractelement <4 x i1> [[TMP16]], i64 0
+; CHECK-NEXT: br i1 [[TMP17]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
+; CHECK: [[PRED_LOAD_IF]]:
; CHECK-NEXT: [[TMP18:%.*]] = load i32, ptr [[TMP4]], align 16
+; CHECK-NEXT: [[TMP21:%.*]] = insertelement <4 x i32> poison, i32 [[TMP18]], i64 0
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE]]
+; CHECK: [[PRED_LOAD_CONTINUE]]:
+; CHECK-NEXT: [[TMP20:%.*]] = phi <4 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP21]], %[[PRED_LOAD_IF]] ]
+; CHECK-NEXT: [[TMP24:%.*]] = extractelement <4 x i1> [[TMP16]], i64 1
+; CHECK-NEXT: br i1 [[TMP24]], label %[[PRED_LOAD_IF1:.*]], label %[[PRED_LOAD_CONTINUE2:.*]]
+; CHECK: [[PRED_LOAD_IF1]]:
; CHECK-NEXT: [[TMP22:%.*]] = load i32, ptr [[TMP5]], align 16
+; CHECK-NEXT: [[TMP27:%.*]] = insertelement <4 x i32> [[TMP20]], i32 [[TMP22]], i64 1
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE2]]
+; CHECK: [[PRED_LOAD_CONTINUE2]]:
+; CHECK-NEXT: [[TMP23:%.*]] = phi <4 x i32> [ [[TMP20]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP27]], %[[PRED_LOAD_IF1]] ]
+; CHECK-NEXT: [[TMP25:%.*]] = extractelement <4 x i1> [[TMP16]], i64 2
+; CHECK-NEXT: br i1 [[TMP25]], label %[[PRED_LOAD_IF3:.*]], label %[[PRED_LOAD_CONTINUE4:.*]]
+; CHECK: [[PRED_LOAD_IF3]]:
; CHECK-NEXT: [[TMP26:%.*]] = load i32, ptr [[TMP6]], align 16
-; CHECK-NEXT: [[TMP30:%.*]] = load i32, ptr [[TMP7]], align 16
-; CHECK-NEXT: [[TMP21:%.*]] = insertelement <4 x i32> poison, i32 [[TMP18]], i64 0
-; CHECK-NEXT: [[TMP23:%.*]] = insertelement <4 x i32> [[TMP21]], i32 [[TMP22]], i64 1
; CHECK-NEXT: [[TMP28:%.*]] = insertelement <4 x i32> [[TMP23]], i32 [[TMP26]], i64 2
-; CHECK-NEXT: [[TMP31:%.*]] = insertelement <4 x i32> [[TMP28]], i32 [[TMP30]], i64 3
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE4]]
+; CHECK: [[PRED_LOAD_CONTINUE4]]:
+; CHECK-NEXT: [[TMP32:%.*]] = phi <4 x i32> [ [[TMP23]], %[[PRED_LOAD_CONTINUE2]] ], [ [[TMP28]], %[[PRED_LOAD_IF3]] ]
+; CHECK-NEXT: [[TMP29:%.*]] = extractelement <4 x i1> [[TMP16]], i64 3
+; CHECK-NEXT: br i1 [[TMP29]], label %[[PRED_LOAD_IF5:.*]], label %[[PRED_LOAD_CONTINUE6]]
+; CHECK: [[PRED_LOAD_IF5]]:
+; CHECK-NEXT: [[TMP30:%.*]] = load i32, ptr [[TMP7]], align 16
+; CHECK-NEXT: [[TMP35:%.*]] = insertelement <4 x i32> [[TMP32]], i32 [[TMP30]], i64 3
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE6]]
+; CHECK: [[PRED_LOAD_CONTINUE6]]:
+; CHECK-NEXT: [[TMP31:%.*]] = phi <4 x i32> [ [[TMP32]], %[[PRED_LOAD_CONTINUE4]] ], [ [[TMP35]], %[[PRED_LOAD_IF5]] ]
; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP16]], <4 x i32> [[TMP31]], <4 x i32> zeroinitializer
; CHECK-NEXT: [[TMP33:%.*]] = getelementptr i32, ptr [[DST]], i64 [[INDEX]]
; CHECK-NEXT: store <4 x i32> [[PREDPHI]], ptr [[TMP33]], align 1
More information about the llvm-commits
mailing list