[llvm] [LV] Respect access type and alignment when speculating loads (PR #229379)
Dávid Bolvanský via llvm-commits
llvm-commits at lists.llvm.org
Tue Oct 6 04:43:45 PDT 2026
https://github.com/davidbolvansky updated https://github.com/llvm/llvm-project/pull/229379
>From f6dceee21aa152d2bafe3d78bc5822d1e1f621ec Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?D=C3=A1vid=20Bolvansk=C3=BD?= <david.bolvansky at gmail.com>
Date: Tue, 6 Oct 2026 13:00:42 +0200
Subject: [PATCH 1/4] [LV] Track type and alignment of safe predicated accesses
---
.../Vectorize/LoopVectorizationLegality.h | 11 +-
.../Vectorize/LoopVectorizationLegality.cpp | 31 +-
.../predicated-load-access-safety.ll | 369 ++++++++++++++++++
3 files changed, 398 insertions(+), 13 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/predicated-load-access-safety.ll
diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
index 6dd44f8a71538..8011424bf8f5e 100644
--- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
+++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
@@ -26,8 +26,10 @@
#ifndef LLVM_TRANSFORMS_VECTORIZE_LOOPVECTORIZATIONLEGALITY_H
#define LLVM_TRANSFORMS_VECTORIZE_LOOPVECTORIZATIONLEGALITY_H
+#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/MapVector.h"
#include "llvm/Analysis/LoopAccessAnalysis.h"
+#include "llvm/Support/Alignment.h"
#include "llvm/Support/TypeSize.h"
#include "llvm/Transforms/Utils/LoopUtils.h"
@@ -621,15 +623,18 @@ class LoopVectorizationLegality {
/// for it.
bool canUncountableExitConditionLoadBeMoved(BasicBlock *ExitingBlock);
+ /// Maps a pointer and access type to its known safe alignment.
+ using SafeAccessesTy = DenseMap<std::pair<Value *, Type *>, Align>;
+
/// Return true if all of the instructions in the block can be speculatively
/// executed, and record the loads/stores that require masking.
- /// \p SafePtrs is a list of addresses that are known to be legal and we know
- /// that we can read from them without segfault.
+ /// \p SafePtrs records pointers that can be accessed unconditionally,
+ /// together with the access type and alignment justified for each pointer.
/// \p MaskedOp is a list of instructions that have to be transformed into
/// calls to the appropriate masked intrinsic when the loop is vectorized
/// or dropped if the instruction is a conditional assume intrinsic.
bool
- blockCanBePredicated(BasicBlock *BB, SmallPtrSetImpl<Value *> &SafePtrs,
+ blockCanBePredicated(BasicBlock *BB, SafeAccessesTy &SafePtrs,
SmallPtrSetImpl<const Instruction *> &MaskedOp) const;
/// Updates the vectorization state by adding \p Phi to the inductions list.
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 46f082604b5fb..f8239da6cf18c 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -1342,7 +1342,7 @@ bool LoopVectorizationLegality::blockNeedsPredication(
}
bool LoopVectorizationLegality::blockCanBePredicated(
- BasicBlock *BB, SmallPtrSetImpl<Value *> &SafePtrs,
+ BasicBlock *BB, SafeAccessesTy &SafePtrs,
SmallPtrSetImpl<const Instruction *> &MaskedOp) const {
for (Instruction &I : *BB) {
// We can predicate blocks with calls to assume, as long as we drop them in
@@ -1370,7 +1370,8 @@ bool LoopVectorizationLegality::blockCanBePredicated(
// Loads are handled via masking (or speculated if safe to do so.)
if (auto *LI = dyn_cast<LoadInst>(&I)) {
- if (!SafePtrs.count(LI->getPointerOperand()))
+ auto It = SafePtrs.find({LI->getPointerOperand(), LI->getType()});
+ if (It == SafePtrs.end() || It->second < LI->getAlign())
MaskedOp.insert(LI);
continue;
}
@@ -1406,14 +1407,20 @@ bool LoopVectorizationLegality::canVectorizeWithIfConvert() {
// the memory pointed to can be dereferenced (with the access size implied by
// the value's type) unconditionally within the loop header without
// introducing a new fault.
- SmallPtrSet<Value *, 8> SafePointers;
+ SafeAccessesTy SafePointers;
// Collect safe addresses.
for (BasicBlock *BB : TheLoop->blocks()) {
if (!blockNeedsPredication(BB)) {
- for (Instruction &I : *BB)
- if (auto *Ptr = getLoadStorePointerOperand(&I))
- SafePointers.insert(Ptr);
+ for (Instruction &I : *BB) {
+ if (auto *Ptr = getLoadStorePointerOperand(&I)) {
+ auto [It, Inserted] = SafePointers.try_emplace(
+ std::make_pair(Ptr, getLoadStoreType(&I)),
+ getLoadStoreAlignment(&I));
+ if (!Inserted)
+ It->second = std::max(It->second, getLoadStoreAlignment(&I));
+ }
+ }
continue;
}
@@ -1474,8 +1481,12 @@ bool LoopVectorizationLegality::canVectorizeWithIfConvert() {
if (LI && !LI->getType()->isVectorTy() && !mustSuppressSpeculation(*LI) &&
CanSpeculatePointerOp(LI->getPointerOperand()) &&
isDereferenceableAndAlignedInLoop(LI, TheLoop, SE, *DT, AC,
- &Predicates))
- SafePointers.insert(LI->getPointerOperand());
+ &Predicates)) {
+ auto [It, Inserted] = SafePointers.try_emplace(
+ std::make_pair(LI->getPointerOperand(), LI->getType()), LI->getAlign());
+ if (!Inserted)
+ It->second = std::max(It->second, LI->getAlign());
+ }
Predicates.clear();
}
}
@@ -2047,7 +2058,7 @@ bool LoopVectorizationLegality::canFoldTailByMasking() const {
LLVM_DEBUG(dbgs() << "LV: checking if tail can be folded by masking.\n");
// The list of pointers that we can safely read and write to remains empty.
- SmallPtrSet<Value *, 8> SafePointers;
+ SafeAccessesTy SafePointers;
// Check all blocks for predication, including those that ordinarily do not
// need predication such as the header block.
@@ -2066,7 +2077,7 @@ bool LoopVectorizationLegality::canFoldTailByMasking() const {
void LoopVectorizationLegality::prepareToFoldTailByMasking() {
// The list of pointers that we can safely read and write to remains empty.
- SmallPtrSet<Value *, 8> SafePointers;
+ SafeAccessesTy SafePointers;
// Mark all blocks for predication, including those that ordinarily do not
// need predication such as the header block, and collect instructions needing
diff --git a/llvm/test/Transforms/LoopVectorize/predicated-load-access-safety.ll b/llvm/test/Transforms/LoopVectorize/predicated-load-access-safety.ll
new file mode 100644
index 0000000000000..51b945b58ba5b
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/predicated-load-access-safety.ll
@@ -0,0 +1,369 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes=loop-vectorize -mtriple=aarch64-unknown-linux-gnu -force-vector-width=4 -force-vector-interleave=1 -S < %s | FileCheck %s
+
+; A byte load does not make a wider load at the same pointer safe.
+define void @wider_predicated_load(ptr noalias noundef %src, ptr noalias %dst, i64 %n) {
+; CHECK-LABEL: define void @wider_predicated_load(
+; CHECK-SAME: ptr noalias noundef [[SRC:%.*]], ptr noalias [[DST:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[EMPTY:%.*]] = icmp eq i64 [[N]], 0
+; CHECK-NEXT: br i1 [[EMPTY]], label %[[EXIT:.*]], label %[[PREHEADER:.*]]
+; CHECK: [[PREHEADER]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE6:.*]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[TMP1]]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[TMP3]]
+; CHECK-NEXT: [[TMP8:%.*]] = load i8, ptr [[TMP4]], align 1
+; CHECK-NEXT: [[TMP9:%.*]] = load i8, ptr [[TMP5]], align 1
+; CHECK-NEXT: [[TMP10:%.*]] = load i8, ptr [[TMP6]], align 1
+; CHECK-NEXT: [[TMP11:%.*]] = load i8, ptr [[TMP7]], align 1
+; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x i8> poison, i8 [[TMP8]], i64 0
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i8> [[TMP12]], i8 [[TMP9]], i64 1
+; CHECK-NEXT: [[TMP14:%.*]] = insertelement <4 x i8> [[TMP13]], i8 [[TMP10]], i64 2
+; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x i8> [[TMP14]], i8 [[TMP11]], i64 3
+; CHECK-NEXT: [[TMP16:%.*]] = icmp ne <4 x i8> [[TMP15]], zeroinitializer
+; CHECK-NEXT: [[TMP17:%.*]] = extractelement <4 x i1> [[TMP16]], i64 0
+; CHECK-NEXT: br i1 [[TMP17]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
+; CHECK: [[PRED_LOAD_IF]]:
+; CHECK-NEXT: [[TMP18:%.*]] = load i64, ptr [[TMP4]], align 1
+; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x i64> poison, i64 [[TMP18]], i64 0
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE]]
+; CHECK: [[PRED_LOAD_CONTINUE]]:
+; CHECK-NEXT: [[TMP20:%.*]] = phi <4 x i64> [ poison, %[[VECTOR_BODY]] ], [ [[TMP19]], %[[PRED_LOAD_IF]] ]
+; CHECK-NEXT: [[TMP21:%.*]] = extractelement <4 x i1> [[TMP16]], i64 1
+; CHECK-NEXT: br i1 [[TMP21]], label %[[PRED_LOAD_IF1:.*]], label %[[PRED_LOAD_CONTINUE2:.*]]
+; CHECK: [[PRED_LOAD_IF1]]:
+; CHECK-NEXT: [[TMP22:%.*]] = load i64, ptr [[TMP5]], align 1
+; CHECK-NEXT: [[TMP23:%.*]] = insertelement <4 x i64> [[TMP20]], i64 [[TMP22]], i64 1
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE2]]
+; CHECK: [[PRED_LOAD_CONTINUE2]]:
+; CHECK-NEXT: [[TMP24:%.*]] = phi <4 x i64> [ [[TMP20]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP23]], %[[PRED_LOAD_IF1]] ]
+; CHECK-NEXT: [[TMP25:%.*]] = extractelement <4 x i1> [[TMP16]], i64 2
+; CHECK-NEXT: br i1 [[TMP25]], label %[[PRED_LOAD_IF3:.*]], label %[[PRED_LOAD_CONTINUE4:.*]]
+; CHECK: [[PRED_LOAD_IF3]]:
+; CHECK-NEXT: [[TMP26:%.*]] = load i64, ptr [[TMP6]], align 1
+; CHECK-NEXT: [[TMP27:%.*]] = insertelement <4 x i64> [[TMP24]], i64 [[TMP26]], i64 2
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE4]]
+; CHECK: [[PRED_LOAD_CONTINUE4]]:
+; CHECK-NEXT: [[TMP28:%.*]] = phi <4 x i64> [ [[TMP24]], %[[PRED_LOAD_CONTINUE2]] ], [ [[TMP27]], %[[PRED_LOAD_IF3]] ]
+; CHECK-NEXT: [[TMP29:%.*]] = extractelement <4 x i1> [[TMP16]], i64 3
+; CHECK-NEXT: br i1 [[TMP29]], label %[[PRED_LOAD_IF5:.*]], label %[[PRED_LOAD_CONTINUE6]]
+; CHECK: [[PRED_LOAD_IF5]]:
+; CHECK-NEXT: [[TMP30:%.*]] = load i64, ptr [[TMP7]], align 1
+; CHECK-NEXT: [[TMP31:%.*]] = insertelement <4 x i64> [[TMP28]], i64 [[TMP30]], i64 3
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE6]]
+; CHECK: [[PRED_LOAD_CONTINUE6]]:
+; CHECK-NEXT: [[TMP32:%.*]] = phi <4 x i64> [ [[TMP28]], %[[PRED_LOAD_CONTINUE4]] ], [ [[TMP31]], %[[PRED_LOAD_IF5]] ]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP16]], <4 x i64> [[TMP32]], <4 x i64> zeroinitializer
+; CHECK-NEXT: [[TMP33:%.*]] = getelementptr i64, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i64> [[PREDPHI]], ptr [[TMP33]], align 1
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP34:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP34]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[PREHEADER]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[NEXT:%.*]], %[[MERGE:.*]] ]
+; CHECK-NEXT: [[P:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[GUARD:%.*]] = load i8, ptr [[P]], align 1
+; CHECK-NEXT: [[KEEP:%.*]] = icmp ne i8 [[GUARD]], 0
+; CHECK-NEXT: br i1 [[KEEP]], label %[[READ:.*]], label %[[MERGE]]
+; CHECK: [[READ]]:
+; CHECK-NEXT: [[VALUE:%.*]] = load i64, ptr [[P]], align 1
+; CHECK-NEXT: br label %[[MERGE]]
+; CHECK: [[MERGE]]:
+; CHECK-NEXT: [[RESULT:%.*]] = phi i64 [ [[VALUE]], %[[READ]] ], [ 0, %[[LOOP]] ]
+; CHECK-NEXT: [[OUT:%.*]] = getelementptr i64, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT: store i64 [[RESULT]], ptr [[OUT]], align 1
+; CHECK-NEXT: [[NEXT]] = add nuw i64 [[IV]], 1
+; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT_LOOPEXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[EXIT_LOOPEXIT]]:
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ %empty = icmp eq i64 %n, 0
+ br i1 %empty, label %exit, label %preheader
+
+preheader:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %preheader ], [ %next, %merge ]
+ %p = getelementptr i64, ptr %src, i64 %iv
+ %guard = load i8, ptr %p, align 1
+ %keep = icmp ne i8 %guard, 0
+ br i1 %keep, label %read, label %merge
+
+read:
+ %value = load i64, ptr %p, align 1
+ br label %merge
+
+merge:
+ %result = phi i64 [ %value, %read ], [ 0, %loop ]
+ %out = getelementptr i64, ptr %dst, i64 %iv
+ store i64 %result, ptr %out, align 1
+ %next = add nuw i64 %iv, 1
+ %done = icmp eq i64 %next, %n
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; A weaker alignment does not justify the predicated load alignment.
+define void @stronger_predicated_alignment(ptr noalias noundef %src, ptr noalias %dst, i64 %n) {
+; CHECK-LABEL: define void @stronger_predicated_alignment(
+; CHECK-SAME: ptr noalias noundef [[SRC:%.*]], ptr noalias [[DST:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[EMPTY:%.*]] = icmp eq i64 [[N]], 0
+; CHECK-NEXT: br i1 [[EMPTY]], label %[[EXIT:.*]], label %[[PREHEADER:.*]]
+; CHECK: [[PREHEADER]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE6:.*]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr [4 x i32], ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr [4 x i32], ptr [[SRC]], i64 [[TMP1]]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr [4 x i32], ptr [[SRC]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr [4 x i32], ptr [[SRC]], i64 [[TMP3]]
+; CHECK-NEXT: [[TMP8:%.*]] = load i32, ptr [[TMP4]], align 4
+; CHECK-NEXT: [[TMP9:%.*]] = load i32, ptr [[TMP5]], align 4
+; CHECK-NEXT: [[TMP10:%.*]] = load i32, ptr [[TMP6]], align 4
+; CHECK-NEXT: [[TMP11:%.*]] = load i32, ptr [[TMP7]], align 4
+; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x i32> poison, i32 [[TMP8]], i64 0
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP9]], i64 1
+; CHECK-NEXT: [[TMP14:%.*]] = insertelement <4 x i32> [[TMP13]], i32 [[TMP10]], i64 2
+; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x i32> [[TMP14]], i32 [[TMP11]], i64 3
+; CHECK-NEXT: [[TMP16:%.*]] = icmp ne <4 x i32> [[TMP15]], zeroinitializer
+; CHECK-NEXT: [[TMP17:%.*]] = extractelement <4 x i1> [[TMP16]], i64 0
+; CHECK-NEXT: br i1 [[TMP17]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
+; CHECK: [[PRED_LOAD_IF]]:
+; CHECK-NEXT: [[TMP18:%.*]] = load i32, ptr [[TMP4]], align 16
+; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x i32> poison, i32 [[TMP18]], i64 0
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE]]
+; CHECK: [[PRED_LOAD_CONTINUE]]:
+; CHECK-NEXT: [[TMP20:%.*]] = phi <4 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP19]], %[[PRED_LOAD_IF]] ]
+; CHECK-NEXT: [[TMP21:%.*]] = extractelement <4 x i1> [[TMP16]], i64 1
+; CHECK-NEXT: br i1 [[TMP21]], label %[[PRED_LOAD_IF1:.*]], label %[[PRED_LOAD_CONTINUE2:.*]]
+; CHECK: [[PRED_LOAD_IF1]]:
+; CHECK-NEXT: [[TMP22:%.*]] = load i32, ptr [[TMP5]], align 16
+; CHECK-NEXT: [[TMP23:%.*]] = insertelement <4 x i32> [[TMP20]], i32 [[TMP22]], i64 1
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE2]]
+; CHECK: [[PRED_LOAD_CONTINUE2]]:
+; CHECK-NEXT: [[TMP24:%.*]] = phi <4 x i32> [ [[TMP20]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP23]], %[[PRED_LOAD_IF1]] ]
+; CHECK-NEXT: [[TMP25:%.*]] = extractelement <4 x i1> [[TMP16]], i64 2
+; CHECK-NEXT: br i1 [[TMP25]], label %[[PRED_LOAD_IF3:.*]], label %[[PRED_LOAD_CONTINUE4:.*]]
+; CHECK: [[PRED_LOAD_IF3]]:
+; CHECK-NEXT: [[TMP26:%.*]] = load i32, ptr [[TMP6]], align 16
+; CHECK-NEXT: [[TMP27:%.*]] = insertelement <4 x i32> [[TMP24]], i32 [[TMP26]], i64 2
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE4]]
+; CHECK: [[PRED_LOAD_CONTINUE4]]:
+; CHECK-NEXT: [[TMP28:%.*]] = phi <4 x i32> [ [[TMP24]], %[[PRED_LOAD_CONTINUE2]] ], [ [[TMP27]], %[[PRED_LOAD_IF3]] ]
+; CHECK-NEXT: [[TMP29:%.*]] = extractelement <4 x i1> [[TMP16]], i64 3
+; CHECK-NEXT: br i1 [[TMP29]], label %[[PRED_LOAD_IF5:.*]], label %[[PRED_LOAD_CONTINUE6]]
+; CHECK: [[PRED_LOAD_IF5]]:
+; CHECK-NEXT: [[TMP30:%.*]] = load i32, ptr [[TMP7]], align 16
+; CHECK-NEXT: [[TMP31:%.*]] = insertelement <4 x i32> [[TMP28]], i32 [[TMP30]], i64 3
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE6]]
+; CHECK: [[PRED_LOAD_CONTINUE6]]:
+; CHECK-NEXT: [[TMP32:%.*]] = phi <4 x i32> [ [[TMP28]], %[[PRED_LOAD_CONTINUE4]] ], [ [[TMP31]], %[[PRED_LOAD_IF5]] ]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP16]], <4 x i32> [[TMP32]], <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP33:%.*]] = getelementptr i32, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[PREDPHI]], ptr [[TMP33]], align 1
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP34:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP34]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[PREHEADER]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[NEXT:%.*]], %[[MERGE:.*]] ]
+; CHECK-NEXT: [[P:%.*]] = getelementptr [4 x i32], ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[GUARD:%.*]] = load i32, ptr [[P]], align 4
+; CHECK-NEXT: [[KEEP:%.*]] = icmp ne i32 [[GUARD]], 0
+; CHECK-NEXT: br i1 [[KEEP]], label %[[READ:.*]], label %[[MERGE]]
+; CHECK: [[READ]]:
+; CHECK-NEXT: [[VALUE:%.*]] = load i32, ptr [[P]], align 16
+; CHECK-NEXT: br label %[[MERGE]]
+; CHECK: [[MERGE]]:
+; CHECK-NEXT: [[RESULT:%.*]] = phi i32 [ [[VALUE]], %[[READ]] ], [ 0, %[[LOOP]] ]
+; CHECK-NEXT: [[OUT:%.*]] = getelementptr i32, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT: store i32 [[RESULT]], ptr [[OUT]], align 1
+; CHECK-NEXT: [[NEXT]] = add nuw i64 [[IV]], 1
+; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT_LOOPEXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK: [[EXIT_LOOPEXIT]]:
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ %empty = icmp eq i64 %n, 0
+ br i1 %empty, label %exit, label %preheader
+
+preheader:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %preheader ], [ %next, %merge ]
+ %p = getelementptr [4 x i32], ptr %src, i64 %iv
+ %guard = load i32, ptr %p, align 4
+ %keep = icmp ne i32 %guard, 0
+ br i1 %keep, label %read, label %merge
+
+read:
+ %value = load i32, ptr %p, align 16
+ br label %merge
+
+merge:
+ %result = phi i32 [ %value, %read ], [ 0, %loop ]
+ %out = getelementptr i32, ptr %dst, i64 %iv
+ store i32 %result, ptr %out, align 1
+ %next = add nuw i64 %iv, 1
+ %done = icmp eq i64 %next, %n
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; An unconditional access with sufficient alignment still permits speculation.
+define void @sufficient_alignment(ptr noalias noundef %src, ptr noalias %dst, i64 %n) {
+; CHECK-LABEL: define void @sufficient_alignment(
+; CHECK-SAME: ptr noalias noundef [[SRC:%.*]], ptr noalias [[DST:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[EMPTY:%.*]] = icmp eq i64 [[N]], 0
+; CHECK-NEXT: br i1 [[EMPTY]], label %[[EXIT:.*]], label %[[PREHEADER:.*]]
+; CHECK: [[PREHEADER]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr [4 x i32], ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr [4 x i32], ptr [[SRC]], i64 [[TMP1]]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr [4 x i32], ptr [[SRC]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr [4 x i32], ptr [[SRC]], i64 [[TMP3]]
+; CHECK-NEXT: [[TMP8:%.*]] = load i32, ptr [[TMP4]], align 16
+; CHECK-NEXT: [[TMP9:%.*]] = load i32, ptr [[TMP5]], align 16
+; CHECK-NEXT: [[TMP10:%.*]] = load i32, ptr [[TMP6]], align 16
+; CHECK-NEXT: [[TMP11:%.*]] = load i32, ptr [[TMP7]], align 16
+; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x i32> poison, i32 [[TMP8]], i64 0
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP9]], i64 1
+; CHECK-NEXT: [[TMP14:%.*]] = insertelement <4 x i32> [[TMP13]], i32 [[TMP10]], i64 2
+; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x i32> [[TMP14]], i32 [[TMP11]], i64 3
+; CHECK-NEXT: [[TMP16:%.*]] = icmp ne <4 x i32> [[TMP15]], zeroinitializer
+; CHECK-NEXT: [[TMP17:%.*]] = load i32, ptr [[TMP4]], align 4
+; CHECK-NEXT: [[TMP18:%.*]] = load i32, ptr [[TMP5]], align 4
+; CHECK-NEXT: [[TMP19:%.*]] = load i32, ptr [[TMP6]], align 4
+; CHECK-NEXT: [[TMP20:%.*]] = load i32, ptr [[TMP7]], align 4
+; CHECK-NEXT: [[TMP21:%.*]] = insertelement <4 x i32> poison, i32 [[TMP17]], i64 0
+; CHECK-NEXT: [[TMP22:%.*]] = insertelement <4 x i32> [[TMP21]], i32 [[TMP18]], i64 1
+; CHECK-NEXT: [[TMP23:%.*]] = insertelement <4 x i32> [[TMP22]], i32 [[TMP19]], i64 2
+; CHECK-NEXT: [[TMP24:%.*]] = insertelement <4 x i32> [[TMP23]], i32 [[TMP20]], i64 3
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP16]], <4 x i32> [[TMP24]], <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP25:%.*]] = getelementptr i32, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[PREDPHI]], ptr [[TMP25]], align 1
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP26]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[PREHEADER]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[NEXT:%.*]], %[[MERGE:.*]] ]
+; CHECK-NEXT: [[P:%.*]] = getelementptr [4 x i32], ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[GUARD:%.*]] = load i32, ptr [[P]], align 16
+; CHECK-NEXT: [[KEEP:%.*]] = icmp ne i32 [[GUARD]], 0
+; CHECK-NEXT: br i1 [[KEEP]], label %[[READ:.*]], label %[[MERGE]]
+; CHECK: [[READ]]:
+; CHECK-NEXT: [[VALUE:%.*]] = load i32, ptr [[P]], align 4
+; CHECK-NEXT: br label %[[MERGE]]
+; CHECK: [[MERGE]]:
+; CHECK-NEXT: [[RESULT:%.*]] = phi i32 [ [[VALUE]], %[[READ]] ], [ 0, %[[LOOP]] ]
+; CHECK-NEXT: [[OUT:%.*]] = getelementptr i32, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT: store i32 [[RESULT]], ptr [[OUT]], align 1
+; CHECK-NEXT: [[NEXT]] = add nuw i64 [[IV]], 1
+; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT_LOOPEXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK: [[EXIT_LOOPEXIT]]:
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ %empty = icmp eq i64 %n, 0
+ br i1 %empty, label %exit, label %preheader
+
+preheader:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %preheader ], [ %next, %merge ]
+ %p = getelementptr [4 x i32], ptr %src, i64 %iv
+ %guard = load i32, ptr %p, align 16
+ %keep = icmp ne i32 %guard, 0
+ br i1 %keep, label %read, label %merge
+
+read:
+ %value = load i32, ptr %p, align 4
+ br label %merge
+
+merge:
+ %result = phi i32 [ %value, %read ], [ 0, %loop ]
+ %out = getelementptr i32, ptr %dst, i64 %iv
+ store i32 %result, ptr %out, align 1
+ %next = add nuw i64 %iv, 1
+ %done = icmp eq i64 %next, %n
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+;.
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
+; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
+; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]}
+; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]}
+; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META2]], [[META1]]}
+;.
>From e3adcd9bf1d780fc77d85fd018c98b538b810a95 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?D=C3=A1vid=20Bolvansk=C3=BD?= <david.bolvansky at gmail.com>
Date: Tue, 6 Oct 2026 13:11:33 +0200
Subject: [PATCH 2/4] [LV] Share safe access recording and simplify regression
tests
---
.../Vectorize/LoopVectorizationLegality.h | 2 +-
.../Vectorize/LoopVectorizationLegality.cpp | 29 +++--
.../predicated-load-access-safety.ll | 108 ++++++++----------
3 files changed, 62 insertions(+), 77 deletions(-)
diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
index 8011424bf8f5e..a02d18af0c3bb 100644
--- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
+++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
@@ -624,7 +624,7 @@ class LoopVectorizationLegality {
bool canUncountableExitConditionLoadBeMoved(BasicBlock *ExitingBlock);
/// Maps a pointer and access type to its known safe alignment.
- using SafeAccessesTy = DenseMap<std::pair<Value *, Type *>, Align>;
+ using SafeAccessesTy = SmallDenseMap<std::pair<Value *, Type *>, Align, 8>;
/// Return true if all of the instructions in the block can be speculatively
/// executed, and record the loads/stores that require masking.
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index f8239da6cf18c..be02b08274999 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -1408,19 +1408,22 @@ bool LoopVectorizationLegality::canVectorizeWithIfConvert() {
// the value's type) unconditionally within the loop header without
// introducing a new fault.
SafeAccessesTy SafePointers;
+ auto RecordSafeAccess = [&](Instruction &I) {
+ Value *Ptr = getLoadStorePointerOperand(&I);
+ if (!Ptr)
+ return;
+ Align Alignment = getLoadStoreAlignment(&I);
+ auto [It, Inserted] = SafePointers.try_emplace(
+ std::make_pair(Ptr, getLoadStoreType(&I)), Alignment);
+ if (!Inserted)
+ It->second = std::max(It->second, Alignment);
+ };
// Collect safe addresses.
for (BasicBlock *BB : TheLoop->blocks()) {
if (!blockNeedsPredication(BB)) {
- for (Instruction &I : *BB) {
- if (auto *Ptr = getLoadStorePointerOperand(&I)) {
- auto [It, Inserted] = SafePointers.try_emplace(
- std::make_pair(Ptr, getLoadStoreType(&I)),
- getLoadStoreAlignment(&I));
- if (!Inserted)
- It->second = std::max(It->second, getLoadStoreAlignment(&I));
- }
- }
+ for (Instruction &I : *BB)
+ RecordSafeAccess(I);
continue;
}
@@ -1481,12 +1484,8 @@ bool LoopVectorizationLegality::canVectorizeWithIfConvert() {
if (LI && !LI->getType()->isVectorTy() && !mustSuppressSpeculation(*LI) &&
CanSpeculatePointerOp(LI->getPointerOperand()) &&
isDereferenceableAndAlignedInLoop(LI, TheLoop, SE, *DT, AC,
- &Predicates)) {
- auto [It, Inserted] = SafePointers.try_emplace(
- std::make_pair(LI->getPointerOperand(), LI->getType()), LI->getAlign());
- if (!Inserted)
- It->second = std::max(It->second, LI->getAlign());
- }
+ &Predicates))
+ RecordSafeAccess(*LI);
Predicates.clear();
}
}
diff --git a/llvm/test/Transforms/LoopVectorize/predicated-load-access-safety.ll b/llvm/test/Transforms/LoopVectorize/predicated-load-access-safety.ll
index 51b945b58ba5b..891059e0643dc 100644
--- a/llvm/test/Transforms/LoopVectorize/predicated-load-access-safety.ll
+++ b/llvm/test/Transforms/LoopVectorize/predicated-load-access-safety.ll
@@ -5,10 +5,7 @@
define void @wider_predicated_load(ptr noalias noundef %src, ptr noalias %dst, i64 %n) {
; CHECK-LABEL: define void @wider_predicated_load(
; CHECK-SAME: ptr noalias noundef [[SRC:%.*]], ptr noalias [[DST:%.*]], i64 [[N:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[EMPTY:%.*]] = icmp eq i64 [[N]], 0
-; CHECK-NEXT: br i1 [[EMPTY]], label %[[EXIT:.*]], label %[[PREHEADER:.*]]
-; CHECK: [[PREHEADER]]:
+; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
@@ -73,9 +70,9 @@ define void @wider_predicated_load(ptr noalias noundef %src, ptr noalias %dst, i
; CHECK-NEXT: br i1 [[TMP34]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[PREHEADER]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[NEXT:%.*]], %[[MERGE:.*]] ]
@@ -92,21 +89,15 @@ define void @wider_predicated_load(ptr noalias noundef %src, ptr noalias %dst, i
; CHECK-NEXT: store i64 [[RESULT]], ptr [[OUT]], align 1
; CHECK-NEXT: [[NEXT]] = add nuw i64 [[IV]], 1
; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT_LOOPEXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
-; CHECK: [[EXIT_LOOPEXIT]]:
-; CHECK-NEXT: br label %[[EXIT]]
+; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
entry:
- %empty = icmp eq i64 %n, 0
- br i1 %empty, label %exit, label %preheader
-
-preheader:
br label %loop
loop:
- %iv = phi i64 [ 0, %preheader ], [ %next, %merge ]
+ %iv = phi i64 [ 0, %entry ], [ %next, %merge ]
%p = getelementptr i64, ptr %src, i64 %iv
%guard = load i8, ptr %p, align 1
%keep = icmp ne i8 %guard, 0
@@ -132,10 +123,7 @@ exit:
define void @stronger_predicated_alignment(ptr noalias noundef %src, ptr noalias %dst, i64 %n) {
; CHECK-LABEL: define void @stronger_predicated_alignment(
; CHECK-SAME: ptr noalias noundef [[SRC:%.*]], ptr noalias [[DST:%.*]], i64 [[N:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[EMPTY:%.*]] = icmp eq i64 [[N]], 0
-; CHECK-NEXT: br i1 [[EMPTY]], label %[[EXIT:.*]], label %[[PREHEADER:.*]]
-; CHECK: [[PREHEADER]]:
+; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
@@ -200,9 +188,9 @@ define void @stronger_predicated_alignment(ptr noalias noundef %src, ptr noalias
; CHECK-NEXT: br i1 [[TMP34]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[PREHEADER]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[NEXT:%.*]], %[[MERGE:.*]] ]
@@ -219,21 +207,15 @@ define void @stronger_predicated_alignment(ptr noalias noundef %src, ptr noalias
; CHECK-NEXT: store i32 [[RESULT]], ptr [[OUT]], align 1
; CHECK-NEXT: [[NEXT]] = add nuw i64 [[IV]], 1
; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT_LOOPEXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
-; CHECK: [[EXIT_LOOPEXIT]]:
-; CHECK-NEXT: br label %[[EXIT]]
+; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
entry:
- %empty = icmp eq i64 %n, 0
- br i1 %empty, label %exit, label %preheader
-
-preheader:
br label %loop
loop:
- %iv = phi i64 [ 0, %preheader ], [ %next, %merge ]
+ %iv = phi i64 [ 0, %entry ], [ %next, %merge ]
%p = getelementptr [4 x i32], ptr %src, i64 %iv
%guard = load i32, ptr %p, align 4
%keep = icmp ne i32 %guard, 0
@@ -255,14 +237,11 @@ exit:
ret void
}
-; An unconditional access with sufficient alignment still permits speculation.
+; Keep the strongest known alignment when several accesses share a pointer.
define void @sufficient_alignment(ptr noalias noundef %src, ptr noalias %dst, i64 %n) {
; CHECK-LABEL: define void @sufficient_alignment(
; CHECK-SAME: ptr noalias noundef [[SRC:%.*]], ptr noalias [[DST:%.*]], i64 [[N:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[EMPTY:%.*]] = icmp eq i64 [[N]], 0
-; CHECK-NEXT: br i1 [[EMPTY]], label %[[EXIT:.*]], label %[[PREHEADER:.*]]
-; CHECK: [[PREHEADER]]:
+; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
@@ -286,35 +265,46 @@ define void @sufficient_alignment(ptr noalias noundef %src, ptr noalias %dst, i6
; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP9]], i64 1
; CHECK-NEXT: [[TMP14:%.*]] = insertelement <4 x i32> [[TMP13]], i32 [[TMP10]], i64 2
; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x i32> [[TMP14]], i32 [[TMP11]], i64 3
-; CHECK-NEXT: [[TMP16:%.*]] = icmp ne <4 x i32> [[TMP15]], zeroinitializer
-; CHECK-NEXT: [[TMP17:%.*]] = load i32, ptr [[TMP4]], align 4
-; CHECK-NEXT: [[TMP18:%.*]] = load i32, ptr [[TMP5]], align 4
-; CHECK-NEXT: [[TMP19:%.*]] = load i32, ptr [[TMP6]], align 4
-; CHECK-NEXT: [[TMP20:%.*]] = load i32, ptr [[TMP7]], align 4
-; CHECK-NEXT: [[TMP21:%.*]] = insertelement <4 x i32> poison, i32 [[TMP17]], i64 0
-; CHECK-NEXT: [[TMP22:%.*]] = insertelement <4 x i32> [[TMP21]], i32 [[TMP18]], i64 1
-; CHECK-NEXT: [[TMP23:%.*]] = insertelement <4 x i32> [[TMP22]], i32 [[TMP19]], i64 2
-; CHECK-NEXT: [[TMP24:%.*]] = insertelement <4 x i32> [[TMP23]], i32 [[TMP20]], i64 3
-; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP16]], <4 x i32> [[TMP24]], <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP25:%.*]] = getelementptr i32, ptr [[DST]], i64 [[INDEX]]
-; CHECK-NEXT: store <4 x i32> [[PREDPHI]], ptr [[TMP25]], align 1
+; CHECK-NEXT: [[TMP16:%.*]] = load i32, ptr [[TMP4]], align 4
+; CHECK-NEXT: [[TMP17:%.*]] = load i32, ptr [[TMP5]], align 4
+; CHECK-NEXT: [[TMP18:%.*]] = load i32, ptr [[TMP6]], align 4
+; CHECK-NEXT: [[TMP19:%.*]] = load i32, ptr [[TMP7]], align 4
+; CHECK-NEXT: [[TMP20:%.*]] = insertelement <4 x i32> poison, i32 [[TMP16]], i64 0
+; CHECK-NEXT: [[TMP21:%.*]] = insertelement <4 x i32> [[TMP20]], i32 [[TMP17]], i64 1
+; CHECK-NEXT: [[TMP22:%.*]] = insertelement <4 x i32> [[TMP21]], i32 [[TMP18]], i64 2
+; CHECK-NEXT: [[TMP23:%.*]] = insertelement <4 x i32> [[TMP22]], i32 [[TMP19]], i64 3
+; CHECK-NEXT: [[TMP24:%.*]] = add <4 x i32> [[TMP15]], [[TMP23]]
+; CHECK-NEXT: [[TMP25:%.*]] = icmp ne <4 x i32> [[TMP24]], zeroinitializer
+; CHECK-NEXT: [[TMP26:%.*]] = load i32, ptr [[TMP4]], align 16
+; CHECK-NEXT: [[TMP27:%.*]] = load i32, ptr [[TMP5]], align 16
+; CHECK-NEXT: [[TMP28:%.*]] = load i32, ptr [[TMP6]], align 16
+; CHECK-NEXT: [[TMP29:%.*]] = load i32, ptr [[TMP7]], align 16
+; CHECK-NEXT: [[TMP30:%.*]] = insertelement <4 x i32> poison, i32 [[TMP26]], i64 0
+; CHECK-NEXT: [[TMP31:%.*]] = insertelement <4 x i32> [[TMP30]], i32 [[TMP27]], i64 1
+; CHECK-NEXT: [[TMP32:%.*]] = insertelement <4 x i32> [[TMP31]], i32 [[TMP28]], i64 2
+; CHECK-NEXT: [[TMP33:%.*]] = insertelement <4 x i32> [[TMP32]], i32 [[TMP29]], i64 3
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP25]], <4 x i32> [[TMP33]], <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP34:%.*]] = getelementptr i32, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[PREDPHI]], ptr [[TMP34]], align 1
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP26]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT: [[TMP35:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP35]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[PREHEADER]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[NEXT:%.*]], %[[MERGE:.*]] ]
; CHECK-NEXT: [[P:%.*]] = getelementptr [4 x i32], ptr [[SRC]], i64 [[IV]]
; CHECK-NEXT: [[GUARD:%.*]] = load i32, ptr [[P]], align 16
-; CHECK-NEXT: [[KEEP:%.*]] = icmp ne i32 [[GUARD]], 0
+; CHECK-NEXT: [[WEAK:%.*]] = load i32, ptr [[P]], align 4
+; CHECK-NEXT: [[SUM:%.*]] = add i32 [[GUARD]], [[WEAK]]
+; CHECK-NEXT: [[KEEP:%.*]] = icmp ne i32 [[SUM]], 0
; CHECK-NEXT: br i1 [[KEEP]], label %[[READ:.*]], label %[[MERGE]]
; CHECK: [[READ]]:
-; CHECK-NEXT: [[VALUE:%.*]] = load i32, ptr [[P]], align 4
+; CHECK-NEXT: [[VALUE:%.*]] = load i32, ptr [[P]], align 16
; CHECK-NEXT: br label %[[MERGE]]
; CHECK: [[MERGE]]:
; CHECK-NEXT: [[RESULT:%.*]] = phi i32 [ [[VALUE]], %[[READ]] ], [ 0, %[[LOOP]] ]
@@ -322,28 +312,24 @@ define void @sufficient_alignment(ptr noalias noundef %src, ptr noalias %dst, i6
; CHECK-NEXT: store i32 [[RESULT]], ptr [[OUT]], align 1
; CHECK-NEXT: [[NEXT]] = add nuw i64 [[IV]], 1
; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT_LOOPEXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
-; CHECK: [[EXIT_LOOPEXIT]]:
-; CHECK-NEXT: br label %[[EXIT]]
+; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
entry:
- %empty = icmp eq i64 %n, 0
- br i1 %empty, label %exit, label %preheader
-
-preheader:
br label %loop
loop:
- %iv = phi i64 [ 0, %preheader ], [ %next, %merge ]
+ %iv = phi i64 [ 0, %entry ], [ %next, %merge ]
%p = getelementptr [4 x i32], ptr %src, i64 %iv
%guard = load i32, ptr %p, align 16
- %keep = icmp ne i32 %guard, 0
+ %weak = load i32, ptr %p, align 4
+ %sum = add i32 %guard, %weak
+ %keep = icmp ne i32 %sum, 0
br i1 %keep, label %read, label %merge
read:
- %value = load i32, ptr %p, align 4
+ %value = load i32, ptr %p, align 16
br label %merge
merge:
>From 15ad27c98589070af61bb2c1fa302ddab8876480 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?D=C3=A1vid=20Bolvansk=C3=BD?= <david.bolvansky at gmail.com>
Date: Tue, 6 Oct 2026 13:12:37 +0200
Subject: [PATCH 3/4] [LV] Limit access safety checks to vectorized code
---
.../predicated-load-access-safety.ll | 88 ++-----------------
1 file changed, 8 insertions(+), 80 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/predicated-load-access-safety.ll b/llvm/test/Transforms/LoopVectorize/predicated-load-access-safety.ll
index 891059e0643dc..8efd5a2aacc4e 100644
--- a/llvm/test/Transforms/LoopVectorize/predicated-load-access-safety.ll
+++ b/llvm/test/Transforms/LoopVectorize/predicated-load-access-safety.ll
@@ -1,11 +1,11 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
-; RUN: opt -passes=loop-vectorize -mtriple=aarch64-unknown-linux-gnu -force-vector-width=4 -force-vector-interleave=1 -S < %s | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph" --version 6
+; RUN: opt -passes=loop-vectorize -mtriple=aarch64 -force-vector-width=4 -force-vector-interleave=1 -S %s | FileCheck %s
; A byte load does not make a wider load at the same pointer safe.
define void @wider_predicated_load(ptr noalias noundef %src, ptr noalias %dst, i64 %n) {
; CHECK-LABEL: define void @wider_predicated_load(
; CHECK-SAME: ptr noalias noundef [[SRC:%.*]], ptr noalias [[DST:%.*]], i64 [[N:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
@@ -70,28 +70,8 @@ define void @wider_predicated_load(ptr noalias noundef %src, ptr noalias %dst, i
; CHECK-NEXT: br i1 [[TMP34]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[NEXT:%.*]], %[[MERGE:.*]] ]
-; CHECK-NEXT: [[P:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT: [[GUARD:%.*]] = load i8, ptr [[P]], align 1
-; CHECK-NEXT: [[KEEP:%.*]] = icmp ne i8 [[GUARD]], 0
-; CHECK-NEXT: br i1 [[KEEP]], label %[[READ:.*]], label %[[MERGE]]
-; CHECK: [[READ]]:
-; CHECK-NEXT: [[VALUE:%.*]] = load i64, ptr [[P]], align 1
-; CHECK-NEXT: br label %[[MERGE]]
-; CHECK: [[MERGE]]:
-; CHECK-NEXT: [[RESULT:%.*]] = phi i64 [ [[VALUE]], %[[READ]] ], [ 0, %[[LOOP]] ]
-; CHECK-NEXT: [[OUT:%.*]] = getelementptr i64, ptr [[DST]], i64 [[IV]]
-; CHECK-NEXT: store i64 [[RESULT]], ptr [[OUT]], align 1
-; CHECK-NEXT: [[NEXT]] = add nuw i64 [[IV]], 1
-; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret void
;
entry:
br label %loop
@@ -123,7 +103,7 @@ exit:
define void @stronger_predicated_alignment(ptr noalias noundef %src, ptr noalias %dst, i64 %n) {
; CHECK-LABEL: define void @stronger_predicated_alignment(
; CHECK-SAME: ptr noalias noundef [[SRC:%.*]], ptr noalias [[DST:%.*]], i64 [[N:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
@@ -188,28 +168,8 @@ define void @stronger_predicated_alignment(ptr noalias noundef %src, ptr noalias
; CHECK-NEXT: br i1 [[TMP34]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[NEXT:%.*]], %[[MERGE:.*]] ]
-; CHECK-NEXT: [[P:%.*]] = getelementptr [4 x i32], ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT: [[GUARD:%.*]] = load i32, ptr [[P]], align 4
-; CHECK-NEXT: [[KEEP:%.*]] = icmp ne i32 [[GUARD]], 0
-; CHECK-NEXT: br i1 [[KEEP]], label %[[READ:.*]], label %[[MERGE]]
-; CHECK: [[READ]]:
-; CHECK-NEXT: [[VALUE:%.*]] = load i32, ptr [[P]], align 16
-; CHECK-NEXT: br label %[[MERGE]]
-; CHECK: [[MERGE]]:
-; CHECK-NEXT: [[RESULT:%.*]] = phi i32 [ [[VALUE]], %[[READ]] ], [ 0, %[[LOOP]] ]
-; CHECK-NEXT: [[OUT:%.*]] = getelementptr i32, ptr [[DST]], i64 [[IV]]
-; CHECK-NEXT: store i32 [[RESULT]], ptr [[OUT]], align 1
-; CHECK-NEXT: [[NEXT]] = add nuw i64 [[IV]], 1
-; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret void
;
entry:
br label %loop
@@ -241,7 +201,7 @@ exit:
define void @sufficient_alignment(ptr noalias noundef %src, ptr noalias %dst, i64 %n) {
; CHECK-LABEL: define void @sufficient_alignment(
; CHECK-SAME: ptr noalias noundef [[SRC:%.*]], ptr noalias [[DST:%.*]], i64 [[N:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
@@ -291,30 +251,8 @@ define void @sufficient_alignment(ptr noalias noundef %src, ptr noalias %dst, i6
; CHECK-NEXT: br i1 [[TMP35]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[NEXT:%.*]], %[[MERGE:.*]] ]
-; CHECK-NEXT: [[P:%.*]] = getelementptr [4 x i32], ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT: [[GUARD:%.*]] = load i32, ptr [[P]], align 16
-; CHECK-NEXT: [[WEAK:%.*]] = load i32, ptr [[P]], align 4
-; CHECK-NEXT: [[SUM:%.*]] = add i32 [[GUARD]], [[WEAK]]
-; CHECK-NEXT: [[KEEP:%.*]] = icmp ne i32 [[SUM]], 0
-; CHECK-NEXT: br i1 [[KEEP]], label %[[READ:.*]], label %[[MERGE]]
-; CHECK: [[READ]]:
-; CHECK-NEXT: [[VALUE:%.*]] = load i32, ptr [[P]], align 16
-; CHECK-NEXT: br label %[[MERGE]]
-; CHECK: [[MERGE]]:
-; CHECK-NEXT: [[RESULT:%.*]] = phi i32 [ [[VALUE]], %[[READ]] ], [ 0, %[[LOOP]] ]
-; CHECK-NEXT: [[OUT:%.*]] = getelementptr i32, ptr [[DST]], i64 [[IV]]
-; CHECK-NEXT: store i32 [[RESULT]], ptr [[OUT]], align 1
-; CHECK-NEXT: [[NEXT]] = add nuw i64 [[IV]], 1
-; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret void
;
entry:
br label %loop
@@ -343,13 +281,3 @@ merge:
exit:
ret void
}
-;.
-; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
-; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
-; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
-; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
-; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
-; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]}
-; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]}
-; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META2]], [[META1]]}
-;.
>From 0054d8c7a14148de7dc4052972d8254175c02c2b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?D=C3=A1vid=20Bolvansk=C3=BD?= <david.bolvansky at gmail.com>
Date: Tue, 6 Oct 2026 13:18:10 +0200
Subject: [PATCH 4/4] [LV] Clarify safe access names and lambda capture
---
.../Vectorize/LoopVectorizationLegality.h | 4 ++--
.../Vectorize/LoopVectorizationLegality.cpp | 24 +++++++++----------
2 files changed, 14 insertions(+), 14 deletions(-)
diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
index a02d18af0c3bb..afa4b59fcb544 100644
--- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
+++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
@@ -628,13 +628,13 @@ class LoopVectorizationLegality {
/// Return true if all of the instructions in the block can be speculatively
/// executed, and record the loads/stores that require masking.
- /// \p SafePtrs records pointers that can be accessed unconditionally,
+ /// \p SafeAccesses records pointers that can be accessed unconditionally,
/// together with the access type and alignment justified for each pointer.
/// \p MaskedOp is a list of instructions that have to be transformed into
/// calls to the appropriate masked intrinsic when the loop is vectorized
/// or dropped if the instruction is a conditional assume intrinsic.
bool
- blockCanBePredicated(BasicBlock *BB, SafeAccessesTy &SafePtrs,
+ blockCanBePredicated(BasicBlock *BB, SafeAccessesTy &SafeAccesses,
SmallPtrSetImpl<const Instruction *> &MaskedOp) const;
/// Updates the vectorization state by adding \p Phi to the inductions list.
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index be02b08274999..fc1bc50e096fb 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -1342,7 +1342,7 @@ bool LoopVectorizationLegality::blockNeedsPredication(
}
bool LoopVectorizationLegality::blockCanBePredicated(
- BasicBlock *BB, SafeAccessesTy &SafePtrs,
+ BasicBlock *BB, SafeAccessesTy &SafeAccesses,
SmallPtrSetImpl<const Instruction *> &MaskedOp) const {
for (Instruction &I : *BB) {
// We can predicate blocks with calls to assume, as long as we drop them in
@@ -1370,8 +1370,8 @@ bool LoopVectorizationLegality::blockCanBePredicated(
// Loads are handled via masking (or speculated if safe to do so.)
if (auto *LI = dyn_cast<LoadInst>(&I)) {
- auto It = SafePtrs.find({LI->getPointerOperand(), LI->getType()});
- if (It == SafePtrs.end() || It->second < LI->getAlign())
+ auto It = SafeAccesses.find({LI->getPointerOperand(), LI->getType()});
+ if (It == SafeAccesses.end() || It->second < LI->getAlign())
MaskedOp.insert(LI);
continue;
}
@@ -1407,14 +1407,14 @@ bool LoopVectorizationLegality::canVectorizeWithIfConvert() {
// the memory pointed to can be dereferenced (with the access size implied by
// the value's type) unconditionally within the loop header without
// introducing a new fault.
- SafeAccessesTy SafePointers;
- auto RecordSafeAccess = [&](Instruction &I) {
+ SafeAccessesTy SafeAccesses;
+ auto RecordSafeAccess = [&SafeAccesses](Instruction &I) {
Value *Ptr = getLoadStorePointerOperand(&I);
if (!Ptr)
return;
Align Alignment = getLoadStoreAlignment(&I);
- auto [It, Inserted] = SafePointers.try_emplace(
- std::make_pair(Ptr, getLoadStoreType(&I)), Alignment);
+ auto [It, Inserted] =
+ SafeAccesses.try_emplace({Ptr, getLoadStoreType(&I)}, Alignment);
if (!Inserted)
It->second = std::max(It->second, Alignment);
};
@@ -1510,7 +1510,7 @@ bool LoopVectorizationLegality::canVectorizeWithIfConvert() {
// We must be able to predicate all blocks that need to be predicated.
if (blockNeedsPredication(BB) &&
- !blockCanBePredicated(BB, SafePointers, ConditionallyExecutedOps)) {
+ !blockCanBePredicated(BB, SafeAccesses, ConditionallyExecutedOps)) {
reportVectorizationFailure(
"Control flow cannot be substituted for a select", "NoCFGForSelect",
ORE, TheLoop, BB->getTerminator());
@@ -2057,13 +2057,13 @@ bool LoopVectorizationLegality::canFoldTailByMasking() const {
LLVM_DEBUG(dbgs() << "LV: checking if tail can be folded by masking.\n");
// The list of pointers that we can safely read and write to remains empty.
- SafeAccessesTy SafePointers;
+ SafeAccessesTy SafeAccesses;
// Check all blocks for predication, including those that ordinarily do not
// need predication such as the header block.
SmallPtrSet<const Instruction *, 8> TmpMaskedOp;
for (BasicBlock *BB : TheLoop->blocks()) {
- if (!blockCanBePredicated(BB, SafePointers, TmpMaskedOp)) {
+ if (!blockCanBePredicated(BB, SafeAccesses, TmpMaskedOp)) {
LLVM_DEBUG(dbgs() << "LV: Cannot fold tail by masking.\n");
return false;
}
@@ -2076,14 +2076,14 @@ bool LoopVectorizationLegality::canFoldTailByMasking() const {
void LoopVectorizationLegality::prepareToFoldTailByMasking() {
// The list of pointers that we can safely read and write to remains empty.
- SafeAccessesTy SafePointers;
+ SafeAccessesTy SafeAccesses;
// Mark all blocks for predication, including those that ordinarily do not
// need predication such as the header block, and collect instructions needing
// predication in TailFoldedMaskedOp.
for (BasicBlock *BB : TheLoop->blocks()) {
[[maybe_unused]] bool R =
- blockCanBePredicated(BB, SafePointers, TailFoldedMaskedOp);
+ blockCanBePredicated(BB, SafeAccesses, TailFoldedMaskedOp);
assert(R && "Must be able to predicate block when tail-folding.");
}
}
More information about the llvm-commits
mailing list