[llvm] [LV] Respect access type and alignment when speculating loads (PR #229379)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Oct 6 04:05:40 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-vectorizers
Author: Dávid Bolvanský (davidbolvansky)
<details>
<summary>Changes</summary>
An unconditional access currently marks its pointer as safe for any predicated load, regardless of the load's type or alignment. This can speculate a wider load past the end of an object, or a load requiring stronger alignment than the unconditional access provides.
Track the access type and maximum known safe alignment along with the pointer. Add regression tests for both cases and a positive case with sufficient alignment.
Split from #<!-- -->228797 following review. Implemented with assistance from OpenAI Codex.
The `wider_predicated_load` test also reproduces a crash with this harness. The loop reads bytes at offsets 0, 8, 16 and 24, and only reads an `i64` at that address when the byte is nonzero. All bytes below are zero, so the scalar program never performs the wider load.
```c
#include <sys/mman.h>
#include <unistd.h>
#include <stdint.h>
#include <stdio.h>
// Generated from wider_predicated_load in the LLVM regression test.
// It reads one byte per iteration, and reads an i64 only if that byte is nonzero.
extern void wider_predicated_load(void *, void *, uint64_t);
int main(void) {
enum { Iterations = 4, Stride = 8,
AccessibleBytes = (Iterations - 1) * Stride + 1 }; // 25 bytes
long size = sysconf(_SC_PAGESIZE);
if (size <= 0) return 2;
// Anonymous mappings start zeroed. Protect the second page so that an
// out-of-bounds read fails deterministically instead of reading adjacent data.
char *p = mmap(0, size * 2, PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANON, -1, 0);
if (p == MAP_FAILED || mprotect(p + size, size, PROT_NONE)) return 2;
// Place the last accessible byte immediately before the protected page.
// The byte loads at offsets 0, 8, 16 and 24 are all valid and return zero,
// so none of the conditional i64 loads should execute.
// Speculating the last i64 load reads offsets 24..31: seven protected bytes.
int64_t out[Iterations] = {1, 1, 1, 1};
wider_predicated_load(p + size - AccessibleBytes, out, Iterations);
for (int i = 0; i < Iterations; i++) if (out[i]) return 3;
munmap(p, size * 2);
puts("PASS: no out-of-bounds speculative load");
return 0;
}
```
On Apple Silicon, linking this harness with the vectorized test produces SIGBUS before the fix and prints `PASS` after it. Both versions use the same backend; only the vectorizer legality change differs.
---
Patch is 26.06 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/229379.diff
3 Files Affected:
- (modified) llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h (+8-3)
- (modified) llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp (+21-10)
- (added) llvm/test/Transforms/LoopVectorize/predicated-load-access-safety.ll (+369)
``````````diff
diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
index 6dd44f8a71538..8011424bf8f5e 100644
--- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
+++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
@@ -26,8 +26,10 @@
#ifndef LLVM_TRANSFORMS_VECTORIZE_LOOPVECTORIZATIONLEGALITY_H
#define LLVM_TRANSFORMS_VECTORIZE_LOOPVECTORIZATIONLEGALITY_H
+#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/MapVector.h"
#include "llvm/Analysis/LoopAccessAnalysis.h"
+#include "llvm/Support/Alignment.h"
#include "llvm/Support/TypeSize.h"
#include "llvm/Transforms/Utils/LoopUtils.h"
@@ -621,15 +623,18 @@ class LoopVectorizationLegality {
/// for it.
bool canUncountableExitConditionLoadBeMoved(BasicBlock *ExitingBlock);
+ /// Maps a pointer and access type to its known safe alignment.
+ using SafeAccessesTy = DenseMap<std::pair<Value *, Type *>, Align>;
+
/// Return true if all of the instructions in the block can be speculatively
/// executed, and record the loads/stores that require masking.
- /// \p SafePtrs is a list of addresses that are known to be legal and we know
- /// that we can read from them without segfault.
+ /// \p SafePtrs records pointers that can be accessed unconditionally,
+ /// together with the access type and alignment justified for each pointer.
/// \p MaskedOp is a list of instructions that have to be transformed into
/// calls to the appropriate masked intrinsic when the loop is vectorized
/// or dropped if the instruction is a conditional assume intrinsic.
bool
- blockCanBePredicated(BasicBlock *BB, SmallPtrSetImpl<Value *> &SafePtrs,
+ blockCanBePredicated(BasicBlock *BB, SafeAccessesTy &SafePtrs,
SmallPtrSetImpl<const Instruction *> &MaskedOp) const;
/// Updates the vectorization state by adding \p Phi to the inductions list.
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 46f082604b5fb..f8239da6cf18c 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -1342,7 +1342,7 @@ bool LoopVectorizationLegality::blockNeedsPredication(
}
bool LoopVectorizationLegality::blockCanBePredicated(
- BasicBlock *BB, SmallPtrSetImpl<Value *> &SafePtrs,
+ BasicBlock *BB, SafeAccessesTy &SafePtrs,
SmallPtrSetImpl<const Instruction *> &MaskedOp) const {
for (Instruction &I : *BB) {
// We can predicate blocks with calls to assume, as long as we drop them in
@@ -1370,7 +1370,8 @@ bool LoopVectorizationLegality::blockCanBePredicated(
// Loads are handled via masking (or speculated if safe to do so.)
if (auto *LI = dyn_cast<LoadInst>(&I)) {
- if (!SafePtrs.count(LI->getPointerOperand()))
+ auto It = SafePtrs.find({LI->getPointerOperand(), LI->getType()});
+ if (It == SafePtrs.end() || It->second < LI->getAlign())
MaskedOp.insert(LI);
continue;
}
@@ -1406,14 +1407,20 @@ bool LoopVectorizationLegality::canVectorizeWithIfConvert() {
// the memory pointed to can be dereferenced (with the access size implied by
// the value's type) unconditionally within the loop header without
// introducing a new fault.
- SmallPtrSet<Value *, 8> SafePointers;
+ SafeAccessesTy SafePointers;
// Collect safe addresses.
for (BasicBlock *BB : TheLoop->blocks()) {
if (!blockNeedsPredication(BB)) {
- for (Instruction &I : *BB)
- if (auto *Ptr = getLoadStorePointerOperand(&I))
- SafePointers.insert(Ptr);
+ for (Instruction &I : *BB) {
+ if (auto *Ptr = getLoadStorePointerOperand(&I)) {
+ auto [It, Inserted] = SafePointers.try_emplace(
+ std::make_pair(Ptr, getLoadStoreType(&I)),
+ getLoadStoreAlignment(&I));
+ if (!Inserted)
+ It->second = std::max(It->second, getLoadStoreAlignment(&I));
+ }
+ }
continue;
}
@@ -1474,8 +1481,12 @@ bool LoopVectorizationLegality::canVectorizeWithIfConvert() {
if (LI && !LI->getType()->isVectorTy() && !mustSuppressSpeculation(*LI) &&
CanSpeculatePointerOp(LI->getPointerOperand()) &&
isDereferenceableAndAlignedInLoop(LI, TheLoop, SE, *DT, AC,
- &Predicates))
- SafePointers.insert(LI->getPointerOperand());
+ &Predicates)) {
+ auto [It, Inserted] = SafePointers.try_emplace(
+ std::make_pair(LI->getPointerOperand(), LI->getType()), LI->getAlign());
+ if (!Inserted)
+ It->second = std::max(It->second, LI->getAlign());
+ }
Predicates.clear();
}
}
@@ -2047,7 +2058,7 @@ bool LoopVectorizationLegality::canFoldTailByMasking() const {
LLVM_DEBUG(dbgs() << "LV: checking if tail can be folded by masking.\n");
// The list of pointers that we can safely read and write to remains empty.
- SmallPtrSet<Value *, 8> SafePointers;
+ SafeAccessesTy SafePointers;
// Check all blocks for predication, including those that ordinarily do not
// need predication such as the header block.
@@ -2066,7 +2077,7 @@ bool LoopVectorizationLegality::canFoldTailByMasking() const {
void LoopVectorizationLegality::prepareToFoldTailByMasking() {
// The list of pointers that we can safely read and write to remains empty.
- SmallPtrSet<Value *, 8> SafePointers;
+ SafeAccessesTy SafePointers;
// Mark all blocks for predication, including those that ordinarily do not
// need predication such as the header block, and collect instructions needing
diff --git a/llvm/test/Transforms/LoopVectorize/predicated-load-access-safety.ll b/llvm/test/Transforms/LoopVectorize/predicated-load-access-safety.ll
new file mode 100644
index 0000000000000..51b945b58ba5b
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/predicated-load-access-safety.ll
@@ -0,0 +1,369 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes=loop-vectorize -mtriple=aarch64-unknown-linux-gnu -force-vector-width=4 -force-vector-interleave=1 -S < %s | FileCheck %s
+
+; A byte load does not make a wider load at the same pointer safe.
+define void @wider_predicated_load(ptr noalias noundef %src, ptr noalias %dst, i64 %n) {
+; CHECK-LABEL: define void @wider_predicated_load(
+; CHECK-SAME: ptr noalias noundef [[SRC:%.*]], ptr noalias [[DST:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[EMPTY:%.*]] = icmp eq i64 [[N]], 0
+; CHECK-NEXT: br i1 [[EMPTY]], label %[[EXIT:.*]], label %[[PREHEADER:.*]]
+; CHECK: [[PREHEADER]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE6:.*]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[TMP1]]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[TMP3]]
+; CHECK-NEXT: [[TMP8:%.*]] = load i8, ptr [[TMP4]], align 1
+; CHECK-NEXT: [[TMP9:%.*]] = load i8, ptr [[TMP5]], align 1
+; CHECK-NEXT: [[TMP10:%.*]] = load i8, ptr [[TMP6]], align 1
+; CHECK-NEXT: [[TMP11:%.*]] = load i8, ptr [[TMP7]], align 1
+; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x i8> poison, i8 [[TMP8]], i64 0
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i8> [[TMP12]], i8 [[TMP9]], i64 1
+; CHECK-NEXT: [[TMP14:%.*]] = insertelement <4 x i8> [[TMP13]], i8 [[TMP10]], i64 2
+; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x i8> [[TMP14]], i8 [[TMP11]], i64 3
+; CHECK-NEXT: [[TMP16:%.*]] = icmp ne <4 x i8> [[TMP15]], zeroinitializer
+; CHECK-NEXT: [[TMP17:%.*]] = extractelement <4 x i1> [[TMP16]], i64 0
+; CHECK-NEXT: br i1 [[TMP17]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
+; CHECK: [[PRED_LOAD_IF]]:
+; CHECK-NEXT: [[TMP18:%.*]] = load i64, ptr [[TMP4]], align 1
+; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x i64> poison, i64 [[TMP18]], i64 0
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE]]
+; CHECK: [[PRED_LOAD_CONTINUE]]:
+; CHECK-NEXT: [[TMP20:%.*]] = phi <4 x i64> [ poison, %[[VECTOR_BODY]] ], [ [[TMP19]], %[[PRED_LOAD_IF]] ]
+; CHECK-NEXT: [[TMP21:%.*]] = extractelement <4 x i1> [[TMP16]], i64 1
+; CHECK-NEXT: br i1 [[TMP21]], label %[[PRED_LOAD_IF1:.*]], label %[[PRED_LOAD_CONTINUE2:.*]]
+; CHECK: [[PRED_LOAD_IF1]]:
+; CHECK-NEXT: [[TMP22:%.*]] = load i64, ptr [[TMP5]], align 1
+; CHECK-NEXT: [[TMP23:%.*]] = insertelement <4 x i64> [[TMP20]], i64 [[TMP22]], i64 1
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE2]]
+; CHECK: [[PRED_LOAD_CONTINUE2]]:
+; CHECK-NEXT: [[TMP24:%.*]] = phi <4 x i64> [ [[TMP20]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP23]], %[[PRED_LOAD_IF1]] ]
+; CHECK-NEXT: [[TMP25:%.*]] = extractelement <4 x i1> [[TMP16]], i64 2
+; CHECK-NEXT: br i1 [[TMP25]], label %[[PRED_LOAD_IF3:.*]], label %[[PRED_LOAD_CONTINUE4:.*]]
+; CHECK: [[PRED_LOAD_IF3]]:
+; CHECK-NEXT: [[TMP26:%.*]] = load i64, ptr [[TMP6]], align 1
+; CHECK-NEXT: [[TMP27:%.*]] = insertelement <4 x i64> [[TMP24]], i64 [[TMP26]], i64 2
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE4]]
+; CHECK: [[PRED_LOAD_CONTINUE4]]:
+; CHECK-NEXT: [[TMP28:%.*]] = phi <4 x i64> [ [[TMP24]], %[[PRED_LOAD_CONTINUE2]] ], [ [[TMP27]], %[[PRED_LOAD_IF3]] ]
+; CHECK-NEXT: [[TMP29:%.*]] = extractelement <4 x i1> [[TMP16]], i64 3
+; CHECK-NEXT: br i1 [[TMP29]], label %[[PRED_LOAD_IF5:.*]], label %[[PRED_LOAD_CONTINUE6]]
+; CHECK: [[PRED_LOAD_IF5]]:
+; CHECK-NEXT: [[TMP30:%.*]] = load i64, ptr [[TMP7]], align 1
+; CHECK-NEXT: [[TMP31:%.*]] = insertelement <4 x i64> [[TMP28]], i64 [[TMP30]], i64 3
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE6]]
+; CHECK: [[PRED_LOAD_CONTINUE6]]:
+; CHECK-NEXT: [[TMP32:%.*]] = phi <4 x i64> [ [[TMP28]], %[[PRED_LOAD_CONTINUE4]] ], [ [[TMP31]], %[[PRED_LOAD_IF5]] ]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP16]], <4 x i64> [[TMP32]], <4 x i64> zeroinitializer
+; CHECK-NEXT: [[TMP33:%.*]] = getelementptr i64, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i64> [[PREDPHI]], ptr [[TMP33]], align 1
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP34:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP34]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[PREHEADER]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[NEXT:%.*]], %[[MERGE:.*]] ]
+; CHECK-NEXT: [[P:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[GUARD:%.*]] = load i8, ptr [[P]], align 1
+; CHECK-NEXT: [[KEEP:%.*]] = icmp ne i8 [[GUARD]], 0
+; CHECK-NEXT: br i1 [[KEEP]], label %[[READ:.*]], label %[[MERGE]]
+; CHECK: [[READ]]:
+; CHECK-NEXT: [[VALUE:%.*]] = load i64, ptr [[P]], align 1
+; CHECK-NEXT: br label %[[MERGE]]
+; CHECK: [[MERGE]]:
+; CHECK-NEXT: [[RESULT:%.*]] = phi i64 [ [[VALUE]], %[[READ]] ], [ 0, %[[LOOP]] ]
+; CHECK-NEXT: [[OUT:%.*]] = getelementptr i64, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT: store i64 [[RESULT]], ptr [[OUT]], align 1
+; CHECK-NEXT: [[NEXT]] = add nuw i64 [[IV]], 1
+; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT_LOOPEXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[EXIT_LOOPEXIT]]:
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ %empty = icmp eq i64 %n, 0
+ br i1 %empty, label %exit, label %preheader
+
+preheader:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %preheader ], [ %next, %merge ]
+ %p = getelementptr i64, ptr %src, i64 %iv
+ %guard = load i8, ptr %p, align 1
+ %keep = icmp ne i8 %guard, 0
+ br i1 %keep, label %read, label %merge
+
+read:
+ %value = load i64, ptr %p, align 1
+ br label %merge
+
+merge:
+ %result = phi i64 [ %value, %read ], [ 0, %loop ]
+ %out = getelementptr i64, ptr %dst, i64 %iv
+ store i64 %result, ptr %out, align 1
+ %next = add nuw i64 %iv, 1
+ %done = icmp eq i64 %next, %n
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; A weaker alignment does not justify the predicated load alignment.
+define void @stronger_predicated_alignment(ptr noalias noundef %src, ptr noalias %dst, i64 %n) {
+; CHECK-LABEL: define void @stronger_predicated_alignment(
+; CHECK-SAME: ptr noalias noundef [[SRC:%.*]], ptr noalias [[DST:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[EMPTY:%.*]] = icmp eq i64 [[N]], 0
+; CHECK-NEXT: br i1 [[EMPTY]], label %[[EXIT:.*]], label %[[PREHEADER:.*]]
+; CHECK: [[PREHEADER]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE6:.*]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr [4 x i32], ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr [4 x i32], ptr [[SRC]], i64 [[TMP1]]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr [4 x i32], ptr [[SRC]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr [4 x i32], ptr [[SRC]], i64 [[TMP3]]
+; CHECK-NEXT: [[TMP8:%.*]] = load i32, ptr [[TMP4]], align 4
+; CHECK-NEXT: [[TMP9:%.*]] = load i32, ptr [[TMP5]], align 4
+; CHECK-NEXT: [[TMP10:%.*]] = load i32, ptr [[TMP6]], align 4
+; CHECK-NEXT: [[TMP11:%.*]] = load i32, ptr [[TMP7]], align 4
+; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x i32> poison, i32 [[TMP8]], i64 0
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP9]], i64 1
+; CHECK-NEXT: [[TMP14:%.*]] = insertelement <4 x i32> [[TMP13]], i32 [[TMP10]], i64 2
+; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x i32> [[TMP14]], i32 [[TMP11]], i64 3
+; CHECK-NEXT: [[TMP16:%.*]] = icmp ne <4 x i32> [[TMP15]], zeroinitializer
+; CHECK-NEXT: [[TMP17:%.*]] = extractelement <4 x i1> [[TMP16]], i64 0
+; CHECK-NEXT: br i1 [[TMP17]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
+; CHECK: [[PRED_LOAD_IF]]:
+; CHECK-NEXT: [[TMP18:%.*]] = load i32, ptr [[TMP4]], align 16
+; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x i32> poison, i32 [[TMP18]], i64 0
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE]]
+; CHECK: [[PRED_LOAD_CONTINUE]]:
+; CHECK-NEXT: [[TMP20:%.*]] = phi <4 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP19]], %[[PRED_LOAD_IF]] ]
+; CHECK-NEXT: [[TMP21:%.*]] = extractelement <4 x i1> [[TMP16]], i64 1
+; CHECK-NEXT: br i1 [[TMP21]], label %[[PRED_LOAD_IF1:.*]], label %[[PRED_LOAD_CONTINUE2:.*]]
+; CHECK: [[PRED_LOAD_IF1]]:
+; CHECK-NEXT: [[TMP22:%.*]] = load i32, ptr [[TMP5]], align 16
+; CHECK-NEXT: [[TMP23:%.*]] = insertelement <4 x i32> [[TMP20]], i32 [[TMP22]], i64 1
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE2]]
+; CHECK: [[PRED_LOAD_CONTINUE2]]:
+; CHECK-NEXT: [[TMP24:%.*]] = phi <4 x i32> [ [[TMP20]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP23]], %[[PRED_LOAD_IF1]] ]
+; CHECK-NEXT: [[TMP25:%.*]] = extractelement <4 x i1> [[TMP16]], i64 2
+; CHECK-NEXT: br i1 [[TMP25]], label %[[PRED_LOAD_IF3:.*]], label %[[PRED_LOAD_CONTINUE4:.*]]
+; CHECK: [[PRED_LOAD_IF3]]:
+; CHECK-NEXT: [[TMP26:%.*]] = load i32, ptr [[TMP6]], align 16
+; CHECK-NEXT: [[TMP27:%.*]] = insertelement <4 x i32> [[TMP24]], i32 [[TMP26]], i64 2
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE4]]
+; CHECK: [[PRED_LOAD_CONTINUE4]]:
+; CHECK-NEXT: [[TMP28:%.*]] = phi <4 x i32> [ [[TMP24]], %[[PRED_LOAD_CONTINUE2]] ], [ [[TMP27]], %[[PRED_LOAD_IF3]] ]
+; CHECK-NEXT: [[TMP29:%.*]] = extractelement <4 x i1> [[TMP16]], i64 3
+; CHECK-NEXT: br i1 [[TMP29]], label %[[PRED_LOAD_IF5:.*]], label %[[PRED_LOAD_CONTINUE6]]
+; CHECK: [[PRED_LOAD_IF5]]:
+; CHECK-NEXT: [[TMP30:%.*]] = load i32, ptr [[TMP7]], align 16
+; CHECK-NEXT: [[TMP31:%.*]] = insertelement <4 x i32> [[TMP28]], i32 [[TMP30]], i64 3
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE6]]
+; CHECK: [[PRED_LOAD_CONTINUE6]]:
+; CHECK-NEXT: [[TMP32:%.*]] = phi <4 x i32> [ [[TMP28]], %[[PRED_LOAD_CONTINUE4]] ], [ [[TMP31]], %[[PRED_LOAD_IF5]] ]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP16]], <4 x i32> [[TMP32]], <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP33:%.*]] = getelementptr i32, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[PREDPHI]], ptr [[TMP33]], align 1
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP34:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP34]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[PREHEADER]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[NEXT:%.*]], %[[MERGE:.*]] ]
+; CHECK-NEXT: [[P:%.*]] = getelementptr [4 x i32], ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[GUARD:%.*]] = load i32, ptr [[P]], align 4
+; CHECK-NEXT: [[KEEP:%.*]] = icmp ne i32 [[GUARD]], 0
+; CHECK-NEXT: br i1 [[KEEP]], label %[[READ:.*]], label %[[MERGE]]
+; CHECK: [[READ]]:
+; CHECK-NEXT: [[VALUE:%.*]] = load i32, ptr [[P]], align 16
+; CHECK-NEXT: br label %[[MERGE]]
+; CHECK: [[MERGE]]:
+; CHECK-NEXT: [[RESULT:%.*]] = phi i32 [ [[VALUE]], %[[READ]] ], [ 0, %[[LOOP]] ]
+; CHECK-NEXT: [[OUT:%.*]] = getelementptr i32, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT: store i32 [[RESULT]], ptr [[OUT]], align 1
+; CHECK-NEXT: [[NEXT]] = add nuw i64 [[IV]], 1
+; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT_LOOPEXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK: [[EXIT_LOOPEXIT]]:
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ %empty = icmp eq i64 %n, 0
+ br i1 %empty, label %exit, label %preheader
+
+preheader:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %preheader ], [ %next, %merge ]
+ %p = getelementptr [4 x i32], ptr %src, i64 %iv
+ %guard = load i32, ptr %p, align 4
+ %keep = icmp ne i32 %guard, 0
+ br i1 %keep, label %read, label %merge
+
+read:
+ %value = load i32, ptr %p, align 16
+ br label %merge
+
+merge:
+ %result = phi i32 [ %value, %read ], [ 0, %loop ]
+ %out = getelementptr i32, ptr %dst, i64 %i...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/229379
More information about the llvm-commits
mailing list