[llvm] [LV] Recognize safe loads through equivalent pointers (PR #228797)
via llvm-commits
llvm-commits at lists.llvm.org
Sat Oct 3 18:13:42 PDT 2026
=?utf-8?q?Dávid_Bolvanský?= <david.bolvansky at gmail.com>,
=?utf-8?q?Dávid_Bolvanský?= <david.bolvansky at gmail.com>
Message-ID:
In-Reply-To: <llvm.org/llvm/llvm-project/pull/228797 at github.com>
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-transforms
Author: Dávid Bolvanský (davidbolvansky)
<details>
<summary>Changes</summary>
Recognize predicated loads whose pointer SCEV matches an unconditional access of the same type and sufficient alignment. Identical GEPs also reuse that access's non-poison guarantee; other equivalent expressions retain the existing speculation checks.
Motivated by LAME's `count_bits` substep loop, where separate identical GEPs prevent unmasked vector loads. On an M1 Pro, the actual LLVM patch reduces full WAV encoding from 7.166 s to 6.017 s (five-round medians, **16.03% less time**), with byte-identical MP3 output.
Implemented with assistance from OpenAI Codex.
---
Patch is 32.24 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/228797.diff
2 Files Affected:
- (modified) llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp (+48-11)
- (added) llvm/test/Transforms/LoopVectorize/AArch64/equivalent-pointers-predication.ll (+470)
``````````diff
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 46f082604b5fb..1b07aa31a6cbb 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -1407,25 +1407,45 @@ bool LoopVectorizationLegality::canVectorizeWithIfConvert() {
// the value's type) unconditionally within the loop header without
// introducing a new fault.
SmallPtrSet<Value *, 8> SafePointers;
+ ScalarEvolution &SE = *PSE.getSE();
+ DenseMap<std::pair<const SCEV *, Type *>, Instruction *> SafeAccesses;
// Collect safe addresses.
for (BasicBlock *BB : TheLoop->blocks()) {
if (!blockNeedsPredication(BB)) {
- for (Instruction &I : *BB)
- if (auto *Ptr = getLoadStorePointerOperand(&I))
- SafePointers.insert(Ptr);
- continue;
+ for (Instruction &I : *BB) {
+ auto *Ptr = getLoadStorePointerOperand(&I);
+ if (!Ptr)
+ continue;
+ SafePointers.insert(Ptr);
+ bool IsSimple = isa<LoadInst>(I) ? cast<LoadInst>(I).isSimple()
+ : cast<StoreInst>(I).isSimple();
+ Type *Ty = getLoadStoreType(&I);
+ if (!IsSimple || Ty->isVectorTy())
+ continue;
+ auto Key = std::make_pair(SE.getSCEV(Ptr), Ty);
+ auto [It, Inserted] = SafeAccesses.try_emplace(Key, &I);
+ if (!Inserted &&
+ getLoadStoreAlignment(&I) > getLoadStoreAlignment(It->second))
+ It->second = &I;
+ }
}
+ }
+
+ for (BasicBlock *BB : TheLoop->blocks()) {
+ if (!blockNeedsPredication(BB))
+ continue;
// For a block which requires predication, a address may be safe to access
// in the loop w/o predication if we can prove dereferenceability facts
// sufficient to ensure it'll never fault within the loop. For the moment,
// we restrict this to loads; stores are more complicated due to
// concurrency restrictions.
- ScalarEvolution &SE = *PSE.getSE();
SmallVector<const SCEVPredicate *, 4> Predicates;
for (Instruction &I : *BB) {
LoadInst *LI = dyn_cast<LoadInst>(&I);
+ if (!LI || SafePointers.contains(LI->getPointerOperand()))
+ continue;
// Make sure we can execute all computations feeding into Ptr in the loop
// w/o triggering UB and that none of the out-of-loop operands are poison.
@@ -1434,7 +1454,14 @@ bool LoopVectorizationLegality::canVectorizeWithIfConvert() {
// because flags will be dropped when executing them unconditionally.
// TODO: Results could be improved by considering poison-propagation
// properties of visited ops.
- auto CanSpeculatePointerOp = [this](Value *Ptr) {
+ auto CanSpeculatePointerOp = [this](Value *Ptr, Value *KnownSafePtr) {
+ auto *GEP = dyn_cast<GetElementPtrInst>(Ptr);
+ auto *SafeGEP = dyn_cast_or_null<GetElementPtrInst>(KnownSafePtr);
+ // Identical operands and flags inherit the unconditional access's
+ // non-poison guarantee, including operands defined outside the loop.
+ if (GEP && SafeGEP && GEP->isIdenticalTo(SafeGEP))
+ return true;
+
SmallVector<Value *> Worklist = {Ptr};
SmallPtrSet<Value *, 4> Visited;
while (!Worklist.empty()) {
@@ -1471,11 +1498,21 @@ bool LoopVectorizationLegality::canVectorizeWithIfConvert() {
// Pass the Predicates pointer to isDereferenceableAndAlignedInLoop so
// that it will consider loops that need guarding by SCEV checks. The
// vectoriser will generate these checks if we decide to vectorise.
- if (LI && !LI->getType()->isVectorTy() && !mustSuppressSpeculation(*LI) &&
- CanSpeculatePointerOp(LI->getPointerOperand()) &&
- isDereferenceableAndAlignedInLoop(LI, TheLoop, SE, *DT, AC,
- &Predicates))
- SafePointers.insert(LI->getPointerOperand());
+ if (!LI->getType()->isVectorTy() && !mustSuppressSpeculation(*LI)) {
+ // Equivalent pointers need not be represented by the same SSA value.
+ // Require the same access type and sufficient alignment.
+ auto It = SafeAccesses.find(
+ {SE.getSCEV(LI->getPointerOperand()), LI->getType()});
+ bool HasSafeAccess =
+ LI->isSimple() && It != SafeAccesses.end() &&
+ getLoadStoreAlignment(It->second) >= LI->getAlign();
+ Value *KnownSafePtr =
+ HasSafeAccess ? getLoadStorePointerOperand(It->second) : nullptr;
+ if (CanSpeculatePointerOp(LI->getPointerOperand(), KnownSafePtr) &&
+ (HasSafeAccess || isDereferenceableAndAlignedInLoop(
+ LI, TheLoop, SE, *DT, AC, &Predicates)))
+ SafePointers.insert(LI->getPointerOperand());
+ }
Predicates.clear();
}
}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/equivalent-pointers-predication.ll b/llvm/test/Transforms/LoopVectorize/AArch64/equivalent-pointers-predication.ll
new file mode 100644
index 0000000000000..5a0ebcdef1080
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/equivalent-pointers-predication.ll
@@ -0,0 +1,470 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -passes=loop-vectorize -mtriple=aarch64-unknown-linux-gnu -force-vector-width=4 -force-vector-interleave=1 -S < %s | FileCheck %s
+
+define void @equivalent_geps(ptr noalias noundef %xr, ptr noalias %ix, i64 %n, float %threshold) {
+; CHECK-LABEL: @equivalent_geps(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[EMPTY:%.*]] = icmp eq i64 [[N:%.*]], 0
+; CHECK-NEXT: br i1 [[EMPTY]], label [[EXIT:%.*]], label [[PREHEADER:%.*]]
+; CHECK: preheader:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[THRESHOLD:%.*]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr float, ptr [[XR:%.*]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = fcmp oge <4 x float> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP28:%.*]] = getelementptr i32, ptr [[IX:%.*]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP25:%.*]] = load <4 x i32>, ptr [[TMP28]], align 4
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP2]], <4 x i32> [[TMP25]], <4 x i32> zeroinitializer
+; CHECK-NEXT: store <4 x i32> [[PREDPHI]], ptr [[TMP28]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP27:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP27]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT_LOOPEXIT:%.*]], label [[SCALAR_PH]]
+; CHECK: scalar.ph:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[PREHEADER]] ]
+; CHECK-NEXT: br label [[LOOP:%.*]]
+; CHECK: loop:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXT:%.*]], [[MERGE:%.*]] ]
+; CHECK-NEXT: [[XP:%.*]] = getelementptr float, ptr [[XR]], i64 [[IV]]
+; CHECK-NEXT: [[X:%.*]] = load float, ptr [[XP]], align 4
+; CHECK-NEXT: [[KEEP:%.*]] = fcmp oge float [[X]], [[THRESHOLD]]
+; CHECK-NEXT: br i1 [[KEEP]], label [[READ:%.*]], label [[MERGE]]
+; CHECK: read:
+; CHECK-NEXT: [[LOAD_PTR:%.*]] = getelementptr i32, ptr [[IX]], i64 [[IV]]
+; CHECK-NEXT: [[OLD:%.*]] = load i32, ptr [[LOAD_PTR]], align 4
+; CHECK-NEXT: br label [[MERGE]]
+; CHECK: merge:
+; CHECK-NEXT: [[V:%.*]] = phi i32 [ [[OLD]], [[READ]] ], [ 0, [[LOOP]] ]
+; CHECK-NEXT: [[STORE_PTR:%.*]] = getelementptr i32, ptr [[IX]], i64 [[IV]]
+; CHECK-NEXT: store i32 [[V]], ptr [[STORE_PTR]], align 4
+; CHECK-NEXT: [[NEXT]] = add nuw i64 [[IV]], 1
+; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[DONE]], label [[EXIT_LOOPEXIT]], label [[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: exit.loopexit:
+; CHECK-NEXT: br label [[EXIT]]
+; CHECK: exit:
+; CHECK-NEXT: ret void
+;
+entry:
+ %empty = icmp eq i64 %n, 0
+ br i1 %empty, label %exit, label %preheader
+preheader:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %preheader ], [ %next, %merge ]
+ %xp = getelementptr float, ptr %xr, i64 %iv
+ %x = load float, ptr %xp, align 4
+ %keep = fcmp oge float %x, %threshold
+ br i1 %keep, label %read, label %merge
+read:
+ %load.ptr = getelementptr i32, ptr %ix, i64 %iv
+ %old = load i32, ptr %load.ptr, align 4
+ br label %merge
+merge:
+ %v = phi i32 [ %old, %read ], [ 0, %loop ]
+ %store.ptr = getelementptr i32, ptr %ix, i64 %iv
+ store i32 %v, ptr %store.ptr, align 4
+ %next = add nuw i64 %iv, 1
+ %done = icmp eq i64 %next, %n
+ br i1 %done, label %exit, label %loop
+exit:
+ ret void
+}
+
+define void @stronger_load_alignment(ptr noalias noundef %xr, ptr noalias noundef %ix, i64 %n, float %threshold) {
+; CHECK-LABEL: @stronger_load_alignment(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[EMPTY:%.*]] = icmp eq i64 [[N:%.*]], 0
+; CHECK-NEXT: br i1 [[EMPTY]], label [[EXIT:%.*]], label [[PREHEADER:%.*]]
+; CHECK: preheader:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[THRESHOLD:%.*]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE6:%.*]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr float, ptr [[XR:%.*]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = fcmp oge <4 x float> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i1> [[TMP2]], i64 0
+; CHECK-NEXT: br i1 [[TMP3]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
+; CHECK: pred.load.if:
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i32, ptr [[IX:%.*]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr [[TMP4]], align 16
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x i32> poison, i32 [[TMP5]], i64 0
+; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE]]
+; CHECK: pred.load.continue:
+; CHECK-NEXT: [[TMP7:%.*]] = phi <4 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP6]], [[PRED_LOAD_IF]] ]
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i1> [[TMP2]], i64 1
+; CHECK-NEXT: br i1 [[TMP8]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2:%.*]]
+; CHECK: pred.load.if1:
+; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i32, ptr [[IX]], i64 [[TMP9]]
+; CHECK-NEXT: [[TMP11:%.*]] = load i32, ptr [[TMP10]], align 16
+; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x i32> [[TMP7]], i32 [[TMP11]], i64 1
+; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE2]]
+; CHECK: pred.load.continue2:
+; CHECK-NEXT: [[TMP13:%.*]] = phi <4 x i32> [ [[TMP7]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP12]], [[PRED_LOAD_IF1]] ]
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x i1> [[TMP2]], i64 2
+; CHECK-NEXT: br i1 [[TMP14]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]]
+; CHECK: pred.load.if3:
+; CHECK-NEXT: [[TMP15:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP16:%.*]] = getelementptr i32, ptr [[IX]], i64 [[TMP15]]
+; CHECK-NEXT: [[TMP17:%.*]] = load i32, ptr [[TMP16]], align 16
+; CHECK-NEXT: [[TMP18:%.*]] = insertelement <4 x i32> [[TMP13]], i32 [[TMP17]], i64 2
+; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE4]]
+; CHECK: pred.load.continue4:
+; CHECK-NEXT: [[TMP19:%.*]] = phi <4 x i32> [ [[TMP13]], [[PRED_LOAD_CONTINUE2]] ], [ [[TMP18]], [[PRED_LOAD_IF3]] ]
+; CHECK-NEXT: [[TMP20:%.*]] = extractelement <4 x i1> [[TMP2]], i64 3
+; CHECK-NEXT: br i1 [[TMP20]], label [[PRED_LOAD_IF5:%.*]], label [[PRED_LOAD_CONTINUE6]]
+; CHECK: pred.load.if5:
+; CHECK-NEXT: [[TMP21:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT: [[TMP22:%.*]] = getelementptr i32, ptr [[IX]], i64 [[TMP21]]
+; CHECK-NEXT: [[TMP23:%.*]] = load i32, ptr [[TMP22]], align 16
+; CHECK-NEXT: [[TMP24:%.*]] = insertelement <4 x i32> [[TMP19]], i32 [[TMP23]], i64 3
+; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE6]]
+; CHECK: pred.load.continue6:
+; CHECK-NEXT: [[TMP25:%.*]] = phi <4 x i32> [ [[TMP19]], [[PRED_LOAD_CONTINUE4]] ], [ [[TMP24]], [[PRED_LOAD_IF5]] ]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP2]], <4 x i32> [[TMP25]], <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP26:%.*]] = getelementptr i32, ptr [[IX]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[PREDPHI]], ptr [[TMP26]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP27:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP27]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT_LOOPEXIT:%.*]], label [[SCALAR_PH]]
+; CHECK: scalar.ph:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[PREHEADER]] ]
+; CHECK-NEXT: br label [[LOOP:%.*]]
+; CHECK: loop:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXT:%.*]], [[MERGE:%.*]] ]
+; CHECK-NEXT: [[XP:%.*]] = getelementptr float, ptr [[XR]], i64 [[IV]]
+; CHECK-NEXT: [[X:%.*]] = load float, ptr [[XP]], align 4
+; CHECK-NEXT: [[KEEP:%.*]] = fcmp oge float [[X]], [[THRESHOLD]]
+; CHECK-NEXT: br i1 [[KEEP]], label [[READ:%.*]], label [[MERGE]]
+; CHECK: read:
+; CHECK-NEXT: [[LOAD_PTR:%.*]] = getelementptr i32, ptr [[IX]], i64 [[IV]]
+; CHECK-NEXT: [[OLD:%.*]] = load i32, ptr [[LOAD_PTR]], align 16
+; CHECK-NEXT: br label [[MERGE]]
+; CHECK: merge:
+; CHECK-NEXT: [[V:%.*]] = phi i32 [ [[OLD]], [[READ]] ], [ 0, [[LOOP]] ]
+; CHECK-NEXT: [[STORE_PTR:%.*]] = getelementptr i32, ptr [[IX]], i64 [[IV]]
+; CHECK-NEXT: store i32 [[V]], ptr [[STORE_PTR]], align 4
+; CHECK-NEXT: [[NEXT]] = add nuw i64 [[IV]], 1
+; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[DONE]], label [[EXIT_LOOPEXIT]], label [[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK: exit.loopexit:
+; CHECK-NEXT: br label [[EXIT]]
+; CHECK: exit:
+; CHECK-NEXT: ret void
+;
+entry:
+ %empty = icmp eq i64 %n, 0
+ br i1 %empty, label %exit, label %preheader
+preheader:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %preheader ], [ %next, %merge ]
+ %xp = getelementptr float, ptr %xr, i64 %iv
+ %x = load float, ptr %xp, align 4
+ %keep = fcmp oge float %x, %threshold
+ br i1 %keep, label %read, label %merge
+read:
+ %load.ptr = getelementptr i32, ptr %ix, i64 %iv
+ %old = load i32, ptr %load.ptr, align 16
+ br label %merge
+merge:
+ %v = phi i32 [ %old, %read ], [ 0, %loop ]
+ %store.ptr = getelementptr i32, ptr %ix, i64 %iv
+ store i32 %v, ptr %store.ptr, align 4
+ %next = add nuw i64 %iv, 1
+ %done = icmp eq i64 %next, %n
+ br i1 %done, label %exit, label %loop
+exit:
+ ret void
+}
+
+define void @narrower_store(ptr noalias noundef %xr, ptr noalias noundef %ix, i64 %n, float %threshold) {
+; CHECK-LABEL: @narrower_store(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[EMPTY:%.*]] = icmp eq i64 [[N:%.*]], 0
+; CHECK-NEXT: br i1 [[EMPTY]], label [[EXIT:%.*]], label [[PREHEADER:%.*]]
+; CHECK: preheader:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]]
+; CHECK: vector.scevcheck:
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], -1
+; CHECK-NEXT: [[MUL:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 4, i64 [[TMP0]])
+; CHECK-NEXT: [[MUL_RESULT:%.*]] = extractvalue { i64, i1 } [[MUL]], 0
+; CHECK-NEXT: [[MUL_OVERFLOW:%.*]] = extractvalue { i64, i1 } [[MUL]], 1
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[IX:%.*]], i64 [[MUL_RESULT]]
+; CHECK-NEXT: [[TMP2:%.*]] = icmp ult ptr [[TMP1]], [[IX]]
+; CHECK-NEXT: [[TMP3:%.*]] = or i1 [[TMP2]], [[MUL_OVERFLOW]]
+; CHECK-NEXT: br i1 [[TMP3]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[TMP4:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP4]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[THRESHOLD:%.*]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE6:%.*]] ]
+; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr float, ptr [[XR:%.*]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP8]], align 4
+; CHECK-NEXT: [[TMP9:%.*]] = fcmp oge <4 x float> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i1> [[TMP9]], i64 0
+; CHECK-NEXT: br i1 [[TMP10]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
+; CHECK: pred.load.if:
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i32, ptr [[IX]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP11]], align 4
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> poison, i32 [[TMP12]], i64 0
+; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE]]
+; CHECK: pred.load.continue:
+; CHECK-NEXT: [[TMP14:%.*]] = phi <4 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP13]], [[PRED_LOAD_IF]] ]
+; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x i1> [[TMP9]], i64 1
+; CHECK-NEXT: br i1 [[TMP15]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2:%.*]]
+; CHECK: pred.load.if1:
+; CHECK-NEXT: [[TMP16:%.*]] = getelementptr i32, ptr [[IX]], i64 [[TMP5]]
+; CHECK-NEXT: [[TMP17:%.*]] = load i32, ptr [[TMP16]], align 4
+; CHECK-NEXT: [[TMP18:%.*]] = insertelement <4 x i32> [[TMP14]], i32 [[TMP17]], i64 1
+; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE2]]
+; CHECK: pred.load.continue2:
+; CHECK-NEXT: [[TMP19:%.*]] = phi <4 x i32> [ [[TMP14]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP18]], [[PRED_LOAD_IF1]] ]
+; CHECK-NEXT: [[TMP20:%.*]] = extractelement <4 x i1> [[TMP9]], i64 2
+; CHECK-NEXT: br i1 [[TMP20]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]]
+; CHECK: pred.load.if3:
+; CHECK-NEXT: [[TMP21:%.*]] = getelementptr i32, ptr [[IX]], i64 [[TMP6]]
+; CHECK-NEXT: [[TMP22:%.*]] = load i32, ptr [[TMP21]], align 4
+; CHECK-NEXT: [[TMP23:%.*]] = insertelement <4 x i32> [[TMP19]], i32 [[TMP22]], i64 2
+; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE4]]
+; CHECK: pred.load.continue4:
+; CHECK-NEXT: [[TMP24:%.*]] = ...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/228797
More information about the llvm-commits
mailing list