[llvm] [LV] Recognize safe loads through equivalent pointers (PR #228797)
Dávid Bolvanský via llvm-commits
llvm-commits at lists.llvm.org
Sat Oct 3 18:10:35 PDT 2026
https://github.com/davidbolvansky created https://github.com/llvm/llvm-project/pull/228797
Recognize predicated loads whose pointer SCEV matches an unconditional access of the same type and sufficient alignment. Identical GEPs also reuse that access's non-poison guarantee; other equivalent expressions retain the existing speculation checks.
Motivated by LAME's `count_bits` substep loop, where separate identical GEPs prevent unmasked vector loads. On an M1 Pro, the actual LLVM patch reduces full WAV encoding from 7.166 s to 6.017 s (five-round medians, **16.03% less time**), with byte-identical MP3 output.
Draft pending correctness review and compile-time measurements. Implemented with assistance from OpenAI Codex.
>From 9378c08cae51e55c7aebc1ebcd1c61126b49b421 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?D=C3=A1vid=20Bolvansk=C3=BD?= <david.bolvansky at gmail.com>
Date: Sun, 4 Oct 2026 02:55:08 +0200
Subject: [PATCH 1/3] [LV] Add tests for predicated loads through equivalent
pointers
---
.../equivalent-pointers-predication.ll | 379 ++++++++++++++++++
1 file changed, 379 insertions(+)
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/equivalent-pointers-predication.ll
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/equivalent-pointers-predication.ll b/llvm/test/Transforms/LoopVectorize/AArch64/equivalent-pointers-predication.ll
new file mode 100644
index 0000000000000..1fe0db86e508b
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/equivalent-pointers-predication.ll
@@ -0,0 +1,379 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -passes=loop-vectorize -mtriple=aarch64-unknown-linux-gnu -force-vector-width=4 -force-vector-interleave=1 -S < %s | FileCheck %s
+
+define void @equivalent_geps(ptr noalias noundef %xr, ptr noalias noundef %ix, i64 %n, float %threshold) {
+; CHECK-LABEL: @equivalent_geps(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[EMPTY:%.*]] = icmp eq i64 [[N:%.*]], 0
+; CHECK-NEXT: br i1 [[EMPTY]], label [[EXIT:%.*]], label [[PREHEADER:%.*]]
+; CHECK: preheader:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[THRESHOLD:%.*]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE6:%.*]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr float, ptr [[XR:%.*]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = fcmp oge <4 x float> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i1> [[TMP2]], i64 0
+; CHECK-NEXT: br i1 [[TMP3]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
+; CHECK: pred.load.if:
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i32, ptr [[IX:%.*]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr [[TMP4]], align 4
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x i32> poison, i32 [[TMP5]], i64 0
+; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE]]
+; CHECK: pred.load.continue:
+; CHECK-NEXT: [[TMP7:%.*]] = phi <4 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP6]], [[PRED_LOAD_IF]] ]
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i1> [[TMP2]], i64 1
+; CHECK-NEXT: br i1 [[TMP8]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2:%.*]]
+; CHECK: pred.load.if1:
+; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i32, ptr [[IX]], i64 [[TMP9]]
+; CHECK-NEXT: [[TMP11:%.*]] = load i32, ptr [[TMP10]], align 4
+; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x i32> [[TMP7]], i32 [[TMP11]], i64 1
+; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE2]]
+; CHECK: pred.load.continue2:
+; CHECK-NEXT: [[TMP13:%.*]] = phi <4 x i32> [ [[TMP7]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP12]], [[PRED_LOAD_IF1]] ]
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x i1> [[TMP2]], i64 2
+; CHECK-NEXT: br i1 [[TMP14]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]]
+; CHECK: pred.load.if3:
+; CHECK-NEXT: [[TMP15:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP16:%.*]] = getelementptr i32, ptr [[IX]], i64 [[TMP15]]
+; CHECK-NEXT: [[TMP17:%.*]] = load i32, ptr [[TMP16]], align 4
+; CHECK-NEXT: [[TMP18:%.*]] = insertelement <4 x i32> [[TMP13]], i32 [[TMP17]], i64 2
+; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE4]]
+; CHECK: pred.load.continue4:
+; CHECK-NEXT: [[TMP19:%.*]] = phi <4 x i32> [ [[TMP13]], [[PRED_LOAD_CONTINUE2]] ], [ [[TMP18]], [[PRED_LOAD_IF3]] ]
+; CHECK-NEXT: [[TMP20:%.*]] = extractelement <4 x i1> [[TMP2]], i64 3
+; CHECK-NEXT: br i1 [[TMP20]], label [[PRED_LOAD_IF5:%.*]], label [[PRED_LOAD_CONTINUE6]]
+; CHECK: pred.load.if5:
+; CHECK-NEXT: [[TMP21:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT: [[TMP22:%.*]] = getelementptr i32, ptr [[IX]], i64 [[TMP21]]
+; CHECK-NEXT: [[TMP23:%.*]] = load i32, ptr [[TMP22]], align 4
+; CHECK-NEXT: [[TMP24:%.*]] = insertelement <4 x i32> [[TMP19]], i32 [[TMP23]], i64 3
+; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE6]]
+; CHECK: pred.load.continue6:
+; CHECK-NEXT: [[TMP25:%.*]] = phi <4 x i32> [ [[TMP19]], [[PRED_LOAD_CONTINUE4]] ], [ [[TMP24]], [[PRED_LOAD_IF5]] ]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP2]], <4 x i32> [[TMP25]], <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP26:%.*]] = getelementptr i32, ptr [[IX]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[PREDPHI]], ptr [[TMP26]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP27:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP27]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT_LOOPEXIT:%.*]], label [[SCALAR_PH]]
+; CHECK: scalar.ph:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[PREHEADER]] ]
+; CHECK-NEXT: br label [[LOOP:%.*]]
+; CHECK: loop:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXT:%.*]], [[MERGE:%.*]] ]
+; CHECK-NEXT: [[XP:%.*]] = getelementptr float, ptr [[XR]], i64 [[IV]]
+; CHECK-NEXT: [[X:%.*]] = load float, ptr [[XP]], align 4
+; CHECK-NEXT: [[KEEP:%.*]] = fcmp oge float [[X]], [[THRESHOLD]]
+; CHECK-NEXT: br i1 [[KEEP]], label [[READ:%.*]], label [[MERGE]]
+; CHECK: read:
+; CHECK-NEXT: [[LOAD_PTR:%.*]] = getelementptr i32, ptr [[IX]], i64 [[IV]]
+; CHECK-NEXT: [[OLD:%.*]] = load i32, ptr [[LOAD_PTR]], align 4
+; CHECK-NEXT: br label [[MERGE]]
+; CHECK: merge:
+; CHECK-NEXT: [[V:%.*]] = phi i32 [ [[OLD]], [[READ]] ], [ 0, [[LOOP]] ]
+; CHECK-NEXT: [[STORE_PTR:%.*]] = getelementptr i32, ptr [[IX]], i64 [[IV]]
+; CHECK-NEXT: store i32 [[V]], ptr [[STORE_PTR]], align 4
+; CHECK-NEXT: [[NEXT]] = add nuw i64 [[IV]], 1
+; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[DONE]], label [[EXIT_LOOPEXIT]], label [[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: exit.loopexit:
+; CHECK-NEXT: br label [[EXIT]]
+; CHECK: exit:
+; CHECK-NEXT: ret void
+;
+entry:
+ %empty = icmp eq i64 %n, 0
+ br i1 %empty, label %exit, label %preheader
+preheader:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %preheader ], [ %next, %merge ]
+ %xp = getelementptr float, ptr %xr, i64 %iv
+ %x = load float, ptr %xp, align 4
+ %keep = fcmp oge float %x, %threshold
+ br i1 %keep, label %read, label %merge
+read:
+ %load.ptr = getelementptr i32, ptr %ix, i64 %iv
+ %old = load i32, ptr %load.ptr, align 4
+ br label %merge
+merge:
+ %v = phi i32 [ %old, %read ], [ 0, %loop ]
+ %store.ptr = getelementptr i32, ptr %ix, i64 %iv
+ store i32 %v, ptr %store.ptr, align 4
+ %next = add nuw i64 %iv, 1
+ %done = icmp eq i64 %next, %n
+ br i1 %done, label %exit, label %loop
+exit:
+ ret void
+}
+
+define void @stronger_load_alignment(ptr noalias noundef %xr, ptr noalias noundef %ix, i64 %n, float %threshold) {
+; CHECK-LABEL: @stronger_load_alignment(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[EMPTY:%.*]] = icmp eq i64 [[N:%.*]], 0
+; CHECK-NEXT: br i1 [[EMPTY]], label [[EXIT:%.*]], label [[PREHEADER:%.*]]
+; CHECK: preheader:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[THRESHOLD:%.*]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE6:%.*]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr float, ptr [[XR:%.*]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = fcmp oge <4 x float> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i1> [[TMP2]], i64 0
+; CHECK-NEXT: br i1 [[TMP3]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
+; CHECK: pred.load.if:
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i32, ptr [[IX:%.*]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr [[TMP4]], align 16
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x i32> poison, i32 [[TMP5]], i64 0
+; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE]]
+; CHECK: pred.load.continue:
+; CHECK-NEXT: [[TMP7:%.*]] = phi <4 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP6]], [[PRED_LOAD_IF]] ]
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i1> [[TMP2]], i64 1
+; CHECK-NEXT: br i1 [[TMP8]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2:%.*]]
+; CHECK: pred.load.if1:
+; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i32, ptr [[IX]], i64 [[TMP9]]
+; CHECK-NEXT: [[TMP11:%.*]] = load i32, ptr [[TMP10]], align 16
+; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x i32> [[TMP7]], i32 [[TMP11]], i64 1
+; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE2]]
+; CHECK: pred.load.continue2:
+; CHECK-NEXT: [[TMP13:%.*]] = phi <4 x i32> [ [[TMP7]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP12]], [[PRED_LOAD_IF1]] ]
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x i1> [[TMP2]], i64 2
+; CHECK-NEXT: br i1 [[TMP14]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]]
+; CHECK: pred.load.if3:
+; CHECK-NEXT: [[TMP15:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP16:%.*]] = getelementptr i32, ptr [[IX]], i64 [[TMP15]]
+; CHECK-NEXT: [[TMP17:%.*]] = load i32, ptr [[TMP16]], align 16
+; CHECK-NEXT: [[TMP18:%.*]] = insertelement <4 x i32> [[TMP13]], i32 [[TMP17]], i64 2
+; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE4]]
+; CHECK: pred.load.continue4:
+; CHECK-NEXT: [[TMP19:%.*]] = phi <4 x i32> [ [[TMP13]], [[PRED_LOAD_CONTINUE2]] ], [ [[TMP18]], [[PRED_LOAD_IF3]] ]
+; CHECK-NEXT: [[TMP20:%.*]] = extractelement <4 x i1> [[TMP2]], i64 3
+; CHECK-NEXT: br i1 [[TMP20]], label [[PRED_LOAD_IF5:%.*]], label [[PRED_LOAD_CONTINUE6]]
+; CHECK: pred.load.if5:
+; CHECK-NEXT: [[TMP21:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT: [[TMP22:%.*]] = getelementptr i32, ptr [[IX]], i64 [[TMP21]]
+; CHECK-NEXT: [[TMP23:%.*]] = load i32, ptr [[TMP22]], align 16
+; CHECK-NEXT: [[TMP24:%.*]] = insertelement <4 x i32> [[TMP19]], i32 [[TMP23]], i64 3
+; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE6]]
+; CHECK: pred.load.continue6:
+; CHECK-NEXT: [[TMP25:%.*]] = phi <4 x i32> [ [[TMP19]], [[PRED_LOAD_CONTINUE4]] ], [ [[TMP24]], [[PRED_LOAD_IF5]] ]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP2]], <4 x i32> [[TMP25]], <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP26:%.*]] = getelementptr i32, ptr [[IX]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[PREDPHI]], ptr [[TMP26]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP27:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP27]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT_LOOPEXIT:%.*]], label [[SCALAR_PH]]
+; CHECK: scalar.ph:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[PREHEADER]] ]
+; CHECK-NEXT: br label [[LOOP:%.*]]
+; CHECK: loop:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXT:%.*]], [[MERGE:%.*]] ]
+; CHECK-NEXT: [[XP:%.*]] = getelementptr float, ptr [[XR]], i64 [[IV]]
+; CHECK-NEXT: [[X:%.*]] = load float, ptr [[XP]], align 4
+; CHECK-NEXT: [[KEEP:%.*]] = fcmp oge float [[X]], [[THRESHOLD]]
+; CHECK-NEXT: br i1 [[KEEP]], label [[READ:%.*]], label [[MERGE]]
+; CHECK: read:
+; CHECK-NEXT: [[LOAD_PTR:%.*]] = getelementptr i32, ptr [[IX]], i64 [[IV]]
+; CHECK-NEXT: [[OLD:%.*]] = load i32, ptr [[LOAD_PTR]], align 16
+; CHECK-NEXT: br label [[MERGE]]
+; CHECK: merge:
+; CHECK-NEXT: [[V:%.*]] = phi i32 [ [[OLD]], [[READ]] ], [ 0, [[LOOP]] ]
+; CHECK-NEXT: [[STORE_PTR:%.*]] = getelementptr i32, ptr [[IX]], i64 [[IV]]
+; CHECK-NEXT: store i32 [[V]], ptr [[STORE_PTR]], align 4
+; CHECK-NEXT: [[NEXT]] = add nuw i64 [[IV]], 1
+; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[DONE]], label [[EXIT_LOOPEXIT]], label [[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK: exit.loopexit:
+; CHECK-NEXT: br label [[EXIT]]
+; CHECK: exit:
+; CHECK-NEXT: ret void
+;
+entry:
+ %empty = icmp eq i64 %n, 0
+ br i1 %empty, label %exit, label %preheader
+preheader:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %preheader ], [ %next, %merge ]
+ %xp = getelementptr float, ptr %xr, i64 %iv
+ %x = load float, ptr %xp, align 4
+ %keep = fcmp oge float %x, %threshold
+ br i1 %keep, label %read, label %merge
+read:
+ %load.ptr = getelementptr i32, ptr %ix, i64 %iv
+ %old = load i32, ptr %load.ptr, align 16
+ br label %merge
+merge:
+ %v = phi i32 [ %old, %read ], [ 0, %loop ]
+ %store.ptr = getelementptr i32, ptr %ix, i64 %iv
+ store i32 %v, ptr %store.ptr, align 4
+ %next = add nuw i64 %iv, 1
+ %done = icmp eq i64 %next, %n
+ br i1 %done, label %exit, label %loop
+exit:
+ ret void
+}
+
+define void @narrower_store(ptr noalias noundef %xr, ptr noalias noundef %ix, i64 %n, float %threshold) {
+; CHECK-LABEL: @narrower_store(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[EMPTY:%.*]] = icmp eq i64 [[N:%.*]], 0
+; CHECK-NEXT: br i1 [[EMPTY]], label [[EXIT:%.*]], label [[PREHEADER:%.*]]
+; CHECK: preheader:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]]
+; CHECK: vector.scevcheck:
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], -1
+; CHECK-NEXT: [[MUL:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 4, i64 [[TMP0]])
+; CHECK-NEXT: [[MUL_RESULT:%.*]] = extractvalue { i64, i1 } [[MUL]], 0
+; CHECK-NEXT: [[MUL_OVERFLOW:%.*]] = extractvalue { i64, i1 } [[MUL]], 1
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[IX:%.*]], i64 [[MUL_RESULT]]
+; CHECK-NEXT: [[TMP2:%.*]] = icmp ult ptr [[TMP1]], [[IX]]
+; CHECK-NEXT: [[TMP3:%.*]] = or i1 [[TMP2]], [[MUL_OVERFLOW]]
+; CHECK-NEXT: br i1 [[TMP3]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[TMP4:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP4]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[THRESHOLD:%.*]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE6:%.*]] ]
+; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr float, ptr [[XR:%.*]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP8]], align 4
+; CHECK-NEXT: [[TMP9:%.*]] = fcmp oge <4 x float> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i1> [[TMP9]], i64 0
+; CHECK-NEXT: br i1 [[TMP10]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
+; CHECK: pred.load.if:
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i32, ptr [[IX]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP11]], align 4
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> poison, i32 [[TMP12]], i64 0
+; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE]]
+; CHECK: pred.load.continue:
+; CHECK-NEXT: [[TMP14:%.*]] = phi <4 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP13]], [[PRED_LOAD_IF]] ]
+; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x i1> [[TMP9]], i64 1
+; CHECK-NEXT: br i1 [[TMP15]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2:%.*]]
+; CHECK: pred.load.if1:
+; CHECK-NEXT: [[TMP16:%.*]] = getelementptr i32, ptr [[IX]], i64 [[TMP5]]
+; CHECK-NEXT: [[TMP17:%.*]] = load i32, ptr [[TMP16]], align 4
+; CHECK-NEXT: [[TMP18:%.*]] = insertelement <4 x i32> [[TMP14]], i32 [[TMP17]], i64 1
+; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE2]]
+; CHECK: pred.load.continue2:
+; CHECK-NEXT: [[TMP19:%.*]] = phi <4 x i32> [ [[TMP14]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP18]], [[PRED_LOAD_IF1]] ]
+; CHECK-NEXT: [[TMP20:%.*]] = extractelement <4 x i1> [[TMP9]], i64 2
+; CHECK-NEXT: br i1 [[TMP20]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]]
+; CHECK: pred.load.if3:
+; CHECK-NEXT: [[TMP21:%.*]] = getelementptr i32, ptr [[IX]], i64 [[TMP6]]
+; CHECK-NEXT: [[TMP22:%.*]] = load i32, ptr [[TMP21]], align 4
+; CHECK-NEXT: [[TMP23:%.*]] = insertelement <4 x i32> [[TMP19]], i32 [[TMP22]], i64 2
+; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE4]]
+; CHECK: pred.load.continue4:
+; CHECK-NEXT: [[TMP24:%.*]] = phi <4 x i32> [ [[TMP19]], [[PRED_LOAD_CONTINUE2]] ], [ [[TMP23]], [[PRED_LOAD_IF3]] ]
+; CHECK-NEXT: [[TMP25:%.*]] = extractelement <4 x i1> [[TMP9]], i64 3
+; CHECK-NEXT: br i1 [[TMP25]], label [[PRED_LOAD_IF5:%.*]], label [[PRED_LOAD_CONTINUE6]]
+; CHECK: pred.load.if5:
+; CHECK-NEXT: [[TMP26:%.*]] = getelementptr i32, ptr [[IX]], i64 [[TMP7]]
+; CHECK-NEXT: [[TMP27:%.*]] = load i32, ptr [[TMP26]], align 4
+; CHECK-NEXT: [[TMP28:%.*]] = insertelement <4 x i32> [[TMP24]], i32 [[TMP27]], i64 3
+; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE6]]
+; CHECK: pred.load.continue6:
+; CHECK-NEXT: [[TMP29:%.*]] = phi <4 x i32> [ [[TMP24]], [[PRED_LOAD_CONTINUE4]] ], [ [[TMP28]], [[PRED_LOAD_IF5]] ]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP9]], <4 x i32> [[TMP29]], <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP30:%.*]] = trunc <4 x i32> [[PREDPHI]] to <4 x i16>
+; CHECK-NEXT: [[TMP31:%.*]] = getelementptr i32, ptr [[IX]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP32:%.*]] = getelementptr i32, ptr [[IX]], i64 [[TMP5]]
+; CHECK-NEXT: [[TMP33:%.*]] = getelementptr i32, ptr [[IX]], i64 [[TMP6]]
+; CHECK-NEXT: [[TMP34:%.*]] = getelementptr i32, ptr [[IX]], i64 [[TMP7]]
+; CHECK-NEXT: [[TMP35:%.*]] = extractelement <4 x i16> [[TMP30]], i64 0
+; CHECK-NEXT: store i16 [[TMP35]], ptr [[TMP31]], align 4
+; CHECK-NEXT: [[TMP36:%.*]] = extractelement <4 x i16> [[TMP30]], i64 1
+; CHECK-NEXT: store i16 [[TMP36]], ptr [[TMP32]], align 4
+; CHECK-NEXT: [[TMP37:%.*]] = extractelement <4 x i16> [[TMP30]], i64 2
+; CHECK-NEXT: store i16 [[TMP37]], ptr [[TMP33]], align 4
+; CHECK-NEXT: [[TMP38:%.*]] = extractelement <4 x i16> [[TMP30]], i64 3
+; CHECK-NEXT: store i16 [[TMP38]], ptr [[TMP34]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP39:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP39]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT_LOOPEXIT:%.*]], label [[SCALAR_PH]]
+; CHECK: scalar.ph:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[PREHEADER]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
+; CHECK-NEXT: br label [[LOOP:%.*]]
+; CHECK: loop:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXT:%.*]], [[MERGE:%.*]] ]
+; CHECK-NEXT: [[XP:%.*]] = getelementptr float, ptr [[XR]], i64 [[IV]]
+; CHECK-NEXT: [[X:%.*]] = load float, ptr [[XP]], align 4
+; CHECK-NEXT: [[KEEP:%.*]] = fcmp oge float [[X]], [[THRESHOLD]]
+; CHECK-NEXT: br i1 [[KEEP]], label [[READ:%.*]], label [[MERGE]]
+; CHECK: read:
+; CHECK-NEXT: [[LOAD_PTR:%.*]] = getelementptr i32, ptr [[IX]], i64 [[IV]]
+; CHECK-NEXT: [[OLD:%.*]] = load i32, ptr [[LOAD_PTR]], align 4
+; CHECK-NEXT: br label [[MERGE]]
+; CHECK: merge:
+; CHECK-NEXT: [[V:%.*]] = phi i32 [ [[OLD]], [[READ]] ], [ 0, [[LOOP]] ]
+; CHECK-NEXT: [[TRUNC:%.*]] = trunc i32 [[V]] to i16
+; CHECK-NEXT: [[STORE_PTR:%.*]] = getelementptr i32, ptr [[IX]], i64 [[IV]]
+; CHECK-NEXT: store i16 [[TRUNC]], ptr [[STORE_PTR]], align 4
+; CHECK-NEXT: [[NEXT]] = add nuw i64 [[IV]], 1
+; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[DONE]], label [[EXIT_LOOPEXIT]], label [[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK: exit.loopexit:
+; CHECK-NEXT: br label [[EXIT]]
+; CHECK: exit:
+; CHECK-NEXT: ret void
+;
+entry:
+ %empty = icmp eq i64 %n, 0
+ br i1 %empty, label %exit, label %preheader
+preheader:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %preheader ], [ %next, %merge ]
+ %xp = getelementptr float, ptr %xr, i64 %iv
+ %x = load float, ptr %xp, align 4
+ %keep = fcmp oge float %x, %threshold
+ br i1 %keep, label %read, label %merge
+read:
+ %load.ptr = getelementptr i32, ptr %ix, i64 %iv
+ %old = load i32, ptr %load.ptr, align 4
+ br label %merge
+merge:
+ %v = phi i32 [ %old, %read ], [ 0, %loop ]
+ %trunc = trunc i32 %v to i16
+ %store.ptr = getelementptr i32, ptr %ix, i64 %iv
+ store i16 %trunc, ptr %store.ptr, align 4
+ %next = add nuw i64 %iv, 1
+ %done = icmp eq i64 %next, %n
+ br i1 %done, label %exit, label %loop
+exit:
+ ret void
+}
+
>From 2dc397ff65f5ccf6abfeb4d3300d1cfa02e17331 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?D=C3=A1vid=20Bolvansk=C3=BD?= <david.bolvansky at gmail.com>
Date: Sun, 4 Oct 2026 03:03:40 +0200
Subject: [PATCH 2/3] [LV] Cover unknown pointer poison and equivalent poisoned
offsets
---
.../equivalent-pointers-predication.ll | 139 +++++++++++++++++-
1 file changed, 134 insertions(+), 5 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/equivalent-pointers-predication.ll b/llvm/test/Transforms/LoopVectorize/AArch64/equivalent-pointers-predication.ll
index 1fe0db86e508b..4289598bf3bff 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/equivalent-pointers-predication.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/equivalent-pointers-predication.ll
@@ -1,7 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt -passes=loop-vectorize -mtriple=aarch64-unknown-linux-gnu -force-vector-width=4 -force-vector-interleave=1 -S < %s | FileCheck %s
-define void @equivalent_geps(ptr noalias noundef %xr, ptr noalias noundef %ix, i64 %n, float %threshold) {
+define void @equivalent_geps(ptr noalias noundef %xr, ptr noalias %ix, i64 %n, float %threshold) {
; CHECK-LABEL: @equivalent_geps(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[EMPTY:%.*]] = icmp eq i64 [[N:%.*]], 0
@@ -23,8 +23,8 @@ define void @equivalent_geps(ptr noalias noundef %xr, ptr noalias noundef %ix, i
; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i1> [[TMP2]], i64 0
; CHECK-NEXT: br i1 [[TMP3]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
; CHECK: pred.load.if:
-; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i32, ptr [[IX:%.*]], i64 [[INDEX]]
-; CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr [[TMP4]], align 4
+; CHECK-NEXT: [[TMP26:%.*]] = getelementptr i32, ptr [[IX:%.*]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr [[TMP26]], align 4
; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x i32> poison, i32 [[TMP5]], i64 0
; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE]]
; CHECK: pred.load.continue:
@@ -60,8 +60,8 @@ define void @equivalent_geps(ptr noalias noundef %xr, ptr noalias noundef %ix, i
; CHECK: pred.load.continue6:
; CHECK-NEXT: [[TMP25:%.*]] = phi <4 x i32> [ [[TMP19]], [[PRED_LOAD_CONTINUE4]] ], [ [[TMP24]], [[PRED_LOAD_IF5]] ]
; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP2]], <4 x i32> [[TMP25]], <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP26:%.*]] = getelementptr i32, ptr [[IX]], i64 [[INDEX]]
-; CHECK-NEXT: store <4 x i32> [[PREDPHI]], ptr [[TMP26]], align 4
+; CHECK-NEXT: [[TMP28:%.*]] = getelementptr i32, ptr [[IX]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[PREDPHI]], ptr [[TMP28]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP27:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP27]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
@@ -377,3 +377,132 @@ exit:
ret void
}
+
+; SCEV equality does not prove that extra pointer operands cannot be poison.
+define void @same_scev_poison_offset(ptr noundef %src, ptr noundef %dst, i64 %n, float %threshold, i64 %possibly_poison) {
+; CHECK-LABEL: @same_scev_poison_offset(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[EMPTY:%.*]] = icmp eq i64 [[N:%.*]], 0
+; CHECK-NEXT: br i1 [[EMPTY]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK: loop.preheader:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
+; CHECK: vector.memcheck:
+; CHECK-NEXT: [[TMP0:%.*]] = shl i64 [[N]], 2
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[DST:%.*]], i64 [[TMP0]]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[SRC:%.*]], i64 [[TMP0]]
+; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[DST]], [[TMP2]]
+; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[SRC]], [[TMP1]]
+; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[TMP3:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP3]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[THRESHOLD:%.*]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE7:%.*]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP4]], align 4, !alias.scope [[META8:![0-9]+]]
+; CHECK-NEXT: [[TMP5:%.*]] = fcmp oge <4 x float> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP5]], i64 0
+; CHECK-NEXT: br i1 [[TMP6]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
+; CHECK: pred.load.if:
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4, !alias.scope [[META11:![0-9]+]], !noalias [[META8]]
+; CHECK-NEXT: [[TMP9:%.*]] = insertelement <4 x i32> poison, i32 [[TMP8]], i64 0
+; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE]]
+; CHECK: pred.load.continue:
+; CHECK-NEXT: [[TMP10:%.*]] = phi <4 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP9]], [[PRED_LOAD_IF]] ]
+; CHECK-NEXT: [[TMP11:%.*]] = extractelement <4 x i1> [[TMP5]], i64 1
+; CHECK-NEXT: br i1 [[TMP11]], label [[PRED_LOAD_IF2:%.*]], label [[PRED_LOAD_CONTINUE3:%.*]]
+; CHECK: pred.load.if2:
+; CHECK-NEXT: [[TMP12:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP12]]
+; CHECK-NEXT: [[TMP14:%.*]] = load i32, ptr [[TMP13]], align 4, !alias.scope [[META11]], !noalias [[META8]]
+; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x i32> [[TMP10]], i32 [[TMP14]], i64 1
+; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE3]]
+; CHECK: pred.load.continue3:
+; CHECK-NEXT: [[TMP16:%.*]] = phi <4 x i32> [ [[TMP10]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP15]], [[PRED_LOAD_IF2]] ]
+; CHECK-NEXT: [[TMP17:%.*]] = extractelement <4 x i1> [[TMP5]], i64 2
+; CHECK-NEXT: br i1 [[TMP17]], label [[PRED_LOAD_IF4:%.*]], label [[PRED_LOAD_CONTINUE5:%.*]]
+; CHECK: pred.load.if4:
+; CHECK-NEXT: [[TMP18:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP18]]
+; CHECK-NEXT: [[TMP20:%.*]] = load i32, ptr [[TMP19]], align 4, !alias.scope [[META11]], !noalias [[META8]]
+; CHECK-NEXT: [[TMP21:%.*]] = insertelement <4 x i32> [[TMP16]], i32 [[TMP20]], i64 2
+; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE5]]
+; CHECK: pred.load.continue5:
+; CHECK-NEXT: [[TMP22:%.*]] = phi <4 x i32> [ [[TMP16]], [[PRED_LOAD_CONTINUE3]] ], [ [[TMP21]], [[PRED_LOAD_IF4]] ]
+; CHECK-NEXT: [[TMP23:%.*]] = extractelement <4 x i1> [[TMP5]], i64 3
+; CHECK-NEXT: br i1 [[TMP23]], label [[PRED_LOAD_IF6:%.*]], label [[PRED_LOAD_CONTINUE7]]
+; CHECK: pred.load.if6:
+; CHECK-NEXT: [[TMP24:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT: [[TMP25:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP24]]
+; CHECK-NEXT: [[TMP26:%.*]] = load i32, ptr [[TMP25]], align 4, !alias.scope [[META11]], !noalias [[META8]]
+; CHECK-NEXT: [[TMP27:%.*]] = insertelement <4 x i32> [[TMP22]], i32 [[TMP26]], i64 3
+; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE7]]
+; CHECK: pred.load.continue7:
+; CHECK-NEXT: [[TMP28:%.*]] = phi <4 x i32> [ [[TMP22]], [[PRED_LOAD_CONTINUE5]] ], [ [[TMP27]], [[PRED_LOAD_IF6]] ]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP5]], <4 x i32> [[TMP28]], <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP29:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[PREDPHI]], ptr [[TMP29]], align 4, !alias.scope [[META11]], !noalias [[META8]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP30:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP30]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT_LOOPEXIT:%.*]], label [[SCALAR_PH]]
+; CHECK: scalar.ph:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[LOOP_PREHEADER]] ], [ 0, [[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT: br label [[LOOP:%.*]]
+; CHECK: loop:
+; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[NEXT:%.*]], [[MERGE:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT: [[SRCP:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[I]]
+; CHECK-NEXT: [[X:%.*]] = load float, ptr [[SRCP]], align 4
+; CHECK-NEXT: [[KEEP:%.*]] = fcmp oge float [[X]], [[THRESHOLD]]
+; CHECK-NEXT: br i1 [[KEEP]], label [[READ:%.*]], label [[MERGE]]
+; CHECK: read:
+; CHECK-NEXT: [[ZERO:%.*]] = mul i64 [[POSSIBLY_POISON:%.*]], 0
+; CHECK-NEXT: [[CONDITIONAL_INDEX:%.*]] = add i64 [[I]], [[ZERO]]
+; CHECK-NEXT: [[LOADP:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[CONDITIONAL_INDEX]]
+; CHECK-NEXT: [[OLD:%.*]] = load i32, ptr [[LOADP]], align 4
+; CHECK-NEXT: br label [[MERGE]]
+; CHECK: merge:
+; CHECK-NEXT: [[RESULT:%.*]] = phi i32 [ [[OLD]], [[READ]] ], [ 0, [[LOOP]] ]
+; CHECK-NEXT: [[STOREP:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[I]]
+; CHECK-NEXT: store i32 [[RESULT]], ptr [[STOREP]], align 4
+; CHECK-NEXT: [[NEXT]] = add nuw i64 [[I]], 1
+; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[DONE]], label [[EXIT_LOOPEXIT]], label [[LOOP]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK: exit.loopexit:
+; CHECK-NEXT: br label [[EXIT]]
+; CHECK: exit:
+; CHECK-NEXT: ret void
+;
+entry:
+ %empty = icmp eq i64 %n, 0
+ br i1 %empty, label %exit, label %loop
+loop:
+ %i = phi i64 [ 0, %entry ], [ %next, %merge ]
+ %srcp = getelementptr inbounds float, ptr %src, i64 %i
+ %x = load float, ptr %srcp, align 4
+ %keep = fcmp oge float %x, %threshold
+ br i1 %keep, label %read, label %merge
+read:
+ %zero = mul i64 %possibly_poison, 0
+ %conditional.index = add i64 %i, %zero
+ %loadp = getelementptr inbounds i32, ptr %dst, i64 %conditional.index
+ %old = load i32, ptr %loadp, align 4
+ br label %merge
+merge:
+ %result = phi i32 [ %old, %read ], [ 0, %loop ]
+ %storep = getelementptr inbounds i32, ptr %dst, i64 %i
+ store i32 %result, ptr %storep, align 4
+ %next = add nuw i64 %i, 1
+ %done = icmp eq i64 %next, %n
+ br i1 %done, label %exit, label %loop
+exit:
+ ret void
+}
>From 2a4d3a7087a3ff17e509b93a4c6f23215c0fed16 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?D=C3=A1vid=20Bolvansk=C3=BD?= <david.bolvansky at gmail.com>
Date: Sun, 4 Oct 2026 03:10:07 +0200
Subject: [PATCH 3/3] [LV] Recognize safe loads through equivalent pointers
---
.../Vectorize/LoopVectorizationLegality.cpp | 59 +++++++++++++++----
.../equivalent-pointers-predication.ll | 44 +-------------
2 files changed, 51 insertions(+), 52 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 46f082604b5fb..1b07aa31a6cbb 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -1407,25 +1407,45 @@ bool LoopVectorizationLegality::canVectorizeWithIfConvert() {
// the value's type) unconditionally within the loop header without
// introducing a new fault.
SmallPtrSet<Value *, 8> SafePointers;
+ ScalarEvolution &SE = *PSE.getSE();
+ DenseMap<std::pair<const SCEV *, Type *>, Instruction *> SafeAccesses;
// Collect safe addresses.
for (BasicBlock *BB : TheLoop->blocks()) {
if (!blockNeedsPredication(BB)) {
- for (Instruction &I : *BB)
- if (auto *Ptr = getLoadStorePointerOperand(&I))
- SafePointers.insert(Ptr);
- continue;
+ for (Instruction &I : *BB) {
+ auto *Ptr = getLoadStorePointerOperand(&I);
+ if (!Ptr)
+ continue;
+ SafePointers.insert(Ptr);
+ bool IsSimple = isa<LoadInst>(I) ? cast<LoadInst>(I).isSimple()
+ : cast<StoreInst>(I).isSimple();
+ Type *Ty = getLoadStoreType(&I);
+ if (!IsSimple || Ty->isVectorTy())
+ continue;
+ auto Key = std::make_pair(SE.getSCEV(Ptr), Ty);
+ auto [It, Inserted] = SafeAccesses.try_emplace(Key, &I);
+ if (!Inserted &&
+ getLoadStoreAlignment(&I) > getLoadStoreAlignment(It->second))
+ It->second = &I;
+ }
}
+ }
+
+ for (BasicBlock *BB : TheLoop->blocks()) {
+ if (!blockNeedsPredication(BB))
+ continue;
// For a block which requires predication, a address may be safe to access
// in the loop w/o predication if we can prove dereferenceability facts
// sufficient to ensure it'll never fault within the loop. For the moment,
// we restrict this to loads; stores are more complicated due to
// concurrency restrictions.
- ScalarEvolution &SE = *PSE.getSE();
SmallVector<const SCEVPredicate *, 4> Predicates;
for (Instruction &I : *BB) {
LoadInst *LI = dyn_cast<LoadInst>(&I);
+ if (!LI || SafePointers.contains(LI->getPointerOperand()))
+ continue;
// Make sure we can execute all computations feeding into Ptr in the loop
// w/o triggering UB and that none of the out-of-loop operands are poison.
@@ -1434,7 +1454,14 @@ bool LoopVectorizationLegality::canVectorizeWithIfConvert() {
// because flags will be dropped when executing them unconditionally.
// TODO: Results could be improved by considering poison-propagation
// properties of visited ops.
- auto CanSpeculatePointerOp = [this](Value *Ptr) {
+ auto CanSpeculatePointerOp = [this](Value *Ptr, Value *KnownSafePtr) {
+ auto *GEP = dyn_cast<GetElementPtrInst>(Ptr);
+ auto *SafeGEP = dyn_cast_or_null<GetElementPtrInst>(KnownSafePtr);
+ // Identical operands and flags inherit the unconditional access's
+ // non-poison guarantee, including operands defined outside the loop.
+ if (GEP && SafeGEP && GEP->isIdenticalTo(SafeGEP))
+ return true;
+
SmallVector<Value *> Worklist = {Ptr};
SmallPtrSet<Value *, 4> Visited;
while (!Worklist.empty()) {
@@ -1471,11 +1498,21 @@ bool LoopVectorizationLegality::canVectorizeWithIfConvert() {
// Pass the Predicates pointer to isDereferenceableAndAlignedInLoop so
// that it will consider loops that need guarding by SCEV checks. The
// vectoriser will generate these checks if we decide to vectorise.
- if (LI && !LI->getType()->isVectorTy() && !mustSuppressSpeculation(*LI) &&
- CanSpeculatePointerOp(LI->getPointerOperand()) &&
- isDereferenceableAndAlignedInLoop(LI, TheLoop, SE, *DT, AC,
- &Predicates))
- SafePointers.insert(LI->getPointerOperand());
+ if (!LI->getType()->isVectorTy() && !mustSuppressSpeculation(*LI)) {
+ // Equivalent pointers need not be represented by the same SSA value.
+ // Require the same access type and sufficient alignment.
+ auto It = SafeAccesses.find(
+ {SE.getSCEV(LI->getPointerOperand()), LI->getType()});
+ bool HasSafeAccess =
+ LI->isSimple() && It != SafeAccesses.end() &&
+ getLoadStoreAlignment(It->second) >= LI->getAlign();
+ Value *KnownSafePtr =
+ HasSafeAccess ? getLoadStorePointerOperand(It->second) : nullptr;
+ if (CanSpeculatePointerOp(LI->getPointerOperand(), KnownSafePtr) &&
+ (HasSafeAccess || isDereferenceableAndAlignedInLoop(
+ LI, TheLoop, SE, *DT, AC, &Predicates)))
+ SafePointers.insert(LI->getPointerOperand());
+ }
Predicates.clear();
}
}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/equivalent-pointers-predication.ll b/llvm/test/Transforms/LoopVectorize/AArch64/equivalent-pointers-predication.ll
index 4289598bf3bff..5a0ebcdef1080 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/equivalent-pointers-predication.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/equivalent-pointers-predication.ll
@@ -16,51 +16,13 @@ define void @equivalent_geps(ptr noalias noundef %xr, ptr noalias %ix, i64 %n, f
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE6:%.*]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr float, ptr [[XR:%.*]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
; CHECK-NEXT: [[TMP2:%.*]] = fcmp oge <4 x float> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i1> [[TMP2]], i64 0
-; CHECK-NEXT: br i1 [[TMP3]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
-; CHECK: pred.load.if:
-; CHECK-NEXT: [[TMP26:%.*]] = getelementptr i32, ptr [[IX:%.*]], i64 [[INDEX]]
-; CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr [[TMP26]], align 4
-; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x i32> poison, i32 [[TMP5]], i64 0
-; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE]]
-; CHECK: pred.load.continue:
-; CHECK-NEXT: [[TMP7:%.*]] = phi <4 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP6]], [[PRED_LOAD_IF]] ]
-; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i1> [[TMP2]], i64 1
-; CHECK-NEXT: br i1 [[TMP8]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2:%.*]]
-; CHECK: pred.load.if1:
-; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], 1
-; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i32, ptr [[IX]], i64 [[TMP9]]
-; CHECK-NEXT: [[TMP11:%.*]] = load i32, ptr [[TMP10]], align 4
-; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x i32> [[TMP7]], i32 [[TMP11]], i64 1
-; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE2]]
-; CHECK: pred.load.continue2:
-; CHECK-NEXT: [[TMP13:%.*]] = phi <4 x i32> [ [[TMP7]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP12]], [[PRED_LOAD_IF1]] ]
-; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x i1> [[TMP2]], i64 2
-; CHECK-NEXT: br i1 [[TMP14]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]]
-; CHECK: pred.load.if3:
-; CHECK-NEXT: [[TMP15:%.*]] = add i64 [[INDEX]], 2
-; CHECK-NEXT: [[TMP16:%.*]] = getelementptr i32, ptr [[IX]], i64 [[TMP15]]
-; CHECK-NEXT: [[TMP17:%.*]] = load i32, ptr [[TMP16]], align 4
-; CHECK-NEXT: [[TMP18:%.*]] = insertelement <4 x i32> [[TMP13]], i32 [[TMP17]], i64 2
-; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE4]]
-; CHECK: pred.load.continue4:
-; CHECK-NEXT: [[TMP19:%.*]] = phi <4 x i32> [ [[TMP13]], [[PRED_LOAD_CONTINUE2]] ], [ [[TMP18]], [[PRED_LOAD_IF3]] ]
-; CHECK-NEXT: [[TMP20:%.*]] = extractelement <4 x i1> [[TMP2]], i64 3
-; CHECK-NEXT: br i1 [[TMP20]], label [[PRED_LOAD_IF5:%.*]], label [[PRED_LOAD_CONTINUE6]]
-; CHECK: pred.load.if5:
-; CHECK-NEXT: [[TMP21:%.*]] = add i64 [[INDEX]], 3
-; CHECK-NEXT: [[TMP22:%.*]] = getelementptr i32, ptr [[IX]], i64 [[TMP21]]
-; CHECK-NEXT: [[TMP23:%.*]] = load i32, ptr [[TMP22]], align 4
-; CHECK-NEXT: [[TMP24:%.*]] = insertelement <4 x i32> [[TMP19]], i32 [[TMP23]], i64 3
-; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE6]]
-; CHECK: pred.load.continue6:
-; CHECK-NEXT: [[TMP25:%.*]] = phi <4 x i32> [ [[TMP19]], [[PRED_LOAD_CONTINUE4]] ], [ [[TMP24]], [[PRED_LOAD_IF5]] ]
+; CHECK-NEXT: [[TMP28:%.*]] = getelementptr i32, ptr [[IX:%.*]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP25:%.*]] = load <4 x i32>, ptr [[TMP28]], align 4
; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP2]], <4 x i32> [[TMP25]], <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP28:%.*]] = getelementptr i32, ptr [[IX]], i64 [[INDEX]]
; CHECK-NEXT: store <4 x i32> [[PREDPHI]], ptr [[TMP28]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP27:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
More information about the llvm-commits
mailing list