[llvm] [LV] Add tests with GEPs and different pointer casts and scalarization. (PR #195544)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Sun May 3 13:15:07 PDT 2026
https://github.com/fhahn created https://github.com/llvm/llvm-project/pull/195544
None
>From ce1b785564f59714c95d6766e140313a313c1e03 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Fri, 1 May 2026 20:49:36 +0100
Subject: [PATCH] [LV] Add tests with GEPs and different pointer casts and
scalarization.
---
.../scalarize-wide-load-for-address-use.ll | 261 ++++++++++++++++++
...reserve-inbounds-gep-with-pointer-casts.ll | 203 ++++++++++++++
2 files changed, 464 insertions(+)
create mode 100644 llvm/test/Transforms/LoopVectorize/X86/scalarize-wide-load-for-address-use.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/preserve-inbounds-gep-with-pointer-casts.ll
diff --git a/llvm/test/Transforms/LoopVectorize/X86/scalarize-wide-load-for-address-use.ll b/llvm/test/Transforms/LoopVectorize/X86/scalarize-wide-load-for-address-use.ll
new file mode 100644
index 0000000000000..5a6fac3e696bf
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/scalarize-wide-load-for-address-use.ll
@@ -0,0 +1,261 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt -passes=loop-vectorize -force-vector-width=4 -S %s | FileCheck %s
+
+target triple = "x86_64-unknown-linux-gnu"
+
+ at src.arr = external global [15 x double]
+ at tbl.a = external global [1024 x double]
+ at tbl.b = external global [1024 x double]
+
+; Non-masked consecutive load feeding a store address via select: scalarize
+; to per-lane loads. Each lane must use its own address, not lane 0's.
+define void @unmasked_load_feeds_address(ptr %src, i64 %n) #0 {
+; CHECK-LABEL: define void @unmasked_load_feeds_address(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[VECTOR_BODY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK: [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT: [[TMP0:%.*]] = shl i64 [[N]], 3
+; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP0]]
+; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr @tbl.a, [[SCEVGEP]]
+; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[SRC]], getelementptr inbounds nuw (i8, ptr @tbl.a, i64 8)
+; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT: [[BOUND01:%.*]] = icmp ult ptr @tbl.b, [[SCEVGEP]]
+; CHECK-NEXT: [[BOUND12:%.*]] = icmp ult ptr [[SRC]], getelementptr inbounds nuw (i8, ptr @tbl.b, i64 8)
+; CHECK-NEXT: [[FOUND_CONFLICT3:%.*]] = and i1 [[BOUND01]], [[BOUND12]]
+; CHECK-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT3]]
+; CHECK-NEXT: br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY1:.*]]
+; CHECK: [[VECTOR_BODY1]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY1]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr double, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr double, ptr [[SRC]], i64 [[TMP1]]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr double, ptr [[SRC]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr double, ptr [[SRC]], i64 [[TMP3]]
+; CHECK-NEXT: [[TMP8:%.*]] = load double, ptr [[TMP4]], align 8, !alias.scope [[META0:![0-9]+]]
+; CHECK-NEXT: [[TMP9:%.*]] = load double, ptr [[TMP5]], align 8, !alias.scope [[META0]]
+; CHECK-NEXT: [[TMP10:%.*]] = load double, ptr [[TMP6]], align 8, !alias.scope [[META0]]
+; CHECK-NEXT: [[TMP11:%.*]] = load double, ptr [[TMP7]], align 8, !alias.scope [[META0]]
+; CHECK-NEXT: [[TMP12:%.*]] = fcmp oeq double [[TMP8]], 0.000000e+00
+; CHECK-NEXT: [[TMP13:%.*]] = fcmp oeq double [[TMP9]], 0.000000e+00
+; CHECK-NEXT: [[TMP14:%.*]] = fcmp oeq double [[TMP10]], 0.000000e+00
+; CHECK-NEXT: [[TMP15:%.*]] = fcmp oeq double [[TMP11]], 0.000000e+00
+; CHECK-NEXT: [[TMP16:%.*]] = select i1 [[TMP12]], ptr @tbl.a, ptr @tbl.b
+; CHECK-NEXT: [[TMP17:%.*]] = select i1 [[TMP13]], ptr @tbl.a, ptr @tbl.b
+; CHECK-NEXT: [[TMP18:%.*]] = select i1 [[TMP14]], ptr @tbl.a, ptr @tbl.b
+; CHECK-NEXT: [[TMP19:%.*]] = select i1 [[TMP15]], ptr @tbl.a, ptr @tbl.b
+; CHECK-NEXT: store double 1.000000e+00, ptr [[TMP16]], align 8, !alias.scope [[META3:![0-9]+]], !noalias [[META0]]
+; CHECK-NEXT: store double 1.000000e+00, ptr [[TMP17]], align 8, !alias.scope [[META3]], !noalias [[META0]]
+; CHECK-NEXT: store double 1.000000e+00, ptr [[TMP18]], align 8, !alias.scope [[META3]], !noalias [[META0]]
+; CHECK-NEXT: store double 1.000000e+00, ptr [[TMP19]], align 8, !alias.scope [[META3]], !noalias [[META0]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY1]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_BODY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr double, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[VAL:%.*]] = load double, ptr [[GEP]], align 8
+; CHECK-NEXT: [[CMP:%.*]] = fcmp oeq double [[VAL]], 0.000000e+00
+; CHECK-NEXT: [[PTR_SEL:%.*]] = select i1 [[CMP]], ptr @tbl.a, ptr @tbl.b
+; CHECK-NEXT: store double 1.000000e+00, ptr [[PTR_SEL]], align 8
+; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep = getelementptr double, ptr %src, i64 %iv
+ %val = load double, ptr %gep, align 8
+ %cmp = fcmp oeq double %val, 0.0
+ %ptr.sel = select i1 %cmp, ptr @tbl.a, ptr @tbl.b
+ store double 1.0, ptr %ptr.sel, align 8
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Tail-folded masked consecutive load feeding a store address via select.
+define void @masked_load_feeds_address_must_not_scalarize() #0 {
+; CHECK-LABEL: define void @masked_load_feeds_address_must_not_scalarize(
+; CHECK-SAME: ) #[[ATTR0]] {
+; CHECK-NEXT: [[VECTOR_BODY:.*:]]
+; CHECK-NEXT: br label %[[PRED_STORE_IF:.*]]
+; CHECK: [[PRED_STORE_IF]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY1:.*]]
+; CHECK: [[VECTOR_BODY1]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[PRED_STORE_IF]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE6:.*]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i8> [ <i8 0, i8 1, i8 2, i8 3>, %[[PRED_STORE_IF]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE6]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 1, [[INDEX]]
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ule <4 x i8> [[VEC_IND]], splat (i8 10)
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr double, ptr @src.arr, i64 [[TMP0]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0(ptr align 8 [[TMP2]], <4 x i1> [[TMP1]], <4 x double> poison)
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i1> [[TMP1]], i64 0
+; CHECK-NEXT: br i1 [[TMP3]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; CHECK: [[PRED_STORE_IF2]]:
+; CHECK-NEXT: [[EX0:%.*]] = extractelement <4 x double> [[WIDE_MASKED_LOAD]], i64 0
+; CHECK-NEXT: [[TMP5:%.*]] = fcmp oeq double [[EX0]], 0.000000e+00
+; CHECK-NEXT: [[TMP6:%.*]] = select i1 [[TMP5]], ptr @tbl.a, ptr @tbl.b
+; CHECK-NEXT: store double 0.000000e+00, ptr [[TMP6]], align 8
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; CHECK: [[PRED_STORE_CONTINUE]]:
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i1> [[TMP1]], i64 1
+; CHECK-NEXT: br i1 [[TMP7]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
+; CHECK: [[PRED_STORE_IF1]]:
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x double> [[WIDE_MASKED_LOAD]], i64 1
+; CHECK-NEXT: [[TMP9:%.*]] = fcmp oeq double [[TMP8]], 0.000000e+00
+; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], ptr @tbl.a, ptr @tbl.b
+; CHECK-NEXT: store double 0.000000e+00, ptr [[TMP10]], align 8
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE2]]
+; CHECK: [[PRED_STORE_CONTINUE2]]:
+; CHECK-NEXT: [[TMP11:%.*]] = extractelement <4 x i1> [[TMP1]], i64 2
+; CHECK-NEXT: br i1 [[TMP11]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
+; CHECK: [[PRED_STORE_IF3]]:
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <4 x double> [[WIDE_MASKED_LOAD]], i64 2
+; CHECK-NEXT: [[TMP13:%.*]] = fcmp oeq double [[TMP12]], 0.000000e+00
+; CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], ptr @tbl.a, ptr @tbl.b
+; CHECK-NEXT: store double 0.000000e+00, ptr [[TMP14]], align 8
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE4]]
+; CHECK: [[PRED_STORE_CONTINUE4]]:
+; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x i1> [[TMP1]], i64 3
+; CHECK-NEXT: br i1 [[TMP15]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6]]
+; CHECK: [[PRED_STORE_IF5]]:
+; CHECK-NEXT: [[TMP16:%.*]] = extractelement <4 x double> [[WIDE_MASKED_LOAD]], i64 3
+; CHECK-NEXT: [[TMP17:%.*]] = fcmp oeq double [[TMP16]], 0.000000e+00
+; CHECK-NEXT: [[TMP18:%.*]] = select i1 [[TMP17]], ptr @tbl.a, ptr @tbl.b
+; CHECK-NEXT: store double 0.000000e+00, ptr [[TMP18]], align 8
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE6]]
+; CHECK: [[PRED_STORE_CONTINUE6]]:
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i8> [[VEC_IND]], splat (i8 4)
+; CHECK-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 12
+; CHECK-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY1]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 1, %entry ], [ %iv.next, %loop ]
+ %gep.src = getelementptr double, ptr @src.arr, i64 %iv
+ %val = load double, ptr %gep.src, align 8
+ %cmp = fcmp oeq double %val, 0.000000e+00
+ %dst.select = select i1 %cmp, ptr @tbl.a, ptr @tbl.b
+ store double 0.000000e+00, ptr %dst.select, align 8
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 12
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Same pattern as @unmasked_load_feeds_address, but the source load carries
+; !tbaa and !alias.scope metadata.
+define void @unmasked_load_with_metadata_feeds_address(ptr noalias %src, i64 %n) #0 {
+; CHECK-LABEL: define void @unmasked_load_with_metadata_feeds_address(
+; CHECK-SAME: ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr double, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr double, ptr [[SRC]], i64 [[TMP0]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr double, ptr [[SRC]], i64 [[TMP1]]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr double, ptr [[SRC]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP7:%.*]] = load double, ptr [[TMP3]], align 8, !tbaa [[DOUBLE_TBAA10:![0-9]+]], !alias.scope [[META14:![0-9]+]]
+; CHECK-NEXT: [[TMP8:%.*]] = load double, ptr [[TMP4]], align 8, !tbaa [[DOUBLE_TBAA10]], !alias.scope [[META14]]
+; CHECK-NEXT: [[TMP9:%.*]] = load double, ptr [[TMP5]], align 8, !tbaa [[DOUBLE_TBAA10]], !alias.scope [[META14]]
+; CHECK-NEXT: [[TMP10:%.*]] = load double, ptr [[TMP6]], align 8, !tbaa [[DOUBLE_TBAA10]], !alias.scope [[META14]]
+; CHECK-NEXT: [[TMP11:%.*]] = fcmp oeq double [[TMP7]], 0.000000e+00
+; CHECK-NEXT: [[TMP12:%.*]] = fcmp oeq double [[TMP8]], 0.000000e+00
+; CHECK-NEXT: [[TMP13:%.*]] = fcmp oeq double [[TMP9]], 0.000000e+00
+; CHECK-NEXT: [[TMP14:%.*]] = fcmp oeq double [[TMP10]], 0.000000e+00
+; CHECK-NEXT: [[TMP15:%.*]] = select i1 [[TMP11]], ptr @tbl.a, ptr @tbl.b
+; CHECK-NEXT: [[TMP16:%.*]] = select i1 [[TMP12]], ptr @tbl.a, ptr @tbl.b
+; CHECK-NEXT: [[TMP17:%.*]] = select i1 [[TMP13]], ptr @tbl.a, ptr @tbl.b
+; CHECK-NEXT: [[TMP18:%.*]] = select i1 [[TMP14]], ptr @tbl.a, ptr @tbl.b
+; CHECK-NEXT: store double 1.000000e+00, ptr [[TMP15]], align 8
+; CHECK-NEXT: store double 1.000000e+00, ptr [[TMP16]], align 8
+; CHECK-NEXT: store double 1.000000e+00, ptr [[TMP17]], align 8
+; CHECK-NEXT: store double 1.000000e+00, ptr [[TMP18]], align 8
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr double, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[VAL:%.*]] = load double, ptr [[GEP]], align 8, !tbaa [[DOUBLE_TBAA10]], !alias.scope [[META14]]
+; CHECK-NEXT: [[CMP:%.*]] = fcmp oeq double [[VAL]], 0.000000e+00
+; CHECK-NEXT: [[PTR_SEL:%.*]] = select i1 [[CMP]], ptr @tbl.a, ptr @tbl.b
+; CHECK-NEXT: store double 1.000000e+00, ptr [[PTR_SEL]], align 8
+; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP18:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep = getelementptr double, ptr %src, i64 %iv
+ %val = load double, ptr %gep, align 8, !tbaa !10, !alias.scope !20
+ %cmp = fcmp oeq double %val, 0.0
+ %ptr.sel = select i1 %cmp, ptr @tbl.a, ptr @tbl.b
+ store double 1.0, ptr %ptr.sel, align 8
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+!10 = !{!11, !11, i64 0}
+!11 = !{!"double", !12, i64 0}
+!12 = !{!"omnipotent char", !13, i64 0}
+!13 = !{!"Simple C/C++ TBAA"}
+!20 = !{!21}
+!21 = distinct !{!21, !22, !"scope.a"}
+!22 = distinct !{!22, !"domain.a"}
+
+attributes #0 = { "target-cpu"="haswell" }
diff --git a/llvm/test/Transforms/LoopVectorize/preserve-inbounds-gep-with-pointer-casts.ll b/llvm/test/Transforms/LoopVectorize/preserve-inbounds-gep-with-pointer-casts.ll
new file mode 100644
index 0000000000000..a91a4819a042c
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/preserve-inbounds-gep-with-pointer-casts.ll
@@ -0,0 +1,203 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=2 -S %s | FileCheck %s
+
+define void @preserve_inbounds_through_noop_gep(ptr %p, i32 %n) {
+; CHECK-LABEL: define void @preserve_inbounds_through_noop_gep(
+; CHECK-SAME: ptr [[P:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[VECTOR_BODY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 8
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N]], 8
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY1:.*]]
+; CHECK: [[VECTOR_BODY1]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY1]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[INDEX]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[TMP0]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 4
+; CHECK-NEXT: store <4 x i32> zeroinitializer, ptr [[TMP1]], align 4
+; CHECK-NEXT: store <4 x i32> zeroinitializer, ptr [[TMP2]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY1]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_BODY]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[IV]]
+; CHECK-NEXT: [[NOOP:%.*]] = getelementptr i8, ptr [[GEP]], i64 0
+; CHECK-NEXT: store i32 0, ptr [[NOOP]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nsw i32 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep = getelementptr inbounds i32, ptr %p, i32 %iv
+ %noop = getelementptr i8, ptr %gep, i64 0
+ store i32 0, ptr %noop
+ %iv.next = add nsw i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @bitcast_in_ptr_chain(ptr %src, ptr noalias %dst, i64 %n) {
+; CHECK-LABEL: define void @bitcast_in_ptr_chain(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr noalias [[DST:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast ptr [[TMP0]] to ptr
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds double, ptr [[TMP1]], i64 4
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x double>, ptr [[TMP1]], align 8
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x double>, ptr [[TMP2]], align 8
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds double, ptr [[TMP3]], i64 4
+; CHECK-NEXT: store <4 x double> [[WIDE_LOAD]], ptr [[TMP3]], align 8
+; CHECK-NEXT: store <4 x double> [[WIDE_LOAD1]], ptr [[TMP4]], align 8
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[BC:%.*]] = bitcast ptr [[GEP_SRC]] to ptr
+; CHECK-NEXT: [[VAL:%.*]] = load double, ptr [[BC]], align 8
+; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT: store double [[VAL]], ptr [[GEP_DST]], align 8
+; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.src = getelementptr inbounds double, ptr %src, i64 %iv
+ %bc = bitcast ptr %gep.src to ptr
+ %val = load double, ptr %bc, align 8
+ %gep.dst = getelementptr inbounds double, ptr %dst, i64 %iv
+ store double %val, ptr %gep.dst, align 8
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @addrspacecast_in_ptr_chain(ptr addrspace(1) %src, ptr noalias %dst, i64 %n) {
+; CHECK-LABEL: define void @addrspacecast_in_ptr_chain(
+; CHECK-SAME: ptr addrspace(1) [[SRC:%.*]], ptr noalias [[DST:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[STEP_ADD:%.*]] = add nuw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds double, ptr addrspace(1) [[SRC]], <4 x i64> [[VEC_IND]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds double, ptr addrspace(1) [[SRC]], <4 x i64> [[STEP_ADD]]
+; CHECK-NEXT: [[TMP2:%.*]] = addrspacecast <4 x ptr addrspace(1)> [[TMP0]] to <4 x ptr>
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x ptr> [[TMP2]], i64 0
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x ptr> [[TMP2]], i64 1
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x ptr> [[TMP2]], i64 2
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x ptr> [[TMP2]], i64 3
+; CHECK-NEXT: [[TMP7:%.*]] = addrspacecast <4 x ptr addrspace(1)> [[TMP1]] to <4 x ptr>
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x ptr> [[TMP7]], i64 0
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x ptr> [[TMP7]], i64 1
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x ptr> [[TMP7]], i64 2
+; CHECK-NEXT: [[TMP11:%.*]] = extractelement <4 x ptr> [[TMP7]], i64 3
+; CHECK-NEXT: [[TMP12:%.*]] = load double, ptr [[TMP3]], align 8
+; CHECK-NEXT: [[TMP13:%.*]] = load double, ptr [[TMP4]], align 8
+; CHECK-NEXT: [[TMP14:%.*]] = load double, ptr [[TMP5]], align 8
+; CHECK-NEXT: [[TMP15:%.*]] = load double, ptr [[TMP6]], align 8
+; CHECK-NEXT: [[TMP16:%.*]] = insertelement <4 x double> poison, double [[TMP12]], i32 0
+; CHECK-NEXT: [[TMP17:%.*]] = insertelement <4 x double> [[TMP16]], double [[TMP13]], i32 1
+; CHECK-NEXT: [[TMP18:%.*]] = insertelement <4 x double> [[TMP17]], double [[TMP14]], i32 2
+; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x double> [[TMP18]], double [[TMP15]], i32 3
+; CHECK-NEXT: [[TMP20:%.*]] = load double, ptr [[TMP8]], align 8
+; CHECK-NEXT: [[TMP21:%.*]] = load double, ptr [[TMP9]], align 8
+; CHECK-NEXT: [[TMP22:%.*]] = load double, ptr [[TMP10]], align 8
+; CHECK-NEXT: [[TMP23:%.*]] = load double, ptr [[TMP11]], align 8
+; CHECK-NEXT: [[TMP24:%.*]] = insertelement <4 x double> poison, double [[TMP20]], i32 0
+; CHECK-NEXT: [[TMP25:%.*]] = insertelement <4 x double> [[TMP24]], double [[TMP21]], i32 1
+; CHECK-NEXT: [[TMP26:%.*]] = insertelement <4 x double> [[TMP25]], double [[TMP22]], i32 2
+; CHECK-NEXT: [[TMP27:%.*]] = insertelement <4 x double> [[TMP26]], double [[TMP23]], i32 3
+; CHECK-NEXT: [[TMP28:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP29:%.*]] = getelementptr inbounds double, ptr [[TMP28]], i64 4
+; CHECK-NEXT: store <4 x double> [[TMP19]], ptr [[TMP28]], align 8
+; CHECK-NEXT: store <4 x double> [[TMP27]], ptr [[TMP29]], align 8
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[STEP_ADD]], splat (i64 4)
+; CHECK-NEXT: [[TMP30:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP30]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds double, ptr addrspace(1) [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[AC:%.*]] = addrspacecast ptr addrspace(1) [[GEP_SRC]] to ptr
+; CHECK-NEXT: [[VAL:%.*]] = load double, ptr [[AC]], align 8
+; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT: store double [[VAL]], ptr [[GEP_DST]], align 8
+; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.src = getelementptr inbounds double, ptr addrspace(1) %src, i64 %iv
+ %ac = addrspacecast ptr addrspace(1) %gep.src to ptr
+ %val = load double, ptr %ac, align 8
+ %gep.dst = getelementptr inbounds double, ptr %dst, i64 %iv
+ store double %val, ptr %gep.dst, align 8
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
More information about the llvm-commits
mailing list