[llvm] 5a1635c - [LV] Add tests for load/store scalarization and ptrcasts (NFC) (#196839)
via llvm-commits
llvm-commits at lists.llvm.org
Sun May 10 13:57:09 PDT 2026
Author: Florian Hahn
Date: 2026-05-10T22:57:04+02:00
New Revision: 5a1635c6bb90e72c285527c285ed1d6596857085
URL: https://github.com/llvm/llvm-project/commit/5a1635c6bb90e72c285527c285ed1d6596857085
DIFF: https://github.com/llvm/llvm-project/commit/5a1635c6bb90e72c285527c285ed1d6596857085.diff
LOG: [LV] Add tests for load/store scalarization and ptrcasts (NFC) (#196839)
Add missing test coverage for range of pointer casts and load/store
scalarization.
Added:
llvm/test/Transforms/LoopVectorize/VPlan/X86/scalarize-wide-load-for-address-use.ll
llvm/test/Transforms/LoopVectorize/preserve-inbounds-gep-with-pointer-casts.ll
Modified:
llvm/test/Transforms/LoopVectorize/version-mem-access.ll
Removed:
################################################################################
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/X86/scalarize-wide-load-for-address-use.ll b/llvm/test/Transforms/LoopVectorize/VPlan/X86/scalarize-wide-load-for-address-use.ll
new file mode 100644
index 0000000000000..02efbf0ac738b
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/X86/scalarize-wide-load-for-address-use.ll
@@ -0,0 +1,227 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter-out-after "middle.block:" --version 6
+; RUN: opt -passes=loop-vectorize -disable-output \
+; RUN: -vplan-print-after=makeMemOpWideningDecisions \
+; RUN: -force-vector-width=4 -mtriple=x86_64-unknown-linux-gnu -mcpu=haswell \
+; RUN: %s 2>&1 | FileCheck %s
+
+ at tbl.a = external global [1024 x double]
+ at tbl.b = external global [1024 x double]
+
+; Reverse-consecutive load feeding a store address.
+define void @reverse_unmasked_load_feeds_address(ptr noalias %src, i64 %n) {
+; CHECK-LABEL: VPlan for loop in 'reverse_unmasked_load_feeds_address'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<%n> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: IR %n.minus.1 = sub i64 %n, 1
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<%n.minus.1>, ir<-1>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%src>, ir<%iv>
+; CHECK-NEXT: REPLICATE ir<%val> = load ir<%gep>
+; CHECK-NEXT: EMIT ir<%cmp> = fcmp oeq ir<%val>, ir<0.000000e+00>
+; CHECK-NEXT: EMIT ir<%ptr.sel> = select ir<%cmp>, ir<@tbl.a>, ir<@tbl.b>
+; CHECK-NEXT: REPLICATE store ir<1.000000e+00>, ir<%ptr.sel>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+;
+entry:
+ %n.minus.1 = sub i64 %n, 1
+ br label %loop
+
+loop:
+ %iv = phi i64 [ %n.minus.1, %entry ], [ %iv.next, %loop ]
+ %gep = getelementptr double, ptr %src, i64 %iv
+ %val = load double, ptr %gep, align 8
+ %cmp = fcmp oeq double %val, 0.0
+ %ptr.sel = select i1 %cmp, ptr @tbl.a, ptr @tbl.b
+ store double 1.0, ptr %ptr.sel, align 8
+ %iv.next = add i64 %iv, -1
+ %ec = icmp eq i64 %iv, 0
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Widened load whose value is used both as an address (via select) and as
+; a vector arithmetic operand (fmul).
+define void @mixed_address_and_vector_uses(ptr noalias %src, ptr noalias %dst, i64 %n) {
+; CHECK-LABEL: VPlan for loop in 'mixed_address_and_vector_uses'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<%n> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT ir<%gep.src> = getelementptr ir<%src>, ir<%iv>
+; CHECK-NEXT: REPLICATE ir<%val> = load ir<%gep.src>
+; CHECK-NEXT: EMIT ir<%cmp> = fcmp oeq ir<%val>, ir<0.000000e+00>
+; CHECK-NEXT: EMIT ir<%ptr.sel> = select ir<%cmp>, ir<@tbl.a>, ir<@tbl.b>
+; CHECK-NEXT: REPLICATE store ir<1.000000e+00>, ir<%ptr.sel>
+; CHECK-NEXT: EMIT ir<%doubled> = fmul ir<%val>, ir<2.000000e+00>
+; CHECK-NEXT: EMIT ir<%gep.dst> = getelementptr ir<%dst>, ir<%iv>
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-pointer ir<%gep.dst>
+; CHECK-NEXT: WIDEN store vp<[[VP4]]>, ir<%doubled>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.src = getelementptr double, ptr %src, i64 %iv
+ %val = load double, ptr %gep.src, align 8
+ %cmp = fcmp oeq double %val, 0.0
+ %ptr.sel = select i1 %cmp, ptr @tbl.a, ptr @tbl.b
+ store double 1.0, ptr %ptr.sel, align 8
+ %doubled = fmul double %val, 2.0
+ %gep.dst = getelementptr double, ptr %dst, i64 %iv
+ store double %doubled, ptr %gep.dst, align 8
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Interleave-group candidate where one member's value feeds a store address.
+%struct.pair = type { ptr, double }
+define void @interleave_member_feeds_address(ptr noalias %arr, i64 %n) {
+; CHECK-LABEL: VPlan for loop in 'interleave_member_feeds_address'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<%n> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT ir<%gep.p> = getelementptr ir<%arr>, ir<%iv>, ir<0>
+; CHECK-NEXT: REPLICATE ir<%p> = load ir<%gep.p>
+; CHECK-NEXT: EMIT ir<%gep.v> = getelementptr ir<%arr>, ir<%iv>, ir<1>
+; CHECK-NEXT: REPLICATE ir<%v> = load ir<%gep.v>
+; CHECK-NEXT: REPLICATE store ir<%v>, ir<%p>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.p = getelementptr %struct.pair, ptr %arr, i64 %iv, i32 0
+ %p = load ptr, ptr %gep.p, align 8
+ %gep.v = getelementptr %struct.pair, ptr %arr, i64 %iv, i32 1
+ %v = load double, ptr %gep.v, align 8
+ store double %v, ptr %p, align 8
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Load with symbolic stride that PSE versions to 1 at runtime.
+define void @symbolic_stride_versioned_to_one(ptr noalias %src, ptr noalias %dst, i64 %n, i64 %stride) {
+; CHECK-LABEL: VPlan for loop in 'symbolic_stride_versioned_to_one'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<%n> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%gep.src> = getelementptr ir<%src>, ir<%idx>
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-pointer ir<%gep.src>
+; CHECK-NEXT: WIDEN ir<%val> = load vp<[[VP4]]>
+; CHECK-NEXT: EMIT ir<%doubled> = fmul ir<%val>, ir<2.000000e+00>
+; CHECK-NEXT: EMIT ir<%gep.dst> = getelementptr ir<%dst>, ir<%iv>
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer ir<%gep.dst>
+; CHECK-NEXT: WIDEN store vp<[[VP5]]>, ir<%doubled>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %idx = mul i64 %iv, %stride
+ %gep.src = getelementptr double, ptr %src, i64 %idx
+ %val = load double, ptr %gep.src, align 8
+ %doubled = fmul double %val, 2.0
+ %gep.dst = getelementptr double, ptr %dst, i64 %iv
+ store double %doubled, ptr %gep.dst, align 8
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/preserve-inbounds-gep-with-pointer-casts.ll b/llvm/test/Transforms/LoopVectorize/preserve-inbounds-gep-with-pointer-casts.ll
new file mode 100644
index 0000000000000..a9bb1c8f7f7be
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/preserve-inbounds-gep-with-pointer-casts.ll
@@ -0,0 +1,154 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=2 -S %s | FileCheck %s
+
+define void @preserve_inbounds_through_noop_gep(ptr %p) {
+; CHECK-LABEL: define void @preserve_inbounds_through_noop_gep(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[VECTOR_BODY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY1:.*]]
+; CHECK: [[VECTOR_BODY1]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY1]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[INDEX]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[TMP0]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 4
+; CHECK-NEXT: store <4 x i32> zeroinitializer, ptr [[TMP1]], align 4
+; CHECK-NEXT: store <4 x i32> zeroinitializer, ptr [[TMP2]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1000
+; CHECK-NEXT: br i1 [[TMP3]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY1]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep = getelementptr inbounds i32, ptr %p, i32 %iv
+ %noop = getelementptr i8, ptr %gep, i64 0
+ store i32 0, ptr %noop
+ %iv.next = add nsw i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, 1000
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @bitcast_in_ptr_chain(ptr %src, ptr noalias %dst) {
+; CHECK-LABEL: define void @bitcast_in_ptr_chain(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr noalias [[DST:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast ptr [[TMP0]] to ptr
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds double, ptr [[TMP1]], i64 4
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x double>, ptr [[TMP1]], align 8
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x double>, ptr [[TMP2]], align 8
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds double, ptr [[TMP3]], i64 4
+; CHECK-NEXT: store <4 x double> [[WIDE_LOAD]], ptr [[TMP3]], align 8
+; CHECK-NEXT: store <4 x double> [[WIDE_LOAD1]], ptr [[TMP4]], align 8
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
+; CHECK-NEXT: br i1 [[TMP5]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.src = getelementptr inbounds double, ptr %src, i64 %iv
+ %bc = bitcast ptr %gep.src to ptr
+ %val = load double, ptr %bc, align 8
+ %gep.dst = getelementptr inbounds double, ptr %dst, i64 %iv
+ store double %val, ptr %gep.dst, align 8
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1000
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @addrspacecast_in_ptr_chain(ptr addrspace(1) %src, ptr noalias %dst) {
+; CHECK-LABEL: define void @addrspacecast_in_ptr_chain(
+; CHECK-SAME: ptr addrspace(1) [[SRC:%.*]], ptr noalias [[DST:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[STEP_ADD:%.*]] = add nuw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds double, ptr addrspace(1) [[SRC]], <4 x i64> [[VEC_IND]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds double, ptr addrspace(1) [[SRC]], <4 x i64> [[STEP_ADD]]
+; CHECK-NEXT: [[TMP2:%.*]] = addrspacecast <4 x ptr addrspace(1)> [[TMP0]] to <4 x ptr>
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x ptr> [[TMP2]], i64 0
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x ptr> [[TMP2]], i64 1
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x ptr> [[TMP2]], i64 2
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x ptr> [[TMP2]], i64 3
+; CHECK-NEXT: [[TMP7:%.*]] = addrspacecast <4 x ptr addrspace(1)> [[TMP1]] to <4 x ptr>
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x ptr> [[TMP7]], i64 0
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x ptr> [[TMP7]], i64 1
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x ptr> [[TMP7]], i64 2
+; CHECK-NEXT: [[TMP11:%.*]] = extractelement <4 x ptr> [[TMP7]], i64 3
+; CHECK-NEXT: [[TMP12:%.*]] = load double, ptr [[TMP3]], align 8
+; CHECK-NEXT: [[TMP13:%.*]] = load double, ptr [[TMP4]], align 8
+; CHECK-NEXT: [[TMP14:%.*]] = load double, ptr [[TMP5]], align 8
+; CHECK-NEXT: [[TMP15:%.*]] = load double, ptr [[TMP6]], align 8
+; CHECK-NEXT: [[TMP16:%.*]] = insertelement <4 x double> poison, double [[TMP12]], i32 0
+; CHECK-NEXT: [[TMP17:%.*]] = insertelement <4 x double> [[TMP16]], double [[TMP13]], i32 1
+; CHECK-NEXT: [[TMP18:%.*]] = insertelement <4 x double> [[TMP17]], double [[TMP14]], i32 2
+; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x double> [[TMP18]], double [[TMP15]], i32 3
+; CHECK-NEXT: [[TMP20:%.*]] = load double, ptr [[TMP8]], align 8
+; CHECK-NEXT: [[TMP21:%.*]] = load double, ptr [[TMP9]], align 8
+; CHECK-NEXT: [[TMP22:%.*]] = load double, ptr [[TMP10]], align 8
+; CHECK-NEXT: [[TMP23:%.*]] = load double, ptr [[TMP11]], align 8
+; CHECK-NEXT: [[TMP24:%.*]] = insertelement <4 x double> poison, double [[TMP20]], i32 0
+; CHECK-NEXT: [[TMP25:%.*]] = insertelement <4 x double> [[TMP24]], double [[TMP21]], i32 1
+; CHECK-NEXT: [[TMP26:%.*]] = insertelement <4 x double> [[TMP25]], double [[TMP22]], i32 2
+; CHECK-NEXT: [[TMP27:%.*]] = insertelement <4 x double> [[TMP26]], double [[TMP23]], i32 3
+; CHECK-NEXT: [[TMP28:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP29:%.*]] = getelementptr inbounds double, ptr [[TMP28]], i64 4
+; CHECK-NEXT: store <4 x double> [[TMP19]], ptr [[TMP28]], align 8
+; CHECK-NEXT: store <4 x double> [[TMP27]], ptr [[TMP29]], align 8
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[STEP_ADD]], splat (i64 4)
+; CHECK-NEXT: [[TMP30:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
+; CHECK-NEXT: br i1 [[TMP30]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.src = getelementptr inbounds double, ptr addrspace(1) %src, i64 %iv
+ %ac = addrspacecast ptr addrspace(1) %gep.src to ptr
+ %val = load double, ptr %ac, align 8
+ %gep.dst = getelementptr inbounds double, ptr %dst, i64 %iv
+ store double %val, ptr %gep.dst, align 8
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1000
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/version-mem-access.ll b/llvm/test/Transforms/LoopVectorize/version-mem-access.ll
index c573ebaa51e9f..3d0272bef7dfa 100644
--- a/llvm/test/Transforms/LoopVectorize/version-mem-access.ll
+++ b/llvm/test/Transforms/LoopVectorize/version-mem-access.ll
@@ -168,3 +168,51 @@ loop:
exit:
ret void
}
+
+define void @load_symbolic_stride_versioned_to_one(ptr noalias %src, ptr noalias %dst,
+; CHECK-LABEL: define void @load_symbolic_stride_versioned_to_one(
+; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]], i64 [[N:%.*]], i64 [[STRIDE:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
+; CHECK: [[VECTOR_SCEVCHECK]]:
+; CHECK-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
+; CHECK-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 2
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr double, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x double>, ptr [[TMP0]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = fmul <2 x double> [[WIDE_LOAD]], splat (double 2.000000e+00)
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr double, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT: store <2 x double> [[TMP1]], ptr [[TMP2]], align 8
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+;
+ i64 %n, i64 %stride) {
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %idx = mul i64 %iv, %stride
+ %gep.src = getelementptr double, ptr %src, i64 %idx
+ %val = load double, ptr %gep.src, align 8
+ %doubled = fmul double %val, 2.0
+ %gep.dst = getelementptr double, ptr %dst, i64 %iv
+ store double %doubled, ptr %gep.dst, align 8
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
More information about the llvm-commits
mailing list