[llvm] [LV] Add call widening tests with various attributes. (NFC) (PR #206549)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Mon Jun 29 11:00:42 PDT 2026
https://github.com/fhahn created https://github.com/llvm/llvm-project/pull/206549
Add tests with both wide library and intrinsic calls and various function, return and argument attributes.
>From f3fedb22877b1a5a137bdb9d3fdedcd8fef05176 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Tue, 2 Jun 2026 14:33:26 +0100
Subject: [PATCH] [LV] Add call widening tests with various attributes. (NFC)
---
.../AArch64/widen-call-attrs-masked.ll | 107 +++
.../LoopVectorize/VPlan/print-attributes.ll | 138 ++++
.../widen-call-attrs-interleave.ll | 52 ++
.../LoopVectorize/widen-call-attrs.ll | 637 ++++++++++++++++++
.../LoopVectorize/widen-intrinsic-cse.ll | 211 ++++++
.../widen-intrinsic-tail-fold.ll | 122 ++++
.../LoopVectorize/widen-intrinsic.ll | 230 ++++++-
7 files changed, 1494 insertions(+), 3 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/widen-call-attrs-masked.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/VPlan/print-attributes.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/widen-call-attrs-interleave.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/widen-call-attrs.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/widen-intrinsic-cse.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/widen-intrinsic-tail-fold.ll
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-attrs-masked.ll b/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-attrs-masked.ll
new file mode 100644
index 0000000000000..24d85bfdea0b5
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-attrs-masked.ll
@@ -0,0 +1,107 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -tail-folding-policy=must-fold-tail -S %s | FileCheck %s
+
+target triple = "aarch64-unknown-linux-gnu"
+
+; Tail-folded vectorization to a masked (_ZGVsM) vector-ABI variant.
+
+declare i64 @foo(i64)
+declare <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64>, <vscale x 2 x i1>)
+
+; range on both the argument and the return is poison-on-violation, which will
+; be masked off.
+define void @masked_range_arg_and_ret_propagated(ptr noalias %a, ptr readnone %b) #1 {
+; CHECK-LABEL: define void @masked_range_arg_and_ret_propagated(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr readnone [[B:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i64, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP2]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
+; CHECK-NEXT: [[TMP3:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i64, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP3]], ptr align 8 [[TMP4]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025)
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-NEXT: [[TMP6:%.*]] = xor i1 [[TMP5]], true
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [0, %entry], [%iv.next, %loop]
+ %gep = getelementptr i64, ptr %b, i64 %iv
+ %x = load i64, ptr %gep
+ %r = call range(i64 0, 100) i64 @foo(i64 range(i64 0, 50) %x) #0
+ %a.gep = getelementptr i64, ptr %a, i64 %iv
+ store i64 %r, ptr %a.gep
+ %iv.next = add i64 %iv, 1
+ %done = icmp eq i64 %iv.next, 1025
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; noundef and dereferenceable are UB-implying and cannot be preserved.
+define void @masked_noundef_dereferenceable_stripped(ptr noalias %a, ptr readnone %b) #1 {
+; CHECK-LABEL: define void @masked_noundef_dereferenceable_stripped(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr readnone [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i64, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP2]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
+; CHECK-NEXT: [[TMP3:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i64, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP3]], ptr align 8 [[TMP4]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025)
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-NEXT: [[TMP6:%.*]] = xor i1 [[TMP5]], true
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [0, %entry], [%iv.next, %loop]
+ %gep = getelementptr i64, ptr %b, i64 %iv
+ %x = load i64, ptr %gep
+ %r = call range(i64 0, 100) i64 @foo(i64 noundef range(i64 0, 50) %x) #0
+ %a.gep = getelementptr i64, ptr %a, i64 %iv
+ store i64 %r, ptr %a.gep
+ %iv.next = add i64 %iv, 1
+ %done = icmp eq i64 %iv.next, 1025
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+attributes #0 = { nounwind "vector-function-abi-variant"="_ZGVsMxv_foo(foo_vector)" }
+attributes #1 = { "target-features"="+sve" vscale_range(2,16) }
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/print-attributes.ll b/llvm/test/Transforms/LoopVectorize/VPlan/print-attributes.ll
new file mode 100644
index 0000000000000..d7f46b8a62a5a
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/print-attributes.ll
@@ -0,0 +1,138 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter-out-after "scalar.ph:" --version 6
+; RUN: opt -passes=loop-vectorize -force-vector-width=2 -force-vector-interleave=1 -disable-output -vplan-print-after=makeCallWideningDecisions %s 2>&1 | FileCheck %s
+
+target triple = "arm64-apple-macosx"
+
+declare double @acos(double)
+declare <2 x double> @vec_acos(<2 x double>)
+
+define void @wide_call_attrs(ptr noalias %in.ptr, ptr noalias %out.ptr) {
+; CHECK-LABEL: VPlan for loop in 'wide_call_attrs'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<1000> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: CLONE ir<%in.gep> = getelementptr inbounds ir<%in.ptr>, ir<%iv>
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-pointer inbounds ir<%in.gep>, ir<1>
+; CHECK-NEXT: WIDEN ir<%in> = load vp<[[VP4]]>
+; CHECK-NEXT: WIDEN-CALL ir<%call> = call @acos(ir<%in>) (using library function: vec_acos)
+; CHECK-NEXT: CLONE ir<%out.gep> = getelementptr inbounds ir<%out.ptr>, ir<%iv>
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds ir<%out.gep>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP5]]>, ir<%call>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
+; CHECK-NEXT: CLONE ir<%exitcond> = icmp eq ir<%iv.next>, ir<1000>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<1000>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %in.gep = getelementptr inbounds double, ptr %in.ptr, i64 %iv
+ %in = load double, ptr %in.gep, align 8
+ %call = tail call nofpclass(nan) double @acos(double nofpclass(inf) %in) #0
+ %out.gep = getelementptr inbounds double, ptr %out.ptr, i64 %iv
+ store double %call, ptr %out.gep, align 8
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, 1000
+ br i1 %exitcond, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @test_intrinsic_with_arg_and_ret_attrs(ptr noalias %A, ptr noalias %B, i32 %n) {
+; CHECK-LABEL: VPlan for loop in 'test_intrinsic_with_arg_and_ret_attrs'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<%n> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%i> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: CLONE ir<%gep.A> = getelementptr inbounds ir<%A>, ir<%i>
+; CHECK-NEXT: CLONE ir<%gep.B> = getelementptr inbounds ir<%B>, ir<%i>
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-pointer inbounds ir<%gep.A>, ir<1>
+; CHECK-NEXT: WIDEN ir<%a> = load vp<[[VP4]]>
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds ir<%gep.B>, ir<1>
+; CHECK-NEXT: WIDEN ir<%b> = load vp<[[VP5]]>
+; CHECK-NEXT: WIDEN-INTRINSIC ir<%m> = call llvm.minnum(ir<%a>, ir<%b>)
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds ir<%gep.A>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP6]]>, ir<%m>
+; CHECK-NEXT: EMIT ir<%i.next> = add ir<%i>, ir<1>
+; CHECK-NEXT: CLONE ir<%cond> = icmp eq ir<%i.next>, ir<%n>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = exiting-iv-value ir<%i>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %gep.A = getelementptr inbounds float, ptr %A, i32 %i
+ %gep.B = getelementptr inbounds float, ptr %B, i32 %i
+ %a = load float, ptr %gep.A
+ %b = load float, ptr %gep.B
+ %m = call nofpclass(nan) float @llvm.minnum.f32(float nofpclass(nan) %a, float nofpclass(nan) %b)
+ store float %m, ptr %gep.A
+ %i.next = add i32 %i, 1
+ %cond = icmp eq i32 %i.next, %n
+ br i1 %cond, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+attributes #0 = { "vector-function-abi-variant"="_ZGVnN2v_acos(vec_acos)" }
diff --git a/llvm/test/Transforms/LoopVectorize/widen-call-attrs-interleave.ll b/llvm/test/Transforms/LoopVectorize/widen-call-attrs-interleave.ll
new file mode 100644
index 0000000000000..b68f1545aad6a
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/widen-call-attrs-interleave.ll
@@ -0,0 +1,52 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt -passes=loop-vectorize -force-vector-width=2 -force-vector-interleave=2 -S %s | FileCheck %s --check-prefix=CHECK
+
+declare i32 @scalar_fn_int(i32) #0
+declare <2 x i32> @vec_fn_int(<2 x i32>) #0
+
+define void @range_propagated_uf2(ptr noalias %src, ptr noalias %out) {
+; CHECK-LABEL: define void @range_propagated_uf2(
+; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[OUT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i32, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i32, ptr [[TMP0]], i64 2
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x i32>, ptr [[TMP1]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = call <2 x i32> @vec_fn_int(<2 x i32> [[WIDE_LOAD]])
+; CHECK-NEXT: [[TMP3:%.*]] = call <2 x i32> @vec_fn_int(<2 x i32> [[WIDE_LOAD1]])
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i32, ptr [[OUT]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i32, ptr [[TMP4]], i64 2
+; CHECK-NEXT: store <2 x i32> [[TMP2]], ptr [[TMP4]], align 4
+; CHECK-NEXT: store <2 x i32> [[TMP3]], ptr [[TMP5]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [0, %entry], [%iv.next, %loop]
+ %s.gep = getelementptr i32, ptr %src, i64 %iv
+ %x = load i32, ptr %s.gep
+ %r = call i32 @scalar_fn_int(i32 range(i32 0, 100) %x)
+ %o.gep = getelementptr i32, ptr %out, i64 %iv
+ store i32 %r, ptr %o.gep
+ %iv.next = add i64 %iv, 1
+ %done = icmp eq i64 %iv.next, 1024
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+attributes #0 = { nounwind willreturn memory(none) "vector-function-abi-variant"="_ZGVnN2v_scalar_fn_int(vec_fn_int)" }
diff --git a/llvm/test/Transforms/LoopVectorize/widen-call-attrs.ll b/llvm/test/Transforms/LoopVectorize/widen-call-attrs.ll
new file mode 100644
index 0000000000000..071eb9b001027
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/widen-call-attrs.ll
@@ -0,0 +1,637 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt -passes=loop-vectorize -force-vector-width=2 -force-vector-interleave=1 -S %s | FileCheck %s
+
+declare i32 @scalar_fn_uniform_ptr(ptr) #0
+declare <2 x i32> @vec_fn_uniform_ptr(ptr) #0
+
+declare i32 @scalar_fn_vec_ptr(ptr) #1
+declare <2 x i32> @vec_fn_vec_ptr(<2 x ptr>) #1
+
+declare i32 @scalar_fn_int(i32) #2
+declare <2 x i32> @vec_fn_int(<2 x i32>) #2
+
+declare float @scalar_fn_fp(float) #3
+declare <2 x float> @vec_fn_fp(<2 x float>) #3
+
+declare i32 @scalar_byval_fn(ptr byval(i32)) #4
+declare <2 x i32> @vec_byval_fn(ptr) #4
+
+declare i32 @scalar_fn_int_variant_attrs(i32) #5
+declare <2 x i32> @vec_fn_int_variant_attrs(<2 x i32> range(i32 0, 50)) #5
+
+declare i32 @scalar_fn_uniform_ptr_variant_attrs(ptr) #6
+declare <2 x i32> @vec_fn_uniform_ptr_variant_attrs(ptr align 32) #6
+
+declare float @scalar_fn_fp_variant_attrs(float) #7
+declare <2 x float> @vec_fn_fp_variant_attrs(<2 x float> nofpclass(nan inf)) #7
+
+define void @combined_stripped_uniform_ptr(ptr noalias %p, ptr noalias %out) {
+; CHECK-LABEL: define void @combined_stripped_uniform_ptr(
+; CHECK-SAME: ptr noalias [[P:%.*]], ptr noalias [[OUT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call <2 x i32> @vec_fn_uniform_ptr(ptr [[P]])
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i32, ptr [[OUT]], i64 [[INDEX]]
+; CHECK-NEXT: store <2 x i32> [[TMP0]], ptr [[TMP1]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [0, %entry], [%iv.next, %loop]
+ %r = call i32 @scalar_fn_uniform_ptr(ptr noalias writable captures(none) dead_on_return dead_on_unwind noundef dereferenceable(16) dereferenceable_or_null(16) %p)
+ %o.gep = getelementptr i32, ptr %out, i64 %iv
+ store i32 %r, ptr %o.gep
+ %iv.next = add i64 %iv, 1
+ %done = icmp eq i64 %iv.next, 1024
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; noalias on a vector-of-pointers parameter is dropped (per-iteration
+; disjointness does not imply lane-disjointness within a single call).
+define void @noalias_vec_ptr_stripped(ptr noalias %dst, ptr noalias %out) {
+; CHECK-LABEL: define void @noalias_vec_ptr_stripped(
+; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[OUT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr ptr, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x ptr>, ptr [[TMP0]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = call <2 x i32> @vec_fn_vec_ptr(<2 x ptr> [[WIDE_LOAD]])
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i32, ptr [[OUT]], i64 [[INDEX]]
+; CHECK-NEXT: store <2 x i32> [[TMP1]], ptr [[TMP2]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [0, %entry], [%iv.next, %loop]
+ %p.gep = getelementptr ptr, ptr %dst, i64 %iv
+ %x = load ptr, ptr %p.gep
+ %r = call i32 @scalar_fn_vec_ptr(ptr noalias %x)
+ %o.gep = getelementptr i32, ptr %out, i64 %iv
+ store i32 %r, ptr %o.gep
+ %iv.next = add i64 %iv, 1
+ %done = icmp eq i64 %iv.next, 1024
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; noundef on a parameter and on the return value is UB-implying and must
+; be stripped from the widened call: masked-off lanes can be poison.
+define void @noundef_arg_and_ret_stripped(ptr noalias %src, ptr noalias %out) {
+; CHECK-LABEL: define void @noundef_arg_and_ret_stripped(
+; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[OUT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i32, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = call <2 x i32> @vec_fn_int(<2 x i32> [[WIDE_LOAD]])
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i32, ptr [[OUT]], i64 [[INDEX]]
+; CHECK-NEXT: store <2 x i32> [[TMP1]], ptr [[TMP2]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [0, %entry], [%iv.next, %loop]
+ %s.gep = getelementptr i32, ptr %src, i64 %iv
+ %x = load i32, ptr %s.gep
+ %r = call noundef i32 @scalar_fn_int(i32 noundef %x)
+ %o.gep = getelementptr i32, ptr %out, i64 %iv
+ store i32 %r, ptr %o.gep
+ %iv.next = add i64 %iv, 1
+ %done = icmp eq i64 %iv.next, 1024
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; range(...) is poison-on-violation. It should propagate.
+define void @range_arg_propagated(ptr noalias %src, ptr noalias %out) {
+; CHECK-LABEL: define void @range_arg_propagated(
+; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[OUT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i32, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = call <2 x i32> @vec_fn_int(<2 x i32> [[WIDE_LOAD]])
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i32, ptr [[OUT]], i64 [[INDEX]]
+; CHECK-NEXT: store <2 x i32> [[TMP1]], ptr [[TMP2]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [0, %entry], [%iv.next, %loop]
+ %s.gep = getelementptr i32, ptr %src, i64 %iv
+ %x = load i32, ptr %s.gep
+ %r = call i32 @scalar_fn_int(i32 range(i32 0, 100) %x)
+ %o.gep = getelementptr i32, ptr %out, i64 %iv
+ store i32 %r, ptr %o.gep
+ %iv.next = add i64 %iv, 1
+ %done = icmp eq i64 %iv.next, 1024
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; nofpclass is poison-on-violation. It should propagate.
+define void @nofpclass_propagated(ptr noalias %src, ptr noalias %out) {
+; CHECK-LABEL: define void @nofpclass_propagated(
+; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[OUT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr float, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x float>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = call <2 x float> @vec_fn_fp(<2 x float> [[WIDE_LOAD]])
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr float, ptr [[OUT]], i64 [[INDEX]]
+; CHECK-NEXT: store <2 x float> [[TMP1]], ptr [[TMP2]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [0, %entry], [%iv.next, %loop]
+ %s.gep = getelementptr float, ptr %src, i64 %iv
+ %x = load float, ptr %s.gep
+ %r = call nofpclass(nan) float @scalar_fn_fp(float nofpclass(nan) %x)
+ %o.gep = getelementptr float, ptr %out, i64 %iv
+ store float %r, ptr %o.gep
+ %iv.next = add i64 %iv, 1
+ %done = icmp eq i64 %iv.next, 1024
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; align on a uniform pointer arg should propagate.
+define void @align_uniform_ptr_propagated(ptr noalias %p, ptr noalias %out) {
+; CHECK-LABEL: define void @align_uniform_ptr_propagated(
+; CHECK-SAME: ptr noalias [[P:%.*]], ptr noalias [[OUT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call <2 x i32> @vec_fn_uniform_ptr(ptr [[P]])
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i32, ptr [[OUT]], i64 [[INDEX]]
+; CHECK-NEXT: store <2 x i32> [[TMP0]], ptr [[TMP1]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [0, %entry], [%iv.next, %loop]
+ %r = call i32 @scalar_fn_uniform_ptr(ptr align 16 %p)
+ %o.gep = getelementptr i32, ptr %out, i64 %iv
+ store i32 %r, ptr %o.gep
+ %iv.next = add i64 %iv, 1
+ %done = icmp eq i64 %iv.next, 1024
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; nonnull is poison-on-violation. It should propagate.
+define void @nonnull_uniform_ptr_propagated(ptr noalias %p, ptr noalias %out) {
+; CHECK-LABEL: define void @nonnull_uniform_ptr_propagated(
+; CHECK-SAME: ptr noalias [[P:%.*]], ptr noalias [[OUT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call <2 x i32> @vec_fn_uniform_ptr(ptr [[P]])
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i32, ptr [[OUT]], i64 [[INDEX]]
+; CHECK-NEXT: store <2 x i32> [[TMP0]], ptr [[TMP1]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [0, %entry], [%iv.next, %loop]
+ %r = call i32 @scalar_fn_uniform_ptr(ptr nonnull %p)
+ %o.gep = getelementptr i32, ptr %out, i64 %iv
+ store i32 %r, ptr %o.gep
+ %iv.next = add i64 %iv, 1
+ %done = icmp eq i64 %iv.next, 1024
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; byval(T) is an ABI parameter attribute and cannot be preserved for different
+; types.
+define void @byval_stripped(ptr noalias %p, ptr noalias %out) {
+; CHECK-LABEL: define void @byval_stripped(
+; CHECK-SAME: ptr noalias [[P:%.*]], ptr noalias [[OUT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call <2 x i32> @vec_byval_fn(ptr [[P]])
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i32, ptr [[OUT]], i64 [[INDEX]]
+; CHECK-NEXT: store <2 x i32> [[TMP0]], ptr [[TMP1]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [0, %entry], [%iv.next, %loop]
+ %r = call i32 @scalar_byval_fn(ptr byval(i32) %p)
+ %o.gep = getelementptr i32, ptr %out, i64 %iv
+ store i32 %r, ptr %o.gep
+ %iv.next = add i64 %iv, 1
+ %done = icmp eq i64 %iv.next, 1024
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+
+; The variant declaration carries range(i32 0, 50) on its parameter; the
+; scalar call site's range(i32 0, 100). Should keep the stricter range.
+define void @range_arg_variant_decl_wins(ptr noalias %src, ptr noalias %out) {
+; CHECK-LABEL: define void @range_arg_variant_decl_wins(
+; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[OUT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i32, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = call <2 x i32> @vec_fn_int_variant_attrs(<2 x i32> [[WIDE_LOAD]])
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i32, ptr [[OUT]], i64 [[INDEX]]
+; CHECK-NEXT: store <2 x i32> [[TMP1]], ptr [[TMP2]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [0, %entry], [%iv.next, %loop]
+ %s.gep = getelementptr i32, ptr %src, i64 %iv
+ %x = load i32, ptr %s.gep
+ %r = call i32 @scalar_fn_int_variant_attrs(i32 range(i32 0, 100) %x)
+ %o.gep = getelementptr i32, ptr %out, i64 %iv
+ store i32 %r, ptr %o.gep
+ %iv.next = add i64 %iv, 1
+ %done = icmp eq i64 %iv.next, 1024
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; The variant declaration carries align 32 on its parameter; the scalar
+; call site's align 16. Should keep the stricter alignment.
+; should keep the variant's align 32.
+define void @align_uniform_ptr_variant_decl_wins(ptr noalias %p, ptr noalias %out) {
+; CHECK-LABEL: define void @align_uniform_ptr_variant_decl_wins(
+; CHECK-SAME: ptr noalias [[P:%.*]], ptr noalias [[OUT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call <2 x i32> @vec_fn_uniform_ptr_variant_attrs(ptr [[P]])
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i32, ptr [[OUT]], i64 [[INDEX]]
+; CHECK-NEXT: store <2 x i32> [[TMP0]], ptr [[TMP1]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [0, %entry], [%iv.next, %loop]
+ %r = call i32 @scalar_fn_uniform_ptr_variant_attrs(ptr align 16 %p)
+ %o.gep = getelementptr i32, ptr %out, i64 %iv
+ store i32 %r, ptr %o.gep
+ %iv.next = add i64 %iv, 1
+ %done = icmp eq i64 %iv.next, 1024
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; The variant declaration carries nofpclass(nan inf) on its parameter; the
+; scalar call site's weaker nofpclass(nan). Should keep the stricter mask.
+define void @nofpclass_arg_variant_decl_wins(ptr noalias %src, ptr noalias %out) {
+; CHECK-LABEL: define void @nofpclass_arg_variant_decl_wins(
+; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[OUT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr float, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x float>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = call <2 x float> @vec_fn_fp_variant_attrs(<2 x float> [[WIDE_LOAD]])
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr float, ptr [[OUT]], i64 [[INDEX]]
+; CHECK-NEXT: store <2 x float> [[TMP1]], ptr [[TMP2]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [0, %entry], [%iv.next, %loop]
+ %s.gep = getelementptr float, ptr %src, i64 %iv
+ %x = load float, ptr %s.gep
+ %r = call float @scalar_fn_fp_variant_attrs(float nofpclass(nan) %x)
+ %o.gep = getelementptr float, ptr %out, i64 %iv
+ store float %r, ptr %o.gep
+ %iv.next = add i64 %iv, 1
+ %done = icmp eq i64 %iv.next, 1024
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; range on the return value is poison-on-violation. Should be propagated.
+define void @range_ret_propagated(ptr noalias %src, ptr noalias %out) {
+; CHECK-LABEL: define void @range_ret_propagated(
+; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[OUT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i32, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = call <2 x i32> @vec_fn_int(<2 x i32> [[WIDE_LOAD]])
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i32, ptr [[OUT]], i64 [[INDEX]]
+; CHECK-NEXT: store <2 x i32> [[TMP1]], ptr [[TMP2]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [0, %entry], [%iv.next, %loop]
+ %s.gep = getelementptr i32, ptr %src, i64 %iv
+ %x = load i32, ptr %s.gep
+ %r = call range(i32 0, 100) i32 @scalar_fn_int(i32 %x)
+ %o.gep = getelementptr i32, ptr %out, i64 %iv
+ store i32 %r, ptr %o.gep
+ %iv.next = add i64 %iv, 1
+ %done = icmp eq i64 %iv.next, 1024
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; align N can be propagated to widened pointers.
+define void @align_vec_ptr_propagated(ptr noalias %dst, ptr noalias %out) {
+; CHECK-LABEL: define void @align_vec_ptr_propagated(
+; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[OUT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr ptr, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x ptr>, ptr [[TMP0]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = call <2 x i32> @vec_fn_vec_ptr(<2 x ptr> [[WIDE_LOAD]])
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i32, ptr [[OUT]], i64 [[INDEX]]
+; CHECK-NEXT: store <2 x i32> [[TMP1]], ptr [[TMP2]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [0, %entry], [%iv.next, %loop]
+ %p.gep = getelementptr ptr, ptr %dst, i64 %iv
+ %x = load ptr, ptr %p.gep
+ %r = call i32 @scalar_fn_vec_ptr(ptr align 16 %x)
+ %o.gep = getelementptr i32, ptr %out, i64 %iv
+ store i32 %r, ptr %o.gep
+ %iv.next = add i64 %iv, 1
+ %done = icmp eq i64 %iv.next, 1024
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; nonnull cannot be propagated to widened pointer vectors currently.
+define void @nonnull_vec_ptr_dropped(ptr noalias %dst, ptr noalias %out) {
+; CHECK-LABEL: define void @nonnull_vec_ptr_dropped(
+; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[OUT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr ptr, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x ptr>, ptr [[TMP0]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = call <2 x i32> @vec_fn_vec_ptr(<2 x ptr> [[WIDE_LOAD]])
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i32, ptr [[OUT]], i64 [[INDEX]]
+; CHECK-NEXT: store <2 x i32> [[TMP1]], ptr [[TMP2]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [0, %entry], [%iv.next, %loop]
+ %p.gep = getelementptr ptr, ptr %dst, i64 %iv
+ %x = load ptr, ptr %p.gep
+ %r = call i32 @scalar_fn_vec_ptr(ptr nonnull %x)
+ %o.gep = getelementptr i32, ptr %out, i64 %iv
+ store i32 %r, ptr %o.gep
+ %iv.next = add i64 %iv, 1
+ %done = icmp eq i64 %iv.next, 1024
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; signext/zeroext are ABI parameter attributes, Should not be propagated.
+define void @signext_zeroext_stripped(ptr noalias %src, ptr noalias %out) {
+; CHECK-LABEL: define void @signext_zeroext_stripped(
+; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[OUT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i32, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = call <2 x i32> @vec_fn_int(<2 x i32> [[WIDE_LOAD]])
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i32, ptr [[OUT]], i64 [[INDEX]]
+; CHECK-NEXT: store <2 x i32> [[TMP1]], ptr [[TMP2]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [0, %entry], [%iv.next, %loop]
+ %s.gep = getelementptr i32, ptr %src, i64 %iv
+ %x = load i32, ptr %s.gep
+ %r = call signext i32 @scalar_fn_int(i32 zeroext %x)
+ %o.gep = getelementptr i32, ptr %out, i64 %iv
+ store i32 %r, ptr %o.gep
+ %iv.next = add i64 %iv, 1
+ %done = icmp eq i64 %iv.next, 1024
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+attributes #0 = { nounwind willreturn memory(none) "vector-function-abi-variant"="_ZGVnN2u_scalar_fn_uniform_ptr(vec_fn_uniform_ptr)" }
+attributes #1 = { nounwind willreturn memory(none) "vector-function-abi-variant"="_ZGVnN2v_scalar_fn_vec_ptr(vec_fn_vec_ptr)" }
+attributes #2 = { nounwind willreturn memory(none) "vector-function-abi-variant"="_ZGVnN2v_scalar_fn_int(vec_fn_int)" }
+attributes #3 = { nounwind willreturn memory(none) "vector-function-abi-variant"="_ZGVnN2v_scalar_fn_fp(vec_fn_fp)" }
+attributes #4 = { nounwind willreturn memory(none) "vector-function-abi-variant"="_ZGVnN2u_scalar_byval_fn(vec_byval_fn)" }
+attributes #5 = { nounwind willreturn memory(none) "vector-function-abi-variant"="_ZGVnN2v_scalar_fn_int_variant_attrs(vec_fn_int_variant_attrs)" }
+attributes #6 = { nounwind willreturn memory(none) "vector-function-abi-variant"="_ZGVnN2u_scalar_fn_uniform_ptr_variant_attrs(vec_fn_uniform_ptr_variant_attrs)" }
+attributes #7 = { nounwind willreturn memory(none) "vector-function-abi-variant"="_ZGVnN2v_scalar_fn_fp_variant_attrs(vec_fn_fp_variant_attrs)" }
diff --git a/llvm/test/Transforms/LoopVectorize/widen-intrinsic-cse.ll b/llvm/test/Transforms/LoopVectorize/widen-intrinsic-cse.ll
new file mode 100644
index 0000000000000..0a4731898ab96
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/widen-intrinsic-cse.ll
@@ -0,0 +1,211 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt -passes=loop-vectorize -force-vector-width=4 -S %s | FileCheck %s
+
+; Two widened intrinsic calls with identical operands but different call site
+; attributes.
+define void @cse_merges_across_differing_fn_attrs(ptr %p, ptr %q, ptr noalias %r1, ptr noalias %r2) {
+; CHECK-LABEL: define void @cse_merges_across_differing_fn_attrs(
+; CHECK-SAME: ptr [[P:%.*]], ptr [[Q:%.*]], ptr noalias [[R1:%.*]], ptr noalias [[R2:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr float, ptr [[P]], i32 [[INDEX]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr float, ptr [[Q]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT: [[TMP7:%.*]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[WIDE_LOAD1]])
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr float, ptr [[R1]], i32 [[INDEX]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr float, ptr [[R2]], i32 [[INDEX]]
+; CHECK-NEXT: store <4 x float> [[TMP7]], ptr [[TMP4]], align 4
+; CHECK-NEXT: store <4 x float> [[TMP7]], ptr [[TMP5]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [0, %entry], [%iv.next, %loop]
+ %pa = getelementptr float, ptr %p, i32 %iv
+ %pb = getelementptr float, ptr %q, i32 %iv
+ %a = load float, ptr %pa
+ %b = load float, ptr %pb
+ %m1 = call float @llvm.minnum.f32(float %a, float %b) #0
+ %m2 = call float @llvm.minnum.f32(float %a, float %b) #1
+ %gep.r1 = getelementptr float, ptr %r1, i32 %iv
+ %gep.r2 = getelementptr float, ptr %r2, i32 %iv
+ store float %m1, ptr %gep.r1
+ store float %m2, ptr %gep.r2
+ %iv.next = add i32 %iv, 1
+ %done = icmp eq i32 %iv.next, 1024
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Same loop with both calls carrying identical function attributes.
+define void @cse_succeeds_same_fn_attrs(ptr %p, ptr %q, ptr noalias %r1, ptr noalias %r2) {
+; CHECK-LABEL: define void @cse_succeeds_same_fn_attrs(
+; CHECK-SAME: ptr [[P:%.*]], ptr [[Q:%.*]], ptr noalias [[R1:%.*]], ptr noalias [[R2:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr float, ptr [[P]], i32 [[INDEX]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr float, ptr [[Q]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[WIDE_LOAD1]])
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr float, ptr [[R1]], i32 [[INDEX]]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr float, ptr [[R2]], i32 [[INDEX]]
+; CHECK-NEXT: store <4 x float> [[TMP2]], ptr [[TMP3]], align 4
+; CHECK-NEXT: store <4 x float> [[TMP2]], ptr [[TMP4]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [0, %entry], [%iv.next, %loop]
+ %pa = getelementptr float, ptr %p, i32 %iv
+ %pb = getelementptr float, ptr %q, i32 %iv
+ %a = load float, ptr %pa
+ %b = load float, ptr %pb
+ %m1 = call float @llvm.minnum.f32(float %a, float %b) #0
+ %m2 = call float @llvm.minnum.f32(float %a, float %b) #0
+ %gep.r1 = getelementptr float, ptr %r1, i32 %iv
+ %gep.r2 = getelementptr float, ptr %r2, i32 %iv
+ store float %m1, ptr %gep.r1
+ store float %m2, ptr %gep.r2
+ %iv.next = add i32 %iv, 1
+ %done = icmp eq i32 %iv.next, 1024
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Two calls with identical operands and different per-arg attributes.
+define void @cse_param_nofpclass(ptr %p, ptr %q, ptr noalias %r1, ptr noalias %r2) {
+; CHECK-LABEL: define void @cse_param_nofpclass(
+; CHECK-SAME: ptr [[P:%.*]], ptr [[Q:%.*]], ptr noalias [[R1:%.*]], ptr noalias [[R2:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr float, ptr [[P]], i32 [[INDEX]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr float, ptr [[Q]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[WIDE_LOAD1]])
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr float, ptr [[R1]], i32 [[INDEX]]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr float, ptr [[R2]], i32 [[INDEX]]
+; CHECK-NEXT: store <4 x float> [[TMP2]], ptr [[TMP3]], align 4
+; CHECK-NEXT: store <4 x float> [[TMP2]], ptr [[TMP4]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [0, %entry], [%iv.next, %loop]
+ %pa = getelementptr float, ptr %p, i32 %iv
+ %pb = getelementptr float, ptr %q, i32 %iv
+ %a = load float, ptr %pa
+ %b = load float, ptr %pb
+ %m1 = call float @llvm.minnum.f32(float nofpclass(nan) %a, float nofpclass(nan) %b)
+ %m2 = call float @llvm.minnum.f32(float %a, float %b)
+ %gep.r1 = getelementptr float, ptr %r1, i32 %iv
+ %gep.r2 = getelementptr float, ptr %r2, i32 %iv
+ store float %m1, ptr %gep.r1
+ store float %m2, ptr %gep.r2
+ %iv.next = add i32 %iv, 1
+ %done = icmp eq i32 %iv.next, 1024
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Three calls with identical operands but differing string per-arg attributes.
+define void @cse_differing_param_string_attrs(ptr %p, ptr %q, ptr noalias %r1, ptr noalias %r2, ptr noalias %r3) {
+; CHECK-LABEL: define void @cse_differing_param_string_attrs(
+; CHECK-SAME: ptr [[P:%.*]], ptr [[Q:%.*]], ptr noalias [[R1:%.*]], ptr noalias [[R2:%.*]], ptr noalias [[R3:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr float, ptr [[P]], i32 [[INDEX]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr float, ptr [[Q]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[WIDE_LOAD1]])
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr float, ptr [[R1]], i32 [[INDEX]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr float, ptr [[R2]], i32 [[INDEX]]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr float, ptr [[R3]], i32 [[INDEX]]
+; CHECK-NEXT: store <4 x float> [[TMP3]], ptr [[TMP4]], align 4
+; CHECK-NEXT: store <4 x float> [[TMP3]], ptr [[TMP5]], align 4
+; CHECK-NEXT: store <4 x float> [[TMP3]], ptr [[TMP6]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [0, %entry], [%iv.next, %loop]
+ %pa = getelementptr float, ptr %p, i32 %iv
+ %pb = getelementptr float, ptr %q, i32 %iv
+ %a = load float, ptr %pa
+ %b = load float, ptr %pb
+ %m1 = call float @llvm.minnum.f32(float "foo"="A" %a, float %b)
+ %m2 = call float @llvm.minnum.f32(float "foo"="B" %a, float %b)
+ %m3 = call float @llvm.minnum.f32(float "foo"="B" %a, float %b)
+ %gep.r1 = getelementptr float, ptr %r1, i32 %iv
+ %gep.r2 = getelementptr float, ptr %r2, i32 %iv
+ %gep.r3 = getelementptr float, ptr %r3, i32 %iv
+ store float %m1, ptr %gep.r1
+ store float %m2, ptr %gep.r2
+ store float %m3, ptr %gep.r3
+ %iv.next = add i32 %iv, 1
+ %done = icmp eq i32 %iv.next, 1024
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+attributes #0 = { memory(none) "target-cpu"="generic" }
+attributes #1 = { memory(none) "target-cpu"="skylake" }
diff --git a/llvm/test/Transforms/LoopVectorize/widen-intrinsic-tail-fold.ll b/llvm/test/Transforms/LoopVectorize/widen-intrinsic-tail-fold.ll
new file mode 100644
index 0000000000000..69a50456657ea
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/widen-intrinsic-tail-fold.ll
@@ -0,0 +1,122 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -force-target-supports-masked-memory-ops -force-tail-folding-style=data-and-control -tail-folding-policy=prefer-fold-tail -S %s | FileCheck %s
+
+define void @tf_ctlz_range(ptr %p, i32 %n) {
+; CHECK-LABEL: define void @tf_ctlz_range(
+; CHECK-SAME: ptr [[P:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
+; CHECK: [[VECTOR_SCEVCHECK]]:
+; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
+; CHECK-NEXT: [[TMP1:%.*]] = icmp slt i32 [[TMP0]], 0
+; CHECK-NEXT: br i1 [[TMP1]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 0, i32 [[N]])
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i32, ptr [[P]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP2]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> poison)
+; CHECK-NEXT: [[TMP3:%.*]] = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> [[WIDE_MASKED_LOAD]], i1 false)
+; CHECK-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> [[TMP3]], ptr align 4 [[TMP2]], <4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 [[INDEX_NEXT]], i32 [[N]])
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-NEXT: [[TMP5:%.*]] = xor i1 [[TMP4]], true
+; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P_GEP:%.*]] = getelementptr i32, ptr [[P]], i32 [[IV]]
+; CHECK-NEXT: [[X:%.*]] = load i32, ptr [[P_GEP]], align 4
+; CHECK-NEXT: [[Y:%.*]] = call i32 @llvm.ctlz.i32(i32 range(i32 1, 0) [[X]], i1 false)
+; CHECK-NEXT: store i32 [[Y]], ptr [[P_GEP]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT: [[DONE:%.*]] = icmp eq i32 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [0, %entry], [%iv.next, %loop]
+ %p.gep = getelementptr i32, ptr %p, i32 %iv
+ %x = load i32, ptr %p.gep
+ %y = call i32 @llvm.ctlz.i32(i32 range(i32 1, 0) %x, i1 false)
+ store i32 %y, ptr %p.gep
+ %iv.next = add i32 %iv, 1
+ %done = icmp eq i32 %iv.next, %n
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @tf_minnum_noundef(ptr noalias %p, ptr noalias %q, i32 %n) {
+; CHECK-LABEL: define void @tf_minnum_noundef(
+; CHECK-SAME: ptr noalias [[P:%.*]], ptr noalias [[Q:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
+; CHECK: [[VECTOR_SCEVCHECK]]:
+; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
+; CHECK-NEXT: [[TMP1:%.*]] = icmp slt i32 [[TMP0]], 0
+; CHECK-NEXT: br i1 [[TMP1]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 0, i32 [[N]])
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr float, ptr [[P]], i32 [[INDEX]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr float, ptr [[Q]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x float> @llvm.masked.load.v4f32.p0(ptr align 4 [[TMP2]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x float> poison)
+; CHECK-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <4 x float> @llvm.masked.load.v4f32.p0(ptr align 4 [[TMP3]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x float> poison)
+; CHECK-NEXT: [[TMP4:%.*]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[WIDE_MASKED_LOAD]], <4 x float> [[WIDE_MASKED_LOAD1]])
+; CHECK-NEXT: call void @llvm.masked.store.v4f32.p0(<4 x float> [[TMP4]], ptr align 4 [[TMP2]], <4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 [[INDEX_NEXT]], i32 [[N]])
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-NEXT: [[TMP6:%.*]] = xor i1 [[TMP5]], true
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P_GEP:%.*]] = getelementptr float, ptr [[P]], i32 [[IV]]
+; CHECK-NEXT: [[Q_GEP:%.*]] = getelementptr float, ptr [[Q]], i32 [[IV]]
+; CHECK-NEXT: [[A:%.*]] = load float, ptr [[P_GEP]], align 4
+; CHECK-NEXT: [[B:%.*]] = load float, ptr [[Q_GEP]], align 4
+; CHECK-NEXT: [[Y:%.*]] = call noundef float @llvm.minnum.f32(float [[A]], float [[B]])
+; CHECK-NEXT: store float [[Y]], ptr [[P_GEP]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT: [[DONE:%.*]] = icmp eq i32 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [0, %entry], [%iv.next, %loop]
+ %p.gep = getelementptr float, ptr %p, i32 %iv
+ %q.gep = getelementptr float, ptr %q, i32 %iv
+ %a = load float, ptr %p.gep
+ %b = load float, ptr %q.gep
+ %y = call noundef float @llvm.minnum.f32(float %a, float %b)
+ store float %y, ptr %p.gep
+ %iv.next = add i32 %iv, 1
+ %done = icmp eq i32 %iv.next, %n
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/widen-intrinsic.ll b/llvm/test/Transforms/LoopVectorize/widen-intrinsic.ll
index 922ebe7211b6e..8343f7656e246 100644
--- a/llvm/test/Transforms/LoopVectorize/widen-intrinsic.ll
+++ b/llvm/test/Transforms/LoopVectorize/widen-intrinsic.ll
@@ -1,7 +1,6 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 5
-; RUN: opt < %s -passes=loop-vectorize -force-vector-width=4 -S | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt -passes=loop-vectorize -force-vector-width=4 -S %s | FileCheck %s
-; Check that we don't unnecessarily broadcast %pow
define void @powi_only_first_lane_used_of_second_arg(ptr %p, i32 %pow) {
; CHECK-LABEL: define void @powi_only_first_lane_used_of_second_arg(
; CHECK-SAME: ptr [[P:%.*]], i32 [[POW:%.*]]) {
@@ -39,3 +38,228 @@ loop:
exit:
ret void
}
+
+; Per-arg and return nofpclass(nan) on the scalar call.
+define void @minnum_nofpclass_arg_and_ret(ptr %p, ptr %q) {
+; CHECK-LABEL: define void @minnum_nofpclass_arg_and_ret(
+; CHECK-SAME: ptr [[P:%.*]], ptr [[Q:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
+; CHECK: [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[P]], i64 4096
+; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[Q]], i64 4096
+; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[P]], [[SCEVGEP1]]
+; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[Q]], [[SCEVGEP]]
+; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr float, ptr [[P]], i32 [[INDEX]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr float, ptr [[Q]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP0]], align 4, !alias.scope [[META3:![0-9]+]], !noalias [[META6:![0-9]+]]
+; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x float>, ptr [[TMP1]], align 4, !alias.scope [[META6]]
+; CHECK-NEXT: [[TMP2:%.*]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[WIDE_LOAD2]])
+; CHECK-NEXT: store <4 x float> [[TMP2]], ptr [[TMP0]], align 4, !alias.scope [[META3]], !noalias [[META6]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P_GEP:%.*]] = getelementptr float, ptr [[P]], i32 [[IV]]
+; CHECK-NEXT: [[Q_GEP:%.*]] = getelementptr float, ptr [[Q]], i32 [[IV]]
+; CHECK-NEXT: [[A:%.*]] = load float, ptr [[P_GEP]], align 4
+; CHECK-NEXT: [[B:%.*]] = load float, ptr [[Q_GEP]], align 4
+; CHECK-NEXT: [[Y:%.*]] = call nofpclass(nan) float @llvm.minnum.f32(float nofpclass(nan) [[A]], float nofpclass(nan) [[B]])
+; CHECK-NEXT: store float [[Y]], ptr [[P_GEP]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT: [[DONE:%.*]] = icmp eq i32 [[IV_NEXT]], 1024
+; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [0, %entry], [%iv.next, %loop]
+ %p.gep = getelementptr float, ptr %p, i32 %iv
+ %q.gep = getelementptr float, ptr %q, i32 %iv
+ %a = load float, ptr %p.gep
+ %b = load float, ptr %q.gep
+ %y = call nofpclass(nan) float @llvm.minnum.f32(float nofpclass(nan) %a, float nofpclass(nan) %b)
+ store float %y, ptr %p.gep
+ %iv.next = add i32 %iv, 1
+ %done = icmp eq i32 %iv.next, 1024
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Call wih range(...) argument attribute.
+define void @ctlz_range(ptr %p) {
+; CHECK-LABEL: define void @ctlz_range(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i32, ptr [[P]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> [[WIDE_LOAD]], i1 false)
+; CHECK-NEXT: store <4 x i32> [[TMP1]], ptr [[TMP0]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [0, %entry], [%iv.next, %loop]
+ %p.gep = getelementptr i32, ptr %p, i32 %iv
+ %x = load i32, ptr %p.gep
+ %y = call i32 @llvm.ctlz.i32(i32 range(i32 1, 0) %x, i1 false)
+ store i32 %y, ptr %p.gep
+ %iv.next = add i32 %iv, 1
+ %done = icmp eq i32 %iv.next, 1024
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; noundef on a parameter and the return.
+define void @smin_noundef_arg_and_ret(ptr noalias %p, ptr noalias %q) {
+; CHECK-LABEL: define void @smin_noundef_arg_and_ret(
+; CHECK-SAME: ptr noalias [[P:%.*]], ptr noalias [[Q:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i32, ptr [[P]], i32 [[INDEX]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i32, ptr [[Q]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = call <4 x i32> @llvm.smin.v4i32(<4 x i32> [[WIDE_LOAD]], <4 x i32> [[WIDE_LOAD1]])
+; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[TMP0]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [0, %entry], [%iv.next, %loop]
+ %p.gep = getelementptr i32, ptr %p, i32 %iv
+ %q.gep = getelementptr i32, ptr %q, i32 %iv
+ %a = load i32, ptr %p.gep
+ %b = load i32, ptr %q.gep
+ %m = call noundef i32 @llvm.smin.i32(i32 noundef %a, i32 noundef %b)
+ store i32 %m, ptr %p.gep
+ %iv.next = add i32 %iv, 1
+ %done = icmp eq i32 %iv.next, 1024
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Call wih signext argument attribute.
+define void @ctlz_signext_arg(ptr %p) {
+; CHECK-LABEL: define void @ctlz_signext_arg(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i32, ptr [[P]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> [[WIDE_LOAD]], i1 false)
+; CHECK-NEXT: store <4 x i32> [[TMP1]], ptr [[TMP0]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [0, %entry], [%iv.next, %loop]
+ %p.gep = getelementptr i32, ptr %p, i32 %iv
+ %x = load i32, ptr %p.gep
+ %y = call i32 @llvm.ctlz.i32(i32 signext %x, i1 false)
+ store i32 %y, ptr %p.gep
+ %iv.next = add i32 %iv, 1
+ %done = icmp eq i32 %iv.next, 1024
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; A loop-invariant intrinsic call with a "deopt" operand bundle.
+define void @loop_invariant_intrinsic_with_deopt_bundle_not_hoisted(ptr %dst, float %a, float %b) {
+; CHECK-LABEL: define void @loop_invariant_intrinsic_with_deopt_bundle_not_hoisted(
+; CHECK-SAME: ptr [[DST:%.*]], float [[A:%.*]], float [[B:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x float> poison, float [[A]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT1]], <4 x float> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <4 x float> poison, float [[B]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT2]], <4 x float> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = call <4 x float> @llvm.pow.v4f32(<4 x float> [[BROADCAST_SPLAT2]], <4 x float> [[BROADCAST_SPLAT3]]) [ "deopt"(float 1.000000e+01) ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr float, ptr [[DST]], i32 [[INDEX]]
+; CHECK-NEXT: store <4 x float> [[TMP0]], ptr [[TMP1]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [0, %entry], [%iv.next, %loop]
+ %r = call float @llvm.pow.f32(float %a, float %b) [ "deopt"(float 1.000000e+01) ]
+ %dst.gep = getelementptr float, ptr %dst, i32 %iv
+ store float %r, ptr %dst.gep
+ %iv.next = add i32 %iv, 1
+ %done = icmp eq i32 %iv.next, 1024
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
More information about the llvm-commits
mailing list