[llvm] [VPlan] Stop outerloop vectorization from vectorizing nonvector intrinsics (PR #185347)
Henry Jiang via llvm-commits
llvm-commits at lists.llvm.org
Tue Mar 24 22:48:47 PDT 2026
https://github.com/mustartt updated https://github.com/llvm/llvm-project/pull/185347
>From a4cc4e47ced6ee837f984b8058bc1c836c2cc976 Mon Sep 17 00:00:00 2001
From: Henry Jiang <henry_jiang2 at apple.com>
Date: Sun, 8 Mar 2026 22:25:27 -0700
Subject: [PATCH 1/8] Fix psuedoprobe being treated a vector intrinsic
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 8 ++++-
.../LoopVectorize/pseudo-probe-crash.ll | 36 +++++++++++++++++++
2 files changed, 43 insertions(+), 1 deletion(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/pseudo-probe-crash.ll
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index dc0edb178efbb..80f0b5d6a3b8b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -92,7 +92,13 @@ bool VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(
Ingredient.getDebugLoc());
} else if (CallInst *CI = dyn_cast<CallInst>(Inst)) {
Intrinsic::ID VectorID = getVectorIntrinsicIDForCall(CI, &TLI);
- if (VectorID == Intrinsic::not_intrinsic)
+ if (VectorID == Intrinsic::not_intrinsic ||
+ VectorID == Intrinsic::assume ||
+ VectorID == Intrinsic::lifetime_end ||
+ VectorID == Intrinsic::lifetime_start ||
+ VectorID == Intrinsic::sideeffect ||
+ VectorID == Intrinsic::pseudoprobe ||
+ VectorID == Intrinsic::experimental_noalias_scope_decl)
return false;
NewRecipe = new VPWidenIntrinsicRecipe(
*CI, getVectorIntrinsicIDForCall(CI, &TLI),
diff --git a/llvm/test/Transforms/LoopVectorize/pseudo-probe-crash.ll b/llvm/test/Transforms/LoopVectorize/pseudo-probe-crash.ll
new file mode 100644
index 0000000000000..f4298d3c5d770
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/pseudo-probe-crash.ll
@@ -0,0 +1,36 @@
+; REQUIRES: asserts
+; RUN: opt -S -passes=loop-vectorize -enable-vplan-native-path -force-vector-width=2 -debug-only=loop-vectorize -mtriple=aarch64 < %s 2>&1 | FileCheck %s
+
+; Verify that llvm.pseudoprobe present in the outer loop does not crash the
+; VPlan native path and is not treated as a widenable intrinsic.
+; CHECK-NOT: WIDEN-INTRINSIC
+
+define i32 @main() {
+entry:
+ br label %outer.header
+
+outer.header:
+ %j = phi i64 [ 0, %entry ], [ %j.next, %outer.latch ]
+ br label %inner
+
+outer.latch:
+ %j.next = add i64 %j, 1
+ call void @llvm.pseudoprobe(i64 0, i64 0, i32 0, i64 0)
+ %outer.cond = icmp eq i64 %j, 100
+ br i1 %outer.cond, label %exit, label %outer.header, !llvm.loop !0
+
+inner:
+ %k = phi i64 [ 0, %outer.header ], [ %k.next, %inner ]
+ %k.next = add i64 %k, 1
+ %inner.cond = icmp eq i64 %k.next, 0
+ br i1 %inner.cond, label %outer.latch, label %inner
+
+exit:
+ ret i32 0
+}
+
+declare void @llvm.pseudoprobe(i64, i64, i32, i64)
+
+!0 = distinct !{!0, !1, !2}
+!1 = !{!"llvm.loop.mustprogress"}
+!2 = !{!"llvm.loop.vectorize.enable", i1 true}
>From 794cbc5619267a9b9f7fa9d0cef781bd7a134608 Mon Sep 17 00:00:00 2001
From: Henry Jiang <henry_jiang2 at apple.com>
Date: Mon, 9 Mar 2026 09:56:08 -0700
Subject: [PATCH 2/8] Add scalable crash
---
.../Transforms/Vectorize/LoopVectorize.cpp | 2 ++
.../LoopVectorize/pseudo-probe-crash.ll | 30 ++++++++++++++++++-
2 files changed, 31 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index bb4eef5a41c09..13a4cc55ca1d8 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3213,6 +3213,7 @@ void LoopVectorizationCostModel::collectLoopUniforms(ElementCount VF) {
case Intrinsic::assume:
case Intrinsic::lifetime_start:
case Intrinsic::lifetime_end:
+ case Intrinsic::pseudoprobe:
if (TheLoop->hasLoopInvariantOperands(&I))
AddToWorklistIfAllowed(&I);
break;
@@ -8008,6 +8009,7 @@ VPReplicateRecipe *VPRecipeBuilder::handleReplication(VPInstruction *VPI,
case Intrinsic::assume:
case Intrinsic::lifetime_start:
case Intrinsic::lifetime_end:
+ case Intrinsic::pseudoprobe:
// For scalable vectors if one of the operands is variant then we still
// want to mark as uniform, which will generate one instruction for just
// the first lane of the vector. We can't scalarize the call in the same
diff --git a/llvm/test/Transforms/LoopVectorize/pseudo-probe-crash.ll b/llvm/test/Transforms/LoopVectorize/pseudo-probe-crash.ll
index f4298d3c5d770..cb74821d482c0 100644
--- a/llvm/test/Transforms/LoopVectorize/pseudo-probe-crash.ll
+++ b/llvm/test/Transforms/LoopVectorize/pseudo-probe-crash.ll
@@ -1,8 +1,10 @@
; REQUIRES: asserts
-; RUN: opt -S -passes=loop-vectorize -enable-vplan-native-path -force-vector-width=2 -debug-only=loop-vectorize -mtriple=aarch64 < %s 2>&1 | FileCheck %s
+; RUN: opt -S -passes=loop-vectorize -enable-vplan-native-path -force-vector-width=2 -debug-only=loop-vectorize < %s 2>&1 | FileCheck %s
+; RUN: opt -S -passes=loop-vectorize -scalable-vectorization=on -force-vector-width=4 -force-vector-interleave=1 -force-target-supports-scalable-vectors=true -debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=CHECK-SCALABLE
; Verify that llvm.pseudoprobe present in the outer loop does not crash the
; VPlan native path and is not treated as a widenable intrinsic.
+;
; CHECK-NOT: WIDEN-INTRINSIC
define i32 @main() {
@@ -29,6 +31,32 @@ exit:
ret i32 0
}
+; Verify that pseudoprobe does not crash with scalable vector vectorization
+;
+; CHECK-SCALABLE: LV: Found uniform instruction:{{.*}}pseudoprobe
+; CHECK-SCALABLE: CLONE call @llvm.pseudoprobe
+; CHECK-SCALABLE-NOT: WIDEN-INTRINSIC{{.*}}pseudoprobe
+;
+; CHECK-SCALABLE-LABEL: @test_scalable
+; CHECK-SCALABLE: vector.body:
+; CHECK-SCALABLE: call void @llvm.pseudoprobe
+; CHECK-SCALABLE: br i1
+
+define void @test_scalable() {
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 1, %entry ], [ %iv.next, %loop ]
+ %iv.next = add i64 %iv, 1
+ call void @llvm.pseudoprobe(i64 0, i64 0, i32 0, i64 0)
+ %done = icmp eq i64 %iv.next, 0
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
declare void @llvm.pseudoprobe(i64, i64, i32, i64)
!0 = distinct !{!0, !1, !2}
>From dd8ee270750d324a1c88f17a3584ca2b6e888035 Mon Sep 17 00:00:00 2001
From: Henry Jiang <henry_jiang2 at apple.com>
Date: Tue, 10 Mar 2026 11:32:18 -0700
Subject: [PATCH 3/8] Add test for more intrinsics
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 8 +-
.../non-widenable-intrinsics-outer-loop.ll | 252 ++++++++++++++++++
.../LoopVectorize/pseudo-probe-crash.ll | 64 -----
.../LoopVectorize/pseudoprobe-scalable.ll | 57 ++++
4 files changed, 316 insertions(+), 65 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/non-widenable-intrinsics-outer-loop.ll
delete mode 100644 llvm/test/Transforms/LoopVectorize/pseudo-probe-crash.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/pseudoprobe-scalable.ll
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 80f0b5d6a3b8b..a41e102c6b79f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -92,14 +92,20 @@ bool VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(
Ingredient.getDebugLoc());
} else if (CallInst *CI = dyn_cast<CallInst>(Inst)) {
Intrinsic::ID VectorID = getVectorIntrinsicIDForCall(CI, &TLI);
+ // These intrinsics are not widenable and should not be converted to
+ // VPWidenIntrinsicRecipe
if (VectorID == Intrinsic::not_intrinsic ||
- VectorID == Intrinsic::assume ||
VectorID == Intrinsic::lifetime_end ||
VectorID == Intrinsic::lifetime_start ||
VectorID == Intrinsic::sideeffect ||
VectorID == Intrinsic::pseudoprobe ||
VectorID == Intrinsic::experimental_noalias_scope_decl)
return false;
+ // llvm.assume is safe to drop since it is only an optimization hint
+ if (VectorID == Intrinsic::assume) {
+ Ingredient.eraseFromParent();
+ continue;
+ }
NewRecipe = new VPWidenIntrinsicRecipe(
*CI, getVectorIntrinsicIDForCall(CI, &TLI),
drop_end(Ingredient.operands()), CI->getType(), VPIRFlags(*CI),
diff --git a/llvm/test/Transforms/LoopVectorize/non-widenable-intrinsics-outer-loop.ll b/llvm/test/Transforms/LoopVectorize/non-widenable-intrinsics-outer-loop.ll
new file mode 100644
index 0000000000000..963342c4dd6a1
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/non-widenable-intrinsics-outer-loop.ll
@@ -0,0 +1,252 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -passes=loop-vectorize -enable-vplan-native-path -force-vector-width=2 < %s | FileCheck %s
+
+; Test non-widenable intrinsics in the outer loop do not crash the VPlan native path
+
+; llvm.assume is safe to drop during vectorization
+define i32 @test_assume() {
+; CHECK-LABEL: define i32 @test_assume() {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_LATCH:.*]] ]
+; CHECK-NEXT: br label %[[INNER1:.*]]
+; CHECK: [[INNER1]]:
+; CHECK-NEXT: [[TMP0:%.*]] = phi <2 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP1:%.*]], %[[INNER1]] ]
+; CHECK-NEXT: [[TMP1]] = add <2 x i64> [[TMP0]], splat (i64 1)
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq <2 x i64> [[TMP1]], zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i1> [[TMP2]], i32 0
+; CHECK-NEXT: br i1 [[TMP3]], label %[[VECTOR_LATCH]], label %[[INNER1]]
+; CHECK: [[VECTOR_LATCH]]:
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br i1 false, label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[OUTER_HEADER:.*]]
+; CHECK: [[OUTER_HEADER]]:
+; CHECK-NEXT: [[J:%.*]] = phi i64 [ 100, %[[SCALAR_PH]] ], [ [[J_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
+; CHECK-NEXT: br label %[[INNER:.*]]
+; CHECK: [[OUTER_LATCH]]:
+; CHECK-NEXT: [[J_NEXT]] = add i64 [[J]], 1
+; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i64 [[J]], 0
+; CHECK-NEXT: call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT: [[OUTER_COND:%.*]] = icmp eq i64 [[J]], 100
+; CHECK-NEXT: br i1 [[OUTER_COND]], label %[[EXIT]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[INNER]]:
+; CHECK-NEXT: [[K:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[K_NEXT:%.*]], %[[INNER]] ]
+; CHECK-NEXT: [[K_NEXT]] = add i64 [[K]], 1
+; CHECK-NEXT: [[INNER_COND:%.*]] = icmp eq i64 [[K_NEXT]], 0
+; CHECK-NEXT: br i1 [[INNER_COND]], label %[[OUTER_LATCH]], label %[[INNER]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret i32 0
+;
+entry:
+ br label %outer.header
+outer.header:
+ %j = phi i64 [ 0, %entry ], [ %j.next, %outer.latch ]
+ br label %inner
+outer.latch:
+ %j.next = add i64 %j, 1
+ %cmp = icmp sgt i64 %j, 0
+ call void @llvm.assume(i1 %cmp)
+ %outer.cond = icmp eq i64 %j, 100
+ br i1 %outer.cond, label %exit, label %outer.header, !llvm.loop !0
+inner:
+ %k = phi i64 [ 0, %outer.header ], [ %k.next, %inner ]
+ %k.next = add i64 %k, 1
+ %inner.cond = icmp eq i64 %k.next, 0
+ br i1 %inner.cond, label %outer.latch, label %inner
+exit:
+ ret i32 0
+}
+
+; Bail out the remaining intrinsics
+define i32 @test_lifetime() {
+; CHECK-LABEL: define i32 @test_lifetime() {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[A:%.*]] = alloca i32, align 4
+; CHECK-NEXT: br label %[[OUTER_HEADER:.*]]
+; CHECK: [[OUTER_HEADER]]:
+; CHECK-NEXT: [[J:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[J_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
+; CHECK-NEXT: br label %[[INNER:.*]]
+; CHECK: [[OUTER_LATCH]]:
+; CHECK-NEXT: [[J_NEXT]] = add i64 [[J]], 1
+; CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[A]])
+; CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[A]])
+; CHECK-NEXT: [[OUTER_COND:%.*]] = icmp eq i64 [[J]], 100
+; CHECK-NEXT: br i1 [[OUTER_COND]], label %[[EXIT:.*]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK: [[INNER]]:
+; CHECK-NEXT: [[K:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[K_NEXT:%.*]], %[[INNER]] ]
+; CHECK-NEXT: [[K_NEXT]] = add i64 [[K]], 1
+; CHECK-NEXT: [[INNER_COND:%.*]] = icmp eq i64 [[K_NEXT]], 0
+; CHECK-NEXT: br i1 [[INNER_COND]], label %[[OUTER_LATCH]], label %[[INNER]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret i32 0
+;
+entry:
+ %a = alloca i32
+ br label %outer.header
+outer.header:
+ %j = phi i64 [ 0, %entry ], [ %j.next, %outer.latch ]
+ br label %inner
+outer.latch:
+ %j.next = add i64 %j, 1
+ call void @llvm.lifetime.start.p0(i64 4, ptr %a)
+ call void @llvm.lifetime.end.p0(i64 4, ptr %a)
+ %outer.cond = icmp eq i64 %j, 100
+ br i1 %outer.cond, label %exit, label %outer.header, !llvm.loop !0
+inner:
+ %k = phi i64 [ 0, %outer.header ], [ %k.next, %inner ]
+ %k.next = add i64 %k, 1
+ %inner.cond = icmp eq i64 %k.next, 0
+ br i1 %inner.cond, label %outer.latch, label %inner
+exit:
+ ret i32 0
+}
+
+define i32 @test_sideeffect() {
+; CHECK-LABEL: define i32 @test_sideeffect() {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[OUTER_HEADER:.*]]
+; CHECK: [[OUTER_HEADER]]:
+; CHECK-NEXT: [[J:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[J_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
+; CHECK-NEXT: br label %[[INNER:.*]]
+; CHECK: [[OUTER_LATCH]]:
+; CHECK-NEXT: [[J_NEXT]] = add i64 [[J]], 1
+; CHECK-NEXT: call void @llvm.sideeffect()
+; CHECK-NEXT: [[OUTER_COND:%.*]] = icmp eq i64 [[J]], 100
+; CHECK-NEXT: br i1 [[OUTER_COND]], label %[[EXIT:.*]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP5]]
+; CHECK: [[INNER]]:
+; CHECK-NEXT: [[K:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[K_NEXT:%.*]], %[[INNER]] ]
+; CHECK-NEXT: [[K_NEXT]] = add i64 [[K]], 1
+; CHECK-NEXT: [[INNER_COND:%.*]] = icmp eq i64 [[K_NEXT]], 0
+; CHECK-NEXT: br i1 [[INNER_COND]], label %[[OUTER_LATCH]], label %[[INNER]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret i32 0
+;
+entry:
+ br label %outer.header
+outer.header:
+ %j = phi i64 [ 0, %entry ], [ %j.next, %outer.latch ]
+ br label %inner
+outer.latch:
+ %j.next = add i64 %j, 1
+ call void @llvm.sideeffect()
+ %outer.cond = icmp eq i64 %j, 100
+ br i1 %outer.cond, label %exit, label %outer.header, !llvm.loop !0
+inner:
+ %k = phi i64 [ 0, %outer.header ], [ %k.next, %inner ]
+ %k.next = add i64 %k, 1
+ %inner.cond = icmp eq i64 %k.next, 0
+ br i1 %inner.cond, label %outer.latch, label %inner
+exit:
+ ret i32 0
+}
+
+define i32 @test_pseudoprobe() {
+; CHECK-LABEL: define i32 @test_pseudoprobe() {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[OUTER_HEADER:.*]]
+; CHECK: [[OUTER_HEADER]]:
+; CHECK-NEXT: [[J:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[J_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
+; CHECK-NEXT: br label %[[INNER:.*]]
+; CHECK: [[OUTER_LATCH]]:
+; CHECK-NEXT: [[J_NEXT]] = add i64 [[J]], 1
+; CHECK-NEXT: call void @llvm.pseudoprobe(i64 0, i64 0, i32 0, i64 0)
+; CHECK-NEXT: [[OUTER_COND:%.*]] = icmp eq i64 [[J]], 100
+; CHECK-NEXT: br i1 [[OUTER_COND]], label %[[EXIT:.*]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP5]]
+; CHECK: [[INNER]]:
+; CHECK-NEXT: [[K:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[K_NEXT:%.*]], %[[INNER]] ]
+; CHECK-NEXT: [[K_NEXT]] = add i64 [[K]], 1
+; CHECK-NEXT: [[INNER_COND:%.*]] = icmp eq i64 [[K_NEXT]], 0
+; CHECK-NEXT: br i1 [[INNER_COND]], label %[[OUTER_LATCH]], label %[[INNER]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret i32 0
+;
+entry:
+ br label %outer.header
+outer.header:
+ %j = phi i64 [ 0, %entry ], [ %j.next, %outer.latch ]
+ br label %inner
+outer.latch:
+ %j.next = add i64 %j, 1
+ call void @llvm.pseudoprobe(i64 0, i64 0, i32 0, i64 0)
+ %outer.cond = icmp eq i64 %j, 100
+ br i1 %outer.cond, label %exit, label %outer.header, !llvm.loop !0
+inner:
+ %k = phi i64 [ 0, %outer.header ], [ %k.next, %inner ]
+ %k.next = add i64 %k, 1
+ %inner.cond = icmp eq i64 %k.next, 0
+ br i1 %inner.cond, label %outer.latch, label %inner
+exit:
+ ret i32 0
+}
+
+define i32 @test_noalias_scope_decl() {
+; CHECK-LABEL: define i32 @test_noalias_scope_decl() {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[OUTER_HEADER:.*]]
+; CHECK: [[OUTER_HEADER]]:
+; CHECK-NEXT: [[J:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[J_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
+; CHECK-NEXT: br label %[[INNER:.*]]
+; CHECK: [[OUTER_LATCH]]:
+; CHECK-NEXT: [[J_NEXT]] = add i64 [[J]], 1
+; CHECK-NEXT: call void @llvm.experimental.noalias.scope.decl(metadata [[META7:![0-9]+]])
+; CHECK-NEXT: [[OUTER_COND:%.*]] = icmp eq i64 [[J]], 100
+; CHECK-NEXT: br i1 [[OUTER_COND]], label %[[EXIT:.*]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP5]]
+; CHECK: [[INNER]]:
+; CHECK-NEXT: [[K:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[K_NEXT:%.*]], %[[INNER]] ]
+; CHECK-NEXT: [[K_NEXT]] = add i64 [[K]], 1
+; CHECK-NEXT: [[INNER_COND:%.*]] = icmp eq i64 [[K_NEXT]], 0
+; CHECK-NEXT: br i1 [[INNER_COND]], label %[[OUTER_LATCH]], label %[[INNER]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret i32 0
+;
+entry:
+ br label %outer.header
+outer.header:
+ %j = phi i64 [ 0, %entry ], [ %j.next, %outer.latch ]
+ br label %inner
+outer.latch:
+ %j.next = add i64 %j, 1
+ call void @llvm.experimental.noalias.scope.decl(metadata !3)
+ %outer.cond = icmp eq i64 %j, 100
+ br i1 %outer.cond, label %exit, label %outer.header, !llvm.loop !0
+inner:
+ %k = phi i64 [ 0, %outer.header ], [ %k.next, %inner ]
+ %k.next = add i64 %k, 1
+ %inner.cond = icmp eq i64 %k.next, 0
+ br i1 %inner.cond, label %outer.latch, label %inner
+exit:
+ ret i32 0
+}
+
+declare void @llvm.assume(i1)
+declare void @llvm.lifetime.start.p0(i64, ptr)
+declare void @llvm.lifetime.end.p0(i64, ptr)
+declare void @llvm.sideeffect()
+declare void @llvm.pseudoprobe(i64, i64, i32, i64)
+declare void @llvm.experimental.noalias.scope.decl(metadata)
+
+!0 = distinct !{!0, !1, !2}
+!1 = !{!"llvm.loop.mustprogress"}
+!2 = !{!"llvm.loop.vectorize.enable", i1 true}
+!3 = !{!4}
+!4 = distinct !{!4, !5}
+!5 = distinct !{!5}
+;.
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]], [[META3:![0-9]+]]}
+; CHECK: [[META1]] = !{!"llvm.loop.mustprogress"}
+; CHECK: [[META2]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META3]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META3]], [[META2]]}
+; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]], [[META6:![0-9]+]]}
+; CHECK: [[META6]] = !{!"llvm.loop.vectorize.enable", i1 true}
+; CHECK: [[META7]] = !{[[META8:![0-9]+]]}
+; CHECK: [[META8]] = distinct !{[[META8]], [[META9:![0-9]+]]}
+; CHECK: [[META9]] = distinct !{[[META9]]}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/pseudo-probe-crash.ll b/llvm/test/Transforms/LoopVectorize/pseudo-probe-crash.ll
deleted file mode 100644
index cb74821d482c0..0000000000000
--- a/llvm/test/Transforms/LoopVectorize/pseudo-probe-crash.ll
+++ /dev/null
@@ -1,64 +0,0 @@
-; REQUIRES: asserts
-; RUN: opt -S -passes=loop-vectorize -enable-vplan-native-path -force-vector-width=2 -debug-only=loop-vectorize < %s 2>&1 | FileCheck %s
-; RUN: opt -S -passes=loop-vectorize -scalable-vectorization=on -force-vector-width=4 -force-vector-interleave=1 -force-target-supports-scalable-vectors=true -debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=CHECK-SCALABLE
-
-; Verify that llvm.pseudoprobe present in the outer loop does not crash the
-; VPlan native path and is not treated as a widenable intrinsic.
-;
-; CHECK-NOT: WIDEN-INTRINSIC
-
-define i32 @main() {
-entry:
- br label %outer.header
-
-outer.header:
- %j = phi i64 [ 0, %entry ], [ %j.next, %outer.latch ]
- br label %inner
-
-outer.latch:
- %j.next = add i64 %j, 1
- call void @llvm.pseudoprobe(i64 0, i64 0, i32 0, i64 0)
- %outer.cond = icmp eq i64 %j, 100
- br i1 %outer.cond, label %exit, label %outer.header, !llvm.loop !0
-
-inner:
- %k = phi i64 [ 0, %outer.header ], [ %k.next, %inner ]
- %k.next = add i64 %k, 1
- %inner.cond = icmp eq i64 %k.next, 0
- br i1 %inner.cond, label %outer.latch, label %inner
-
-exit:
- ret i32 0
-}
-
-; Verify that pseudoprobe does not crash with scalable vector vectorization
-;
-; CHECK-SCALABLE: LV: Found uniform instruction:{{.*}}pseudoprobe
-; CHECK-SCALABLE: CLONE call @llvm.pseudoprobe
-; CHECK-SCALABLE-NOT: WIDEN-INTRINSIC{{.*}}pseudoprobe
-;
-; CHECK-SCALABLE-LABEL: @test_scalable
-; CHECK-SCALABLE: vector.body:
-; CHECK-SCALABLE: call void @llvm.pseudoprobe
-; CHECK-SCALABLE: br i1
-
-define void @test_scalable() {
-entry:
- br label %loop
-
-loop:
- %iv = phi i64 [ 1, %entry ], [ %iv.next, %loop ]
- %iv.next = add i64 %iv, 1
- call void @llvm.pseudoprobe(i64 0, i64 0, i32 0, i64 0)
- %done = icmp eq i64 %iv.next, 0
- br i1 %done, label %exit, label %loop
-
-exit:
- ret void
-}
-
-declare void @llvm.pseudoprobe(i64, i64, i32, i64)
-
-!0 = distinct !{!0, !1, !2}
-!1 = !{!"llvm.loop.mustprogress"}
-!2 = !{!"llvm.loop.vectorize.enable", i1 true}
diff --git a/llvm/test/Transforms/LoopVectorize/pseudoprobe-scalable.ll b/llvm/test/Transforms/LoopVectorize/pseudoprobe-scalable.ll
new file mode 100644
index 0000000000000..9a9d2ef6f7c13
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/pseudoprobe-scalable.ll
@@ -0,0 +1,57 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -passes=loop-vectorize -scalable-vectorization=on -force-vector-width=2 -force-vector-interleave=1 -force-target-supports-scalable-vectors=true < %s | FileCheck %s
+
+; Verify that pseudoprobe does not crash with scalable vector vectorization.
+
+define void @test_scalable_pseudoprobe() {
+; CHECK-LABEL: define void @test_scalable_pseudoprobe() {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 -1, [[TMP1]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 -1, [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP2:%.*]] = add i64 1, [[N_VEC]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: call void @llvm.pseudoprobe(i64 0, i64 0, i32 0, i64 0)
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 -1, [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[TMP2]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT: call void @llvm.pseudoprobe(i64 0, i64 0, i32 0, i64 0)
+; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[IV_NEXT]], 0
+; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 1, %entry ], [ %iv.next, %loop ]
+ %iv.next = add i64 %iv, 1
+ call void @llvm.pseudoprobe(i64 0, i64 0, i32 0, i64 0)
+ %done = icmp eq i64 %iv.next, 0
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+declare void @llvm.pseudoprobe(i64, i64, i32, i64)
+;.
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
+;.
>From ca7f8d3e823608f13ab6b109caf3f88650a6c73d Mon Sep 17 00:00:00 2001
From: Henry Jiang <henry_jiang2 at apple.com>
Date: Tue, 10 Mar 2026 12:42:31 -0700
Subject: [PATCH 4/8] Emit VPReplicateRecipe for pseudo intrinsics
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 32 ++--
.../non-widenable-intrinsics-outer-loop.ll | 149 ++++++++++++++----
2 files changed, 137 insertions(+), 44 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index a41e102c6b79f..1f48a89ae7a71 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -92,24 +92,30 @@ bool VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(
Ingredient.getDebugLoc());
} else if (CallInst *CI = dyn_cast<CallInst>(Inst)) {
Intrinsic::ID VectorID = getVectorIntrinsicIDForCall(CI, &TLI);
- // These intrinsics are not widenable and should not be converted to
- // VPWidenIntrinsicRecipe
- if (VectorID == Intrinsic::not_intrinsic ||
- VectorID == Intrinsic::lifetime_end ||
- VectorID == Intrinsic::lifetime_start ||
- VectorID == Intrinsic::sideeffect ||
- VectorID == Intrinsic::pseudoprobe ||
- VectorID == Intrinsic::experimental_noalias_scope_decl)
+ if (VectorID == Intrinsic::not_intrinsic)
return false;
- // llvm.assume is safe to drop since it is only an optimization hint
+ // llvm.assume is safe to drop
if (VectorID == Intrinsic::assume) {
Ingredient.eraseFromParent();
continue;
}
- NewRecipe = new VPWidenIntrinsicRecipe(
- *CI, getVectorIntrinsicIDForCall(CI, &TLI),
- drop_end(Ingredient.operands()), CI->getType(), VPIRFlags(*CI),
- *VPI, CI->getDebugLoc());
+ // These intrinsics are recognized by getVectorIntrinsicIDForCall
+ // but are not widenable. Emit them as single-scalar replicate
+ // instead of widening
+ if (VectorID == Intrinsic::lifetime_end ||
+ VectorID == Intrinsic::lifetime_start ||
+ VectorID == Intrinsic::sideeffect ||
+ VectorID == Intrinsic::pseudoprobe ||
+ VectorID == Intrinsic::experimental_noalias_scope_decl) {
+ NewRecipe = new VPReplicateRecipe(CI, Ingredient.operands(),
+ /*IsSingleScalar=*/true,
+ /*Mask=*/nullptr, *VPI, *VPI,
+ Ingredient.getDebugLoc());
+ } else {
+ NewRecipe = new VPWidenIntrinsicRecipe(
+ *CI, VectorID, drop_end(Ingredient.operands()), CI->getType(),
+ VPIRFlags(*CI), *VPI, CI->getDebugLoc());
+ }
} else if (auto *CI = dyn_cast<CastInst>(Inst)) {
NewRecipe = new VPWidenCastRecipe(
CI->getOpcode(), Ingredient.getOperand(0), CI->getType(), CI,
diff --git a/llvm/test/Transforms/LoopVectorize/non-widenable-intrinsics-outer-loop.ll b/llvm/test/Transforms/LoopVectorize/non-widenable-intrinsics-outer-loop.ll
index 963342c4dd6a1..d8751095790c5 100644
--- a/llvm/test/Transforms/LoopVectorize/non-widenable-intrinsics-outer-loop.ll
+++ b/llvm/test/Transforms/LoopVectorize/non-widenable-intrinsics-outer-loop.ll
@@ -1,7 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -S -passes=loop-vectorize -enable-vplan-native-path -force-vector-width=2 < %s | FileCheck %s
-; Test non-widenable intrinsics in the outer loop do not crash the VPlan native path
+; Test non-widenable intrinsics in the outer loop do not crash the VPlan native path
; llvm.assume is safe to drop during vectorization
define i32 @test_assume() {
@@ -67,23 +67,44 @@ exit:
; Bail out the remaining intrinsics
define i32 @test_lifetime() {
; CHECK-LABEL: define i32 @test_lifetime() {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[A:%.*]] = alloca i32, align 4
; CHECK-NEXT: br label %[[OUTER_HEADER:.*]]
; CHECK: [[OUTER_HEADER]]:
-; CHECK-NEXT: [[J:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[J_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
; CHECK-NEXT: br label %[[INNER:.*]]
+; CHECK: [[INNER]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_LATCH:.*]] ]
+; CHECK-NEXT: br label %[[OUTER_LATCH:.*]]
; CHECK: [[OUTER_LATCH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = phi <2 x i64> [ zeroinitializer, %[[INNER]] ], [ [[TMP1:%.*]], %[[OUTER_LATCH]] ]
+; CHECK-NEXT: [[TMP1]] = add <2 x i64> [[TMP0]], splat (i64 1)
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq <2 x i64> [[TMP1]], zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i1> [[TMP2]], i32 0
+; CHECK-NEXT: br i1 [[TMP3]], label %[[VECTOR_LATCH]], label %[[OUTER_LATCH]]
+; CHECK: [[VECTOR_LATCH]]:
+; CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[A]])
+; CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[A]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[INNER]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br i1 false, label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[OUTER_HEADER1:.*]]
+; CHECK: [[OUTER_HEADER1]]:
+; CHECK-NEXT: [[J:%.*]] = phi i64 [ 100, %[[SCALAR_PH]] ], [ [[J_NEXT:%.*]], %[[OUTER_LATCH1:.*]] ]
+; CHECK-NEXT: br label %[[INNER1:.*]]
+; CHECK: [[OUTER_LATCH1]]:
; CHECK-NEXT: [[J_NEXT]] = add i64 [[J]], 1
; CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[A]])
; CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[A]])
; CHECK-NEXT: [[OUTER_COND:%.*]] = icmp eq i64 [[J]], 100
-; CHECK-NEXT: br i1 [[OUTER_COND]], label %[[EXIT:.*]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP5:![0-9]+]]
-; CHECK: [[INNER]]:
-; CHECK-NEXT: [[K:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[K_NEXT:%.*]], %[[INNER]] ]
+; CHECK-NEXT: br i1 [[OUTER_COND]], label %[[EXIT]], label %[[OUTER_HEADER1]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: [[INNER1]]:
+; CHECK-NEXT: [[K:%.*]] = phi i64 [ 0, %[[OUTER_HEADER1]] ], [ [[K_NEXT:%.*]], %[[INNER1]] ]
; CHECK-NEXT: [[K_NEXT]] = add i64 [[K]], 1
; CHECK-NEXT: [[INNER_COND:%.*]] = icmp eq i64 [[K_NEXT]], 0
-; CHECK-NEXT: br i1 [[INNER_COND]], label %[[OUTER_LATCH]], label %[[INNER]]
+; CHECK-NEXT: br i1 [[INNER_COND]], label %[[OUTER_LATCH1]], label %[[INNER1]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret i32 0
;
@@ -110,21 +131,41 @@ exit:
define i32 @test_sideeffect() {
; CHECK-LABEL: define i32 @test_sideeffect() {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[OUTER_HEADER:.*]]
; CHECK: [[OUTER_HEADER]]:
-; CHECK-NEXT: [[J:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[J_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
; CHECK-NEXT: br label %[[INNER:.*]]
+; CHECK: [[INNER]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_LATCH:.*]] ]
+; CHECK-NEXT: br label %[[OUTER_LATCH:.*]]
; CHECK: [[OUTER_LATCH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = phi <2 x i64> [ zeroinitializer, %[[INNER]] ], [ [[TMP1:%.*]], %[[OUTER_LATCH]] ]
+; CHECK-NEXT: [[TMP1]] = add <2 x i64> [[TMP0]], splat (i64 1)
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq <2 x i64> [[TMP1]], zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i1> [[TMP2]], i32 0
+; CHECK-NEXT: br i1 [[TMP3]], label %[[VECTOR_LATCH]], label %[[OUTER_LATCH]]
+; CHECK: [[VECTOR_LATCH]]:
+; CHECK-NEXT: call void @llvm.sideeffect()
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[INNER]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br i1 false, label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[OUTER_HEADER1:.*]]
+; CHECK: [[OUTER_HEADER1]]:
+; CHECK-NEXT: [[J:%.*]] = phi i64 [ 100, %[[SCALAR_PH]] ], [ [[J_NEXT:%.*]], %[[OUTER_LATCH1:.*]] ]
+; CHECK-NEXT: br label %[[INNER1:.*]]
+; CHECK: [[OUTER_LATCH1]]:
; CHECK-NEXT: [[J_NEXT]] = add i64 [[J]], 1
; CHECK-NEXT: call void @llvm.sideeffect()
; CHECK-NEXT: [[OUTER_COND:%.*]] = icmp eq i64 [[J]], 100
-; CHECK-NEXT: br i1 [[OUTER_COND]], label %[[EXIT:.*]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP5]]
-; CHECK: [[INNER]]:
-; CHECK-NEXT: [[K:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[K_NEXT:%.*]], %[[INNER]] ]
+; CHECK-NEXT: br i1 [[OUTER_COND]], label %[[EXIT]], label %[[OUTER_HEADER1]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK: [[INNER1]]:
+; CHECK-NEXT: [[K:%.*]] = phi i64 [ 0, %[[OUTER_HEADER1]] ], [ [[K_NEXT:%.*]], %[[INNER1]] ]
; CHECK-NEXT: [[K_NEXT]] = add i64 [[K]], 1
; CHECK-NEXT: [[INNER_COND:%.*]] = icmp eq i64 [[K_NEXT]], 0
-; CHECK-NEXT: br i1 [[INNER_COND]], label %[[OUTER_LATCH]], label %[[INNER]]
+; CHECK-NEXT: br i1 [[INNER_COND]], label %[[OUTER_LATCH1]], label %[[INNER1]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret i32 0
;
@@ -149,21 +190,41 @@ exit:
define i32 @test_pseudoprobe() {
; CHECK-LABEL: define i32 @test_pseudoprobe() {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[OUTER_HEADER:.*]]
; CHECK: [[OUTER_HEADER]]:
-; CHECK-NEXT: [[J:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[J_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
; CHECK-NEXT: br label %[[INNER:.*]]
+; CHECK: [[INNER]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_LATCH:.*]] ]
+; CHECK-NEXT: br label %[[OUTER_LATCH:.*]]
; CHECK: [[OUTER_LATCH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = phi <2 x i64> [ zeroinitializer, %[[INNER]] ], [ [[TMP1:%.*]], %[[OUTER_LATCH]] ]
+; CHECK-NEXT: [[TMP1]] = add <2 x i64> [[TMP0]], splat (i64 1)
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq <2 x i64> [[TMP1]], zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i1> [[TMP2]], i32 0
+; CHECK-NEXT: br i1 [[TMP3]], label %[[VECTOR_LATCH]], label %[[OUTER_LATCH]]
+; CHECK: [[VECTOR_LATCH]]:
+; CHECK-NEXT: call void @llvm.pseudoprobe(i64 0, i64 0, i32 0, i64 0)
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[INNER]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br i1 false, label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[OUTER_HEADER1:.*]]
+; CHECK: [[OUTER_HEADER1]]:
+; CHECK-NEXT: [[J:%.*]] = phi i64 [ 100, %[[SCALAR_PH]] ], [ [[J_NEXT:%.*]], %[[OUTER_LATCH1:.*]] ]
+; CHECK-NEXT: br label %[[INNER1:.*]]
+; CHECK: [[OUTER_LATCH1]]:
; CHECK-NEXT: [[J_NEXT]] = add i64 [[J]], 1
; CHECK-NEXT: call void @llvm.pseudoprobe(i64 0, i64 0, i32 0, i64 0)
; CHECK-NEXT: [[OUTER_COND:%.*]] = icmp eq i64 [[J]], 100
-; CHECK-NEXT: br i1 [[OUTER_COND]], label %[[EXIT:.*]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP5]]
-; CHECK: [[INNER]]:
-; CHECK-NEXT: [[K:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[K_NEXT:%.*]], %[[INNER]] ]
+; CHECK-NEXT: br i1 [[OUTER_COND]], label %[[EXIT]], label %[[OUTER_HEADER1]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK: [[INNER1]]:
+; CHECK-NEXT: [[K:%.*]] = phi i64 [ 0, %[[OUTER_HEADER1]] ], [ [[K_NEXT:%.*]], %[[INNER1]] ]
; CHECK-NEXT: [[K_NEXT]] = add i64 [[K]], 1
; CHECK-NEXT: [[INNER_COND:%.*]] = icmp eq i64 [[K_NEXT]], 0
-; CHECK-NEXT: br i1 [[INNER_COND]], label %[[OUTER_LATCH]], label %[[INNER]]
+; CHECK-NEXT: br i1 [[INNER_COND]], label %[[OUTER_LATCH1]], label %[[INNER1]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret i32 0
;
@@ -188,21 +249,41 @@ exit:
define i32 @test_noalias_scope_decl() {
; CHECK-LABEL: define i32 @test_noalias_scope_decl() {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[OUTER_HEADER:.*]]
; CHECK: [[OUTER_HEADER]]:
-; CHECK-NEXT: [[J:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[J_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
; CHECK-NEXT: br label %[[INNER:.*]]
+; CHECK: [[INNER]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_LATCH:.*]] ]
+; CHECK-NEXT: br label %[[OUTER_LATCH:.*]]
; CHECK: [[OUTER_LATCH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = phi <2 x i64> [ zeroinitializer, %[[INNER]] ], [ [[TMP1:%.*]], %[[OUTER_LATCH]] ]
+; CHECK-NEXT: [[TMP1]] = add <2 x i64> [[TMP0]], splat (i64 1)
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq <2 x i64> [[TMP1]], zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i1> [[TMP2]], i32 0
+; CHECK-NEXT: br i1 [[TMP3]], label %[[VECTOR_LATCH]], label %[[OUTER_LATCH]]
+; CHECK: [[VECTOR_LATCH]]:
+; CHECK-NEXT: call void @llvm.experimental.noalias.scope.decl(metadata [[META11:![0-9]+]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[INNER]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br i1 false, label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[OUTER_HEADER1:.*]]
+; CHECK: [[OUTER_HEADER1]]:
+; CHECK-NEXT: [[J:%.*]] = phi i64 [ 100, %[[SCALAR_PH]] ], [ [[J_NEXT:%.*]], %[[OUTER_LATCH1:.*]] ]
+; CHECK-NEXT: br label %[[INNER1:.*]]
+; CHECK: [[OUTER_LATCH1]]:
; CHECK-NEXT: [[J_NEXT]] = add i64 [[J]], 1
-; CHECK-NEXT: call void @llvm.experimental.noalias.scope.decl(metadata [[META7:![0-9]+]])
+; CHECK-NEXT: call void @llvm.experimental.noalias.scope.decl(metadata [[META11]])
; CHECK-NEXT: [[OUTER_COND:%.*]] = icmp eq i64 [[J]], 100
-; CHECK-NEXT: br i1 [[OUTER_COND]], label %[[EXIT:.*]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP5]]
-; CHECK: [[INNER]]:
-; CHECK-NEXT: [[K:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[K_NEXT:%.*]], %[[INNER]] ]
+; CHECK-NEXT: br i1 [[OUTER_COND]], label %[[EXIT]], label %[[OUTER_HEADER1]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK: [[INNER1]]:
+; CHECK-NEXT: [[K:%.*]] = phi i64 [ 0, %[[OUTER_HEADER1]] ], [ [[K_NEXT:%.*]], %[[INNER1]] ]
; CHECK-NEXT: [[K_NEXT]] = add i64 [[K]], 1
; CHECK-NEXT: [[INNER_COND:%.*]] = icmp eq i64 [[K_NEXT]], 0
-; CHECK-NEXT: br i1 [[INNER_COND]], label %[[OUTER_LATCH]], label %[[INNER]]
+; CHECK-NEXT: br i1 [[INNER_COND]], label %[[OUTER_LATCH1]], label %[[INNER1]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret i32 0
;
@@ -244,9 +325,15 @@ declare void @llvm.experimental.noalias.scope.decl(metadata)
; CHECK: [[META2]] = !{!"llvm.loop.isvectorized", i32 1}
; CHECK: [[META3]] = !{!"llvm.loop.unroll.runtime.disable"}
; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META3]], [[META2]]}
-; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]], [[META6:![0-9]+]]}
-; CHECK: [[META6]] = !{!"llvm.loop.vectorize.enable", i1 true}
-; CHECK: [[META7]] = !{[[META8:![0-9]+]]}
-; CHECK: [[META8]] = distinct !{[[META8]], [[META9:![0-9]+]]}
-; CHECK: [[META9]] = distinct !{[[META9]]}
+; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]], [[META2]], [[META3]]}
+; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META3]], [[META2]]}
+; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META1]], [[META2]], [[META3]]}
+; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META3]], [[META2]]}
+; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META1]], [[META2]], [[META3]]}
+; CHECK: [[LOOP10]] = distinct !{[[LOOP10]], [[META1]], [[META3]], [[META2]]}
+; CHECK: [[META11]] = !{[[META12:![0-9]+]]}
+; CHECK: [[META12]] = distinct !{[[META12]], [[META13:![0-9]+]]}
+; CHECK: [[META13]] = distinct !{[[META13]]}
+; CHECK: [[LOOP14]] = distinct !{[[LOOP14]], [[META1]], [[META2]], [[META3]]}
+; CHECK: [[LOOP15]] = distinct !{[[LOOP15]], [[META1]], [[META3]], [[META2]]}
;.
>From e325174756084ee1d4dc5f3baf377159ada032d4 Mon Sep 17 00:00:00 2001
From: Henry Jiang <henry_jiang2 at apple.com>
Date: Thu, 19 Mar 2026 21:16:26 -0700
Subject: [PATCH 5/8] Drop noalias.scope.decl and add back assume
---
.../Transforms/Vectorize/LoopVectorize.cpp | 1 +
.../Transforms/Vectorize/VPlanTransforms.cpp | 16 +-
.../noalias-scope-decl-outer-loop.ll | 109 +++++
.../non-widenable-intrinsics-outer-loop.ll | 437 +++++++++---------
4 files changed, 343 insertions(+), 220 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/noalias-scope-decl-outer-loop.ll
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 13a4cc55ca1d8..75fbbf4606635 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8009,6 +8009,7 @@ VPReplicateRecipe *VPRecipeBuilder::handleReplication(VPInstruction *VPI,
case Intrinsic::assume:
case Intrinsic::lifetime_start:
case Intrinsic::lifetime_end:
+ case Intrinsic::sideeffect:
case Intrinsic::pseudoprobe:
// For scalable vectors if one of the operands is variant then we still
// want to mark as uniform, which will generate one instruction for just
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 1f48a89ae7a71..aaa9d8ac9202f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -94,19 +94,23 @@ bool VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(
Intrinsic::ID VectorID = getVectorIntrinsicIDForCall(CI, &TLI);
if (VectorID == Intrinsic::not_intrinsic)
return false;
- // llvm.assume is safe to drop
- if (VectorID == Intrinsic::assume) {
+ // The noalias.scope.decl intrinsic declares a noalias scope that
+ // is valid for a single iteration. Emitting it as a single-scalar
+ // replicate would incorrectly extend the scope across multiple
+ // original iterations packed into one vector iteration. Taking the
+ // conservative approach and drop it, but still allow vectorization.
+ if (VectorID == Intrinsic::experimental_noalias_scope_decl) {
Ingredient.eraseFromParent();
continue;
}
// These intrinsics are recognized by getVectorIntrinsicIDForCall
// but are not widenable. Emit them as single-scalar replicate
- // instead of widening
- if (VectorID == Intrinsic::lifetime_end ||
+ // instead of widening.
+ if (VectorID == Intrinsic::assume ||
+ VectorID == Intrinsic::lifetime_end ||
VectorID == Intrinsic::lifetime_start ||
VectorID == Intrinsic::sideeffect ||
- VectorID == Intrinsic::pseudoprobe ||
- VectorID == Intrinsic::experimental_noalias_scope_decl) {
+ VectorID == Intrinsic::pseudoprobe) {
NewRecipe = new VPReplicateRecipe(CI, Ingredient.operands(),
/*IsSingleScalar=*/true,
/*Mask=*/nullptr, *VPI, *VPI,
diff --git a/llvm/test/Transforms/LoopVectorize/noalias-scope-decl-outer-loop.ll b/llvm/test/Transforms/LoopVectorize/noalias-scope-decl-outer-loop.ll
new file mode 100644
index 0000000000000..3d9f9e9851612
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/noalias-scope-decl-outer-loop.ll
@@ -0,0 +1,109 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -passes='loop-vectorize,verify' -enable-vplan-native-path -force-vector-width=2 < %s | FileCheck %s
+
+; The noalias.scope.decl intrinsic declares a noalias scope valid for a single iteration.
+; During outer loop vectorization, emitting it as a single-scalar replicate would incorrectly
+; extend the scope across multiple original iterations packed into one vector iteration.
+; We can drop the intrinsics but still allow outerloop vectorization to happen.
+
+define void @test_noalias_scope_decl(ptr %a, ptr %b, i64 %n) {
+; CHECK-LABEL: define void @test_noalias_scope_decl(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 2
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[N]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_LATCH:.*]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_LATCH]] ]
+; CHECK-NEXT: br label %[[INNER1:.*]]
+; CHECK: [[INNER1]]:
+; CHECK-NEXT: [[TMP0:%.*]] = phi <2 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP4:%.*]], %[[INNER1]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = add <2 x i64> [[VEC_IND]], [[TMP0]]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[A]], <2 x i64> [[TMP1]]
+; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <2 x i32> @llvm.masked.gather.v2i32.v2p0(<2 x ptr> align 4 [[TMP2]], <2 x i1> splat (i1 true), <2 x i32> poison), !alias.scope [[META0:![0-9]+]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[B]], <2 x i64> [[TMP1]]
+; CHECK-NEXT: call void @llvm.masked.scatter.v2i32.v2p0(<2 x i32> [[WIDE_MASKED_GATHER]], <2 x ptr> align 4 [[TMP3]], <2 x i1> splat (i1 true)), !noalias [[META0]]
+; CHECK-NEXT: [[TMP4]] = add <2 x i64> [[TMP0]], splat (i64 1)
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq <2 x i64> [[TMP4]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x i1> [[TMP5]], i32 0
+; CHECK-NEXT: br i1 [[TMP6]], label %[[VECTOR_LATCH]], label %[[INNER1]]
+; CHECK: [[VECTOR_LATCH]]:
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 2)
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[OUTER_HEADER:.*]]
+; CHECK: [[OUTER_HEADER]]:
+; CHECK-NEXT: [[J:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[J_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
+; CHECK-NEXT: call void @llvm.experimental.noalias.scope.decl(metadata [[META0]])
+; CHECK-NEXT: br label %[[INNER:.*]]
+; CHECK: [[INNER]]:
+; CHECK-NEXT: [[K:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[K_NEXT:%.*]], %[[INNER]] ]
+; CHECK-NEXT: [[IDX:%.*]] = add i64 [[J]], [[K]]
+; CHECK-NEXT: [[PTR_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IDX]]
+; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[PTR_A]], align 4, !alias.scope [[META0]]
+; CHECK-NEXT: [[PTR_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IDX]]
+; CHECK-NEXT: store i32 [[VAL]], ptr [[PTR_B]], align 4, !noalias [[META0]]
+; CHECK-NEXT: [[K_NEXT]] = add i64 [[K]], 1
+; CHECK-NEXT: [[INNER_COND:%.*]] = icmp eq i64 [[K_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[INNER_COND]], label %[[OUTER_LATCH]], label %[[INNER]]
+; CHECK: [[OUTER_LATCH]]:
+; CHECK-NEXT: [[J_NEXT]] = add i64 [[J]], 1
+; CHECK-NEXT: [[OUTER_COND:%.*]] = icmp eq i64 [[J_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[OUTER_COND]], label %[[EXIT]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %outer.header
+outer.header:
+ %j = phi i64 [ 0, %entry ], [ %j.next, %outer.latch ]
+ call void @llvm.experimental.noalias.scope.decl(metadata !3)
+ br label %inner
+inner:
+ %k = phi i64 [ 0, %outer.header ], [ %k.next, %inner ]
+ %idx = add i64 %j, %k
+ %ptr.a = getelementptr inbounds i32, ptr %a, i64 %idx
+ %val = load i32, ptr %ptr.a, align 4, !alias.scope !3
+ %ptr.b = getelementptr inbounds i32, ptr %b, i64 %idx
+ store i32 %val, ptr %ptr.b, align 4, !noalias !3
+ %k.next = add i64 %k, 1
+ %inner.cond = icmp eq i64 %k.next, %n
+ br i1 %inner.cond, label %outer.latch, label %inner
+outer.latch:
+ %j.next = add i64 %j, 1
+ %outer.cond = icmp eq i64 %j.next, %n
+ br i1 %outer.cond, label %exit, label %outer.header, !llvm.loop !0
+exit:
+ ret void
+}
+
+declare void @llvm.experimental.noalias.scope.decl(metadata)
+
+!0 = distinct !{!0, !1, !2}
+!1 = !{!"llvm.loop.mustprogress"}
+!2 = !{!"llvm.loop.vectorize.enable", i1 true}
+!3 = !{!4}
+!4 = distinct !{!4, !5}
+!5 = distinct !{!5}
+;.
+; CHECK: [[META0]] = !{[[META1:![0-9]+]]}
+; CHECK: [[META1]] = distinct !{[[META1]], [[META2:![0-9]+]]}
+; CHECK: [[META2]] = distinct !{[[META2]]}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META4:![0-9]+]], [[META5:![0-9]+]], [[META6:![0-9]+]]}
+; CHECK: [[META4]] = !{!"llvm.loop.mustprogress"}
+; CHECK: [[META5]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META6]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META4]], [[META6]], [[META5]]}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/non-widenable-intrinsics-outer-loop.ll b/llvm/test/Transforms/LoopVectorize/non-widenable-intrinsics-outer-loop.ll
index d8751095790c5..bf0a118e3add7 100644
--- a/llvm/test/Transforms/LoopVectorize/non-widenable-intrinsics-outer-loop.ll
+++ b/llvm/test/Transforms/LoopVectorize/non-widenable-intrinsics-outer-loop.ll
@@ -1,309 +1,327 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
-; RUN: opt -S -passes=loop-vectorize -enable-vplan-native-path -force-vector-width=2 < %s | FileCheck %s
+; RUN: opt -S -passes='verify,loop-vectorize,verify' -enable-vplan-native-path -force-vector-width=2 < %s | FileCheck %s
-; Test non-widenable intrinsics in the outer loop do not crash the VPlan native path
+; Test that non-widenable intrinsics in the outer loop are emitted as single
+; scalar calls and do not crash the VPlan native path.
-; llvm.assume is safe to drop during vectorization
-define i32 @test_assume() {
-; CHECK-LABEL: define i32 @test_assume() {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+define void @test_assume(ptr %arr, i64 %n) {
+; CHECK-LABEL: define void @test_assume(
+; CHECK-SAME: ptr [[ARR:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 2
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[N]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_LATCH:.*]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_LATCH]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = icmp sgt i64 [[INDEX]], 0
+; CHECK-NEXT: call void @llvm.assume(i1 [[TMP0]])
; CHECK-NEXT: br label %[[INNER1:.*]]
; CHECK: [[INNER1]]:
-; CHECK-NEXT: [[TMP0:%.*]] = phi <2 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP1:%.*]], %[[INNER1]] ]
-; CHECK-NEXT: [[TMP1]] = add <2 x i64> [[TMP0]], splat (i64 1)
-; CHECK-NEXT: [[TMP2:%.*]] = icmp eq <2 x i64> [[TMP1]], zeroinitializer
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i1> [[TMP2]], i32 0
-; CHECK-NEXT: br i1 [[TMP3]], label %[[VECTOR_LATCH]], label %[[INNER1]]
+; CHECK-NEXT: [[TMP1:%.*]] = phi <2 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP4:%.*]], %[[INNER1]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = add <2 x i64> [[VEC_IND]], [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[ARR]], <2 x i64> [[TMP2]]
+; CHECK-NEXT: call void @llvm.masked.scatter.v2i32.v2p0(<2 x i32> splat (i32 1), <2 x ptr> align 4 [[TMP3]], <2 x i1> splat (i1 true))
+; CHECK-NEXT: [[TMP4]] = add <2 x i64> [[TMP1]], splat (i64 1)
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq <2 x i64> [[TMP4]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x i1> [[TMP5]], i32 0
+; CHECK-NEXT: br i1 [[TMP6]], label %[[VECTOR_LATCH]], label %[[INNER1]]
; CHECK: [[VECTOR_LATCH]]:
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
-; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 2)
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br i1 false, label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[OUTER_HEADER:.*]]
; CHECK: [[OUTER_HEADER]]:
-; CHECK-NEXT: [[J:%.*]] = phi i64 [ 100, %[[SCALAR_PH]] ], [ [[J_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
-; CHECK-NEXT: br label %[[INNER:.*]]
-; CHECK: [[OUTER_LATCH]]:
-; CHECK-NEXT: [[J_NEXT]] = add i64 [[J]], 1
+; CHECK-NEXT: [[J:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[J_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i64 [[J]], 0
; CHECK-NEXT: call void @llvm.assume(i1 [[CMP]])
-; CHECK-NEXT: [[OUTER_COND:%.*]] = icmp eq i64 [[J]], 100
-; CHECK-NEXT: br i1 [[OUTER_COND]], label %[[EXIT]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT: br label %[[INNER:.*]]
; CHECK: [[INNER]]:
; CHECK-NEXT: [[K:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[K_NEXT:%.*]], %[[INNER]] ]
+; CHECK-NEXT: [[IDX:%.*]] = add i64 [[J]], [[K]]
+; CHECK-NEXT: [[PTR:%.*]] = getelementptr inbounds i32, ptr [[ARR]], i64 [[IDX]]
+; CHECK-NEXT: store i32 1, ptr [[PTR]], align 4
; CHECK-NEXT: [[K_NEXT]] = add i64 [[K]], 1
-; CHECK-NEXT: [[INNER_COND:%.*]] = icmp eq i64 [[K_NEXT]], 0
+; CHECK-NEXT: [[INNER_COND:%.*]] = icmp eq i64 [[K_NEXT]], [[N]]
; CHECK-NEXT: br i1 [[INNER_COND]], label %[[OUTER_LATCH]], label %[[INNER]]
+; CHECK: [[OUTER_LATCH]]:
+; CHECK-NEXT: [[J_NEXT]] = add i64 [[J]], 1
+; CHECK-NEXT: [[OUTER_COND:%.*]] = icmp eq i64 [[J_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[OUTER_COND]], label %[[EXIT]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret i32 0
+; CHECK-NEXT: ret void
;
entry:
br label %outer.header
outer.header:
%j = phi i64 [ 0, %entry ], [ %j.next, %outer.latch ]
- br label %inner
-outer.latch:
- %j.next = add i64 %j, 1
%cmp = icmp sgt i64 %j, 0
call void @llvm.assume(i1 %cmp)
- %outer.cond = icmp eq i64 %j, 100
- br i1 %outer.cond, label %exit, label %outer.header, !llvm.loop !0
+ br label %inner
inner:
%k = phi i64 [ 0, %outer.header ], [ %k.next, %inner ]
+ %idx = add i64 %j, %k
+ %ptr = getelementptr inbounds i32, ptr %arr, i64 %idx
+ store i32 1, ptr %ptr, align 4
%k.next = add i64 %k, 1
- %inner.cond = icmp eq i64 %k.next, 0
+ %inner.cond = icmp eq i64 %k.next, %n
br i1 %inner.cond, label %outer.latch, label %inner
+outer.latch:
+ %j.next = add i64 %j, 1
+ %outer.cond = icmp eq i64 %j.next, %n
+ br i1 %outer.cond, label %exit, label %outer.header, !llvm.loop !0
exit:
- ret i32 0
+ ret void
}
-; Bail out the remaining intrinsics
-define i32 @test_lifetime() {
-; CHECK-LABEL: define i32 @test_lifetime() {
-; CHECK-NEXT: [[ENTRY:.*:]]
+define void @test_lifetime(ptr %arr, i64 %n) {
+; CHECK-LABEL: define void @test_lifetime(
+; CHECK-SAME: ptr [[ARR:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[A:%.*]] = alloca i32, align 4
-; CHECK-NEXT: br label %[[OUTER_HEADER:.*]]
-; CHECK: [[OUTER_HEADER]]:
-; CHECK-NEXT: br label %[[INNER:.*]]
-; CHECK: [[INNER]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_LATCH:.*]] ]
-; CHECK-NEXT: br label %[[OUTER_LATCH:.*]]
-; CHECK: [[OUTER_LATCH]]:
-; CHECK-NEXT: [[TMP0:%.*]] = phi <2 x i64> [ zeroinitializer, %[[INNER]] ], [ [[TMP1:%.*]], %[[OUTER_LATCH]] ]
-; CHECK-NEXT: [[TMP1]] = add <2 x i64> [[TMP0]], splat (i64 1)
-; CHECK-NEXT: [[TMP2:%.*]] = icmp eq <2 x i64> [[TMP1]], zeroinitializer
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i1> [[TMP2]], i32 0
-; CHECK-NEXT: br i1 [[TMP3]], label %[[VECTOR_LATCH]], label %[[OUTER_LATCH]]
-; CHECK: [[VECTOR_LATCH]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 2
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[N]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_LATCH:.*]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_LATCH]] ]
; CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[A]])
+; CHECK-NEXT: br label %[[INNER1:.*]]
+; CHECK: [[INNER1]]:
+; CHECK-NEXT: [[TMP0:%.*]] = phi <2 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP3:%.*]], %[[INNER1]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = add <2 x i64> [[VEC_IND]], [[TMP0]]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[ARR]], <2 x i64> [[TMP1]]
+; CHECK-NEXT: call void @llvm.masked.scatter.v2i32.v2p0(<2 x i32> splat (i32 1), <2 x ptr> align 4 [[TMP2]], <2 x i1> splat (i1 true))
+; CHECK-NEXT: [[TMP3]] = add <2 x i64> [[TMP0]], splat (i64 1)
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq <2 x i64> [[TMP3]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x i1> [[TMP4]], i32 0
+; CHECK-NEXT: br i1 [[TMP5]], label %[[VECTOR_LATCH]], label %[[INNER1]]
+; CHECK: [[VECTOR_LATCH]]:
; CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[A]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
-; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[INNER]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 2)
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br i1 false, label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: br label %[[OUTER_HEADER1:.*]]
-; CHECK: [[OUTER_HEADER1]]:
-; CHECK-NEXT: [[J:%.*]] = phi i64 [ 100, %[[SCALAR_PH]] ], [ [[J_NEXT:%.*]], %[[OUTER_LATCH1:.*]] ]
-; CHECK-NEXT: br label %[[INNER1:.*]]
-; CHECK: [[OUTER_LATCH1]]:
-; CHECK-NEXT: [[J_NEXT]] = add i64 [[J]], 1
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[OUTER_HEADER:.*]]
+; CHECK: [[OUTER_HEADER]]:
+; CHECK-NEXT: [[J:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[J_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
; CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[A]])
-; CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[A]])
-; CHECK-NEXT: [[OUTER_COND:%.*]] = icmp eq i64 [[J]], 100
-; CHECK-NEXT: br i1 [[OUTER_COND]], label %[[EXIT]], label %[[OUTER_HEADER1]], !llvm.loop [[LOOP6:![0-9]+]]
-; CHECK: [[INNER1]]:
-; CHECK-NEXT: [[K:%.*]] = phi i64 [ 0, %[[OUTER_HEADER1]] ], [ [[K_NEXT:%.*]], %[[INNER1]] ]
+; CHECK-NEXT: br label %[[INNER:.*]]
+; CHECK: [[INNER]]:
+; CHECK-NEXT: [[K:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[K_NEXT:%.*]], %[[INNER]] ]
+; CHECK-NEXT: [[IDX:%.*]] = add i64 [[J]], [[K]]
+; CHECK-NEXT: [[PTR:%.*]] = getelementptr inbounds i32, ptr [[ARR]], i64 [[IDX]]
+; CHECK-NEXT: store i32 1, ptr [[PTR]], align 4
; CHECK-NEXT: [[K_NEXT]] = add i64 [[K]], 1
-; CHECK-NEXT: [[INNER_COND:%.*]] = icmp eq i64 [[K_NEXT]], 0
-; CHECK-NEXT: br i1 [[INNER_COND]], label %[[OUTER_LATCH1]], label %[[INNER1]]
+; CHECK-NEXT: [[INNER_COND:%.*]] = icmp eq i64 [[K_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[INNER_COND]], label %[[OUTER_LATCH]], label %[[INNER]]
+; CHECK: [[OUTER_LATCH]]:
+; CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[A]])
+; CHECK-NEXT: [[J_NEXT]] = add i64 [[J]], 1
+; CHECK-NEXT: [[OUTER_COND:%.*]] = icmp eq i64 [[J_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[OUTER_COND]], label %[[EXIT]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret i32 0
+; CHECK-NEXT: ret void
;
entry:
%a = alloca i32
br label %outer.header
outer.header:
%j = phi i64 [ 0, %entry ], [ %j.next, %outer.latch ]
- br label %inner
-outer.latch:
- %j.next = add i64 %j, 1
call void @llvm.lifetime.start.p0(i64 4, ptr %a)
- call void @llvm.lifetime.end.p0(i64 4, ptr %a)
- %outer.cond = icmp eq i64 %j, 100
- br i1 %outer.cond, label %exit, label %outer.header, !llvm.loop !0
+ br label %inner
inner:
%k = phi i64 [ 0, %outer.header ], [ %k.next, %inner ]
+ %idx = add i64 %j, %k
+ %ptr = getelementptr inbounds i32, ptr %arr, i64 %idx
+ store i32 1, ptr %ptr, align 4
%k.next = add i64 %k, 1
- %inner.cond = icmp eq i64 %k.next, 0
+ %inner.cond = icmp eq i64 %k.next, %n
br i1 %inner.cond, label %outer.latch, label %inner
+outer.latch:
+ call void @llvm.lifetime.end.p0(i64 4, ptr %a)
+ %j.next = add i64 %j, 1
+ %outer.cond = icmp eq i64 %j.next, %n
+ br i1 %outer.cond, label %exit, label %outer.header, !llvm.loop !0
exit:
- ret i32 0
+ ret void
}
-define i32 @test_sideeffect() {
-; CHECK-LABEL: define i32 @test_sideeffect() {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[OUTER_HEADER:.*]]
-; CHECK: [[OUTER_HEADER]]:
-; CHECK-NEXT: br label %[[INNER:.*]]
-; CHECK: [[INNER]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_LATCH:.*]] ]
-; CHECK-NEXT: br label %[[OUTER_LATCH:.*]]
-; CHECK: [[OUTER_LATCH]]:
-; CHECK-NEXT: [[TMP0:%.*]] = phi <2 x i64> [ zeroinitializer, %[[INNER]] ], [ [[TMP1:%.*]], %[[OUTER_LATCH]] ]
-; CHECK-NEXT: [[TMP1]] = add <2 x i64> [[TMP0]], splat (i64 1)
-; CHECK-NEXT: [[TMP2:%.*]] = icmp eq <2 x i64> [[TMP1]], zeroinitializer
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i1> [[TMP2]], i32 0
-; CHECK-NEXT: br i1 [[TMP3]], label %[[VECTOR_LATCH]], label %[[OUTER_LATCH]]
-; CHECK: [[VECTOR_LATCH]]:
+define void @test_sideeffect(ptr %arr, i64 %n) {
+; CHECK-LABEL: define void @test_sideeffect(
+; CHECK-SAME: ptr [[ARR:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 2
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[N]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_LATCH:.*]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_LATCH]] ]
; CHECK-NEXT: call void @llvm.sideeffect()
+; CHECK-NEXT: br label %[[INNER1:.*]]
+; CHECK: [[INNER1]]:
+; CHECK-NEXT: [[TMP0:%.*]] = phi <2 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP3:%.*]], %[[INNER1]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = add <2 x i64> [[VEC_IND]], [[TMP0]]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[ARR]], <2 x i64> [[TMP1]]
+; CHECK-NEXT: call void @llvm.masked.scatter.v2i32.v2p0(<2 x i32> splat (i32 1), <2 x ptr> align 4 [[TMP2]], <2 x i1> splat (i1 true))
+; CHECK-NEXT: [[TMP3]] = add <2 x i64> [[TMP0]], splat (i64 1)
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq <2 x i64> [[TMP3]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x i1> [[TMP4]], i32 0
+; CHECK-NEXT: br i1 [[TMP5]], label %[[VECTOR_LATCH]], label %[[INNER1]]
+; CHECK: [[VECTOR_LATCH]]:
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
-; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[INNER]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 2)
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br i1 false, label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: br label %[[OUTER_HEADER1:.*]]
-; CHECK: [[OUTER_HEADER1]]:
-; CHECK-NEXT: [[J:%.*]] = phi i64 [ 100, %[[SCALAR_PH]] ], [ [[J_NEXT:%.*]], %[[OUTER_LATCH1:.*]] ]
-; CHECK-NEXT: br label %[[INNER1:.*]]
-; CHECK: [[OUTER_LATCH1]]:
-; CHECK-NEXT: [[J_NEXT]] = add i64 [[J]], 1
-; CHECK-NEXT: call void @llvm.sideeffect()
-; CHECK-NEXT: [[OUTER_COND:%.*]] = icmp eq i64 [[J]], 100
-; CHECK-NEXT: br i1 [[OUTER_COND]], label %[[EXIT]], label %[[OUTER_HEADER1]], !llvm.loop [[LOOP8:![0-9]+]]
-; CHECK: [[INNER1]]:
-; CHECK-NEXT: [[K:%.*]] = phi i64 [ 0, %[[OUTER_HEADER1]] ], [ [[K_NEXT:%.*]], %[[INNER1]] ]
-; CHECK-NEXT: [[K_NEXT]] = add i64 [[K]], 1
-; CHECK-NEXT: [[INNER_COND:%.*]] = icmp eq i64 [[K_NEXT]], 0
-; CHECK-NEXT: br i1 [[INNER_COND]], label %[[OUTER_LATCH1]], label %[[INNER1]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret i32 0
-;
-entry:
- br label %outer.header
-outer.header:
- %j = phi i64 [ 0, %entry ], [ %j.next, %outer.latch ]
- br label %inner
-outer.latch:
- %j.next = add i64 %j, 1
- call void @llvm.sideeffect()
- %outer.cond = icmp eq i64 %j, 100
- br i1 %outer.cond, label %exit, label %outer.header, !llvm.loop !0
-inner:
- %k = phi i64 [ 0, %outer.header ], [ %k.next, %inner ]
- %k.next = add i64 %k, 1
- %inner.cond = icmp eq i64 %k.next, 0
- br i1 %inner.cond, label %outer.latch, label %inner
-exit:
- ret i32 0
-}
-
-define i32 @test_pseudoprobe() {
-; CHECK-LABEL: define i32 @test_pseudoprobe() {
-; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[OUTER_HEADER:.*]]
; CHECK: [[OUTER_HEADER]]:
+; CHECK-NEXT: [[J:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[J_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
+; CHECK-NEXT: call void @llvm.sideeffect()
; CHECK-NEXT: br label %[[INNER:.*]]
; CHECK: [[INNER]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_LATCH:.*]] ]
-; CHECK-NEXT: br label %[[OUTER_LATCH:.*]]
+; CHECK-NEXT: [[K:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[K_NEXT:%.*]], %[[INNER]] ]
+; CHECK-NEXT: [[IDX:%.*]] = add i64 [[J]], [[K]]
+; CHECK-NEXT: [[PTR:%.*]] = getelementptr inbounds i32, ptr [[ARR]], i64 [[IDX]]
+; CHECK-NEXT: store i32 1, ptr [[PTR]], align 4
+; CHECK-NEXT: [[K_NEXT]] = add i64 [[K]], 1
+; CHECK-NEXT: [[INNER_COND:%.*]] = icmp eq i64 [[K_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[INNER_COND]], label %[[OUTER_LATCH]], label %[[INNER]]
; CHECK: [[OUTER_LATCH]]:
-; CHECK-NEXT: [[TMP0:%.*]] = phi <2 x i64> [ zeroinitializer, %[[INNER]] ], [ [[TMP1:%.*]], %[[OUTER_LATCH]] ]
-; CHECK-NEXT: [[TMP1]] = add <2 x i64> [[TMP0]], splat (i64 1)
-; CHECK-NEXT: [[TMP2:%.*]] = icmp eq <2 x i64> [[TMP1]], zeroinitializer
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i1> [[TMP2]], i32 0
-; CHECK-NEXT: br i1 [[TMP3]], label %[[VECTOR_LATCH]], label %[[OUTER_LATCH]]
-; CHECK: [[VECTOR_LATCH]]:
-; CHECK-NEXT: call void @llvm.pseudoprobe(i64 0, i64 0, i32 0, i64 0)
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
-; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[INNER]], !llvm.loop [[LOOP9:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br i1 false, label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: br label %[[OUTER_HEADER1:.*]]
-; CHECK: [[OUTER_HEADER1]]:
-; CHECK-NEXT: [[J:%.*]] = phi i64 [ 100, %[[SCALAR_PH]] ], [ [[J_NEXT:%.*]], %[[OUTER_LATCH1:.*]] ]
-; CHECK-NEXT: br label %[[INNER1:.*]]
-; CHECK: [[OUTER_LATCH1]]:
; CHECK-NEXT: [[J_NEXT]] = add i64 [[J]], 1
-; CHECK-NEXT: call void @llvm.pseudoprobe(i64 0, i64 0, i32 0, i64 0)
-; CHECK-NEXT: [[OUTER_COND:%.*]] = icmp eq i64 [[J]], 100
-; CHECK-NEXT: br i1 [[OUTER_COND]], label %[[EXIT]], label %[[OUTER_HEADER1]], !llvm.loop [[LOOP10:![0-9]+]]
-; CHECK: [[INNER1]]:
-; CHECK-NEXT: [[K:%.*]] = phi i64 [ 0, %[[OUTER_HEADER1]] ], [ [[K_NEXT:%.*]], %[[INNER1]] ]
-; CHECK-NEXT: [[K_NEXT]] = add i64 [[K]], 1
-; CHECK-NEXT: [[INNER_COND:%.*]] = icmp eq i64 [[K_NEXT]], 0
-; CHECK-NEXT: br i1 [[INNER_COND]], label %[[OUTER_LATCH1]], label %[[INNER1]]
+; CHECK-NEXT: [[OUTER_COND:%.*]] = icmp eq i64 [[J_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[OUTER_COND]], label %[[EXIT]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret i32 0
+; CHECK-NEXT: ret void
;
entry:
br label %outer.header
outer.header:
%j = phi i64 [ 0, %entry ], [ %j.next, %outer.latch ]
+ call void @llvm.sideeffect()
br label %inner
-outer.latch:
- %j.next = add i64 %j, 1
- call void @llvm.pseudoprobe(i64 0, i64 0, i32 0, i64 0)
- %outer.cond = icmp eq i64 %j, 100
- br i1 %outer.cond, label %exit, label %outer.header, !llvm.loop !0
inner:
%k = phi i64 [ 0, %outer.header ], [ %k.next, %inner ]
+ %idx = add i64 %j, %k
+ %ptr = getelementptr inbounds i32, ptr %arr, i64 %idx
+ store i32 1, ptr %ptr, align 4
%k.next = add i64 %k, 1
- %inner.cond = icmp eq i64 %k.next, 0
+ %inner.cond = icmp eq i64 %k.next, %n
br i1 %inner.cond, label %outer.latch, label %inner
+outer.latch:
+ %j.next = add i64 %j, 1
+ %outer.cond = icmp eq i64 %j.next, %n
+ br i1 %outer.cond, label %exit, label %outer.header, !llvm.loop !0
exit:
- ret i32 0
+ ret void
}
-define i32 @test_noalias_scope_decl() {
-; CHECK-LABEL: define i32 @test_noalias_scope_decl() {
-; CHECK-NEXT: [[ENTRY:.*:]]
+define void @test_pseudoprobe(ptr %arr, i64 %n) {
+; CHECK-LABEL: define void @test_pseudoprobe(
+; CHECK-SAME: ptr [[ARR:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 2
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[N]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_LATCH:.*]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_LATCH]] ]
+; CHECK-NEXT: call void @llvm.pseudoprobe(i64 6699318081062747564, i64 1, i32 0, i64 -1)
+; CHECK-NEXT: br label %[[INNER1:.*]]
+; CHECK: [[INNER1]]:
+; CHECK-NEXT: [[TMP0:%.*]] = phi <2 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP3:%.*]], %[[INNER1]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = add <2 x i64> [[VEC_IND]], [[TMP0]]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[ARR]], <2 x i64> [[TMP1]]
+; CHECK-NEXT: call void @llvm.masked.scatter.v2i32.v2p0(<2 x i32> splat (i32 1), <2 x ptr> align 4 [[TMP2]], <2 x i1> splat (i1 true))
+; CHECK-NEXT: [[TMP3]] = add <2 x i64> [[TMP0]], splat (i64 1)
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq <2 x i64> [[TMP3]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x i1> [[TMP4]], i32 0
+; CHECK-NEXT: br i1 [[TMP5]], label %[[VECTOR_LATCH]], label %[[INNER1]]
+; CHECK: [[VECTOR_LATCH]]:
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 2)
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[OUTER_HEADER:.*]]
; CHECK: [[OUTER_HEADER]]:
+; CHECK-NEXT: [[J:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[J_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
+; CHECK-NEXT: call void @llvm.pseudoprobe(i64 6699318081062747564, i64 1, i32 0, i64 -1)
; CHECK-NEXT: br label %[[INNER:.*]]
; CHECK: [[INNER]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_LATCH:.*]] ]
-; CHECK-NEXT: br label %[[OUTER_LATCH:.*]]
+; CHECK-NEXT: [[K:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[K_NEXT:%.*]], %[[INNER]] ]
+; CHECK-NEXT: [[IDX:%.*]] = add i64 [[J]], [[K]]
+; CHECK-NEXT: [[PTR:%.*]] = getelementptr inbounds i32, ptr [[ARR]], i64 [[IDX]]
+; CHECK-NEXT: store i32 1, ptr [[PTR]], align 4
+; CHECK-NEXT: [[K_NEXT]] = add i64 [[K]], 1
+; CHECK-NEXT: [[INNER_COND:%.*]] = icmp eq i64 [[K_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[INNER_COND]], label %[[OUTER_LATCH]], label %[[INNER]]
; CHECK: [[OUTER_LATCH]]:
-; CHECK-NEXT: [[TMP0:%.*]] = phi <2 x i64> [ zeroinitializer, %[[INNER]] ], [ [[TMP1:%.*]], %[[OUTER_LATCH]] ]
-; CHECK-NEXT: [[TMP1]] = add <2 x i64> [[TMP0]], splat (i64 1)
-; CHECK-NEXT: [[TMP2:%.*]] = icmp eq <2 x i64> [[TMP1]], zeroinitializer
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i1> [[TMP2]], i32 0
-; CHECK-NEXT: br i1 [[TMP3]], label %[[VECTOR_LATCH]], label %[[OUTER_LATCH]]
-; CHECK: [[VECTOR_LATCH]]:
-; CHECK-NEXT: call void @llvm.experimental.noalias.scope.decl(metadata [[META11:![0-9]+]])
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
-; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[INNER]], !llvm.loop [[LOOP14:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br i1 false, label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: br label %[[OUTER_HEADER1:.*]]
-; CHECK: [[OUTER_HEADER1]]:
-; CHECK-NEXT: [[J:%.*]] = phi i64 [ 100, %[[SCALAR_PH]] ], [ [[J_NEXT:%.*]], %[[OUTER_LATCH1:.*]] ]
-; CHECK-NEXT: br label %[[INNER1:.*]]
-; CHECK: [[OUTER_LATCH1]]:
; CHECK-NEXT: [[J_NEXT]] = add i64 [[J]], 1
-; CHECK-NEXT: call void @llvm.experimental.noalias.scope.decl(metadata [[META11]])
-; CHECK-NEXT: [[OUTER_COND:%.*]] = icmp eq i64 [[J]], 100
-; CHECK-NEXT: br i1 [[OUTER_COND]], label %[[EXIT]], label %[[OUTER_HEADER1]], !llvm.loop [[LOOP15:![0-9]+]]
-; CHECK: [[INNER1]]:
-; CHECK-NEXT: [[K:%.*]] = phi i64 [ 0, %[[OUTER_HEADER1]] ], [ [[K_NEXT:%.*]], %[[INNER1]] ]
-; CHECK-NEXT: [[K_NEXT]] = add i64 [[K]], 1
-; CHECK-NEXT: [[INNER_COND:%.*]] = icmp eq i64 [[K_NEXT]], 0
-; CHECK-NEXT: br i1 [[INNER_COND]], label %[[OUTER_LATCH1]], label %[[INNER1]]
+; CHECK-NEXT: [[OUTER_COND:%.*]] = icmp eq i64 [[J_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[OUTER_COND]], label %[[EXIT]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret i32 0
+; CHECK-NEXT: ret void
;
entry:
br label %outer.header
outer.header:
%j = phi i64 [ 0, %entry ], [ %j.next, %outer.latch ]
+ call void @llvm.pseudoprobe(i64 6699318081062747564, i64 1, i32 0, i64 -1)
br label %inner
-outer.latch:
- %j.next = add i64 %j, 1
- call void @llvm.experimental.noalias.scope.decl(metadata !3)
- %outer.cond = icmp eq i64 %j, 100
- br i1 %outer.cond, label %exit, label %outer.header, !llvm.loop !0
inner:
%k = phi i64 [ 0, %outer.header ], [ %k.next, %inner ]
+ %idx = add i64 %j, %k
+ %ptr = getelementptr inbounds i32, ptr %arr, i64 %idx
+ store i32 1, ptr %ptr, align 4
%k.next = add i64 %k, 1
- %inner.cond = icmp eq i64 %k.next, 0
+ %inner.cond = icmp eq i64 %k.next, %n
br i1 %inner.cond, label %outer.latch, label %inner
+outer.latch:
+ %j.next = add i64 %j, 1
+ %outer.cond = icmp eq i64 %j.next, %n
+ br i1 %outer.cond, label %exit, label %outer.header, !llvm.loop !0
exit:
- ret i32 0
+ ret void
}
declare void @llvm.assume(i1)
@@ -311,14 +329,10 @@ declare void @llvm.lifetime.start.p0(i64, ptr)
declare void @llvm.lifetime.end.p0(i64, ptr)
declare void @llvm.sideeffect()
declare void @llvm.pseudoprobe(i64, i64, i32, i64)
-declare void @llvm.experimental.noalias.scope.decl(metadata)
!0 = distinct !{!0, !1, !2}
!1 = !{!"llvm.loop.mustprogress"}
!2 = !{!"llvm.loop.vectorize.enable", i1 true}
-!3 = !{!4}
-!4 = distinct !{!4, !5}
-!5 = distinct !{!5}
;.
; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]], [[META3:![0-9]+]]}
; CHECK: [[META1]] = !{!"llvm.loop.mustprogress"}
@@ -331,9 +345,4 @@ declare void @llvm.experimental.noalias.scope.decl(metadata)
; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META3]], [[META2]]}
; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META1]], [[META2]], [[META3]]}
; CHECK: [[LOOP10]] = distinct !{[[LOOP10]], [[META1]], [[META3]], [[META2]]}
-; CHECK: [[META11]] = !{[[META12:![0-9]+]]}
-; CHECK: [[META12]] = distinct !{[[META12]], [[META13:![0-9]+]]}
-; CHECK: [[META13]] = distinct !{[[META13]]}
-; CHECK: [[LOOP14]] = distinct !{[[LOOP14]], [[META1]], [[META2]], [[META3]]}
-; CHECK: [[LOOP15]] = distinct !{[[LOOP15]], [[META1]], [[META3]], [[META2]]}
;.
>From cf3553e8ab0e87dc9709b2de4b99d410e61e0845 Mon Sep 17 00:00:00 2001
From: Henry Jiang <henry_jiang2 at apple.com>
Date: Fri, 20 Mar 2026 10:18:20 -0700
Subject: [PATCH 6/8] bail out of no alias decl
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 13 +++--
.../noalias-scope-decl-outer-loop.ll | 58 +++++--------------
.../non-widenable-intrinsics-outer-loop.ll | 4 +-
3 files changed, 22 insertions(+), 53 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index aaa9d8ac9202f..7d3739d805229 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -94,15 +94,16 @@ bool VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(
Intrinsic::ID VectorID = getVectorIntrinsicIDForCall(CI, &TLI);
if (VectorID == Intrinsic::not_intrinsic)
return false;
+
// The noalias.scope.decl intrinsic declares a noalias scope that
// is valid for a single iteration. Emitting it as a single-scalar
// replicate would incorrectly extend the scope across multiple
- // original iterations packed into one vector iteration. Taking the
- // conservative approach and drop it, but still allow vectorization.
- if (VectorID == Intrinsic::experimental_noalias_scope_decl) {
- Ingredient.eraseFromParent();
- continue;
- }
+ // original iterations packed into one vector iteration.
+ // FIXME: If we want to vectorize this this loop, then we have to drop
+ // all the associated !alias.scope and !noalias.
+ if (VectorID == Intrinsic::experimental_noalias_scope_decl)
+ return false;
+
// These intrinsics are recognized by getVectorIntrinsicIDForCall
// but are not widenable. Emit them as single-scalar replicate
// instead of widening.
diff --git a/llvm/test/Transforms/LoopVectorize/noalias-scope-decl-outer-loop.ll b/llvm/test/Transforms/LoopVectorize/noalias-scope-decl-outer-loop.ll
index 3d9f9e9851612..62b6f41d9dbaf 100644
--- a/llvm/test/Transforms/LoopVectorize/noalias-scope-decl-outer-loop.ll
+++ b/llvm/test/Transforms/LoopVectorize/noalias-scope-decl-outer-loop.ll
@@ -1,52 +1,22 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -S -passes='loop-vectorize,verify' -enable-vplan-native-path -force-vector-width=2 < %s | FileCheck %s
-; The noalias.scope.decl intrinsic declares a noalias scope valid for a single iteration.
-; During outer loop vectorization, emitting it as a single-scalar replicate would incorrectly
-; extend the scope across multiple original iterations packed into one vector iteration.
-; We can drop the intrinsics but still allow outerloop vectorization to happen.
+; The noalias.scope.decl intrinsic declares a noalias scope valid for a single
+; iteration. During outer loop vectorization, emitting it as a single-scalar
+; replicate would incorrectly extend the scope across multiple original
+; iterations packed into one vector iteration. Bail out of vectorization for
+; outer loops containing noalias.scope.decl.
+; FIXME: We could still vectorize by dropping the noalias.scope.decl and
+; stripping !alias.scope/!noalias metadata from the loop body.
define void @test_noalias_scope_decl(ptr %a, ptr %b, i64 %n) {
; CHECK-LABEL: define void @test_noalias_scope_decl(
; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 2
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[N]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_LATCH:.*]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_LATCH]] ]
-; CHECK-NEXT: br label %[[INNER1:.*]]
-; CHECK: [[INNER1]]:
-; CHECK-NEXT: [[TMP0:%.*]] = phi <2 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP4:%.*]], %[[INNER1]] ]
-; CHECK-NEXT: [[TMP1:%.*]] = add <2 x i64> [[VEC_IND]], [[TMP0]]
-; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[A]], <2 x i64> [[TMP1]]
-; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <2 x i32> @llvm.masked.gather.v2i32.v2p0(<2 x ptr> align 4 [[TMP2]], <2 x i1> splat (i1 true), <2 x i32> poison), !alias.scope [[META0:![0-9]+]]
-; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[B]], <2 x i64> [[TMP1]]
-; CHECK-NEXT: call void @llvm.masked.scatter.v2i32.v2p0(<2 x i32> [[WIDE_MASKED_GATHER]], <2 x ptr> align 4 [[TMP3]], <2 x i1> splat (i1 true)), !noalias [[META0]]
-; CHECK-NEXT: [[TMP4]] = add <2 x i64> [[TMP0]], splat (i64 1)
-; CHECK-NEXT: [[TMP5:%.*]] = icmp eq <2 x i64> [[TMP4]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x i1> [[TMP5]], i32 0
-; CHECK-NEXT: br i1 [[TMP6]], label %[[VECTOR_LATCH]], label %[[INNER1]]
-; CHECK: [[VECTOR_LATCH]]:
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 2)
-; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[SCALAR_PH:.*]]:
; CHECK-NEXT: br label %[[OUTER_HEADER:.*]]
; CHECK: [[OUTER_HEADER]]:
-; CHECK-NEXT: [[J:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[J_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
-; CHECK-NEXT: call void @llvm.experimental.noalias.scope.decl(metadata [[META0]])
+; CHECK-NEXT: [[J:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[J_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
+; CHECK-NEXT: call void @llvm.experimental.noalias.scope.decl(metadata [[META0:![0-9]+]])
; CHECK-NEXT: br label %[[INNER:.*]]
; CHECK: [[INNER]]:
; CHECK-NEXT: [[K:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[K_NEXT:%.*]], %[[INNER]] ]
@@ -61,7 +31,7 @@ define void @test_noalias_scope_decl(ptr %a, ptr %b, i64 %n) {
; CHECK: [[OUTER_LATCH]]:
; CHECK-NEXT: [[J_NEXT]] = add i64 [[J]], 1
; CHECK-NEXT: [[OUTER_COND:%.*]] = icmp eq i64 [[J_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[OUTER_COND]], label %[[EXIT]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-NEXT: br i1 [[OUTER_COND]], label %[[EXIT:.*]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -101,9 +71,7 @@ declare void @llvm.experimental.noalias.scope.decl(metadata)
; CHECK: [[META0]] = !{[[META1:![0-9]+]]}
; CHECK: [[META1]] = distinct !{[[META1]], [[META2:![0-9]+]]}
; CHECK: [[META2]] = distinct !{[[META2]]}
-; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META4:![0-9]+]], [[META5:![0-9]+]], [[META6:![0-9]+]]}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META4:![0-9]+]], [[META5:![0-9]+]]}
; CHECK: [[META4]] = !{!"llvm.loop.mustprogress"}
-; CHECK: [[META5]] = !{!"llvm.loop.isvectorized", i32 1}
-; CHECK: [[META6]] = !{!"llvm.loop.unroll.runtime.disable"}
-; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META4]], [[META6]], [[META5]]}
+; CHECK: [[META5]] = !{!"llvm.loop.vectorize.enable", i1 true}
;.
diff --git a/llvm/test/Transforms/LoopVectorize/non-widenable-intrinsics-outer-loop.ll b/llvm/test/Transforms/LoopVectorize/non-widenable-intrinsics-outer-loop.ll
index bf0a118e3add7..7b0580c27c7b2 100644
--- a/llvm/test/Transforms/LoopVectorize/non-widenable-intrinsics-outer-loop.ll
+++ b/llvm/test/Transforms/LoopVectorize/non-widenable-intrinsics-outer-loop.ll
@@ -19,8 +19,8 @@ define void @test_assume(ptr %arr, i64 %n) {
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_LATCH:.*]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_LATCH]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = icmp sgt i64 [[INDEX]], 0
-; CHECK-NEXT: call void @llvm.assume(i1 [[TMP0]])
+; CHECK-NEXT: [[TMP9:%.*]] = icmp sgt i64 [[INDEX]], 0
+; CHECK-NEXT: call void @llvm.assume(i1 [[TMP9]])
; CHECK-NEXT: br label %[[INNER1:.*]]
; CHECK: [[INNER1]]:
; CHECK-NEXT: [[TMP1:%.*]] = phi <2 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP4:%.*]], %[[INNER1]] ]
>From eb5de09976980fb814cf2ddbd80e145972f435e9 Mon Sep 17 00:00:00 2001
From: Henry Jiang <henry_jiang2 at apple.com>
Date: Tue, 24 Mar 2026 22:35:33 -0700
Subject: [PATCH 7/8] address comments to isolate change
---
.../Transforms/Vectorize/LoopVectorize.cpp | 3 -
.../Transforms/Vectorize/VPlanTransforms.cpp | 11 +-
.../noalias-scope-decl-outer-loop.ll | 10 +-
.../non-widenable-intrinsics-outer-loop.ll | 129 ++++++++++++++++--
.../LoopVectorize/pseudoprobe-scalable.ll | 57 --------
5 files changed, 128 insertions(+), 82 deletions(-)
delete mode 100644 llvm/test/Transforms/LoopVectorize/pseudoprobe-scalable.ll
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 75fbbf4606635..bb4eef5a41c09 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3213,7 +3213,6 @@ void LoopVectorizationCostModel::collectLoopUniforms(ElementCount VF) {
case Intrinsic::assume:
case Intrinsic::lifetime_start:
case Intrinsic::lifetime_end:
- case Intrinsic::pseudoprobe:
if (TheLoop->hasLoopInvariantOperands(&I))
AddToWorklistIfAllowed(&I);
break;
@@ -8009,8 +8008,6 @@ VPReplicateRecipe *VPRecipeBuilder::handleReplication(VPInstruction *VPI,
case Intrinsic::assume:
case Intrinsic::lifetime_start:
case Intrinsic::lifetime_end:
- case Intrinsic::sideeffect:
- case Intrinsic::pseudoprobe:
// For scalable vectors if one of the operands is variant then we still
// want to mark as uniform, which will generate one instruction for just
// the first lane of the vector. We can't scalarize the call in the same
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 7d3739d805229..a5c2e7c233d6a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -105,15 +105,20 @@ bool VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(
return false;
// These intrinsics are recognized by getVectorIntrinsicIDForCall
- // but are not widenable. Emit them as single-scalar replicate
- // instead of widening.
+ // but are not widenable. Emit them as replicate instead of widening.
if (VectorID == Intrinsic::assume ||
VectorID == Intrinsic::lifetime_end ||
VectorID == Intrinsic::lifetime_start ||
VectorID == Intrinsic::sideeffect ||
VectorID == Intrinsic::pseudoprobe) {
+ // If the operand of llvm.assume holds before vectorization, it will
+ // also holds per lane.
+ // llvm.pseudoprobe requires to be duplicated per lane for accurate
+ // sample count.
+ const bool IsSingleScalar = VectorID != Intrinsic::assume &&
+ VectorID != Intrinsic::pseudoprobe;
NewRecipe = new VPReplicateRecipe(CI, Ingredient.operands(),
- /*IsSingleScalar=*/true,
+ /*IsSingleScalar=*/IsSingleScalar,
/*Mask=*/nullptr, *VPI, *VPI,
Ingredient.getDebugLoc());
} else {
diff --git a/llvm/test/Transforms/LoopVectorize/noalias-scope-decl-outer-loop.ll b/llvm/test/Transforms/LoopVectorize/noalias-scope-decl-outer-loop.ll
index 62b6f41d9dbaf..0bd438560b9eb 100644
--- a/llvm/test/Transforms/LoopVectorize/noalias-scope-decl-outer-loop.ll
+++ b/llvm/test/Transforms/LoopVectorize/noalias-scope-decl-outer-loop.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
; RUN: opt -S -passes='loop-vectorize,verify' -enable-vplan-native-path -force-vector-width=2 < %s | FileCheck %s
; The noalias.scope.decl intrinsic declares a noalias scope valid for a single
@@ -67,11 +67,3 @@ declare void @llvm.experimental.noalias.scope.decl(metadata)
!3 = !{!4}
!4 = distinct !{!4, !5}
!5 = distinct !{!5}
-;.
-; CHECK: [[META0]] = !{[[META1:![0-9]+]]}
-; CHECK: [[META1]] = distinct !{[[META1]], [[META2:![0-9]+]]}
-; CHECK: [[META2]] = distinct !{[[META2]]}
-; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META4:![0-9]+]], [[META5:![0-9]+]]}
-; CHECK: [[META4]] = !{!"llvm.loop.mustprogress"}
-; CHECK: [[META5]] = !{!"llvm.loop.vectorize.enable", i1 true}
-;.
diff --git a/llvm/test/Transforms/LoopVectorize/non-widenable-intrinsics-outer-loop.ll b/llvm/test/Transforms/LoopVectorize/non-widenable-intrinsics-outer-loop.ll
index 7b0580c27c7b2..1397ade51f146 100644
--- a/llvm/test/Transforms/LoopVectorize/non-widenable-intrinsics-outer-loop.ll
+++ b/llvm/test/Transforms/LoopVectorize/non-widenable-intrinsics-outer-loop.ll
@@ -1,9 +1,10 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
-; RUN: opt -S -passes='verify,loop-vectorize,verify' -enable-vplan-native-path -force-vector-width=2 < %s | FileCheck %s
+; RUN: opt -S -passes='loop-vectorize,verify' -enable-vplan-native-path -force-vector-width=2 < %s | FileCheck %s
-; Test that non-widenable intrinsics in the outer loop are emitted as single
-; scalar calls and do not crash the VPlan native path.
+; Test that non-widenable intrinsics in the outer loop are emitted as replicate
+; recipe and do not crash the VPlan native path.
+; Check that llvm.asume is replicated per lane for the assumption.
define void @test_assume(ptr %arr, i64 %n) {
; CHECK-LABEL: define void @test_assume(
; CHECK-SAME: ptr [[ARR:%.*]], i64 [[N:%.*]]) {
@@ -19,8 +20,11 @@ define void @test_assume(ptr %arr, i64 %n) {
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_LATCH:.*]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_LATCH]] ]
-; CHECK-NEXT: [[TMP9:%.*]] = icmp sgt i64 [[INDEX]], 0
+; CHECK-NEXT: [[TMP0:%.*]] = icmp sgt <2 x i64> [[VEC_IND]], zeroinitializer
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x i1> [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x i1> [[TMP0]], i32 1
; CHECK-NEXT: call void @llvm.assume(i1 [[TMP9]])
+; CHECK-NEXT: call void @llvm.assume(i1 [[TMP8]])
; CHECK-NEXT: br label %[[INNER1:.*]]
; CHECK: [[INNER1]]:
; CHECK-NEXT: [[TMP1:%.*]] = phi <2 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP4:%.*]], %[[INNER1]] ]
@@ -85,6 +89,107 @@ exit:
ret void
}
+; Check that llvm.assume with a noncanonical induction variable is replicated per lane.
+define void @test_assume_non_canonical_iv(ptr %arr, i64 %n) {
+; CHECK-LABEL: define void @test_assume_non_canonical_iv(
+; CHECK-SAME: ptr [[ARR:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 2
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP0:%.*]] = mul i64 [[N_VEC]], 3
+; CHECK-NEXT: [[TMP1:%.*]] = add i64 10, [[TMP0]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[N]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_LATCH:.*]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 10, i64 13>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_LATCH]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = icmp sge <2 x i64> [[VEC_IND]], zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i1> [[TMP2]], i32 0
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x i1> [[TMP2]], i32 1
+; CHECK-NEXT: call void @llvm.assume(i1 [[TMP3]])
+; CHECK-NEXT: call void @llvm.assume(i1 [[TMP4]])
+; CHECK-NEXT: br label %[[INNER1:.*]]
+; CHECK: [[INNER1]]:
+; CHECK-NEXT: [[TMP5:%.*]] = phi <2 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP10:%.*]], %[[INNER1]] ]
+; CHECK-NEXT: [[TMP6:%.*]] = add <2 x i64> [[VEC_IND]], [[TMP5]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[ARR]], <2 x i64> [[TMP6]]
+; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <2 x i32> @llvm.masked.gather.v2i32.v2p0(<2 x ptr> align 4 [[TMP7]], <2 x i1> splat (i1 true), <2 x i32> poison)
+; CHECK-NEXT: [[TMP8:%.*]] = select <2 x i1> [[TMP2]], <2 x i32> splat (i32 1), <2 x i32> splat (i32 -1)
+; CHECK-NEXT: [[TMP9:%.*]] = add <2 x i32> [[WIDE_MASKED_GATHER]], [[TMP8]]
+; CHECK-NEXT: call void @llvm.masked.scatter.v2i32.v2p0(<2 x i32> [[TMP9]], <2 x ptr> align 4 [[TMP7]], <2 x i1> splat (i1 true))
+; CHECK-NEXT: [[TMP10]] = add <2 x i64> [[TMP5]], splat (i64 1)
+; CHECK-NEXT: [[TMP11:%.*]] = icmp eq <2 x i64> [[TMP10]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x i1> [[TMP11]], i32 0
+; CHECK-NEXT: br i1 [[TMP12]], label %[[VECTOR_LATCH]], label %[[INNER1]]
+; CHECK: [[VECTOR_LATCH]]:
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 6)
+; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL2:%.*]] = phi i64 [ [[TMP1]], %[[MIDDLE_BLOCK]] ], [ 10, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[OUTER_HEADER:.*]]
+; CHECK: [[OUTER_HEADER]]:
+; CHECK-NEXT: [[J:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[J_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
+; CHECK-NEXT: [[M:%.*]] = phi i64 [ [[BC_RESUME_VAL2]], %[[SCALAR_PH]] ], [ [[M_NEXT:%.*]], %[[OUTER_LATCH]] ]
+; CHECK-NEXT: [[M_NEXT]] = add i64 [[M]], 3
+; CHECK-NEXT: [[CMP:%.*]] = icmp sge i64 [[M]], 0
+; CHECK-NEXT: call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT: br label %[[INNER:.*]]
+; CHECK: [[INNER]]:
+; CHECK-NEXT: [[K:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[K_NEXT:%.*]], %[[INNER]] ]
+; CHECK-NEXT: [[IDX:%.*]] = add i64 [[M]], [[K]]
+; CHECK-NEXT: [[PTR:%.*]] = getelementptr inbounds i32, ptr [[ARR]], i64 [[IDX]]
+; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[PTR]], align 4
+; CHECK-NEXT: [[INC:%.*]] = select i1 [[CMP]], i32 1, i32 -1
+; CHECK-NEXT: [[VAL2:%.*]] = add i32 [[VAL]], [[INC]]
+; CHECK-NEXT: store i32 [[VAL2]], ptr [[PTR]], align 4
+; CHECK-NEXT: [[K_NEXT]] = add i64 [[K]], 1
+; CHECK-NEXT: [[INNER_COND:%.*]] = icmp eq i64 [[K_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[INNER_COND]], label %[[OUTER_LATCH]], label %[[INNER]]
+; CHECK: [[OUTER_LATCH]]:
+; CHECK-NEXT: [[J_NEXT]] = add i64 [[J]], 1
+; CHECK-NEXT: [[OUTER_COND:%.*]] = icmp eq i64 [[J_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[OUTER_COND]], label %[[EXIT]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %outer.header
+outer.header:
+ %j = phi i64 [ 0, %entry ], [ %j.next, %outer.latch ]
+ %m = phi i64 [ 10, %entry ], [ %m.next, %outer.latch ]
+ %m.next = add i64 %m, 3
+ %cmp = icmp sge i64 %m, 0
+ call void @llvm.assume(i1 %cmp)
+ br label %inner
+inner:
+ %k = phi i64 [ 0, %outer.header ], [ %k.next, %inner ]
+ %idx = add i64 %m, %k
+ %ptr = getelementptr inbounds i32, ptr %arr, i64 %idx
+ %val = load i32, ptr %ptr, align 4
+ %inc = select i1 %cmp, i32 1, i32 -1
+ %val2 = add i32 %val, %inc
+ store i32 %val2, ptr %ptr, align 4
+ %k.next = add i64 %k, 1
+ %inner.cond = icmp eq i64 %k.next, %n
+ br i1 %inner.cond, label %outer.latch, label %inner
+outer.latch:
+ %j.next = add i64 %j, 1
+ %outer.cond = icmp eq i64 %j.next, %n
+ br i1 %outer.cond, label %exit, label %outer.header, !llvm.loop !0
+exit:
+ ret void
+}
+
define void @test_lifetime(ptr %arr, i64 %n) {
; CHECK-LABEL: define void @test_lifetime(
; CHECK-SAME: ptr [[ARR:%.*]], i64 [[N:%.*]]) {
@@ -117,7 +222,7 @@ define void @test_lifetime(ptr %arr, i64 %n) {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 2)
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
@@ -140,7 +245,7 @@ define void @test_lifetime(ptr %arr, i64 %n) {
; CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[A]])
; CHECK-NEXT: [[J_NEXT]] = add i64 [[J]], 1
; CHECK-NEXT: [[OUTER_COND:%.*]] = icmp eq i64 [[J_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[OUTER_COND]], label %[[EXIT]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT: br i1 [[OUTER_COND]], label %[[EXIT]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -198,7 +303,7 @@ define void @test_sideeffect(ptr %arr, i64 %n) {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 2)
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
@@ -220,7 +325,7 @@ define void @test_sideeffect(ptr %arr, i64 %n) {
; CHECK: [[OUTER_LATCH]]:
; CHECK-NEXT: [[J_NEXT]] = add i64 [[J]], 1
; CHECK-NEXT: [[OUTER_COND:%.*]] = icmp eq i64 [[J_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[OUTER_COND]], label %[[EXIT]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NEXT: br i1 [[OUTER_COND]], label %[[EXIT]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -246,6 +351,7 @@ exit:
ret void
}
+; Check that llvm.pseudoprobe is replicated once per lane for accurate sample trip count.
define void @test_pseudoprobe(ptr %arr, i64 %n) {
; CHECK-LABEL: define void @test_pseudoprobe(
; CHECK-SAME: ptr [[ARR:%.*]], i64 [[N:%.*]]) {
@@ -262,6 +368,7 @@ define void @test_pseudoprobe(ptr %arr, i64 %n) {
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_LATCH:.*]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_LATCH]] ]
; CHECK-NEXT: call void @llvm.pseudoprobe(i64 6699318081062747564, i64 1, i32 0, i64 -1)
+; CHECK-NEXT: call void @llvm.pseudoprobe(i64 6699318081062747564, i64 1, i32 0, i64 -1)
; CHECK-NEXT: br label %[[INNER1:.*]]
; CHECK: [[INNER1]]:
; CHECK-NEXT: [[TMP0:%.*]] = phi <2 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP3:%.*]], %[[INNER1]] ]
@@ -276,7 +383,7 @@ define void @test_pseudoprobe(ptr %arr, i64 %n) {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 2)
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
@@ -298,7 +405,7 @@ define void @test_pseudoprobe(ptr %arr, i64 %n) {
; CHECK: [[OUTER_LATCH]]:
; CHECK-NEXT: [[J_NEXT]] = add i64 [[J]], 1
; CHECK-NEXT: [[OUTER_COND:%.*]] = icmp eq i64 [[J_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[OUTER_COND]], label %[[EXIT]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-NEXT: br i1 [[OUTER_COND]], label %[[EXIT]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP12:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -345,4 +452,6 @@ declare void @llvm.pseudoprobe(i64, i64, i32, i64)
; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META3]], [[META2]]}
; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META1]], [[META2]], [[META3]]}
; CHECK: [[LOOP10]] = distinct !{[[LOOP10]], [[META1]], [[META3]], [[META2]]}
+; CHECK: [[LOOP11]] = distinct !{[[LOOP11]], [[META1]], [[META2]], [[META3]]}
+; CHECK: [[LOOP12]] = distinct !{[[LOOP12]], [[META1]], [[META3]], [[META2]]}
;.
diff --git a/llvm/test/Transforms/LoopVectorize/pseudoprobe-scalable.ll b/llvm/test/Transforms/LoopVectorize/pseudoprobe-scalable.ll
deleted file mode 100644
index 9a9d2ef6f7c13..0000000000000
--- a/llvm/test/Transforms/LoopVectorize/pseudoprobe-scalable.ll
+++ /dev/null
@@ -1,57 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
-; RUN: opt -S -passes=loop-vectorize -scalable-vectorization=on -force-vector-width=2 -force-vector-interleave=1 -force-target-supports-scalable-vectors=true < %s | FileCheck %s
-
-; Verify that pseudoprobe does not crash with scalable vector vectorization.
-
-define void @test_scalable_pseudoprobe() {
-; CHECK-LABEL: define void @test_scalable_pseudoprobe() {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 -1, [[TMP1]]
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 -1, [[N_MOD_VF]]
-; CHECK-NEXT: [[TMP2:%.*]] = add i64 1, [[N_VEC]]
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: call void @llvm.pseudoprobe(i64 0, i64 0, i32 0, i64 0)
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
-; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 -1, [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[TMP2]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
-; CHECK-NEXT: call void @llvm.pseudoprobe(i64 0, i64 0, i32 0, i64 0)
-; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[IV_NEXT]], 0
-; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret void
-;
-entry:
- br label %loop
-
-loop:
- %iv = phi i64 [ 1, %entry ], [ %iv.next, %loop ]
- %iv.next = add i64 %iv, 1
- call void @llvm.pseudoprobe(i64 0, i64 0, i32 0, i64 0)
- %done = icmp eq i64 %iv.next, 0
- br i1 %done, label %exit, label %loop
-
-exit:
- ret void
-}
-
-declare void @llvm.pseudoprobe(i64, i64, i32, i64)
-;.
-; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
-; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
-; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
-; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
-;.
>From 39c528b0e15c1d98d4c115159dfd18ee58888e88 Mon Sep 17 00:00:00 2001
From: Henry Jiang <henry_jiang2 at apple.com>
Date: Tue, 24 Mar 2026 22:48:15 -0700
Subject: [PATCH 8/8] fix typos
---
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 4 ++--
.../LoopVectorize/non-widenable-intrinsics-outer-loop.ll | 2 +-
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index a5c2e7c233d6a..bb6f9643652c5 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -99,7 +99,7 @@ bool VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(
// is valid for a single iteration. Emitting it as a single-scalar
// replicate would incorrectly extend the scope across multiple
// original iterations packed into one vector iteration.
- // FIXME: If we want to vectorize this this loop, then we have to drop
+ // FIXME: If we want to vectorize this loop, then we have to drop
// all the associated !alias.scope and !noalias.
if (VectorID == Intrinsic::experimental_noalias_scope_decl)
return false;
@@ -112,7 +112,7 @@ bool VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(
VectorID == Intrinsic::sideeffect ||
VectorID == Intrinsic::pseudoprobe) {
// If the operand of llvm.assume holds before vectorization, it will
- // also holds per lane.
+ // also hold per lane.
// llvm.pseudoprobe requires to be duplicated per lane for accurate
// sample count.
const bool IsSingleScalar = VectorID != Intrinsic::assume &&
diff --git a/llvm/test/Transforms/LoopVectorize/non-widenable-intrinsics-outer-loop.ll b/llvm/test/Transforms/LoopVectorize/non-widenable-intrinsics-outer-loop.ll
index 1397ade51f146..42a79f603f02e 100644
--- a/llvm/test/Transforms/LoopVectorize/non-widenable-intrinsics-outer-loop.ll
+++ b/llvm/test/Transforms/LoopVectorize/non-widenable-intrinsics-outer-loop.ll
@@ -4,7 +4,7 @@
; Test that non-widenable intrinsics in the outer loop are emitted as replicate
; recipe and do not crash the VPlan native path.
-; Check that llvm.asume is replicated per lane for the assumption.
+; Check that llvm.assume is replicated per lane for the assumption.
define void @test_assume(ptr %arr, i64 %n) {
; CHECK-LABEL: define void @test_assume(
; CHECK-SAME: ptr [[ARR:%.*]], i64 [[N:%.*]]) {
More information about the llvm-commits
mailing list