[llvm] [VPlan] Constant-fold ActiveLaneMask intrinsics (PR #208852)
Ramkumar Ramachandra via llvm-commits
llvm-commits at lists.llvm.org
Fri Oct 2 07:12:55 PDT 2026
https://github.com/artagnon updated https://github.com/llvm/llvm-project/pull/208852
>From 42df793603776a684f799a70b0fb85efce5f49a9 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Sun, 2 Aug 2026 11:53:43 +0100
Subject: [PATCH 1/2] [LV] Pre-commit ALM tests
---
.../foldable-scalable-active-lane-mask.ll | 73 +++++
.../VPlan/foldable-active-lane-mask.ll | 271 ++++++++++++++++++
2 files changed, 344 insertions(+)
create mode 100644 llvm/test/Transforms/LoopVectorize/VPlan/AArch64/foldable-scalable-active-lane-mask.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/VPlan/foldable-active-lane-mask.ll
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/foldable-scalable-active-lane-mask.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/foldable-scalable-active-lane-mask.ll
new file mode 100644
index 00000000000000..a36b34dba91134
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/foldable-scalable-active-lane-mask.ll
@@ -0,0 +1,73 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "active.lane.mask" --version 6
+; RUN: opt -mtriple=aarch64 -mcpu=neoverse-v1 -passes=loop-vectorize -force-vector-width="vscale x 2" -force-vector-interleave=1 -tail-folding-policy=prefer-fold-tail -disable-output -vplan-print-after=printFinalVPlan %s 2>&1 | FileCheck %s
+
+define void @wide_alm(ptr %dst, ptr noalias %src) vscale_range(1, 2) {
+; CHECK-LABEL: VPlan for loop in 'wide_alm'
+; CHECK: EMIT vp<%active.lane.mask.entry> = wide active lane mask ir<0>, ir<15>, ir<1>
+; CHECK: EMIT vp<%extract.entry.alm.part> = extract-vector-for-part vp<%active.lane.mask.entry>, ir<0>
+; CHECK: EMIT vp<%active.lane.mask.next> = wide active lane mask vp<%index.next>, ir<15>, ir<1>
+; CHECK: EMIT vp<%extract.next.alm.part> = extract-vector-for-part vp<%active.lane.mask.next>, ir<0>
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %x = load i32, ptr %src
+ %gep = getelementptr inbounds i32, ptr %dst, i64 %iv
+ store i32 %x, ptr %gep
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, 15
+ br i1 %exitcond.not, label %for.end, label %for.body
+
+for.end:
+ ret void
+}
+
+define void @all_false(ptr %p) vscale_range(1, 2) {
+; CHECK-LABEL: VPlan for loop in 'all_false'
+; CHECK: EMIT vp<%active.lane.mask.entry> = wide active lane mask ir<0>, ir<7>, ir<1>
+; CHECK: EMIT vp<%extract.entry.alm.part> = extract-vector-for-part vp<%active.lane.mask.entry>, ir<0>
+; CHECK: EMIT vp<%active.lane.mask.next> = wide active lane mask vp<%index.next>, ir<7>, ir<1>
+; CHECK: EMIT vp<%extract.next.alm.part> = extract-vector-for-part vp<%active.lane.mask.next>, ir<0>
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %g = getelementptr i32, ptr %p, i64 %iv
+ store i32 1, ptr %g
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 7
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Check that we don't crash on an unreasonably large vscale_range.
+define void @vscale_range_large(ptr %A, ptr noalias %B) vscale_range(1, 1073741824) {
+; CHECK-LABEL: VPlan for loop in 'vscale_range_large'
+; CHECK: EMIT vp<%active.lane.mask.entry> = wide active lane mask ir<0>, ir<1002>, ir<1>
+; CHECK: EMIT vp<%extract.entry.alm.part> = extract-vector-for-part vp<%active.lane.mask.entry>, ir<0>
+; CHECK: EMIT vp<%active.lane.mask.next> = wide active lane mask vp<%index.next>, ir<1002>, ir<1>
+; CHECK: EMIT vp<%extract.next.alm.part> = extract-vector-for-part vp<%active.lane.mask.next>, ir<0>
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %for = phi i16 [ 0, %entry ], [ %for.next, %loop ]
+ %sext = sext i16 %for to i32
+ %iv.next = add i64 %iv, 1
+ %gep.A = getelementptr i32, ptr %A, i64 %iv.next
+ %for.next = load i16, ptr %gep.A, align 2
+ %gep.B = getelementptr i32, ptr %B, i64 %iv.next
+ store i32 %sext, ptr %gep.B
+ %ec = icmp eq i64 %iv, 1001
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/foldable-active-lane-mask.ll b/llvm/test/Transforms/LoopVectorize/VPlan/foldable-active-lane-mask.ll
new file mode 100644
index 00000000000000..f2434220b97819
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/foldable-active-lane-mask.ll
@@ -0,0 +1,271 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter-out-after "^scalar.ph" --version 6
+; RUN: opt -passes=loop-vectorize -tail-folding-policy=must-fold-tail \
+; RUN: -force-tail-folding-style=data -force-vector-width=4 -force-vector-interleave=2 \
+; RUN: -disable-output -vplan-print-after=printFinalVPlan %s 2>&1 | FileCheck %s
+
+define void @unfoldable(ptr %src, ptr noalias %dst) {
+; CHECK-LABEL: VPlan for loop in 'unfoldable'
+; CHECK: VPlan 'Final VPlan for VF={4},UF={2}' {
+; CHECK-NEXT: Live-in ir<7> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector.body
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: vp<[[VP2:%[0-9]+]]> = SCALAR-STEPS ir<0>, ir<1>, ir<4>, ir<4>
+; CHECK-NEXT: EMIT vp<%active.lane.mask> = active lane mask ir<0>, ir<7>
+; CHECK-NEXT: EMIT vp<%active.lane.mask>.1 = active lane mask vp<[[VP2]]>, ir<7>
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = SCALAR-STEPS ir<0>, ir<1>, ir<4>, ir<1>
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS ir<0>, ir<1>, ir<4>, ir<2>
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS ir<0>, ir<1>, ir<4>, ir<3>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS ir<0>, ir<1>, ir<4>, ir<5>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = SCALAR-STEPS ir<0>, ir<1>, ir<4>, ir<6>
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = SCALAR-STEPS ir<0>, ir<1>, ir<4>, ir<7>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = extractelement vp<%active.lane.mask>, ir<0>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP9]]>
+; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue
+; CHECK-EMPTY:
+; CHECK-NEXT: pred.load.if:
+; CHECK-NEXT: CLONE ir<%l> = load ir<%src>
+; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = insertelement ir<poison>, ir<%l>, ir<0>
+; CHECK-NEXT: Successor(s): pred.load.continue
+; CHECK-EMPTY:
+; CHECK-NEXT: pred.load.continue:
+; CHECK-NEXT: WIDEN-PHI vp<[[VP12:%[0-9]+]]> = phi [ ir<poison>, vector.body ], [ vp<[[VP11]]>, pred.load.if ]
+; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = extractelement vp<%active.lane.mask>, ir<1>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP13]]>
+; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue
+; CHECK-EMPTY:
+; CHECK-NEXT: pred.load.if:
+; CHECK-NEXT: CLONE ir<%p> = getelementptr inbounds ir<%src>, vp<[[VP3]]>
+; CHECK-NEXT: CLONE ir<%l>.1 = load ir<%p>
+; CHECK-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = insertelement vp<[[VP12]]>, ir<%l>.1, ir<1>
+; CHECK-NEXT: Successor(s): pred.load.continue
+; CHECK-EMPTY:
+; CHECK-NEXT: pred.load.continue:
+; CHECK-NEXT: WIDEN-PHI vp<[[VP16:%[0-9]+]]> = phi [ vp<[[VP12]]>, pred.load.continue ], [ vp<[[VP15]]>, pred.load.if ]
+; CHECK-NEXT: EMIT vp<[[VP17:%[0-9]+]]> = extractelement vp<%active.lane.mask>, ir<2>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP17]]>
+; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue
+; CHECK-EMPTY:
+; CHECK-NEXT: pred.load.if:
+; CHECK-NEXT: CLONE ir<%p>.1 = getelementptr inbounds ir<%src>, vp<[[VP4]]>
+; CHECK-NEXT: CLONE ir<%l>.2 = load ir<%p>.1
+; CHECK-NEXT: EMIT vp<[[VP19:%[0-9]+]]> = insertelement vp<[[VP16]]>, ir<%l>.2, ir<2>
+; CHECK-NEXT: Successor(s): pred.load.continue
+; CHECK-EMPTY:
+; CHECK-NEXT: pred.load.continue:
+; CHECK-NEXT: WIDEN-PHI vp<[[VP20:%[0-9]+]]> = phi [ vp<[[VP16]]>, pred.load.continue ], [ vp<[[VP19]]>, pred.load.if ]
+; CHECK-NEXT: EMIT vp<[[VP21:%[0-9]+]]> = extractelement vp<%active.lane.mask>, ir<3>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP21]]>
+; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue
+; CHECK-EMPTY:
+; CHECK-NEXT: pred.load.if:
+; CHECK-NEXT: CLONE ir<%p>.2 = getelementptr inbounds ir<%src>, vp<[[VP5]]>
+; CHECK-NEXT: CLONE ir<%l>.3 = load ir<%p>.2
+; CHECK-NEXT: EMIT vp<[[VP23:%[0-9]+]]> = insertelement vp<[[VP20]]>, ir<%l>.3, ir<3>
+; CHECK-NEXT: Successor(s): pred.load.continue
+; CHECK-EMPTY:
+; CHECK-NEXT: pred.load.continue:
+; CHECK-NEXT: WIDEN-PHI vp<[[VP24:%[0-9]+]]> = phi [ vp<[[VP20]]>, pred.load.continue ], [ vp<[[VP23]]>, pred.load.if ]
+; CHECK-NEXT: EMIT vp<[[VP25:%[0-9]+]]> = extractelement vp<%active.lane.mask>.1, ir<0>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP25]]>
+; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue
+; CHECK-EMPTY:
+; CHECK-NEXT: pred.load.if:
+; CHECK-NEXT: CLONE ir<%p>.3 = getelementptr inbounds ir<%src>, vp<[[VP2]]>
+; CHECK-NEXT: CLONE ir<%l>.4 = load ir<%p>.3
+; CHECK-NEXT: EMIT vp<[[VP27:%[0-9]+]]> = insertelement ir<poison>, ir<%l>.4, ir<0>
+; CHECK-NEXT: Successor(s): pred.load.continue
+; CHECK-EMPTY:
+; CHECK-NEXT: pred.load.continue:
+; CHECK-NEXT: WIDEN-PHI vp<[[VP28:%[0-9]+]]> = phi [ ir<poison>, pred.load.continue ], [ vp<[[VP27]]>, pred.load.if ]
+; CHECK-NEXT: EMIT vp<[[VP29:%[0-9]+]]> = extractelement vp<%active.lane.mask>.1, ir<1>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP29]]>
+; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue
+; CHECK-EMPTY:
+; CHECK-NEXT: pred.load.if:
+; CHECK-NEXT: CLONE ir<%p>.4 = getelementptr inbounds ir<%src>, vp<[[VP6]]>
+; CHECK-NEXT: CLONE ir<%l>.5 = load ir<%p>.4
+; CHECK-NEXT: EMIT vp<[[VP31:%[0-9]+]]> = insertelement vp<[[VP28]]>, ir<%l>.5, ir<1>
+; CHECK-NEXT: Successor(s): pred.load.continue
+; CHECK-EMPTY:
+; CHECK-NEXT: pred.load.continue:
+; CHECK-NEXT: WIDEN-PHI vp<[[VP32:%[0-9]+]]> = phi [ vp<[[VP28]]>, pred.load.continue ], [ vp<[[VP31]]>, pred.load.if ]
+; CHECK-NEXT: EMIT vp<[[VP33:%[0-9]+]]> = extractelement vp<%active.lane.mask>.1, ir<2>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP33]]>
+; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue
+; CHECK-EMPTY:
+; CHECK-NEXT: pred.load.if:
+; CHECK-NEXT: CLONE ir<%p>.5 = getelementptr inbounds ir<%src>, vp<[[VP7]]>
+; CHECK-NEXT: CLONE ir<%l>.6 = load ir<%p>.5
+; CHECK-NEXT: EMIT vp<[[VP35:%[0-9]+]]> = insertelement vp<[[VP32]]>, ir<%l>.6, ir<2>
+; CHECK-NEXT: Successor(s): pred.load.continue
+; CHECK-EMPTY:
+; CHECK-NEXT: pred.load.continue:
+; CHECK-NEXT: WIDEN-PHI vp<[[VP36:%[0-9]+]]> = phi [ vp<[[VP32]]>, pred.load.continue ], [ vp<[[VP35]]>, pred.load.if ]
+; CHECK-NEXT: EMIT vp<[[VP37:%[0-9]+]]> = extractelement vp<%active.lane.mask>.1, ir<3>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP37]]>
+; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue
+; CHECK-EMPTY:
+; CHECK-NEXT: pred.load.if:
+; CHECK-NEXT: CLONE ir<%p>.6 = getelementptr inbounds ir<%src>, vp<[[VP8]]>
+; CHECK-NEXT: CLONE ir<%l>.7 = load ir<%p>.6
+; CHECK-NEXT: EMIT vp<[[VP39:%[0-9]+]]> = insertelement vp<[[VP36]]>, ir<%l>.7, ir<3>
+; CHECK-NEXT: Successor(s): pred.load.continue
+; CHECK-EMPTY:
+; CHECK-NEXT: pred.load.continue:
+; CHECK-NEXT: WIDEN-PHI vp<[[VP40:%[0-9]+]]> = phi [ vp<[[VP36]]>, pred.load.continue ], [ vp<[[VP39]]>, pred.load.if ]
+; CHECK-NEXT: WIDEN ir<%m> = mul vp<[[VP24]]>, ir<3>
+; CHECK-NEXT: WIDEN ir<%m>.1 = mul vp<[[VP40]]>, ir<3>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP9]]>
+; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
+; CHECK-EMPTY:
+; CHECK-NEXT: pred.store.if:
+; CHECK-NEXT: EMIT vp<[[VP42:%[0-9]+]]> = extractelement ir<%m>, ir<0>
+; CHECK-NEXT: CLONE store vp<[[VP42]]>, ir<%dst>
+; CHECK-NEXT: Successor(s): pred.store.continue
+; CHECK-EMPTY:
+; CHECK-NEXT: pred.store.continue:
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP13]]>
+; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
+; CHECK-EMPTY:
+; CHECK-NEXT: pred.store.if:
+; CHECK-NEXT: CLONE ir<%q> = getelementptr inbounds ir<%dst>, vp<[[VP3]]>
+; CHECK-NEXT: EMIT vp<[[VP44:%[0-9]+]]> = extractelement ir<%m>, ir<1>
+; CHECK-NEXT: CLONE store vp<[[VP44]]>, ir<%q>
+; CHECK-NEXT: Successor(s): pred.store.continue
+; CHECK-EMPTY:
+; CHECK-NEXT: pred.store.continue:
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP17]]>
+; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
+; CHECK-EMPTY:
+; CHECK-NEXT: pred.store.if:
+; CHECK-NEXT: CLONE ir<%q>.1 = getelementptr inbounds ir<%dst>, vp<[[VP4]]>
+; CHECK-NEXT: EMIT vp<[[VP46:%[0-9]+]]> = extractelement ir<%m>, ir<2>
+; CHECK-NEXT: CLONE store vp<[[VP46]]>, ir<%q>.1
+; CHECK-NEXT: Successor(s): pred.store.continue
+; CHECK-EMPTY:
+; CHECK-NEXT: pred.store.continue:
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP21]]>
+; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
+; CHECK-EMPTY:
+; CHECK-NEXT: pred.store.if:
+; CHECK-NEXT: CLONE ir<%q>.2 = getelementptr inbounds ir<%dst>, vp<[[VP5]]>
+; CHECK-NEXT: EMIT vp<[[VP48:%[0-9]+]]> = extractelement ir<%m>, ir<3>
+; CHECK-NEXT: CLONE store vp<[[VP48]]>, ir<%q>.2
+; CHECK-NEXT: Successor(s): pred.store.continue
+; CHECK-EMPTY:
+; CHECK-NEXT: pred.store.continue:
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP25]]>
+; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
+; CHECK-EMPTY:
+; CHECK-NEXT: pred.store.if:
+; CHECK-NEXT: CLONE ir<%q>.3 = getelementptr inbounds ir<%dst>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT vp<[[VP50:%[0-9]+]]> = extractelement ir<%m>.1, ir<0>
+; CHECK-NEXT: CLONE store vp<[[VP50]]>, ir<%q>.3
+; CHECK-NEXT: Successor(s): pred.store.continue
+; CHECK-EMPTY:
+; CHECK-NEXT: pred.store.continue:
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP29]]>
+; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
+; CHECK-EMPTY:
+; CHECK-NEXT: pred.store.if:
+; CHECK-NEXT: CLONE ir<%q>.4 = getelementptr inbounds ir<%dst>, vp<[[VP6]]>
+; CHECK-NEXT: EMIT vp<[[VP52:%[0-9]+]]> = extractelement ir<%m>.1, ir<1>
+; CHECK-NEXT: CLONE store vp<[[VP52]]>, ir<%q>.4
+; CHECK-NEXT: Successor(s): pred.store.continue
+; CHECK-EMPTY:
+; CHECK-NEXT: pred.store.continue:
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP33]]>
+; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
+; CHECK-EMPTY:
+; CHECK-NEXT: pred.store.if:
+; CHECK-NEXT: CLONE ir<%q>.5 = getelementptr inbounds ir<%dst>, vp<[[VP7]]>
+; CHECK-NEXT: EMIT vp<[[VP54:%[0-9]+]]> = extractelement ir<%m>.1, ir<2>
+; CHECK-NEXT: CLONE store vp<[[VP54]]>, ir<%q>.5
+; CHECK-NEXT: Successor(s): pred.store.continue
+; CHECK-EMPTY:
+; CHECK-NEXT: pred.store.continue:
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP37]]>
+; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
+; CHECK-EMPTY:
+; CHECK-NEXT: pred.store.if:
+; CHECK-NEXT: CLONE ir<%q>.6 = getelementptr inbounds ir<%dst>, vp<[[VP8]]>
+; CHECK-NEXT: EMIT vp<[[VP56:%[0-9]+]]> = extractelement ir<%m>.1, ir<3>
+; CHECK-NEXT: CLONE store vp<[[VP56]]>, ir<%q>.6
+; CHECK-NEXT: Successor(s): pred.store.continue
+; CHECK-EMPTY:
+; CHECK-NEXT: pred.store.continue:
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: Successor(s): ir-bb<exit>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %p = getelementptr inbounds i64, ptr %src, i64 %iv
+ %l = load i64, ptr %p, align 8
+ %m = mul i64 %l, 3
+ %q = getelementptr inbounds i64, ptr %dst, i64 %iv
+ store i64 %m, ptr %q, align 8
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 7
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Check that we don't crash on an unreasonably large TC.
+define i16 @tc_overflow(i32 %x) {
+; CHECK-LABEL: VPlan for loop in 'tc_overflow'
+; CHECK: VPlan 'Final VPlan for VF={4},UF={2}' {
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector.body
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: EMIT-SCALAR vp<%index> = phi [ ir<0>, vector.ph ], [ vp<%index.next>, vector.body ]
+; CHECK-NEXT: EMIT vp<%index.next> = add vp<%index>, ir<8>
+; CHECK-NEXT: EMIT vp<[[VP2:%[0-9]+]]> = icmp eq vp<%index.next>, ir<1180591620717411303432>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP2]]>
+; CHECK-NEXT: Successor(s): middle.block, vector.body
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP4:%[0-9]+]]> = trunc ir<%x> to i16
+; CHECK-NEXT: Successor(s): ir-bb<exit>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: IR %t.lcssa = phi i16 [ %t, %loop ] (extra operand: vp<[[VP4]]> from middle.block)
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i128 [ 0, %entry ], [ %iv.next, %loop ]
+ %red = phi i32 [ 0, %entry ], [ %add, %loop ]
+ %t = trunc i32 %red to i16
+ %add = add i32 %red, %x
+ %iv.next = add i128 %iv, 1
+ %ec = icmp eq i128 %iv, 1180591620717411303425
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret i16 %t
+}
>From a6023c5ccbf661612d65e5ea7d380841b74291fc Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Mon, 22 Jun 2026 18:18:42 +0100
Subject: [PATCH 2/2] [VPlan] Constant-fold ActiveLaneMask intrinsics
Constant-fold fixed-vector types via llvm::simplifyIntrinsic, making it
accessible from InstSimplifyFolder, and use it for folding
get.active.lane.mask in VPlan.
---
llvm/lib/Transforms/Vectorize/VPlan.h | 4 +
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 30 +++++
.../first-order-recurrence-fold-tail.ll | 3 +-
.../AArch64/fixed-wide-lane-mask.ll | 3 +-
.../LoopVectorize/AArch64/masked-call.ll | 56 +++------
.../LoopVectorize/AArch64/partial-reduce.ll | 9 +-
.../LoopVectorize/AArch64/store-costs-sve.ll | 3 +-
.../AArch64/sve-tail-folding-optsize.ll | 3 +-
.../LoopVectorize/AArch64/sve-vfabi.ll | 6 +-
.../LoopVectorize/AArch64/wide-alm-const.ll | 7 +-
.../AArch64/widen-call-attrs-masked.ll | 6 +-
.../foldable-scalable-active-lane-mask.ll | 2 -
.../VPlan/foldable-active-lane-mask.ll | 119 ++++++++----------
13 files changed, 115 insertions(+), 136 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 55741c2ac0e42d..6d2b8afc9bbb80 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -5035,6 +5035,10 @@ class VPlan {
return getScalarHeader()->getIRBasicBlock()->getDataLayout();
}
+ Function *getIRFunction() const {
+ return getScalarHeader()->getIRBasicBlock()->getParent();
+ }
+
void addVF(ElementCount VF) { VFs.insert(VF); }
void setVF(ElementCount VF) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index b68040911c1429..8aa7b54f144522 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -1355,6 +1355,36 @@ VPIRValue *vputils::tryToFoldLiveIns(VPSingleDefRecipe &R,
case Instruction::ExtractElement:
assert(!Ops[0]->getType()->isVectorTy() && "Live-ins should be scalar");
return Ops[0];
+ case VPInstruction::ActiveLaneMask:
+ case VPInstruction::WideActiveLaneMask: {
+ uint64_t Multiplier = 1;
+ if (Opcode == VPInstruction::WideActiveLaneMask) {
+ // Optimizing WideALM can only happen after the Plan is unrolled.
+ if (!Plan.isUnrolled())
+ return nullptr;
+ Multiplier = cast<ConstantInt>(Ops[2])->getZExtValue();
+ Ops.pop_back();
+ }
+
+ // We rely on the fact that different VPlans are created for the
+ // fixed-vector and scalable-vector cases.
+ ElementCount MaxVF =
+ *max_element(Plan.vectorFactors(), ElementCount::isKnownLT) *
+ Multiplier;
+
+ Type *I1Ty = IntegerType::getInt1Ty(Plan.getContext());
+ if (auto *C = dyn_cast_if_present<Constant>(Folder.FoldIntrinsic(
+ Intrinsic::get_active_lane_mask, Ops,
+ VectorType::get(I1Ty, MaxVF), {}, Plan.getIRFunction()))) {
+ // We cannot handle vector constants that are not all-true or all-false,
+ // because they would not be collapsable to a scalar constant, that
+ // would be necessary for live-in simplification.
+ if (C->isOneValue())
+ return ConstantInt::getTrue(I1Ty);
+ if (C->isNullValue())
+ return ConstantInt::getFalse(I1Ty);
+ }
+ }
}
return nullptr;
};
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/first-order-recurrence-fold-tail.ll b/llvm/test/Transforms/LoopVectorize/AArch64/first-order-recurrence-fold-tail.ll
index 380554dd22b647..ed4fcb31f9ddaf 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/first-order-recurrence-fold-tail.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/first-order-recurrence-fold-tail.ll
@@ -9,11 +9,10 @@ define i32 @test_phi_iterator_invalidation(ptr %A, ptr noalias %B) {
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
-; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 1002)
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE6:%.*]] ]
-; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[PRED_LOAD_CONTINUE6]] ]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <4 x i1> [ splat (i1 true), [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[PRED_LOAD_CONTINUE6]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_LOAD_CONTINUE6]] ]
; CHECK-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ <i16 poison, i16 poison, i16 poison, i16 0>, [[VECTOR_PH]] ], [ [[TMP24:%.*]], [[PRED_LOAD_CONTINUE6]] ]
; CHECK-NEXT: [[TMP0:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 1)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fixed-wide-lane-mask.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fixed-wide-lane-mask.ll
index b59fc73ebbf0f9..e8bb479f0f78c3 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/fixed-wide-lane-mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fixed-wide-lane-mask.ll
@@ -193,11 +193,10 @@ define void @fixed_known_tc(ptr noalias readonly %dst, ptr noalias readonly %src
; CHECK-UF1-NEXT: entry:
; CHECK-UF1-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK-UF1: vector.ph:
-; CHECK-UF1-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 15)
; CHECK-UF1-NEXT: br label [[VECTOR_BODY1:%.*]]
; CHECK-UF1: vector.body:
; CHECK-UF1-NEXT: [[IV:%.*]] = phi i64 [ 0, [[VECTOR_BODY]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY1]] ]
-; CHECK-UF1-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_BODY]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY1]] ]
+; CHECK-UF1-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <4 x i1> [ splat (i1 true), [[VECTOR_BODY]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY1]] ]
; CHECK-UF1-NEXT: [[X:%.*]] = load i32, ptr [[SRC]], align 4
; CHECK-UF1-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i64 0
; CHECK-UF1-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll b/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll
index f7ee11f778cc13..b851b7acf72f3d 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll
@@ -54,11 +54,10 @@ define void @test_widen(ptr noalias %a, ptr readnone %b) #4 {
; TFCOMMON: [[VECTOR_PH]]:
; TFCOMMON-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; TFCOMMON-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
-; TFCOMMON-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
; TFCOMMON-NEXT: br label %[[VECTOR_BODY:.*]]
; TFCOMMON: [[VECTOR_BODY]]:
; TFCOMMON-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TFCOMMON-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TFCOMMON-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ splat (i1 true), %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
; TFCOMMON-NEXT: [[TMP2:%.*]] = getelementptr i64, ptr [[B]], i64 [[INDEX]]
; TFCOMMON-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP2]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
; TFCOMMON-NEXT: [[TMP3:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
@@ -82,14 +81,11 @@ define void @test_widen(ptr noalias %a, ptr readnone %b) #4 {
; TFA_INTERLEAVE-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; TFA_INTERLEAVE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
; TFA_INTERLEAVE-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 1025)
-; TFA_INTERLEAVE-NEXT: [[TMP9:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
-; TFA_INTERLEAVE-NEXT: [[TMP13:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 2)
; TFA_INTERLEAVE-NEXT: br label %[[VECTOR_BODY:.*]]
; TFA_INTERLEAVE: [[VECTOR_BODY]]:
; TFA_INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[TMP9]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 2 x i1> [ [[TMP13]], %[[VECTOR_PH]] ], [ [[TMP12:%.*]], %[[VECTOR_BODY]] ]
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ splat (i1 true), %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 2 x i1> [ splat (i1 true), %[[VECTOR_PH]] ], [ [[TMP12:%.*]], %[[VECTOR_BODY]] ]
; TFA_INTERLEAVE-NEXT: [[TMP3:%.*]] = getelementptr i64, ptr [[B]], i64 [[INDEX]]
; TFA_INTERLEAVE-NEXT: [[TMP4:%.*]] = getelementptr i64, ptr [[TMP3]], i64 [[TMP1]]
; TFA_INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP3]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
@@ -185,11 +181,10 @@ define void @test_if_then(ptr noalias %a, ptr readnone %b) #4 {
; TFCOMMON: [[VECTOR_PH]]:
; TFCOMMON-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; TFCOMMON-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
-; TFCOMMON-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
; TFCOMMON-NEXT: br label %[[VECTOR_BODY:.*]]
; TFCOMMON: [[VECTOR_BODY]]:
; TFCOMMON-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TFCOMMON-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TFCOMMON-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ splat (i1 true), %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
; TFCOMMON-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
; TFCOMMON-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP2]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
; TFCOMMON-NEXT: [[TMP3:%.*]] = icmp ugt <vscale x 2 x i64> [[WIDE_MASKED_LOAD]], splat (i64 50)
@@ -216,14 +211,11 @@ define void @test_if_then(ptr noalias %a, ptr readnone %b) #4 {
; TFA_INTERLEAVE-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; TFA_INTERLEAVE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
; TFA_INTERLEAVE-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 1025)
-; TFA_INTERLEAVE-NEXT: [[TMP13:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
-; TFA_INTERLEAVE-NEXT: [[TMP17:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 2)
; TFA_INTERLEAVE-NEXT: br label %[[VECTOR_BODY:.*]]
; TFA_INTERLEAVE: [[VECTOR_BODY]]:
; TFA_INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[TMP13]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 2 x i1> [ [[TMP17]], %[[VECTOR_PH]] ], [ [[TMP16:%.*]], %[[VECTOR_BODY]] ]
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ splat (i1 true), %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 2 x i1> [ splat (i1 true), %[[VECTOR_PH]] ], [ [[TMP16:%.*]], %[[VECTOR_BODY]] ]
; TFA_INTERLEAVE-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
; TFA_INTERLEAVE-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[TMP3]], i64 [[TMP1]]
; TFA_INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP3]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
@@ -341,11 +333,10 @@ define void @test_widen_if_then_else(ptr noalias %a, ptr readnone %b) #4 {
; TFCOMMON: [[VECTOR_PH]]:
; TFCOMMON-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; TFCOMMON-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
-; TFCOMMON-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
; TFCOMMON-NEXT: br label %[[VECTOR_BODY:.*]]
; TFCOMMON: [[VECTOR_BODY]]:
; TFCOMMON-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TFCOMMON-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TFCOMMON-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ splat (i1 true), %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
; TFCOMMON-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
; TFCOMMON-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP2]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
; TFCOMMON-NEXT: [[TMP3:%.*]] = icmp ugt <vscale x 2 x i64> [[WIDE_MASKED_LOAD]], splat (i64 50)
@@ -375,14 +366,11 @@ define void @test_widen_if_then_else(ptr noalias %a, ptr readnone %b) #4 {
; TFA_INTERLEAVE-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; TFA_INTERLEAVE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
; TFA_INTERLEAVE-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 1025)
-; TFA_INTERLEAVE-NEXT: [[TMP19:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
-; TFA_INTERLEAVE-NEXT: [[TMP23:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 2)
; TFA_INTERLEAVE-NEXT: br label %[[VECTOR_BODY:.*]]
; TFA_INTERLEAVE: [[VECTOR_BODY]]:
; TFA_INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[TMP19]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 2 x i1> [ [[TMP23]], %[[VECTOR_PH]] ], [ [[TMP22:%.*]], %[[VECTOR_BODY]] ]
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ splat (i1 true), %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 2 x i1> [ splat (i1 true), %[[VECTOR_PH]] ], [ [[TMP22:%.*]], %[[VECTOR_BODY]] ]
; TFA_INTERLEAVE-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
; TFA_INTERLEAVE-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[TMP3]], i64 [[TMP1]]
; TFA_INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP3]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
@@ -628,11 +616,10 @@ define void @test_widen_optmask(ptr noalias %a, ptr readnone %b) #4 {
; TFALWAYS: [[VECTOR_PH]]:
; TFALWAYS-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; TFALWAYS-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
-; TFALWAYS-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
; TFALWAYS-NEXT: br label %[[VECTOR_BODY:.*]]
; TFALWAYS: [[VECTOR_BODY]]:
; TFALWAYS-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TFALWAYS-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TFALWAYS-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ splat (i1 true), %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
; TFALWAYS-NEXT: [[TMP2:%.*]] = getelementptr i64, ptr [[B]], i64 [[INDEX]]
; TFALWAYS-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP2]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
; TFALWAYS-NEXT: [[TMP3:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
@@ -655,11 +642,10 @@ define void @test_widen_optmask(ptr noalias %a, ptr readnone %b) #4 {
; TFFALLBACK: [[VECTOR_PH]]:
; TFFALLBACK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; TFFALLBACK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
-; TFFALLBACK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
; TFFALLBACK-NEXT: br label %[[VECTOR_BODY:.*]]
; TFFALLBACK: [[VECTOR_BODY]]:
; TFFALLBACK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TFFALLBACK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TFFALLBACK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ splat (i1 true), %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
; TFFALLBACK-NEXT: [[TMP2:%.*]] = getelementptr i64, ptr [[B]], i64 [[INDEX]]
; TFFALLBACK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP2]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
; TFFALLBACK-NEXT: [[TMP3:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
@@ -683,14 +669,11 @@ define void @test_widen_optmask(ptr noalias %a, ptr readnone %b) #4 {
; TFA_INTERLEAVE-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; TFA_INTERLEAVE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
; TFA_INTERLEAVE-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 1025)
-; TFA_INTERLEAVE-NEXT: [[TMP9:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
-; TFA_INTERLEAVE-NEXT: [[TMP13:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 2)
; TFA_INTERLEAVE-NEXT: br label %[[VECTOR_BODY:.*]]
; TFA_INTERLEAVE: [[VECTOR_BODY]]:
; TFA_INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[TMP9]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 2 x i1> [ [[TMP13]], %[[VECTOR_PH]] ], [ [[TMP12:%.*]], %[[VECTOR_BODY]] ]
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ splat (i1 true), %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 2 x i1> [ splat (i1 true), %[[VECTOR_PH]] ], [ [[TMP12:%.*]], %[[VECTOR_BODY]] ]
; TFA_INTERLEAVE-NEXT: [[TMP3:%.*]] = getelementptr i64, ptr [[B]], i64 [[INDEX]]
; TFA_INTERLEAVE-NEXT: [[TMP4:%.*]] = getelementptr i64, ptr [[TMP3]], i64 [[TMP1]]
; TFA_INTERLEAVE-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP3]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
@@ -790,13 +773,12 @@ define double @test_widen_fmuladd_and_call(ptr noalias %a, ptr readnone %b, doub
; TFALWAYS: [[VECTOR_PH]]:
; TFALWAYS-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; TFALWAYS-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
-; TFALWAYS-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
; TFALWAYS-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x double> poison, double [[M]], i64 0
; TFALWAYS-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x double> [[BROADCAST_SPLATINSERT]], <vscale x 2 x double> poison, <vscale x 2 x i32> zeroinitializer
; TFALWAYS-NEXT: br label %[[VECTOR_BODY:.*]]
; TFALWAYS: [[VECTOR_BODY]]:
; TFALWAYS-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TFALWAYS-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TFALWAYS-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ splat (i1 true), %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
; TFALWAYS-NEXT: [[VEC_PHI:%.*]] = phi double [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
; TFALWAYS-NEXT: [[TMP2:%.*]] = getelementptr double, ptr [[B]], i64 [[INDEX]]
; TFALWAYS-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP2]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x double> poison)
@@ -824,13 +806,12 @@ define double @test_widen_fmuladd_and_call(ptr noalias %a, ptr readnone %b, doub
; TFFALLBACK: [[VECTOR_PH]]:
; TFFALLBACK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; TFFALLBACK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
-; TFFALLBACK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
; TFFALLBACK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x double> poison, double [[M]], i64 0
; TFFALLBACK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x double> [[BROADCAST_SPLATINSERT]], <vscale x 2 x double> poison, <vscale x 2 x i32> zeroinitializer
; TFFALLBACK-NEXT: br label %[[VECTOR_BODY:.*]]
; TFFALLBACK: [[VECTOR_BODY]]:
; TFFALLBACK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TFFALLBACK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TFFALLBACK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ splat (i1 true), %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
; TFFALLBACK-NEXT: [[VEC_PHI:%.*]] = phi double [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
; TFFALLBACK-NEXT: [[TMP2:%.*]] = getelementptr double, ptr [[B]], i64 [[INDEX]]
; TFFALLBACK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP2]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x double> poison)
@@ -859,16 +840,13 @@ define double @test_widen_fmuladd_and_call(ptr noalias %a, ptr readnone %b, doub
; TFA_INTERLEAVE-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; TFA_INTERLEAVE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
; TFA_INTERLEAVE-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 1025)
-; TFA_INTERLEAVE-NEXT: [[TMP17:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
-; TFA_INTERLEAVE-NEXT: [[TMP21:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 2)
; TFA_INTERLEAVE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x double> poison, double [[M]], i64 0
; TFA_INTERLEAVE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x double> [[BROADCAST_SPLATINSERT]], <vscale x 2 x double> poison, <vscale x 2 x i32> zeroinitializer
; TFA_INTERLEAVE-NEXT: br label %[[VECTOR_BODY:.*]]
; TFA_INTERLEAVE: [[VECTOR_BODY]]:
; TFA_INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[TMP17]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 2 x i1> [ [[TMP21]], %[[VECTOR_PH]] ], [ [[TMP20:%.*]], %[[VECTOR_BODY]] ]
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ splat (i1 true), %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 2 x i1> [ splat (i1 true), %[[VECTOR_PH]] ], [ [[TMP20:%.*]], %[[VECTOR_BODY]] ]
; TFA_INTERLEAVE-NEXT: [[VEC_PHI:%.*]] = phi double [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP16:%.*]], %[[VECTOR_BODY]] ]
; TFA_INTERLEAVE-NEXT: [[TMP3:%.*]] = getelementptr double, ptr [[B]], i64 [[INDEX]]
; TFA_INTERLEAVE-NEXT: [[TMP4:%.*]] = getelementptr double, ptr [[TMP3]], i64 [[TMP1]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce.ll
index 56cea9189d8f58..bd741511de8cd0 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce.ll
@@ -520,11 +520,10 @@ define i32 @zext_add_reduc_i8_i32_predicated(ptr %a) #0 {
; CHECK-INTERLEAVE1: vector.ph:
; CHECK-INTERLEAVE1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-INTERLEAVE1-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
-; CHECK-INTERLEAVE1-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 1025)
; CHECK-INTERLEAVE1-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK-INTERLEAVE1: vector.body:
; CHECK-INTERLEAVE1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVE1-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 16 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 16 x i1> [ splat (i1 true), [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-INTERLEAVE1-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP10:%.*]], [[VECTOR_BODY]] ]
; CHECK-INTERLEAVE1-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
; CHECK-INTERLEAVE1-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP6]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i8> poison)
@@ -549,11 +548,10 @@ define i32 @zext_add_reduc_i8_i32_predicated(ptr %a) #0 {
; CHECK-INTERLEAVED: vector.ph:
; CHECK-INTERLEAVED-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-INTERLEAVED-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
-; CHECK-INTERLEAVED-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 1025)
; CHECK-INTERLEAVED-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK-INTERLEAVED: vector.body:
; CHECK-INTERLEAVED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 16 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 16 x i1> [ splat (i1 true), [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP10:%.*]], [[VECTOR_BODY]] ]
; CHECK-INTERLEAVED-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP6]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i8> poison)
@@ -578,11 +576,10 @@ define i32 @zext_add_reduc_i8_i32_predicated(ptr %a) #0 {
; CHECK-MAXBW: vector.ph:
; CHECK-MAXBW-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-MAXBW-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
-; CHECK-MAXBW-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 1025)
; CHECK-MAXBW-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK-MAXBW: vector.body:
; CHECK-MAXBW-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-MAXBW-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 16 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 16 x i1> [ splat (i1 true), [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-MAXBW-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ]
; CHECK-MAXBW-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
; CHECK-MAXBW-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP6]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i8> poison)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/store-costs-sve.ll b/llvm/test/Transforms/LoopVectorize/AArch64/store-costs-sve.ll
index 7ba5eb17bdc27c..d142f1421799f3 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/store-costs-sve.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/store-costs-sve.ll
@@ -202,11 +202,10 @@ define void @trunc_store(ptr %dst, ptr %src, i16 %x) #1 {
; PRED-NEXT: [[TMP3:%.*]] = trunc <vscale x 16 x i64> [[BROADCAST_SPLAT3]] to <vscale x 16 x i8>
; PRED-NEXT: [[TMP2:%.*]] = trunc <vscale x 16 x i16> [[BROADCAST_SPLAT]] to <vscale x 16 x i8>
; PRED-NEXT: [[TMP5:%.*]] = and <vscale x 16 x i8> [[TMP3]], [[TMP2]]
-; PRED-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 1000)
; PRED-NEXT: br label [[VECTOR_BODY:%.*]]
; PRED: vector.body:
; PRED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; PRED-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 16 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
+; PRED-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 16 x i1> [ splat (i1 true), [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
; PRED-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[DST]], i64 [[INDEX]]
; PRED-NEXT: call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[TMP5]], ptr align 1 [[TMP6]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]]), !alias.scope [[META6:![0-9]+]], !noalias [[META3]]
; PRED-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-optsize.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-optsize.ll
index 0f9677d5e4e7d6..8d9f1b34c747f1 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-optsize.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-optsize.ll
@@ -10,11 +10,10 @@ define void @trip1025_i64(ptr noalias nocapture noundef %dst, ptr noalias nocapt
; CHECK: vector.ph:
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
-; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ splat (i1 true), [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[SRC:%.*]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP8]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
; CHECK-NEXT: [[TMP10:%.*]] = shl nsw <vscale x 2 x i64> [[WIDE_MASKED_LOAD]], splat (i64 1)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-vfabi.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-vfabi.ll
index 2dcd125fe6c7a6..082cb2b786c346 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-vfabi.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-vfabi.ll
@@ -11,11 +11,10 @@ define void @test_big_little_params(ptr readonly %a, ptr readonly %b, ptr noalia
; CHECK: vector.ph:
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
-; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 1025)
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ splat (i1 true), [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i32, ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 4 [[TMP2]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison)
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
@@ -61,11 +60,10 @@ define void @test_little_big_params(ptr readonly %a, ptr readonly %b, ptr noalia
; CHECK: vector.ph:
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
-; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ splat (i1 true), [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr float, ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x float> @llvm.masked.load.nxv2f32.p0(ptr align 4 [[TMP2]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x float> poison)
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr double, ptr [[B]], i64 [[INDEX]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/wide-alm-const.ll b/llvm/test/Transforms/LoopVectorize/AArch64/wide-alm-const.ll
index 7243a4b56358d4..fd61f56cbc01c6 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/wide-alm-const.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/wide-alm-const.ll
@@ -12,14 +12,11 @@ define void @scalable_wide_active_lane_mask_double(ptr noalias %dst, ptr readonl
; CHECK-UF4: [[FOR_BODY_PREHEADER]]:
; CHECK-UF4-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK-UF4: [[VECTOR_PH]]:
-; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 0, i64 9)
-; CHECK-UF4-NEXT: [[EXTRACT_ENTRY_ALM_PART:%.*]] = call <4 x i1> @llvm.vector.extract.v4i1.v8i1(<8 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
-; CHECK-UF4-NEXT: [[EXTRACT_ENTRY_ALM_PART1:%.*]] = call <4 x i1> @llvm.vector.extract.v4i1.v8i1(<8 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 4)
; CHECK-UF4-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-UF4: [[VECTOR_BODY]]:
; CHECK-UF4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <4 x i1> [ [[EXTRACT_ENTRY_ALM_PART]], %[[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <4 x i1> [ [[EXTRACT_ENTRY_ALM_PART1]], %[[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <4 x i1> [ splat (i1 true), %[[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-UF4-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <4 x i1> [ splat (i1 true), %[[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART4:%.*]], %[[VECTOR_BODY]] ]
; CHECK-UF4-NEXT: [[TMP0:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[INDEX]]
; CHECK-UF4-NEXT: [[TMP1:%.*]] = getelementptr inbounds double, ptr [[TMP0]], i64 4
; CHECK-UF4-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0(ptr align 8 [[TMP0]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x double> poison)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-attrs-masked.ll b/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-attrs-masked.ll
index 24d85bfdea0b50..10feb3e47497d5 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-attrs-masked.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-attrs-masked.ll
@@ -18,11 +18,10 @@ define void @masked_range_arg_and_ret_propagated(ptr noalias %a, ptr readnone %b
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
-; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ splat (i1 true), %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i64, ptr [[B]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP2]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
; CHECK-NEXT: [[TMP3:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
@@ -65,11 +64,10 @@ define void @masked_noundef_dereferenceable_stripped(ptr noalias %a, ptr readnon
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
-; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ splat (i1 true), %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i64, ptr [[B]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP2]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
; CHECK-NEXT: [[TMP3:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/foldable-scalable-active-lane-mask.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/foldable-scalable-active-lane-mask.ll
index a36b34dba91134..635685337e0513 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/foldable-scalable-active-lane-mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/foldable-scalable-active-lane-mask.ll
@@ -3,8 +3,6 @@
define void @wide_alm(ptr %dst, ptr noalias %src) vscale_range(1, 2) {
; CHECK-LABEL: VPlan for loop in 'wide_alm'
-; CHECK: EMIT vp<%active.lane.mask.entry> = wide active lane mask ir<0>, ir<15>, ir<1>
-; CHECK: EMIT vp<%extract.entry.alm.part> = extract-vector-for-part vp<%active.lane.mask.entry>, ir<0>
; CHECK: EMIT vp<%active.lane.mask.next> = wide active lane mask vp<%index.next>, ir<15>, ir<1>
; CHECK: EMIT vp<%extract.next.alm.part> = extract-vector-for-part vp<%active.lane.mask.next>, ir<0>
;
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/foldable-active-lane-mask.ll b/llvm/test/Transforms/LoopVectorize/VPlan/foldable-active-lane-mask.ll
index f2434220b97819..c317916a5051c6 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/foldable-active-lane-mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/foldable-active-lane-mask.ll
@@ -16,187 +16,170 @@ define void @unfoldable(ptr %src, ptr noalias %dst) {
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: vp<[[VP2:%[0-9]+]]> = SCALAR-STEPS ir<0>, ir<1>, ir<4>, ir<4>
-; CHECK-NEXT: EMIT vp<%active.lane.mask> = active lane mask ir<0>, ir<7>
-; CHECK-NEXT: EMIT vp<%active.lane.mask>.1 = active lane mask vp<[[VP2]]>, ir<7>
+; CHECK-NEXT: EMIT vp<%active.lane.mask> = active lane mask vp<[[VP2]]>, ir<7>
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = SCALAR-STEPS ir<0>, ir<1>, ir<4>, ir<1>
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS ir<0>, ir<1>, ir<4>, ir<2>
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS ir<0>, ir<1>, ir<4>, ir<3>
; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS ir<0>, ir<1>, ir<4>, ir<5>
; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = SCALAR-STEPS ir<0>, ir<1>, ir<4>, ir<6>
; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = SCALAR-STEPS ir<0>, ir<1>, ir<4>, ir<7>
-; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = extractelement vp<%active.lane.mask>, ir<0>
-; CHECK-NEXT: EMIT branch-on-cond vp<[[VP9]]>
-; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue
+; CHECK-NEXT: Successor(s): pred.load.if
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.if:
; CHECK-NEXT: CLONE ir<%l> = load ir<%src>
-; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = insertelement ir<poison>, ir<%l>, ir<0>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = insertelement ir<poison>, ir<%l>, ir<0>
; CHECK-NEXT: Successor(s): pred.load.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.continue:
-; CHECK-NEXT: WIDEN-PHI vp<[[VP12:%[0-9]+]]> = phi [ ir<poison>, vector.body ], [ vp<[[VP11]]>, pred.load.if ]
-; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = extractelement vp<%active.lane.mask>, ir<1>
-; CHECK-NEXT: EMIT branch-on-cond vp<[[VP13]]>
-; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue
+; CHECK-NEXT: Successor(s): pred.load.if
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.if:
; CHECK-NEXT: CLONE ir<%p> = getelementptr inbounds ir<%src>, vp<[[VP3]]>
; CHECK-NEXT: CLONE ir<%l>.1 = load ir<%p>
-; CHECK-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = insertelement vp<[[VP12]]>, ir<%l>.1, ir<1>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = insertelement vp<[[VP9]]>, ir<%l>.1, ir<1>
; CHECK-NEXT: Successor(s): pred.load.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.continue:
-; CHECK-NEXT: WIDEN-PHI vp<[[VP16:%[0-9]+]]> = phi [ vp<[[VP12]]>, pred.load.continue ], [ vp<[[VP15]]>, pred.load.if ]
-; CHECK-NEXT: EMIT vp<[[VP17:%[0-9]+]]> = extractelement vp<%active.lane.mask>, ir<2>
-; CHECK-NEXT: EMIT branch-on-cond vp<[[VP17]]>
-; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue
+; CHECK-NEXT: Successor(s): pred.load.if
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.if:
; CHECK-NEXT: CLONE ir<%p>.1 = getelementptr inbounds ir<%src>, vp<[[VP4]]>
; CHECK-NEXT: CLONE ir<%l>.2 = load ir<%p>.1
-; CHECK-NEXT: EMIT vp<[[VP19:%[0-9]+]]> = insertelement vp<[[VP16]]>, ir<%l>.2, ir<2>
+; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = insertelement vp<[[VP10]]>, ir<%l>.2, ir<2>
; CHECK-NEXT: Successor(s): pred.load.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.continue:
-; CHECK-NEXT: WIDEN-PHI vp<[[VP20:%[0-9]+]]> = phi [ vp<[[VP16]]>, pred.load.continue ], [ vp<[[VP19]]>, pred.load.if ]
-; CHECK-NEXT: EMIT vp<[[VP21:%[0-9]+]]> = extractelement vp<%active.lane.mask>, ir<3>
-; CHECK-NEXT: EMIT branch-on-cond vp<[[VP21]]>
-; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue
+; CHECK-NEXT: Successor(s): pred.load.if
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.if:
; CHECK-NEXT: CLONE ir<%p>.2 = getelementptr inbounds ir<%src>, vp<[[VP5]]>
; CHECK-NEXT: CLONE ir<%l>.3 = load ir<%p>.2
-; CHECK-NEXT: EMIT vp<[[VP23:%[0-9]+]]> = insertelement vp<[[VP20]]>, ir<%l>.3, ir<3>
+; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = insertelement vp<[[VP11]]>, ir<%l>.3, ir<3>
; CHECK-NEXT: Successor(s): pred.load.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.continue:
-; CHECK-NEXT: WIDEN-PHI vp<[[VP24:%[0-9]+]]> = phi [ vp<[[VP20]]>, pred.load.continue ], [ vp<[[VP23]]>, pred.load.if ]
-; CHECK-NEXT: EMIT vp<[[VP25:%[0-9]+]]> = extractelement vp<%active.lane.mask>.1, ir<0>
-; CHECK-NEXT: EMIT branch-on-cond vp<[[VP25]]>
+; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = extractelement vp<%active.lane.mask>, ir<0>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP13]]>
; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.if:
; CHECK-NEXT: CLONE ir<%p>.3 = getelementptr inbounds ir<%src>, vp<[[VP2]]>
; CHECK-NEXT: CLONE ir<%l>.4 = load ir<%p>.3
-; CHECK-NEXT: EMIT vp<[[VP27:%[0-9]+]]> = insertelement ir<poison>, ir<%l>.4, ir<0>
+; CHECK-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = insertelement ir<poison>, ir<%l>.4, ir<0>
; CHECK-NEXT: Successor(s): pred.load.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.continue:
-; CHECK-NEXT: WIDEN-PHI vp<[[VP28:%[0-9]+]]> = phi [ ir<poison>, pred.load.continue ], [ vp<[[VP27]]>, pred.load.if ]
-; CHECK-NEXT: EMIT vp<[[VP29:%[0-9]+]]> = extractelement vp<%active.lane.mask>.1, ir<1>
-; CHECK-NEXT: EMIT branch-on-cond vp<[[VP29]]>
+; CHECK-NEXT: WIDEN-PHI vp<[[VP16:%[0-9]+]]> = phi [ ir<poison>, pred.load.continue ], [ vp<[[VP15]]>, pred.load.if ]
+; CHECK-NEXT: EMIT vp<[[VP17:%[0-9]+]]> = extractelement vp<%active.lane.mask>, ir<1>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP17]]>
; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.if:
; CHECK-NEXT: CLONE ir<%p>.4 = getelementptr inbounds ir<%src>, vp<[[VP6]]>
; CHECK-NEXT: CLONE ir<%l>.5 = load ir<%p>.4
-; CHECK-NEXT: EMIT vp<[[VP31:%[0-9]+]]> = insertelement vp<[[VP28]]>, ir<%l>.5, ir<1>
+; CHECK-NEXT: EMIT vp<[[VP19:%[0-9]+]]> = insertelement vp<[[VP16]]>, ir<%l>.5, ir<1>
; CHECK-NEXT: Successor(s): pred.load.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.continue:
-; CHECK-NEXT: WIDEN-PHI vp<[[VP32:%[0-9]+]]> = phi [ vp<[[VP28]]>, pred.load.continue ], [ vp<[[VP31]]>, pred.load.if ]
-; CHECK-NEXT: EMIT vp<[[VP33:%[0-9]+]]> = extractelement vp<%active.lane.mask>.1, ir<2>
-; CHECK-NEXT: EMIT branch-on-cond vp<[[VP33]]>
+; CHECK-NEXT: WIDEN-PHI vp<[[VP20:%[0-9]+]]> = phi [ vp<[[VP16]]>, pred.load.continue ], [ vp<[[VP19]]>, pred.load.if ]
+; CHECK-NEXT: EMIT vp<[[VP21:%[0-9]+]]> = extractelement vp<%active.lane.mask>, ir<2>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP21]]>
; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.if:
; CHECK-NEXT: CLONE ir<%p>.5 = getelementptr inbounds ir<%src>, vp<[[VP7]]>
; CHECK-NEXT: CLONE ir<%l>.6 = load ir<%p>.5
-; CHECK-NEXT: EMIT vp<[[VP35:%[0-9]+]]> = insertelement vp<[[VP32]]>, ir<%l>.6, ir<2>
+; CHECK-NEXT: EMIT vp<[[VP23:%[0-9]+]]> = insertelement vp<[[VP20]]>, ir<%l>.6, ir<2>
; CHECK-NEXT: Successor(s): pred.load.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.continue:
-; CHECK-NEXT: WIDEN-PHI vp<[[VP36:%[0-9]+]]> = phi [ vp<[[VP32]]>, pred.load.continue ], [ vp<[[VP35]]>, pred.load.if ]
-; CHECK-NEXT: EMIT vp<[[VP37:%[0-9]+]]> = extractelement vp<%active.lane.mask>.1, ir<3>
-; CHECK-NEXT: EMIT branch-on-cond vp<[[VP37]]>
+; CHECK-NEXT: WIDEN-PHI vp<[[VP24:%[0-9]+]]> = phi [ vp<[[VP20]]>, pred.load.continue ], [ vp<[[VP23]]>, pred.load.if ]
+; CHECK-NEXT: EMIT vp<[[VP25:%[0-9]+]]> = extractelement vp<%active.lane.mask>, ir<3>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP25]]>
; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.if:
; CHECK-NEXT: CLONE ir<%p>.6 = getelementptr inbounds ir<%src>, vp<[[VP8]]>
; CHECK-NEXT: CLONE ir<%l>.7 = load ir<%p>.6
-; CHECK-NEXT: EMIT vp<[[VP39:%[0-9]+]]> = insertelement vp<[[VP36]]>, ir<%l>.7, ir<3>
+; CHECK-NEXT: EMIT vp<[[VP27:%[0-9]+]]> = insertelement vp<[[VP24]]>, ir<%l>.7, ir<3>
; CHECK-NEXT: Successor(s): pred.load.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.continue:
-; CHECK-NEXT: WIDEN-PHI vp<[[VP40:%[0-9]+]]> = phi [ vp<[[VP36]]>, pred.load.continue ], [ vp<[[VP39]]>, pred.load.if ]
-; CHECK-NEXT: WIDEN ir<%m> = mul vp<[[VP24]]>, ir<3>
-; CHECK-NEXT: WIDEN ir<%m>.1 = mul vp<[[VP40]]>, ir<3>
-; CHECK-NEXT: EMIT branch-on-cond vp<[[VP9]]>
-; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
+; CHECK-NEXT: WIDEN-PHI vp<[[VP28:%[0-9]+]]> = phi [ vp<[[VP24]]>, pred.load.continue ], [ vp<[[VP27]]>, pred.load.if ]
+; CHECK-NEXT: WIDEN ir<%m> = mul vp<[[VP12]]>, ir<3>
+; CHECK-NEXT: WIDEN ir<%m>.1 = mul vp<[[VP28]]>, ir<3>
+; CHECK-NEXT: Successor(s): pred.store.if
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
-; CHECK-NEXT: EMIT vp<[[VP42:%[0-9]+]]> = extractelement ir<%m>, ir<0>
-; CHECK-NEXT: CLONE store vp<[[VP42]]>, ir<%dst>
+; CHECK-NEXT: EMIT vp<[[VP29:%[0-9]+]]> = extractelement ir<%m>, ir<0>
+; CHECK-NEXT: CLONE store vp<[[VP29]]>, ir<%dst>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.continue:
-; CHECK-NEXT: EMIT branch-on-cond vp<[[VP13]]>
-; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
+; CHECK-NEXT: Successor(s): pred.store.if
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
; CHECK-NEXT: CLONE ir<%q> = getelementptr inbounds ir<%dst>, vp<[[VP3]]>
-; CHECK-NEXT: EMIT vp<[[VP44:%[0-9]+]]> = extractelement ir<%m>, ir<1>
-; CHECK-NEXT: CLONE store vp<[[VP44]]>, ir<%q>
+; CHECK-NEXT: EMIT vp<[[VP30:%[0-9]+]]> = extractelement ir<%m>, ir<1>
+; CHECK-NEXT: CLONE store vp<[[VP30]]>, ir<%q>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.continue:
-; CHECK-NEXT: EMIT branch-on-cond vp<[[VP17]]>
-; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
+; CHECK-NEXT: Successor(s): pred.store.if
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
; CHECK-NEXT: CLONE ir<%q>.1 = getelementptr inbounds ir<%dst>, vp<[[VP4]]>
-; CHECK-NEXT: EMIT vp<[[VP46:%[0-9]+]]> = extractelement ir<%m>, ir<2>
-; CHECK-NEXT: CLONE store vp<[[VP46]]>, ir<%q>.1
+; CHECK-NEXT: EMIT vp<[[VP31:%[0-9]+]]> = extractelement ir<%m>, ir<2>
+; CHECK-NEXT: CLONE store vp<[[VP31]]>, ir<%q>.1
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.continue:
-; CHECK-NEXT: EMIT branch-on-cond vp<[[VP21]]>
-; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
+; CHECK-NEXT: Successor(s): pred.store.if
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
; CHECK-NEXT: CLONE ir<%q>.2 = getelementptr inbounds ir<%dst>, vp<[[VP5]]>
-; CHECK-NEXT: EMIT vp<[[VP48:%[0-9]+]]> = extractelement ir<%m>, ir<3>
-; CHECK-NEXT: CLONE store vp<[[VP48]]>, ir<%q>.2
+; CHECK-NEXT: EMIT vp<[[VP32:%[0-9]+]]> = extractelement ir<%m>, ir<3>
+; CHECK-NEXT: CLONE store vp<[[VP32]]>, ir<%q>.2
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.continue:
-; CHECK-NEXT: EMIT branch-on-cond vp<[[VP25]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP13]]>
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
; CHECK-NEXT: CLONE ir<%q>.3 = getelementptr inbounds ir<%dst>, vp<[[VP2]]>
-; CHECK-NEXT: EMIT vp<[[VP50:%[0-9]+]]> = extractelement ir<%m>.1, ir<0>
-; CHECK-NEXT: CLONE store vp<[[VP50]]>, ir<%q>.3
+; CHECK-NEXT: EMIT vp<[[VP34:%[0-9]+]]> = extractelement ir<%m>.1, ir<0>
+; CHECK-NEXT: CLONE store vp<[[VP34]]>, ir<%q>.3
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.continue:
-; CHECK-NEXT: EMIT branch-on-cond vp<[[VP29]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP17]]>
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
; CHECK-NEXT: CLONE ir<%q>.4 = getelementptr inbounds ir<%dst>, vp<[[VP6]]>
-; CHECK-NEXT: EMIT vp<[[VP52:%[0-9]+]]> = extractelement ir<%m>.1, ir<1>
-; CHECK-NEXT: CLONE store vp<[[VP52]]>, ir<%q>.4
+; CHECK-NEXT: EMIT vp<[[VP36:%[0-9]+]]> = extractelement ir<%m>.1, ir<1>
+; CHECK-NEXT: CLONE store vp<[[VP36]]>, ir<%q>.4
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.continue:
-; CHECK-NEXT: EMIT branch-on-cond vp<[[VP33]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP21]]>
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
; CHECK-NEXT: CLONE ir<%q>.5 = getelementptr inbounds ir<%dst>, vp<[[VP7]]>
-; CHECK-NEXT: EMIT vp<[[VP54:%[0-9]+]]> = extractelement ir<%m>.1, ir<2>
-; CHECK-NEXT: CLONE store vp<[[VP54]]>, ir<%q>.5
+; CHECK-NEXT: EMIT vp<[[VP38:%[0-9]+]]> = extractelement ir<%m>.1, ir<2>
+; CHECK-NEXT: CLONE store vp<[[VP38]]>, ir<%q>.5
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.continue:
-; CHECK-NEXT: EMIT branch-on-cond vp<[[VP37]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP25]]>
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
; CHECK-NEXT: CLONE ir<%q>.6 = getelementptr inbounds ir<%dst>, vp<[[VP8]]>
-; CHECK-NEXT: EMIT vp<[[VP56:%[0-9]+]]> = extractelement ir<%m>.1, ir<3>
-; CHECK-NEXT: CLONE store vp<[[VP56]]>, ir<%q>.6
+; CHECK-NEXT: EMIT vp<[[VP40:%[0-9]+]]> = extractelement ir<%m>.1, ir<3>
+; CHECK-NEXT: CLONE store vp<[[VP40]]>, ir<%q>.6
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.continue:
More information about the llvm-commits
mailing list