[llvm] [AArch64][SVE] Allow scalable factor-3 interleaved accesses (PR #200424)

Thu Jul 30 09:21:38 PDT 2026

https://github.com/Harry-Ramsey updated https://github.com/llvm/llvm-project/pull/200424

>From 542368692247237924e8ee7d1099c23bea02c6ae Mon Sep 17 00:00:00 2001
From: Harry Ramsey <harry.ramsey at arm.com>
Date: Tue, 21 Jul 2026 12:38:15 +0000
Subject: [PATCH 1/2] [AArch64][ISel] Enable factor-3 scalable interleave
 operations

Allow scalable interleaved memory operations with factor 3 in the
AArch64 TTI cost model.

SelectionDAG can lower explicit scalable vector.interleave3 and
vector.deinterleave3 patterns for legal packed SVE vector types. For
vector.interleave3 values declared in seperate blocks must be
deinterleaved first before being used.

As SVE does not have a zip3 instruction, we need to materialize i1
vectors by sign-extending them to a byte.
---
 .../scalable_masked_deinterleaved_loads.ll    |   11 +
 .../scalable_masked_interleaved_stores.ll     |   65 +
 .../AArch64/sve-tail-folding-option.ll        | 2357 +++++++++++++++--
 3 files changed, 2159 insertions(+), 274 deletions(-)

diff --git a/llvm/test/CodeGen/AArch64/scalable_masked_deinterleaved_loads.ll b/llvm/test/CodeGen/AArch64/scalable_masked_deinterleaved_loads.ll
index ee2d482d9ffb5..031509d5ff6c3 100644
--- a/llvm/test/CodeGen/AArch64/scalable_masked_deinterleaved_loads.ll
+++ b/llvm/test/CodeGen/AArch64/scalable_masked_deinterleaved_loads.ll
@@ -45,6 +45,17 @@ define { <vscale x 2 x double>, <vscale x 2 x double> } @foo_ld2_nxv2f64(<vscale
   ret { <vscale x 2 x double>, <vscale x 2 x double> } %deinterleaved.vec
 }
 
+define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @foo_ld3_nxv16i8(<vscale x 16 x i1> %mask, ptr %p) {
+; CHECK-LABEL: foo_ld3_nxv16i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld3b { z0.b - z2.b }, p0/z, [x0]
+; CHECK-NEXT:    ret
+  %interleaved.mask = call <vscale x 48 x i1> @llvm.vector.interleave3.nxv48i1(<vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask)
+  %wide.masked.vec = call <vscale x 48 x i8> @llvm.masked.load.nxv48i8(ptr %p, i32 1, <vscale x 48 x i1> %interleaved.mask, <vscale x 48 x i8> poison)
+  %deinterleaved.vec = call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.vector.deinterleave3.nxv48i8(<vscale x 48 x i8> %wide.masked.vec)
+  ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %deinterleaved.vec
+}
+
 define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @foo_ld4_nxv16i8(<vscale x 16 x i1> %mask, ptr %p) {
 ; CHECK-LABEL: foo_ld4_nxv16i8:
 ; CHECK:       // %bb.0:
diff --git a/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll b/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll
index fc2f64b756959..a322c6e289f06 100644
--- a/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll
+++ b/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll
@@ -54,6 +54,43 @@ define void @foo_st2_nxv2i64(<vscale x 2 x i1> %mask, <vscale x 2 x i64> %val1,
   ret void
 }
 
+define void @foo_st3_nxv16i8(<vscale x 16 x i1> %mask, <vscale x 16 x i8> %val1, <vscale x 16 x i8> %val2, <vscale x 16 x i8> %val3, ptr %p) {
+; CHECK-LABEL: foo_st3_nxv16i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2 def $z0_z1_z2
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2 def $z0_z1_z2
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2 def $z0_z1_z2
+; CHECK-NEXT:    st3b { z0.b - z2.b }, p0, [x0]
+; CHECK-NEXT:    ret
+  %interleaved.mask = call <vscale x 48 x i1> @llvm.vector.interleave3.nxv48i1(<vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask)
+  %interleaved.value = call <vscale x 48 x i8> @llvm.vector.interleave3.nxv48i8(<vscale x 16 x i8> %val1, <vscale x 16 x i8> %val2, <vscale x 16 x i8> %val3)
+  call void @llvm.masked.store.nxv48i8.p0(<vscale x 48 x i8> %interleaved.value, ptr %p, i32 1, <vscale x 48 x i1> %interleaved.mask)
+  ret void
+}
+
+; This test is specific to interleaves of factor 3.
+; SelectionDAG combines are local to a basic blocks. Keep the interleave in the
+; predecessor so the store sees an opaque, target-illegal nxv48i8 value.
+; The DAG should split the vectors back into three legal SVE vectors before type
+; legalisation.
+define void @interleave3_cross_block(<vscale x 16 x i1> %mask, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i8> %c, ptr %p) {
+; CHECK-LABEL: interleave3_cross_block:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2 def $z0_z1_z2
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2 def $z0_z1_z2
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2 def $z0_z1_z2
+; CHECK-NEXT:    st3b { z0.b - z2.b }, p0, [x0]
+; CHECK-NEXT:    ret
+entry:
+  %wide.value = call <vscale x 48 x i8> @llvm.vector.interleave3.nxv48i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i8> %c)
+  br label %store
+
+store:
+  %wide.mask = call <vscale x 48 x i1> @llvm.vector.interleave3.nxv48i1(<vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask)
+  call void @llvm.masked.store.nxv48i8.p0(<vscale x 48 x i8> %wide.value, ptr %p, i32 1, <vscale x 48 x i1> %wide.mask)
+  ret void
+}
+
 define void @foo_st4_nxv16i8(<vscale x 16 x i1> %mask, <vscale x 16 x i8> %val1, <vscale x 16 x i8> %val2, <vscale x 16 x i8> %val3, <vscale x 16 x i8> %val4, ptr %p) {
 ; CHECK-LABEL: foo_st4_nxv16i8:
 ; CHECK:       // %bb.0:
@@ -296,6 +333,19 @@ define void @foo_st2_nxv16i8_zeroinitializer(<vscale x 16 x i1> %mask, ptr %p) {
   ret void
 }
 
+define void @foo_st3_nxv16i8_zeroinitializer(<vscale x 16 x i1> %mask, <vscale x 16 x i8> %val1, <vscale x 16 x i8> %val2, <vscale x 16 x i8> %val3, ptr %p) {
+; CHECK-LABEL: foo_st3_nxv16i8_zeroinitializer:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-NEXT:    mov z1.d, z0.d
+; CHECK-NEXT:    mov z2.d, z0.d
+; CHECK-NEXT:    st3b { z0.b - z2.b }, p0, [x0]
+; CHECK-NEXT:    ret
+  %interleaved.mask = call <vscale x 48 x i1> @llvm.vector.interleave3.nxv48i1(<vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask)
+  call void @llvm.masked.store.nxv48i8.p0(<vscale x 48 x i8> zeroinitializer, ptr %p, i32 1, <vscale x 48 x i1> %interleaved.mask)
+  ret void
+}
+
 define void @foo_st4_nxv16i8_zeroinitializer(<vscale x 16 x i1> %mask, ptr %p) {
 ; CHECK-LABEL: foo_st4_nxv16i8_zeroinitializer:
 ; CHECK:       // %bb.0:
@@ -324,6 +374,21 @@ define void @foo_st2_nxv16i8_splat(<vscale x 16 x i1> %mask, ptr %p) {
   ret void
 }
 
+define void @foo_st3_nxv16i8_splat(<vscale x 16 x i1> %mask, ptr %p) {
+; CHECK-LABEL: foo_st3_nxv16i8_splat:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov z0.b, #1 // =0x1
+; CHECK-NEXT:    mov z1.d, z0.d
+; CHECK-NEXT:    mov z2.d, z0.d
+; CHECK-NEXT:    st3b { z0.b - z2.b }, p0, [x0]
+; CHECK-NEXT:    ret
+  %base = insertelement <vscale x 48 x i8> poison, i8 1, i32 0
+  %interleaved.value = shufflevector <vscale x 48 x i8> %base, <vscale x 48 x i8> poison, <vscale x 48 x i32> zeroinitializer
+  %interleaved.mask = call <vscale x 48 x i1> @llvm.vector.interleave3.nxv48i1(<vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask)
+  call void @llvm.masked.store.nxv48i8.p0( <vscale x 48 x i8> %interleaved.value, ptr %p, i32 1, <vscale x 48 x i1> %interleaved.mask)
+  ret void
+}
+
 define void @foo_st4_nxv16i8_splat(<vscale x 16 x i1> %mask, ptr %p) {
 ; CHECK-LABEL: foo_st4_nxv16i8_splat:
 ; CHECK:       // %bb.0:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-option.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-option.ll
index fb17b9a80d09b..802c0d460eee8 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-option.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-option.ll
@@ -1,9 +1,10 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
 ; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding-insn-threshold=0 -sve-tail-folding=disabled -S | FileCheck %s -check-prefix=CHECK-NOTF
 ; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding-insn-threshold=0 -sve-tail-folding=default -S | FileCheck %s -check-prefix=CHECK-NOTF
 ; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding-insn-threshold=0 -S | FileCheck %s -check-prefix=CHECK-NOTF
-; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding-insn-threshold=0 -sve-tail-folding=all -S | FileCheck %s -check-prefix=CHECK-TF
+; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding-insn-threshold=0 -sve-tail-folding=all -S | FileCheck %s -check-prefix=CHECK-TF-ALL
 ; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding-insn-threshold=0 -sve-tail-folding=simple+reductions+recurrences+reverse -S | FileCheck %s -check-prefix=CHECK-TF
-; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding-insn-threshold=0 -S -mcpu=neoverse-v1 -sve-tail-folding=default+reductions+recurrences+reverse | FileCheck %s -check-prefix=CHECK-TF
+; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding-insn-threshold=0 -S -mcpu=neoverse-v1 -sve-tail-folding=default+reductions+recurrences+reverse | FileCheck %s -check-prefix=CHECK-TF-DEFAULT
 ; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding-insn-threshold=0 -sve-tail-folding=all+noreductions -S | FileCheck %s -check-prefix=CHECK-TF-NORED
 ; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding-insn-threshold=0 -sve-tail-folding=all+norecurrences -S | FileCheck %s -check-prefix=CHECK-TF-NOREC
 ; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding-insn-threshold=0 -sve-tail-folding=all+noreverse -S | FileCheck %s -check-prefix=CHECK-TF-NOREV
@@ -15,61 +16,277 @@
 target triple = "aarch64-unknown-linux-gnu"
 
 define void @simple_memset(i32 %val, ptr %ptr, i64 %n) #0 {
-; CHECK-NOTF-LABEL: @simple_memset(
-; CHECK-NOTF:       vector.ph:
-; CHECK-NOTF:         %[[INSERT:.*]] = insertelement <vscale x 4 x i32> poison, i32 %val, i64 0
-; CHECK-NOTF:         %[[SPLAT:.*]] = shufflevector <vscale x 4 x i32> %[[INSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NOTF:       vector.body:
-; CHECK-NOTF-NOT:     %{{.*}} = phi <vscale x 4 x i1>
-; CHECK-NOTF:         store <vscale x 4 x i32> %[[SPLAT]], ptr
-
-; CHECK-TF-NORED-LABEL: @simple_memset(
-; CHECK-TF-NORED:       vector.ph:
-; CHECK-TF-NORED:         %[[INSERT:.*]] = insertelement <vscale x 4 x i32> poison, i32 %val, i64 0
-; CHECK-TF-NORED:         %[[SPLAT:.*]] = shufflevector <vscale x 4 x i32> %[[INSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-TF-NORED:       vector.body:
-; CHECK-TF-NORED:         %[[ACTIVE_LANE_MASK:.*]] = phi <vscale x 4 x i1>
-; CHECK-TF-NORED:         call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> %[[SPLAT]], {{.*}} %[[ACTIVE_LANE_MASK]]
-
-; CHECK-TF-NOREC-LABEL: @simple_memset(
-; CHECK-TF-NOREC:       vector.ph:
-; CHECK-TF-NOREC:         %[[INSERT:.*]] = insertelement <vscale x 4 x i32> poison, i32 %val, i64 0
-; CHECK-TF-NOREC:         %[[SPLAT:.*]] = shufflevector <vscale x 4 x i32> %[[INSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-TF-NOREC:       vector.body:
-; CHECK-TF-NOREC:         %[[ACTIVE_LANE_MASK:.*]] = phi <vscale x 4 x i1>
-; CHECK-TF-NOREC:         call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> %[[SPLAT]], {{.*}} %[[ACTIVE_LANE_MASK]]
-
-; CHECK-TF-NOREV-LABEL: @simple_memset(
-; CHECK-TF-NOREV:       vector.ph:
-; CHECK-TF-NOREV:         %[[INSERT:.*]] = insertelement <vscale x 4 x i32> poison, i32 %val, i64 0
-; CHECK-TF-NOREV:         %[[SPLAT:.*]] = shufflevector <vscale x 4 x i32> %[[INSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-TF-NOREV:       vector.body:
-; CHECK-TF-NOREV:         %[[ACTIVE_LANE_MASK:.*]] = phi <vscale x 4 x i1>
-; CHECK-TF-NOREV:         call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> %[[SPLAT]], {{.*}} %[[ACTIVE_LANE_MASK]]
-
-; CHECK-TF-LABEL: @simple_memset(
-; CHECK-TF:       vector.ph:
-; CHECK-TF:         %[[INSERT:.*]] = insertelement <vscale x 4 x i32> poison, i32 %val, i64 0
-; CHECK-TF:         %[[SPLAT:.*]] = shufflevector <vscale x 4 x i32> %[[INSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-TF:       vector.body:
-; CHECK-TF:         %[[ACTIVE_LANE_MASK:.*]] = phi <vscale x 4 x i1>
-; CHECK-TF:         call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> %[[SPLAT]], {{.*}} %[[ACTIVE_LANE_MASK]]
-
-; CHECK-TF-ONLYRED-LABEL: @simple_memset(
-; CHECK-TF-ONLYRED:       vector.ph:
-; CHECK-TF-ONLYRED:         %[[INSERT:.*]] = insertelement <vscale x 4 x i32> poison, i32 %val, i64 0
-; CHECK-TF-ONLYRED:         %[[SPLAT:.*]] = shufflevector <vscale x 4 x i32> %[[INSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-TF-ONLYRED:       vector.body:
-; CHECK-TF-ONLYRED-NOT:     %{{.*}} = phi <vscale x 4 x i1>
-; CHECK-TF-ONLYRED:         store <vscale x 4 x i32> %[[SPLAT]], ptr
-
-; CHECK-NEOVERSE-V1-LABEL: @simple_memset(
-; CHECK-NEOVERSE-V1:       vector.ph:
-; CHECK-NEOVERSE-V1:         %[[INSERT:.*]] = insertelement <vscale x 4 x i32> poison, i32 %val, i64 0
-; CHECK-NEOVERSE-V1:         %[[SPLAT:.*]] = shufflevector <vscale x 4 x i32> %[[INSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEOVERSE-V1:       vector.body:
-; CHECK-NEOVERSE-V1:         %[[ACTIVE_LANE_MASK:.*]] = phi <vscale x 4 x i1>
-; CHECK-NEOVERSE-V1:         call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> %[[SPLAT]], {{.*}} %[[ACTIVE_LANE_MASK]]
+; CHECK-NOTF-LABEL: define void @simple_memset(
+; CHECK-NOTF-SAME: i32 [[VAL:%.*]], ptr [[PTR:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NOTF-NEXT:  [[ENTRY:.*]]:
+; CHECK-NOTF-NEXT:    [[TMP0:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
+; CHECK-NOTF-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NOTF-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
+; CHECK-NOTF-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
+; CHECK-NOTF-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NOTF:       [[VECTOR_PH]]:
+; CHECK-NOTF-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP1]], 2
+; CHECK-NOTF-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP3]]
+; CHECK-NOTF-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
+; CHECK-NOTF-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[VAL]], i64 0
+; CHECK-NOTF-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NOTF-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-NOTF:       [[VECTOR_BODY]]:
+; CHECK-NOTF-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NOTF-NEXT:    [[TMP4:%.*]] = getelementptr i32, ptr [[PTR]], i64 [[INDEX1]]
+; CHECK-NOTF-NEXT:    store <vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr [[TMP4]], align 4
+; CHECK-NOTF-NEXT:    [[INDEX_NEXT2]] = add nuw i64 [[INDEX1]], [[TMP3]]
+; CHECK-NOTF-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT2]], [[N_VEC]]
+; CHECK-NOTF-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NOTF:       [[MIDDLE_BLOCK]]:
+; CHECK-NOTF-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
+; CHECK-NOTF-NEXT:    br i1 [[CMP_N]], label %[[WHILE_END_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NOTF:       [[SCALAR_PH]]:
+; CHECK-NOTF-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NOTF-NEXT:    br label %[[WHILE_BODY:.*]]
+; CHECK-NOTF:       [[WHILE_BODY]]:
+; CHECK-NOTF-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], %[[WHILE_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NOTF-NEXT:    [[GEP:%.*]] = getelementptr i32, ptr [[PTR]], i64 [[INDEX]]
+; CHECK-NOTF-NEXT:    store i32 [[VAL]], ptr [[GEP]], align 4
+; CHECK-NOTF-NEXT:    [[INDEX_NEXT]] = add nsw i64 [[INDEX]], 1
+; CHECK-NOTF-NEXT:    [[CMP10:%.*]] = icmp ult i64 [[INDEX_NEXT]], [[N]]
+; CHECK-NOTF-NEXT:    br i1 [[CMP10]], label %[[WHILE_BODY]], label %[[WHILE_END_LOOPEXIT]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NOTF:       [[WHILE_END_LOOPEXIT]]:
+; CHECK-NOTF-NEXT:    ret void
+;
+; CHECK-TF-ALL-LABEL: define void @simple_memset(
+; CHECK-TF-ALL-SAME: i32 [[VAL:%.*]], ptr [[PTR:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-TF-ALL-NEXT:  [[ENTRY:.*:]]
+; CHECK-TF-ALL-NEXT:    [[TMP0:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
+; CHECK-TF-ALL-NEXT:    br label %[[VECTOR_PH:.*]]
+; CHECK-TF-ALL:       [[VECTOR_PH]]:
+; CHECK-TF-ALL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-ALL-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
+; CHECK-TF-ALL-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[TMP0]])
+; CHECK-TF-ALL-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[VAL]], i64 0
+; CHECK-TF-ALL-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-TF-ALL-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-ALL:       [[VECTOR_BODY]]:
+; CHECK-TF-ALL-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-ALL-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-ALL-NEXT:    [[TMP3:%.*]] = getelementptr i32, ptr [[PTR]], i64 [[INDEX1]]
+; CHECK-TF-ALL-NEXT:    call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr align 4 [[TMP3]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-ALL-NEXT:    [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP2]]
+; CHECK-TF-ALL-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT2]], i64 [[TMP0]])
+; CHECK-TF-ALL-NEXT:    [[TMP4:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-ALL-NEXT:    [[TMP5:%.*]] = xor i1 [[TMP4]], true
+; CHECK-TF-ALL-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-TF-ALL:       [[MIDDLE_BLOCK]]:
+; CHECK-TF-ALL-NEXT:    br label %[[WHILE_END_LOOPEXIT:.*]]
+; CHECK-TF-ALL:       [[WHILE_END_LOOPEXIT]]:
+; CHECK-TF-ALL-NEXT:    ret void
+;
+; CHECK-TF-LABEL: define void @simple_memset(
+; CHECK-TF-SAME: i32 [[VAL:%.*]], ptr [[PTR:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-TF-NEXT:  [[ENTRY:.*:]]
+; CHECK-TF-NEXT:    [[TMP0:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
+; CHECK-TF-NEXT:    br label %[[VECTOR_PH:.*]]
+; CHECK-TF:       [[VECTOR_PH]]:
+; CHECK-TF-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[TMP0]])
+; CHECK-TF-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[VAL]], i64 0
+; CHECK-TF-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-TF-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-TF:       [[VECTOR_BODY]]:
+; CHECK-TF-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NEXT:    [[TMP3:%.*]] = getelementptr i32, ptr [[PTR]], i64 [[INDEX1]]
+; CHECK-TF-NEXT:    call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr align 4 [[TMP3]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-NEXT:    [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP2]]
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT2]], i64 [[TMP0]])
+; CHECK-TF-NEXT:    [[TMP4:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-NEXT:    [[TMP5:%.*]] = xor i1 [[TMP4]], true
+; CHECK-TF-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-TF:       [[MIDDLE_BLOCK]]:
+; CHECK-TF-NEXT:    br label %[[WHILE_END_LOOPEXIT:.*]]
+; CHECK-TF:       [[WHILE_END_LOOPEXIT]]:
+; CHECK-TF-NEXT:    ret void
+;
+; CHECK-TF-DEFAULT-LABEL: define void @simple_memset(
+; CHECK-TF-DEFAULT-SAME: i32 [[VAL:%.*]], ptr [[PTR:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-TF-DEFAULT-NEXT:  [[ENTRY:.*:]]
+; CHECK-TF-DEFAULT-NEXT:    [[TMP0:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
+; CHECK-TF-DEFAULT-NEXT:    br label %[[VECTOR_PH:.*]]
+; CHECK-TF-DEFAULT:       [[VECTOR_PH]]:
+; CHECK-TF-DEFAULT-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-DEFAULT-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
+; CHECK-TF-DEFAULT-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[TMP0]])
+; CHECK-TF-DEFAULT-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[VAL]], i64 0
+; CHECK-TF-DEFAULT-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-TF-DEFAULT-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-DEFAULT:       [[VECTOR_BODY]]:
+; CHECK-TF-DEFAULT-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-DEFAULT-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-DEFAULT-NEXT:    [[TMP3:%.*]] = getelementptr i32, ptr [[PTR]], i64 [[INDEX1]]
+; CHECK-TF-DEFAULT-NEXT:    call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr align 4 [[TMP3]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-DEFAULT-NEXT:    [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP2]]
+; CHECK-TF-DEFAULT-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT2]], i64 [[TMP0]])
+; CHECK-TF-DEFAULT-NEXT:    [[TMP4:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-DEFAULT-NEXT:    [[TMP5:%.*]] = xor i1 [[TMP4]], true
+; CHECK-TF-DEFAULT-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-TF-DEFAULT:       [[MIDDLE_BLOCK]]:
+; CHECK-TF-DEFAULT-NEXT:    br label %[[WHILE_END_LOOPEXIT:.*]]
+; CHECK-TF-DEFAULT:       [[WHILE_END_LOOPEXIT]]:
+; CHECK-TF-DEFAULT-NEXT:    ret void
+;
+; CHECK-TF-NORED-LABEL: define void @simple_memset(
+; CHECK-TF-NORED-SAME: i32 [[VAL:%.*]], ptr [[PTR:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-TF-NORED-NEXT:  [[ENTRY:.*:]]
+; CHECK-TF-NORED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
+; CHECK-TF-NORED-NEXT:    br label %[[VECTOR_PH:.*]]
+; CHECK-TF-NORED:       [[VECTOR_PH]]:
+; CHECK-TF-NORED-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NORED-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
+; CHECK-TF-NORED-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[TMP0]])
+; CHECK-TF-NORED-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[VAL]], i64 0
+; CHECK-TF-NORED-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-TF-NORED-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-NORED:       [[VECTOR_BODY]]:
+; CHECK-TF-NORED-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NORED-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NORED-NEXT:    [[TMP3:%.*]] = getelementptr i32, ptr [[PTR]], i64 [[INDEX1]]
+; CHECK-TF-NORED-NEXT:    call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr align 4 [[TMP3]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-NORED-NEXT:    [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP2]]
+; CHECK-TF-NORED-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT2]], i64 [[TMP0]])
+; CHECK-TF-NORED-NEXT:    [[TMP4:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-NORED-NEXT:    [[TMP5:%.*]] = xor i1 [[TMP4]], true
+; CHECK-TF-NORED-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-TF-NORED:       [[MIDDLE_BLOCK]]:
+; CHECK-TF-NORED-NEXT:    br label %[[WHILE_END_LOOPEXIT:.*]]
+; CHECK-TF-NORED:       [[WHILE_END_LOOPEXIT]]:
+; CHECK-TF-NORED-NEXT:    ret void
+;
+; CHECK-TF-NOREC-LABEL: define void @simple_memset(
+; CHECK-TF-NOREC-SAME: i32 [[VAL:%.*]], ptr [[PTR:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-TF-NOREC-NEXT:  [[ENTRY:.*:]]
+; CHECK-TF-NOREC-NEXT:    [[TMP0:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
+; CHECK-TF-NOREC-NEXT:    br label %[[VECTOR_PH:.*]]
+; CHECK-TF-NOREC:       [[VECTOR_PH]]:
+; CHECK-TF-NOREC-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NOREC-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
+; CHECK-TF-NOREC-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[TMP0]])
+; CHECK-TF-NOREC-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[VAL]], i64 0
+; CHECK-TF-NOREC-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-TF-NOREC-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-NOREC:       [[VECTOR_BODY]]:
+; CHECK-TF-NOREC-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NOREC-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NOREC-NEXT:    [[TMP3:%.*]] = getelementptr i32, ptr [[PTR]], i64 [[INDEX1]]
+; CHECK-TF-NOREC-NEXT:    call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr align 4 [[TMP3]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-NOREC-NEXT:    [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP2]]
+; CHECK-TF-NOREC-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT2]], i64 [[TMP0]])
+; CHECK-TF-NOREC-NEXT:    [[TMP4:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-NOREC-NEXT:    [[TMP5:%.*]] = xor i1 [[TMP4]], true
+; CHECK-TF-NOREC-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-TF-NOREC:       [[MIDDLE_BLOCK]]:
+; CHECK-TF-NOREC-NEXT:    br label %[[WHILE_END_LOOPEXIT:.*]]
+; CHECK-TF-NOREC:       [[WHILE_END_LOOPEXIT]]:
+; CHECK-TF-NOREC-NEXT:    ret void
+;
+; CHECK-TF-NOREV-LABEL: define void @simple_memset(
+; CHECK-TF-NOREV-SAME: i32 [[VAL:%.*]], ptr [[PTR:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-TF-NOREV-NEXT:  [[ENTRY:.*:]]
+; CHECK-TF-NOREV-NEXT:    [[TMP0:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
+; CHECK-TF-NOREV-NEXT:    br label %[[VECTOR_PH:.*]]
+; CHECK-TF-NOREV:       [[VECTOR_PH]]:
+; CHECK-TF-NOREV-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NOREV-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
+; CHECK-TF-NOREV-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[TMP0]])
+; CHECK-TF-NOREV-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[VAL]], i64 0
+; CHECK-TF-NOREV-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-TF-NOREV-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-NOREV:       [[VECTOR_BODY]]:
+; CHECK-TF-NOREV-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NOREV-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NOREV-NEXT:    [[TMP3:%.*]] = getelementptr i32, ptr [[PTR]], i64 [[INDEX1]]
+; CHECK-TF-NOREV-NEXT:    call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr align 4 [[TMP3]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-NOREV-NEXT:    [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP2]]
+; CHECK-TF-NOREV-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT2]], i64 [[TMP0]])
+; CHECK-TF-NOREV-NEXT:    [[TMP4:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-NOREV-NEXT:    [[TMP5:%.*]] = xor i1 [[TMP4]], true
+; CHECK-TF-NOREV-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-TF-NOREV:       [[MIDDLE_BLOCK]]:
+; CHECK-TF-NOREV-NEXT:    br label %[[WHILE_END_LOOPEXIT:.*]]
+; CHECK-TF-NOREV:       [[WHILE_END_LOOPEXIT]]:
+; CHECK-TF-NOREV-NEXT:    ret void
+;
+; CHECK-TF-ONLYRED-LABEL: define void @simple_memset(
+; CHECK-TF-ONLYRED-SAME: i32 [[VAL:%.*]], ptr [[PTR:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-TF-ONLYRED-NEXT:  [[ENTRY:.*]]:
+; CHECK-TF-ONLYRED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
+; CHECK-TF-ONLYRED-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-ONLYRED-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
+; CHECK-TF-ONLYRED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
+; CHECK-TF-ONLYRED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-TF-ONLYRED:       [[VECTOR_PH]]:
+; CHECK-TF-ONLYRED-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP1]], 2
+; CHECK-TF-ONLYRED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP3]]
+; CHECK-TF-ONLYRED-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
+; CHECK-TF-ONLYRED-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[VAL]], i64 0
+; CHECK-TF-ONLYRED-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-TF-ONLYRED-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-ONLYRED:       [[VECTOR_BODY]]:
+; CHECK-TF-ONLYRED-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-ONLYRED-NEXT:    [[TMP4:%.*]] = getelementptr i32, ptr [[PTR]], i64 [[INDEX1]]
+; CHECK-TF-ONLYRED-NEXT:    store <vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr [[TMP4]], align 4
+; CHECK-TF-ONLYRED-NEXT:    [[INDEX_NEXT2]] = add nuw i64 [[INDEX1]], [[TMP3]]
+; CHECK-TF-ONLYRED-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT2]], [[N_VEC]]
+; CHECK-TF-ONLYRED-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-TF-ONLYRED:       [[MIDDLE_BLOCK]]:
+; CHECK-TF-ONLYRED-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
+; CHECK-TF-ONLYRED-NEXT:    br i1 [[CMP_N]], label %[[WHILE_END_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-TF-ONLYRED:       [[SCALAR_PH]]:
+; CHECK-TF-ONLYRED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-TF-ONLYRED-NEXT:    br label %[[WHILE_BODY:.*]]
+; CHECK-TF-ONLYRED:       [[WHILE_BODY]]:
+; CHECK-TF-ONLYRED-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], %[[WHILE_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-TF-ONLYRED-NEXT:    [[GEP:%.*]] = getelementptr i32, ptr [[PTR]], i64 [[INDEX]]
+; CHECK-TF-ONLYRED-NEXT:    store i32 [[VAL]], ptr [[GEP]], align 4
+; CHECK-TF-ONLYRED-NEXT:    [[INDEX_NEXT]] = add nsw i64 [[INDEX]], 1
+; CHECK-TF-ONLYRED-NEXT:    [[CMP10:%.*]] = icmp ult i64 [[INDEX_NEXT]], [[N]]
+; CHECK-TF-ONLYRED-NEXT:    br i1 [[CMP10]], label %[[WHILE_BODY]], label %[[WHILE_END_LOOPEXIT]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-TF-ONLYRED:       [[WHILE_END_LOOPEXIT]]:
+; CHECK-TF-ONLYRED-NEXT:    ret void
+;
+; CHECK-NEOVERSE-V1-LABEL: define void @simple_memset(
+; CHECK-NEOVERSE-V1-SAME: i32 [[VAL:%.*]], ptr [[PTR:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEOVERSE-V1-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP0:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
+; CHECK-NEOVERSE-V1-NEXT:    br label %[[VECTOR_PH:.*]]
+; CHECK-NEOVERSE-V1:       [[VECTOR_PH]]:
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
+; CHECK-NEOVERSE-V1-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[TMP0]])
+; CHECK-NEOVERSE-V1-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[VAL]], i64 0
+; CHECK-NEOVERSE-V1-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEOVERSE-V1-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-NEOVERSE-V1:       [[VECTOR_BODY]]:
+; CHECK-NEOVERSE-V1-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEOVERSE-V1-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP3:%.*]] = getelementptr i32, ptr [[PTR]], i64 [[INDEX1]]
+; CHECK-NEOVERSE-V1-NEXT:    call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr align 4 [[TMP3]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-NEOVERSE-V1-NEXT:    [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP2]]
+; CHECK-NEOVERSE-V1-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT2]], i64 [[TMP0]])
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP4:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP5:%.*]] = xor i1 [[TMP4]], true
+; CHECK-NEOVERSE-V1-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEOVERSE-V1:       [[MIDDLE_BLOCK]]:
+; CHECK-NEOVERSE-V1-NEXT:    br label %[[WHILE_END_LOOPEXIT:.*]]
+; CHECK-NEOVERSE-V1:       [[WHILE_END_LOOPEXIT]]:
+; CHECK-NEOVERSE-V1-NEXT:    ret void
+;
+
+
+
+
+
+
 
 entry:
   br label %while.body
@@ -87,69 +304,306 @@ while.end.loopexit:
 }
 
 define float @fadd_red_fast(ptr noalias nocapture readonly %a, i64 %n) #0 {
-; CHECK-NOTF-LABEL: @fadd_red_fast
-; CHECK-NOTF:       vector.body:
-; CHECK-NOTF-NOT:     %{{.*}} = phi <vscale x 4 x i1>
-; CHECK-NOTF:         %[[LOAD:.*]] = load <vscale x 4 x float>
-; CHECK-NOTF:         %[[ADD:.*]] = fadd fast <vscale x 4 x float> %[[LOAD]]
-; CHECK-NOTF:       middle.block:
-; CHECK-NOTF-NEXT:    call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> %[[ADD]])
-
-; CHECK-TF-NORED-LABEL: @fadd_red_fast
-; CHECK-TF-NORED:       vector.body:
-; CHECK-TF-NORED-NOT:     %{{.*}} = phi <vscale x 4 x i1>
-; CHECK-TF-NORED:         %[[LOAD:.*]] = load <vscale x 4 x float>
-; CHECK-TF-NORED:         %[[ADD:.*]] = fadd fast <vscale x 4 x float> %[[LOAD]]
-; CHECK-TF-NORED:       middle.block:
-; CHECK-TF-NORED-NEXT:    call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> %[[ADD]])
-
-; CHECK-TF-NOREC-LABEL: @fadd_red_fast
-; CHECK-TF-NOREC:       vector.body:
-; CHECK-TF-NOREC:         %[[ACTIVE_LANE_MASK:.*]] = phi <vscale x 4 x i1>
-; CHECK-TF-NOREC:         %[[VEC_PHI:.*]] = phi <vscale x 4 x float>
-; CHECK-TF-NOREC:         %[[LOAD:.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0({{.*}} %[[ACTIVE_LANE_MASK]]
-; CHECK-TF-NOREC:         %[[ADD:.*]] = fadd fast <vscale x 4 x float> %[[LOAD]]
-; CHECK-TF-NOREC:         %[[SEL:.*]] = select fast <vscale x 4 x i1> %[[ACTIVE_LANE_MASK]], <vscale x 4 x float> %[[ADD]], <vscale x 4 x float> %[[VEC_PHI]]
-; CHECK-TF-NOREC:       middle.block:
-; CHECK-TF-NOREC-NEXT:    call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> %[[SEL]])
-
-; CHECK-TF-NOREV-LABEL: @fadd_red_fast
-; CHECK-TF-NOREV:       vector.body:
-; CHECK-TF-NOREV:         %[[ACTIVE_LANE_MASK:.*]] = phi <vscale x 4 x i1>
-; CHECK-TF-NOREV:         %[[VEC_PHI:.*]] = phi <vscale x 4 x float>
-; CHECK-TF-NOREV:         %[[LOAD:.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0({{.*}} %[[ACTIVE_LANE_MASK]]
-; CHECK-TF-NOREV:         %[[ADD:.*]] = fadd fast <vscale x 4 x float> %[[LOAD]]
-; CHECK-TF-NOREV:         %[[SEL:.*]] = select fast <vscale x 4 x i1> %[[ACTIVE_LANE_MASK]], <vscale x 4 x float> %[[ADD]], <vscale x 4 x float> %[[VEC_PHI]]
-; CHECK-TF-NOREV:       middle.block:
-; CHECK-TF-NOREV-NEXT:    call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> %[[SEL]])
-
-; CHECK-TF-LABEL: @fadd_red_fast
-; CHECK-TF:       vector.body:
-; CHECK-TF:         %[[ACTIVE_LANE_MASK:.*]] = phi <vscale x 4 x i1>
-; CHECK-TF:         %[[VEC_PHI:.*]] = phi <vscale x 4 x float>
-; CHECK-TF:         %[[LOAD:.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0({{.*}} %[[ACTIVE_LANE_MASK]]
-; CHECK-TF:         %[[ADD:.*]] = fadd fast <vscale x 4 x float> %[[LOAD]]
-; CHECK-TF:         %[[SEL:.*]] = select fast <vscale x 4 x i1> %[[ACTIVE_LANE_MASK]], <vscale x 4 x float> %[[ADD]], <vscale x 4 x float> %[[VEC_PHI]]
-; CHECK-TF:       middle.block:
-; CHECK-TF-NEXT:    call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> %[[SEL]])
-
-; CHECK-TF-ONLYRED-LABEL: @fadd_red_fast
-; CHECK-TF-ONLYRED:       vector.body:
-; CHECK-TF-ONLYRED:         %[[ACTIVE_LANE_MASK:.*]] = phi <vscale x 4 x i1>
-; CHECK-TF-ONLYRED:         %[[VEC_PHI:.*]] = phi <vscale x 4 x float>
-; CHECK-TF-ONLYRED:         %[[LOAD:.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0({{.*}} %[[ACTIVE_LANE_MASK]]
-; CHECK-TF-ONLYRED:         %[[ADD:.*]] = fadd fast <vscale x 4 x float> %[[LOAD]]
-; CHECK-TF-ONLYRED:         %[[SEL:.*]] = select fast <vscale x 4 x i1> %[[ACTIVE_LANE_MASK]], <vscale x 4 x float> %[[ADD]], <vscale x 4 x float> %[[VEC_PHI]]
-; CHECK-TF-ONLYRED:       middle.block:
-; CHECK-TF-ONLYRED-NEXT:    call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> %[[SEL]])
-
-; CHECK-NEOVERSE-V1-LABEL: @fadd_red_fast
-; CHECK-NEOVERSE-V1:       vector.body:
-; CHECK-NEOVERSE-V1-NOT:     %{{.*}} = phi <vscale x 4 x i1>
-; CHECK-NEOVERSE-V1:         %[[LOAD:.*]] = load <vscale x 4 x float>
-; CHECK-NEOVERSE-V1:         %[[ADD:.*]] = fadd fast <vscale x 4 x float> %[[LOAD]]
-; CHECK-NEOVERSE-V1:       middle.block:
-; CHECK-NEOVERSE-V1-NEXT:    call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> %[[ADD]])
+; CHECK-NOTF-LABEL: define float @fadd_red_fast(
+; CHECK-NOTF-SAME: ptr noalias readonly captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NOTF-NEXT:  [[ENTRY:.*]]:
+; CHECK-NOTF-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NOTF-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NOTF-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-NOTF-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NOTF:       [[VECTOR_PH]]:
+; CHECK-NOTF-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NOTF-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP2]]
+; CHECK-NOTF-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NOTF-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-NOTF:       [[VECTOR_BODY]]:
+; CHECK-NOTF-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NOTF-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NOTF-NEXT:    [[TMP3:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-NOTF-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP3]], align 4
+; CHECK-NOTF-NEXT:    [[TMP4]] = fadd fast <vscale x 4 x float> [[WIDE_LOAD]], [[VEC_PHI]]
+; CHECK-NOTF-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; CHECK-NOTF-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NOTF-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NOTF:       [[MIDDLE_BLOCK]]:
+; CHECK-NOTF-NEXT:    [[TMP6:%.*]] = call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> [[TMP4]])
+; CHECK-NOTF-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NOTF-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-NOTF:       [[SCALAR_PH]]:
+; CHECK-NOTF-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NOTF-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; CHECK-NOTF-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK-NOTF:       [[FOR_BODY]]:
+; CHECK-NOTF-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NOTF-NEXT:    [[SUM_07:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[ADD:%.*]], %[[FOR_BODY]] ]
+; CHECK-NOTF-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
+; CHECK-NOTF-NEXT:    [[TMP7:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-NOTF-NEXT:    [[ADD]] = fadd fast float [[TMP7]], [[SUM_07]]
+; CHECK-NOTF-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NOTF-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NOTF-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NOTF:       [[FOR_END]]:
+; CHECK-NOTF-NEXT:    [[ADD_LCSSA:%.*]] = phi float [ [[ADD]], %[[FOR_BODY]] ], [ [[TMP6]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NOTF-NEXT:    ret float [[ADD_LCSSA]]
+;
+; CHECK-TF-ALL-LABEL: define float @fadd_red_fast(
+; CHECK-TF-ALL-SAME: ptr noalias readonly captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-ALL-NEXT:  [[ENTRY:.*:]]
+; CHECK-TF-ALL-NEXT:    br label %[[VECTOR_PH:.*]]
+; CHECK-TF-ALL:       [[VECTOR_PH]]:
+; CHECK-TF-ALL-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-ALL-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-ALL-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
+; CHECK-TF-ALL-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-ALL:       [[VECTOR_BODY]]:
+; CHECK-TF-ALL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-ALL-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-ALL-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-ALL-NEXT:    [[TMP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-TF-ALL-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0(ptr align 4 [[TMP2]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> poison)
+; CHECK-TF-ALL-NEXT:    [[TMP3:%.*]] = fadd fast <vscale x 4 x float> [[WIDE_MASKED_LOAD]], [[VEC_PHI]]
+; CHECK-TF-ALL-NEXT:    [[TMP4]] = select fast <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> [[TMP3]], <vscale x 4 x float> [[VEC_PHI]]
+; CHECK-TF-ALL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-TF-ALL-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-TF-ALL-NEXT:    [[TMP5:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-ALL-NEXT:    [[TMP6:%.*]] = xor i1 [[TMP5]], true
+; CHECK-TF-ALL-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-TF-ALL:       [[MIDDLE_BLOCK]]:
+; CHECK-TF-ALL-NEXT:    [[TMP7:%.*]] = call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> [[TMP4]])
+; CHECK-TF-ALL-NEXT:    br label %[[FOR_END:.*]]
+; CHECK-TF-ALL:       [[FOR_END]]:
+; CHECK-TF-ALL-NEXT:    ret float [[TMP7]]
+;
+; CHECK-TF-LABEL: define float @fadd_red_fast(
+; CHECK-TF-SAME: ptr noalias readonly captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-NEXT:  [[ENTRY:.*:]]
+; CHECK-TF-NEXT:    br label %[[VECTOR_PH:.*]]
+; CHECK-TF:       [[VECTOR_PH]]:
+; CHECK-TF-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
+; CHECK-TF-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-TF:       [[VECTOR_BODY]]:
+; CHECK-TF-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NEXT:    [[TMP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-TF-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0(ptr align 4 [[TMP2]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> poison)
+; CHECK-TF-NEXT:    [[TMP3:%.*]] = fadd fast <vscale x 4 x float> [[WIDE_MASKED_LOAD]], [[VEC_PHI]]
+; CHECK-TF-NEXT:    [[TMP4]] = select fast <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> [[TMP3]], <vscale x 4 x float> [[VEC_PHI]]
+; CHECK-TF-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-TF-NEXT:    [[TMP5:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-NEXT:    [[TMP6:%.*]] = xor i1 [[TMP5]], true
+; CHECK-TF-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-TF:       [[MIDDLE_BLOCK]]:
+; CHECK-TF-NEXT:    [[TMP7:%.*]] = call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> [[TMP4]])
+; CHECK-TF-NEXT:    br label %[[FOR_END:.*]]
+; CHECK-TF:       [[FOR_END]]:
+; CHECK-TF-NEXT:    ret float [[TMP7]]
+;
+; CHECK-TF-DEFAULT-LABEL: define float @fadd_red_fast(
+; CHECK-TF-DEFAULT-SAME: ptr noalias readonly captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-DEFAULT-NEXT:  [[ENTRY:.*:]]
+; CHECK-TF-DEFAULT-NEXT:    br label %[[VECTOR_PH:.*]]
+; CHECK-TF-DEFAULT:       [[VECTOR_PH]]:
+; CHECK-TF-DEFAULT-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-DEFAULT-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-DEFAULT-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
+; CHECK-TF-DEFAULT-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-DEFAULT:       [[VECTOR_BODY]]:
+; CHECK-TF-DEFAULT-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-DEFAULT-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-DEFAULT-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-DEFAULT-NEXT:    [[TMP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-TF-DEFAULT-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0(ptr align 4 [[TMP2]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> poison)
+; CHECK-TF-DEFAULT-NEXT:    [[TMP3:%.*]] = fadd fast <vscale x 4 x float> [[WIDE_MASKED_LOAD]], [[VEC_PHI]]
+; CHECK-TF-DEFAULT-NEXT:    [[TMP4]] = select fast <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> [[TMP3]], <vscale x 4 x float> [[VEC_PHI]]
+; CHECK-TF-DEFAULT-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-TF-DEFAULT-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-TF-DEFAULT-NEXT:    [[TMP5:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-DEFAULT-NEXT:    [[TMP6:%.*]] = xor i1 [[TMP5]], true
+; CHECK-TF-DEFAULT-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-TF-DEFAULT:       [[MIDDLE_BLOCK]]:
+; CHECK-TF-DEFAULT-NEXT:    [[TMP7:%.*]] = call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> [[TMP4]])
+; CHECK-TF-DEFAULT-NEXT:    br label %[[FOR_END:.*]]
+; CHECK-TF-DEFAULT:       [[FOR_END]]:
+; CHECK-TF-DEFAULT-NEXT:    ret float [[TMP7]]
+;
+; CHECK-TF-NORED-LABEL: define float @fadd_red_fast(
+; CHECK-TF-NORED-SAME: ptr noalias readonly captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-NORED-NEXT:  [[ENTRY:.*]]:
+; CHECK-TF-NORED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NORED-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-NORED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-TF-NORED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-TF-NORED:       [[VECTOR_PH]]:
+; CHECK-TF-NORED-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-NORED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP2]]
+; CHECK-TF-NORED-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-TF-NORED-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-NORED:       [[VECTOR_BODY]]:
+; CHECK-TF-NORED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NORED-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NORED-NEXT:    [[TMP3:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-TF-NORED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP3]], align 4
+; CHECK-TF-NORED-NEXT:    [[TMP4]] = fadd fast <vscale x 4 x float> [[WIDE_LOAD]], [[VEC_PHI]]
+; CHECK-TF-NORED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; CHECK-TF-NORED-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-TF-NORED-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-TF-NORED:       [[MIDDLE_BLOCK]]:
+; CHECK-TF-NORED-NEXT:    [[TMP6:%.*]] = call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> [[TMP4]])
+; CHECK-TF-NORED-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-TF-NORED-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-TF-NORED:       [[SCALAR_PH]]:
+; CHECK-TF-NORED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-TF-NORED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; CHECK-TF-NORED-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK-TF-NORED:       [[FOR_BODY]]:
+; CHECK-TF-NORED-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-TF-NORED-NEXT:    [[SUM_07:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[ADD:%.*]], %[[FOR_BODY]] ]
+; CHECK-TF-NORED-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
+; CHECK-TF-NORED-NEXT:    [[TMP7:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-TF-NORED-NEXT:    [[ADD]] = fadd fast float [[TMP7]], [[SUM_07]]
+; CHECK-TF-NORED-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-TF-NORED-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-TF-NORED-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-TF-NORED:       [[FOR_END]]:
+; CHECK-TF-NORED-NEXT:    [[ADD_LCSSA:%.*]] = phi float [ [[ADD]], %[[FOR_BODY]] ], [ [[TMP6]], %[[MIDDLE_BLOCK]] ]
+; CHECK-TF-NORED-NEXT:    ret float [[ADD_LCSSA]]
+;
+; CHECK-TF-NOREC-LABEL: define float @fadd_red_fast(
+; CHECK-TF-NOREC-SAME: ptr noalias readonly captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-NOREC-NEXT:  [[ENTRY:.*:]]
+; CHECK-TF-NOREC-NEXT:    br label %[[VECTOR_PH:.*]]
+; CHECK-TF-NOREC:       [[VECTOR_PH]]:
+; CHECK-TF-NOREC-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NOREC-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-NOREC-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
+; CHECK-TF-NOREC-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-NOREC:       [[VECTOR_BODY]]:
+; CHECK-TF-NOREC-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NOREC-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NOREC-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NOREC-NEXT:    [[TMP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-TF-NOREC-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0(ptr align 4 [[TMP2]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> poison)
+; CHECK-TF-NOREC-NEXT:    [[TMP3:%.*]] = fadd fast <vscale x 4 x float> [[WIDE_MASKED_LOAD]], [[VEC_PHI]]
+; CHECK-TF-NOREC-NEXT:    [[TMP4]] = select fast <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> [[TMP3]], <vscale x 4 x float> [[VEC_PHI]]
+; CHECK-TF-NOREC-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-TF-NOREC-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-TF-NOREC-NEXT:    [[TMP5:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-NOREC-NEXT:    [[TMP6:%.*]] = xor i1 [[TMP5]], true
+; CHECK-TF-NOREC-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-TF-NOREC:       [[MIDDLE_BLOCK]]:
+; CHECK-TF-NOREC-NEXT:    [[TMP7:%.*]] = call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> [[TMP4]])
+; CHECK-TF-NOREC-NEXT:    br label %[[FOR_END:.*]]
+; CHECK-TF-NOREC:       [[FOR_END]]:
+; CHECK-TF-NOREC-NEXT:    ret float [[TMP7]]
+;
+; CHECK-TF-NOREV-LABEL: define float @fadd_red_fast(
+; CHECK-TF-NOREV-SAME: ptr noalias readonly captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-NOREV-NEXT:  [[ENTRY:.*:]]
+; CHECK-TF-NOREV-NEXT:    br label %[[VECTOR_PH:.*]]
+; CHECK-TF-NOREV:       [[VECTOR_PH]]:
+; CHECK-TF-NOREV-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NOREV-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-NOREV-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
+; CHECK-TF-NOREV-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-NOREV:       [[VECTOR_BODY]]:
+; CHECK-TF-NOREV-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NOREV-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NOREV-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NOREV-NEXT:    [[TMP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-TF-NOREV-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0(ptr align 4 [[TMP2]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> poison)
+; CHECK-TF-NOREV-NEXT:    [[TMP3:%.*]] = fadd fast <vscale x 4 x float> [[WIDE_MASKED_LOAD]], [[VEC_PHI]]
+; CHECK-TF-NOREV-NEXT:    [[TMP4]] = select fast <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> [[TMP3]], <vscale x 4 x float> [[VEC_PHI]]
+; CHECK-TF-NOREV-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-TF-NOREV-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-TF-NOREV-NEXT:    [[TMP5:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-NOREV-NEXT:    [[TMP6:%.*]] = xor i1 [[TMP5]], true
+; CHECK-TF-NOREV-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-TF-NOREV:       [[MIDDLE_BLOCK]]:
+; CHECK-TF-NOREV-NEXT:    [[TMP7:%.*]] = call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> [[TMP4]])
+; CHECK-TF-NOREV-NEXT:    br label %[[FOR_END:.*]]
+; CHECK-TF-NOREV:       [[FOR_END]]:
+; CHECK-TF-NOREV-NEXT:    ret float [[TMP7]]
+;
+; CHECK-TF-ONLYRED-LABEL: define float @fadd_red_fast(
+; CHECK-TF-ONLYRED-SAME: ptr noalias readonly captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-ONLYRED-NEXT:  [[ENTRY:.*:]]
+; CHECK-TF-ONLYRED-NEXT:    br label %[[VECTOR_PH:.*]]
+; CHECK-TF-ONLYRED:       [[VECTOR_PH]]:
+; CHECK-TF-ONLYRED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-ONLYRED-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-ONLYRED-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
+; CHECK-TF-ONLYRED-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-ONLYRED:       [[VECTOR_BODY]]:
+; CHECK-TF-ONLYRED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-ONLYRED-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-ONLYRED-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-ONLYRED-NEXT:    [[TMP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-TF-ONLYRED-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0(ptr align 4 [[TMP2]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> poison)
+; CHECK-TF-ONLYRED-NEXT:    [[TMP3:%.*]] = fadd fast <vscale x 4 x float> [[WIDE_MASKED_LOAD]], [[VEC_PHI]]
+; CHECK-TF-ONLYRED-NEXT:    [[TMP4]] = select fast <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> [[TMP3]], <vscale x 4 x float> [[VEC_PHI]]
+; CHECK-TF-ONLYRED-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-TF-ONLYRED-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-TF-ONLYRED-NEXT:    [[TMP5:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-ONLYRED-NEXT:    [[TMP6:%.*]] = xor i1 [[TMP5]], true
+; CHECK-TF-ONLYRED-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-TF-ONLYRED:       [[MIDDLE_BLOCK]]:
+; CHECK-TF-ONLYRED-NEXT:    [[TMP7:%.*]] = call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> [[TMP4]])
+; CHECK-TF-ONLYRED-NEXT:    br label %[[FOR_END:.*]]
+; CHECK-TF-ONLYRED:       [[FOR_END]]:
+; CHECK-TF-ONLYRED-NEXT:    ret float [[TMP7]]
+;
+; CHECK-NEOVERSE-V1-LABEL: define float @fadd_red_fast(
+; CHECK-NEOVERSE-V1-SAME: ptr noalias readonly captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEOVERSE-V1-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NEOVERSE-V1-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-NEOVERSE-V1-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEOVERSE-V1:       [[VECTOR_PH]]:
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NEOVERSE-V1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP2]]
+; CHECK-NEOVERSE-V1-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEOVERSE-V1-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-NEOVERSE-V1:       [[VECTOR_BODY]]:
+; CHECK-NEOVERSE-V1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEOVERSE-V1-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP3:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEOVERSE-V1-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP3]], align 4
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP4]] = fadd fast <vscale x 4 x float> [[WIDE_LOAD]], [[VEC_PHI]]
+; CHECK-NEOVERSE-V1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEOVERSE-V1-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEOVERSE-V1:       [[MIDDLE_BLOCK]]:
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP6:%.*]] = call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> [[TMP4]])
+; CHECK-NEOVERSE-V1-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEOVERSE-V1-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-NEOVERSE-V1:       [[SCALAR_PH]]:
+; CHECK-NEOVERSE-V1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEOVERSE-V1-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; CHECK-NEOVERSE-V1-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK-NEOVERSE-V1:       [[FOR_BODY]]:
+; CHECK-NEOVERSE-V1-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEOVERSE-V1-NEXT:    [[SUM_07:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[ADD:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEOVERSE-V1-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP7:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-NEOVERSE-V1-NEXT:    [[ADD]] = fadd fast float [[TMP7]], [[SUM_07]]
+; CHECK-NEOVERSE-V1-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEOVERSE-V1-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEOVERSE-V1-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEOVERSE-V1:       [[FOR_END]]:
+; CHECK-NEOVERSE-V1-NEXT:    [[ADD_LCSSA:%.*]] = phi float [ [[ADD]], %[[FOR_BODY]] ], [ [[TMP6]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEOVERSE-V1-NEXT:    ret float [[ADD_LCSSA]]
+;
+
+
+
+
+
+
 
 entry:
   br label %for.body
@@ -169,96 +623,407 @@ for.end:
 }
 
 define void @add_recur(ptr noalias %dst, ptr noalias %src, i64 %n) #0 {
-; CHECK-NOTF-LABEL: @add_recur
-; CHECK-NOTF:       entry:
-; CHECK-NOTF:         %[[PRE:.*]] = load i32, ptr %src, align 4
-; CHECK-NOTF:       vector.ph:
-; CHECK-NOTF:         %[[RECUR_INIT:.*]] = insertelement <vscale x 4 x i32> poison, i32 %[[PRE]]
-; CHECK-NOTF:       vector.body:
-; CHECK-NOTF-NOT:     %{{.*}} = phi <vscale x 4 x i1>
-; CHECK-NOTF:         %[[VECTOR_RECUR:.*]] = phi <vscale x 4 x i32> [ %[[RECUR_INIT]], %vector.ph ], [ %[[LOAD:.*]], %vector.body ]
-; CHECK-NOTF:         %[[LOAD]] = load <vscale x 4 x i32>
-; CHECK-NOTF:         %[[SPLICE:.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> %[[VECTOR_RECUR]], <vscale x 4 x i32> %[[LOAD]], i32 1)
-; CHECK-NOTF:         %[[ADD:.*]] = add nsw <vscale x 4 x i32> %[[LOAD]], %[[SPLICE]]
-; CHECK-NOTF:         store <vscale x 4 x i32> %[[ADD]]
-
-; CHECK-TF-NORED-LABEL: @add_recur
-; CHECK-TF-NORED:       entry:
-; CHECK-TF-NORED:         %[[PRE:.*]] = load i32, ptr %src, align 4
-; CHECK-TF-NORED:       vector.ph:
-; CHECK-TF-NORED:         %[[RECUR_INIT:.*]] = insertelement <vscale x 4 x i32> poison, i32 %[[PRE]]
-; CHECK-TF-NORED:       vector.body:
-; CHECK-TF-NORED:         %[[ACTIVE_LANE_MASK:.*]] = phi <vscale x 4 x i1>
-; CHECK-TF-NORED:         %[[VECTOR_RECUR:.*]] = phi <vscale x 4 x i32> [ %[[RECUR_INIT]], %vector.ph ], [ %[[LOAD:.*]], %vector.body ]
-; CHECK-TF-NORED:         %[[LOAD]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0({{.*}} %[[ACTIVE_LANE_MASK]]
-; CHECK-TF-NORED:         %[[SPLICE:.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> %[[VECTOR_RECUR]], <vscale x 4 x i32> %[[LOAD]], i32 1)
-; CHECK-TF-NORED:         %[[ADD:.*]] = add nsw <vscale x 4 x i32> %[[LOAD]], %[[SPLICE]]
-; CHECK-TF-NORED:         call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> %[[ADD]], {{.*}} <vscale x 4 x i1> %[[ACTIVE_LANE_MASK]])
-
-; CHECK-TF-NOREC-LABEL: @add_recur
-; CHECK-TF-NOREC:       entry:
-; CHECK-TF-NOREC:         %[[PRE:.*]] = load i32, ptr %src, align 4
-; CHECK-TF-NOREC:       vector.ph:
-; CHECK-TF-NOREC:         %[[RECUR_INIT:.*]] = insertelement <vscale x 4 x i32> poison, i32 %[[PRE]]
-; CHECK-TF-NOREC:       vector.body:
-; CHECK-TF-NOREC-NOT:     %{{.*}} = phi <vscale x 4 x i1>
-; CHECK-TF-NOREC:         %[[VECTOR_RECUR:.*]] = phi <vscale x 4 x i32> [ %[[RECUR_INIT]], %vector.ph ], [ %[[LOAD:.*]], %vector.body ]
-; CHECK-TF-NOREC:         %[[LOAD]] = load <vscale x 4 x i32>
-; CHECK-TF-NOREC:         %[[SPLICE:.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> %[[VECTOR_RECUR]], <vscale x 4 x i32> %[[LOAD]], i32 1)
-; CHECK-TF-NOREC:         %[[ADD:.*]] = add nsw <vscale x 4 x i32> %[[LOAD]], %[[SPLICE]]
-; CHECK-TF-NOREC:         store <vscale x 4 x i32> %[[ADD]]
-
-; CHECK-TF-NOREV-LABEL: @add_recur
-; CHECK-TF-NOREV:       entry:
-; CHECK-TF-NOREV:         %[[PRE:.*]] = load i32, ptr %src, align 4
-; CHECK-TF-NOREV:       vector.ph:
-; CHECK-TF-NOREV:         %[[RECUR_INIT:.*]] = insertelement <vscale x 4 x i32> poison, i32 %[[PRE]]
-; CHECK-TF-NOREV:       vector.body:
-; CHECK-TF-NOREV:         %[[ACTIVE_LANE_MASK:.*]] = phi <vscale x 4 x i1>
-; CHECK-TF-NOREV:         %[[VECTOR_RECUR:.*]] = phi <vscale x 4 x i32> [ %[[RECUR_INIT]], %vector.ph ], [ %[[LOAD:.*]], %vector.body ]
-; CHECK-TF-NOREV:         %[[LOAD]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0({{.*}} %[[ACTIVE_LANE_MASK]]
-; CHECK-TF-NOREV:         %[[SPLICE:.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> %[[VECTOR_RECUR]], <vscale x 4 x i32> %[[LOAD]], i32 1)
-; CHECK-TF-NOREV:         %[[ADD:.*]] = add nsw <vscale x 4 x i32> %[[LOAD]], %[[SPLICE]]
-; CHECK-TF-NOREV:         call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> %[[ADD]], {{.*}} <vscale x 4 x i1> %[[ACTIVE_LANE_MASK]])
-
-; CHECK-TF-LABEL: @add_recur
-; CHECK-TF:       entry:
-; CHECK-TF:         %[[PRE:.*]] = load i32, ptr %src, align 4
-; CHECK-TF:       vector.ph:
-; CHECK-TF:         %[[RECUR_INIT:.*]] = insertelement <vscale x 4 x i32> poison, i32 %[[PRE]]
-; CHECK-TF:       vector.body:
-; CHECK-TF:         %[[ACTIVE_LANE_MASK:.*]] = phi <vscale x 4 x i1>
-; CHECK-TF:         %[[VECTOR_RECUR:.*]] = phi <vscale x 4 x i32> [ %[[RECUR_INIT]], %vector.ph ], [ %[[LOAD:.*]], %vector.body ]
-; CHECK-TF:         %[[LOAD]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0({{.*}} %[[ACTIVE_LANE_MASK]]
-; CHECK-TF:         %[[SPLICE:.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> %[[VECTOR_RECUR]], <vscale x 4 x i32> %[[LOAD]], i32 1)
-; CHECK-TF:         %[[ADD:.*]] = add nsw <vscale x 4 x i32> %[[LOAD]], %[[SPLICE]]
-; CHECK-TF:         call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> %[[ADD]], {{.*}} <vscale x 4 x i1> %[[ACTIVE_LANE_MASK]])
-
-; CHECK-TF-ONLYRED-LABEL: @add_recur
-; CHECK-TF-ONLYRED:       entry:
-; CHECK-TF-ONLYRED:         %[[PRE:.*]] = load i32, ptr %src, align 4
-; CHECK-TF-ONLYRED:       vector.ph:
-; CHECK-TF-ONLYRED:         %[[RECUR_INIT:.*]] = insertelement <vscale x 4 x i32> poison, i32 %[[PRE]]
-; CHECK-TF-ONLYRED:       vector.body:
-; CHECK-TF-ONLYRED-NOT:     %{{.*}} = phi <vscale x 4 x i1>
-; CHECK-TF-ONLYRED:         %[[VECTOR_RECUR:.*]] = phi <vscale x 4 x i32> [ %[[RECUR_INIT]], %vector.ph ], [ %[[LOAD:.*]], %vector.body ]
-; CHECK-TF-ONLYRED:         %[[LOAD]] = load <vscale x 4 x i32>
-; CHECK-TF-ONLYRED:         %[[SPLICE:.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> %[[VECTOR_RECUR]], <vscale x 4 x i32> %[[LOAD]], i32 1)
-; CHECK-TF-ONLYRED:         %[[ADD:.*]] = add nsw <vscale x 4 x i32> %[[LOAD]], %[[SPLICE]]
-; CHECK-TF-ONLYRED:         store <vscale x 4 x i32> %[[ADD]]
-
-; CHECK-NEOVERSE-V1-LABEL: @add_recur
-; CHECK-NEOVERSE-V1:       entry:
-; CHECK-NEOVERSE-V1:         %[[PRE:.*]] = load i32, ptr %src, align 4
-; CHECK-NEOVERSE-V1:       vector.ph:
-; CHECK-NEOVERSE-V1:         %[[RECUR_INIT:.*]] = insertelement <vscale x 4 x i32> poison, i32 %[[PRE]]
-; CHECK-NEOVERSE-V1:       vector.body:
-; CHECK-NEOVERSE-V1-NOT:     %{{.*}} = phi <vscale x 4 x i1>
-; CHECK-NEOVERSE-V1:         %[[VECTOR_RECUR:.*]] = phi <vscale x 4 x i32> [ %[[RECUR_INIT]], %vector.ph ], [ %[[LOAD:.*]], %vector.body ]
-; CHECK-NEOVERSE-V1:         %[[LOAD]] = load <vscale x 4 x i32>
-; CHECK-NEOVERSE-V1:         %[[SPLICE:.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> %[[VECTOR_RECUR]], <vscale x 4 x i32> %[[LOAD]], i32 1)
-; CHECK-NEOVERSE-V1:         %[[ADD:.*]] = add nsw <vscale x 4 x i32> %[[LOAD]], %[[SPLICE]]
-; CHECK-NEOVERSE-V1:         store <vscale x 4 x i32> %[[ADD]]
+; CHECK-NOTF-LABEL: define void @add_recur(
+; CHECK-NOTF-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NOTF-NEXT:  [[ENTRY:.*]]:
+; CHECK-NOTF-NEXT:    [[DOTPRE:%.*]] = load i32, ptr [[SRC]], align 4
+; CHECK-NOTF-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NOTF-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NOTF-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-NOTF-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NOTF:       [[VECTOR_PH]]:
+; CHECK-NOTF-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NOTF-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP2]]
+; CHECK-NOTF-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NOTF-NEXT:    [[TMP3:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NOTF-NEXT:    [[TMP4:%.*]] = mul nuw i32 [[TMP3]], 4
+; CHECK-NOTF-NEXT:    [[TMP5:%.*]] = sub i32 [[TMP4]], 1
+; CHECK-NOTF-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[DOTPRE]], i32 [[TMP5]]
+; CHECK-NOTF-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-NOTF:       [[VECTOR_BODY]]:
+; CHECK-NOTF-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NOTF-NEXT:    [[VECTOR_RECUR:%.*]] = phi <vscale x 4 x i32> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ [[WIDE_LOAD:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NOTF-NEXT:    [[TMP6:%.*]] = add nuw nsw i64 [[INDEX]], 1
+; CHECK-NOTF-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP6]]
+; CHECK-NOTF-NEXT:    [[WIDE_LOAD]] = load <vscale x 4 x i32>, ptr [[TMP7]], align 4
+; CHECK-NOTF-NEXT:    [[TMP8:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> [[VECTOR_RECUR]], <vscale x 4 x i32> [[WIDE_LOAD]], i32 1)
+; CHECK-NOTF-NEXT:    [[TMP9:%.*]] = add nsw <vscale x 4 x i32> [[WIDE_LOAD]], [[TMP8]]
+; CHECK-NOTF-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NOTF-NEXT:    store <vscale x 4 x i32> [[TMP9]], ptr [[TMP10]], align 4
+; CHECK-NOTF-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; CHECK-NOTF-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NOTF-NEXT:    br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NOTF:       [[MIDDLE_BLOCK]]:
+; CHECK-NOTF-NEXT:    [[TMP12:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NOTF-NEXT:    [[TMP13:%.*]] = mul nuw i32 [[TMP12]], 4
+; CHECK-NOTF-NEXT:    [[TMP14:%.*]] = sub i32 [[TMP13]], 1
+; CHECK-NOTF-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 4 x i32> [[WIDE_LOAD]], i32 [[TMP14]]
+; CHECK-NOTF-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NOTF-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-NOTF:       [[SCALAR_PH]]:
+; CHECK-NOTF-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ [[DOTPRE]], %[[ENTRY]] ]
+; CHECK-NOTF-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NOTF-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK-NOTF:       [[FOR_BODY]]:
+; CHECK-NOTF-NEXT:    [[TMP15:%.*]] = phi i32 [ [[TMP16:%.*]], %[[FOR_BODY]] ], [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ]
+; CHECK-NOTF-NEXT:    [[I_010:%.*]] = phi i64 [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NOTF-NEXT:    [[ADD]] = add nuw nsw i64 [[I_010]], 1
+; CHECK-NOTF-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[ADD]]
+; CHECK-NOTF-NEXT:    [[TMP16]] = load i32, ptr [[ARRAYIDX1]], align 4
+; CHECK-NOTF-NEXT:    [[ADD2:%.*]] = add nsw i32 [[TMP16]], [[TMP15]]
+; CHECK-NOTF-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[I_010]]
+; CHECK-NOTF-NEXT:    store i32 [[ADD2]], ptr [[ARRAYIDX3]], align 4
+; CHECK-NOTF-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[ADD]], [[N]]
+; CHECK-NOTF-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-NOTF:       [[FOR_END]]:
+; CHECK-NOTF-NEXT:    ret void
+;
+; CHECK-TF-ALL-LABEL: define void @add_recur(
+; CHECK-TF-ALL-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-ALL-NEXT:  [[ENTRY:.*:]]
+; CHECK-TF-ALL-NEXT:    [[DOTPRE:%.*]] = load i32, ptr [[SRC]], align 4
+; CHECK-TF-ALL-NEXT:    br label %[[VECTOR_PH:.*]]
+; CHECK-TF-ALL:       [[VECTOR_PH]]:
+; CHECK-TF-ALL-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-ALL-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-ALL-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
+; CHECK-TF-ALL-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-TF-ALL-NEXT:    [[TMP3:%.*]] = mul nuw i32 [[TMP2]], 4
+; CHECK-TF-ALL-NEXT:    [[TMP4:%.*]] = sub i32 [[TMP3]], 1
+; CHECK-TF-ALL-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[DOTPRE]], i32 [[TMP4]]
+; CHECK-TF-ALL-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-ALL:       [[VECTOR_BODY]]:
+; CHECK-TF-ALL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-ALL-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-ALL-NEXT:    [[VECTOR_RECUR:%.*]] = phi <vscale x 4 x i32> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ [[WIDE_MASKED_LOAD:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-ALL-NEXT:    [[TMP5:%.*]] = add nuw nsw i64 [[INDEX]], 1
+; CHECK-TF-ALL-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP5]]
+; CHECK-TF-ALL-NEXT:    [[WIDE_MASKED_LOAD]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 4 [[TMP6]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison)
+; CHECK-TF-ALL-NEXT:    [[TMP7:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> [[VECTOR_RECUR]], <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], i32 1)
+; CHECK-TF-ALL-NEXT:    [[TMP8:%.*]] = add nsw <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], [[TMP7]]
+; CHECK-TF-ALL-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[INDEX]]
+; CHECK-TF-ALL-NEXT:    call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP8]], ptr align 4 [[TMP9]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-ALL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-TF-ALL-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-TF-ALL-NEXT:    [[TMP10:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-ALL-NEXT:    [[TMP11:%.*]] = xor i1 [[TMP10]], true
+; CHECK-TF-ALL-NEXT:    br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-TF-ALL:       [[MIDDLE_BLOCK]]:
+; CHECK-TF-ALL-NEXT:    br label %[[FOR_END:.*]]
+; CHECK-TF-ALL:       [[FOR_END]]:
+; CHECK-TF-ALL-NEXT:    ret void
+;
+; CHECK-TF-LABEL: define void @add_recur(
+; CHECK-TF-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-NEXT:  [[ENTRY:.*:]]
+; CHECK-TF-NEXT:    [[DOTPRE:%.*]] = load i32, ptr [[SRC]], align 4
+; CHECK-TF-NEXT:    br label %[[VECTOR_PH:.*]]
+; CHECK-TF:       [[VECTOR_PH]]:
+; CHECK-TF-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
+; CHECK-TF-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-TF-NEXT:    [[TMP3:%.*]] = mul nuw i32 [[TMP2]], 4
+; CHECK-TF-NEXT:    [[TMP4:%.*]] = sub i32 [[TMP3]], 1
+; CHECK-TF-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[DOTPRE]], i32 [[TMP4]]
+; CHECK-TF-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-TF:       [[VECTOR_BODY]]:
+; CHECK-TF-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NEXT:    [[VECTOR_RECUR:%.*]] = phi <vscale x 4 x i32> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ [[WIDE_MASKED_LOAD:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NEXT:    [[TMP5:%.*]] = add nuw nsw i64 [[INDEX]], 1
+; CHECK-TF-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP5]]
+; CHECK-TF-NEXT:    [[WIDE_MASKED_LOAD]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 4 [[TMP6]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison)
+; CHECK-TF-NEXT:    [[TMP7:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> [[VECTOR_RECUR]], <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], i32 1)
+; CHECK-TF-NEXT:    [[TMP8:%.*]] = add nsw <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], [[TMP7]]
+; CHECK-TF-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[INDEX]]
+; CHECK-TF-NEXT:    call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP8]], ptr align 4 [[TMP9]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-TF-NEXT:    [[TMP10:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-NEXT:    [[TMP11:%.*]] = xor i1 [[TMP10]], true
+; CHECK-TF-NEXT:    br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-TF:       [[MIDDLE_BLOCK]]:
+; CHECK-TF-NEXT:    br label %[[FOR_END:.*]]
+; CHECK-TF:       [[FOR_END]]:
+; CHECK-TF-NEXT:    ret void
+;
+; CHECK-TF-DEFAULT-LABEL: define void @add_recur(
+; CHECK-TF-DEFAULT-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-DEFAULT-NEXT:  [[ENTRY:.*:]]
+; CHECK-TF-DEFAULT-NEXT:    [[DOTPRE:%.*]] = load i32, ptr [[SRC]], align 4
+; CHECK-TF-DEFAULT-NEXT:    br label %[[VECTOR_PH:.*]]
+; CHECK-TF-DEFAULT:       [[VECTOR_PH]]:
+; CHECK-TF-DEFAULT-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-DEFAULT-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-DEFAULT-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
+; CHECK-TF-DEFAULT-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-TF-DEFAULT-NEXT:    [[TMP3:%.*]] = mul nuw i32 [[TMP2]], 4
+; CHECK-TF-DEFAULT-NEXT:    [[TMP4:%.*]] = sub i32 [[TMP3]], 1
+; CHECK-TF-DEFAULT-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[DOTPRE]], i32 [[TMP4]]
+; CHECK-TF-DEFAULT-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-DEFAULT:       [[VECTOR_BODY]]:
+; CHECK-TF-DEFAULT-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-DEFAULT-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-DEFAULT-NEXT:    [[VECTOR_RECUR:%.*]] = phi <vscale x 4 x i32> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ [[WIDE_MASKED_LOAD:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-DEFAULT-NEXT:    [[TMP5:%.*]] = add nuw nsw i64 [[INDEX]], 1
+; CHECK-TF-DEFAULT-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP5]]
+; CHECK-TF-DEFAULT-NEXT:    [[WIDE_MASKED_LOAD]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 4 [[TMP6]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison)
+; CHECK-TF-DEFAULT-NEXT:    [[TMP7:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> [[VECTOR_RECUR]], <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], i32 1)
+; CHECK-TF-DEFAULT-NEXT:    [[TMP8:%.*]] = add nsw <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], [[TMP7]]
+; CHECK-TF-DEFAULT-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[INDEX]]
+; CHECK-TF-DEFAULT-NEXT:    call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP8]], ptr align 4 [[TMP9]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-DEFAULT-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-TF-DEFAULT-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-TF-DEFAULT-NEXT:    [[TMP10:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-DEFAULT-NEXT:    [[TMP11:%.*]] = xor i1 [[TMP10]], true
+; CHECK-TF-DEFAULT-NEXT:    br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-TF-DEFAULT:       [[MIDDLE_BLOCK]]:
+; CHECK-TF-DEFAULT-NEXT:    br label %[[FOR_END:.*]]
+; CHECK-TF-DEFAULT:       [[FOR_END]]:
+; CHECK-TF-DEFAULT-NEXT:    ret void
+;
+; CHECK-TF-NORED-LABEL: define void @add_recur(
+; CHECK-TF-NORED-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-NORED-NEXT:  [[ENTRY:.*:]]
+; CHECK-TF-NORED-NEXT:    [[DOTPRE:%.*]] = load i32, ptr [[SRC]], align 4
+; CHECK-TF-NORED-NEXT:    br label %[[VECTOR_PH:.*]]
+; CHECK-TF-NORED:       [[VECTOR_PH]]:
+; CHECK-TF-NORED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NORED-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-NORED-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
+; CHECK-TF-NORED-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-TF-NORED-NEXT:    [[TMP3:%.*]] = mul nuw i32 [[TMP2]], 4
+; CHECK-TF-NORED-NEXT:    [[TMP4:%.*]] = sub i32 [[TMP3]], 1
+; CHECK-TF-NORED-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[DOTPRE]], i32 [[TMP4]]
+; CHECK-TF-NORED-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-NORED:       [[VECTOR_BODY]]:
+; CHECK-TF-NORED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NORED-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NORED-NEXT:    [[VECTOR_RECUR:%.*]] = phi <vscale x 4 x i32> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ [[WIDE_MASKED_LOAD:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NORED-NEXT:    [[TMP5:%.*]] = add nuw nsw i64 [[INDEX]], 1
+; CHECK-TF-NORED-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP5]]
+; CHECK-TF-NORED-NEXT:    [[WIDE_MASKED_LOAD]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 4 [[TMP6]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison)
+; CHECK-TF-NORED-NEXT:    [[TMP7:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> [[VECTOR_RECUR]], <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], i32 1)
+; CHECK-TF-NORED-NEXT:    [[TMP8:%.*]] = add nsw <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], [[TMP7]]
+; CHECK-TF-NORED-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[INDEX]]
+; CHECK-TF-NORED-NEXT:    call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP8]], ptr align 4 [[TMP9]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-NORED-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-TF-NORED-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-TF-NORED-NEXT:    [[TMP10:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-NORED-NEXT:    [[TMP11:%.*]] = xor i1 [[TMP10]], true
+; CHECK-TF-NORED-NEXT:    br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-TF-NORED:       [[MIDDLE_BLOCK]]:
+; CHECK-TF-NORED-NEXT:    br label %[[FOR_END:.*]]
+; CHECK-TF-NORED:       [[FOR_END]]:
+; CHECK-TF-NORED-NEXT:    ret void
+;
+; CHECK-TF-NOREC-LABEL: define void @add_recur(
+; CHECK-TF-NOREC-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-NOREC-NEXT:  [[ENTRY:.*]]:
+; CHECK-TF-NOREC-NEXT:    [[DOTPRE:%.*]] = load i32, ptr [[SRC]], align 4
+; CHECK-TF-NOREC-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NOREC-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-NOREC-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-TF-NOREC-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-TF-NOREC:       [[VECTOR_PH]]:
+; CHECK-TF-NOREC-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-NOREC-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP2]]
+; CHECK-TF-NOREC-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-TF-NOREC-NEXT:    [[TMP3:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-TF-NOREC-NEXT:    [[TMP4:%.*]] = mul nuw i32 [[TMP3]], 4
+; CHECK-TF-NOREC-NEXT:    [[TMP5:%.*]] = sub i32 [[TMP4]], 1
+; CHECK-TF-NOREC-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[DOTPRE]], i32 [[TMP5]]
+; CHECK-TF-NOREC-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-NOREC:       [[VECTOR_BODY]]:
+; CHECK-TF-NOREC-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NOREC-NEXT:    [[VECTOR_RECUR:%.*]] = phi <vscale x 4 x i32> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ [[WIDE_LOAD:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NOREC-NEXT:    [[TMP6:%.*]] = add nuw nsw i64 [[INDEX]], 1
+; CHECK-TF-NOREC-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP6]]
+; CHECK-TF-NOREC-NEXT:    [[WIDE_LOAD]] = load <vscale x 4 x i32>, ptr [[TMP7]], align 4
+; CHECK-TF-NOREC-NEXT:    [[TMP8:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> [[VECTOR_RECUR]], <vscale x 4 x i32> [[WIDE_LOAD]], i32 1)
+; CHECK-TF-NOREC-NEXT:    [[TMP9:%.*]] = add nsw <vscale x 4 x i32> [[WIDE_LOAD]], [[TMP8]]
+; CHECK-TF-NOREC-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[INDEX]]
+; CHECK-TF-NOREC-NEXT:    store <vscale x 4 x i32> [[TMP9]], ptr [[TMP10]], align 4
+; CHECK-TF-NOREC-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; CHECK-TF-NOREC-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-TF-NOREC-NEXT:    br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-TF-NOREC:       [[MIDDLE_BLOCK]]:
+; CHECK-TF-NOREC-NEXT:    [[TMP12:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-TF-NOREC-NEXT:    [[TMP13:%.*]] = mul nuw i32 [[TMP12]], 4
+; CHECK-TF-NOREC-NEXT:    [[TMP14:%.*]] = sub i32 [[TMP13]], 1
+; CHECK-TF-NOREC-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 4 x i32> [[WIDE_LOAD]], i32 [[TMP14]]
+; CHECK-TF-NOREC-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-TF-NOREC-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-TF-NOREC:       [[SCALAR_PH]]:
+; CHECK-TF-NOREC-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ [[DOTPRE]], %[[ENTRY]] ]
+; CHECK-TF-NOREC-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-TF-NOREC-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK-TF-NOREC:       [[FOR_BODY]]:
+; CHECK-TF-NOREC-NEXT:    [[TMP15:%.*]] = phi i32 [ [[TMP16:%.*]], %[[FOR_BODY]] ], [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ]
+; CHECK-TF-NOREC-NEXT:    [[I_010:%.*]] = phi i64 [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-TF-NOREC-NEXT:    [[ADD]] = add nuw nsw i64 [[I_010]], 1
+; CHECK-TF-NOREC-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[ADD]]
+; CHECK-TF-NOREC-NEXT:    [[TMP16]] = load i32, ptr [[ARRAYIDX1]], align 4
+; CHECK-TF-NOREC-NEXT:    [[ADD2:%.*]] = add nsw i32 [[TMP16]], [[TMP15]]
+; CHECK-TF-NOREC-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[I_010]]
+; CHECK-TF-NOREC-NEXT:    store i32 [[ADD2]], ptr [[ARRAYIDX3]], align 4
+; CHECK-TF-NOREC-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[ADD]], [[N]]
+; CHECK-TF-NOREC-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-TF-NOREC:       [[FOR_END]]:
+; CHECK-TF-NOREC-NEXT:    ret void
+;
+; CHECK-TF-NOREV-LABEL: define void @add_recur(
+; CHECK-TF-NOREV-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-NOREV-NEXT:  [[ENTRY:.*:]]
+; CHECK-TF-NOREV-NEXT:    [[DOTPRE:%.*]] = load i32, ptr [[SRC]], align 4
+; CHECK-TF-NOREV-NEXT:    br label %[[VECTOR_PH:.*]]
+; CHECK-TF-NOREV:       [[VECTOR_PH]]:
+; CHECK-TF-NOREV-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NOREV-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-NOREV-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
+; CHECK-TF-NOREV-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-TF-NOREV-NEXT:    [[TMP3:%.*]] = mul nuw i32 [[TMP2]], 4
+; CHECK-TF-NOREV-NEXT:    [[TMP4:%.*]] = sub i32 [[TMP3]], 1
+; CHECK-TF-NOREV-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[DOTPRE]], i32 [[TMP4]]
+; CHECK-TF-NOREV-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-NOREV:       [[VECTOR_BODY]]:
+; CHECK-TF-NOREV-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NOREV-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NOREV-NEXT:    [[VECTOR_RECUR:%.*]] = phi <vscale x 4 x i32> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ [[WIDE_MASKED_LOAD:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NOREV-NEXT:    [[TMP5:%.*]] = add nuw nsw i64 [[INDEX]], 1
+; CHECK-TF-NOREV-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP5]]
+; CHECK-TF-NOREV-NEXT:    [[WIDE_MASKED_LOAD]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 4 [[TMP6]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison)
+; CHECK-TF-NOREV-NEXT:    [[TMP7:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> [[VECTOR_RECUR]], <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], i32 1)
+; CHECK-TF-NOREV-NEXT:    [[TMP8:%.*]] = add nsw <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], [[TMP7]]
+; CHECK-TF-NOREV-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[INDEX]]
+; CHECK-TF-NOREV-NEXT:    call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP8]], ptr align 4 [[TMP9]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-NOREV-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-TF-NOREV-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-TF-NOREV-NEXT:    [[TMP10:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-NOREV-NEXT:    [[TMP11:%.*]] = xor i1 [[TMP10]], true
+; CHECK-TF-NOREV-NEXT:    br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-TF-NOREV:       [[MIDDLE_BLOCK]]:
+; CHECK-TF-NOREV-NEXT:    br label %[[FOR_END:.*]]
+; CHECK-TF-NOREV:       [[FOR_END]]:
+; CHECK-TF-NOREV-NEXT:    ret void
+;
+; CHECK-TF-ONLYRED-LABEL: define void @add_recur(
+; CHECK-TF-ONLYRED-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-ONLYRED-NEXT:  [[ENTRY:.*]]:
+; CHECK-TF-ONLYRED-NEXT:    [[DOTPRE:%.*]] = load i32, ptr [[SRC]], align 4
+; CHECK-TF-ONLYRED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-ONLYRED-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-ONLYRED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-TF-ONLYRED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-TF-ONLYRED:       [[VECTOR_PH]]:
+; CHECK-TF-ONLYRED-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-ONLYRED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP2]]
+; CHECK-TF-ONLYRED-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-TF-ONLYRED-NEXT:    [[TMP3:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-TF-ONLYRED-NEXT:    [[TMP4:%.*]] = mul nuw i32 [[TMP3]], 4
+; CHECK-TF-ONLYRED-NEXT:    [[TMP5:%.*]] = sub i32 [[TMP4]], 1
+; CHECK-TF-ONLYRED-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[DOTPRE]], i32 [[TMP5]]
+; CHECK-TF-ONLYRED-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-ONLYRED:       [[VECTOR_BODY]]:
+; CHECK-TF-ONLYRED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-ONLYRED-NEXT:    [[VECTOR_RECUR:%.*]] = phi <vscale x 4 x i32> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ [[WIDE_LOAD:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-ONLYRED-NEXT:    [[TMP6:%.*]] = add nuw nsw i64 [[INDEX]], 1
+; CHECK-TF-ONLYRED-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP6]]
+; CHECK-TF-ONLYRED-NEXT:    [[WIDE_LOAD]] = load <vscale x 4 x i32>, ptr [[TMP7]], align 4
+; CHECK-TF-ONLYRED-NEXT:    [[TMP8:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> [[VECTOR_RECUR]], <vscale x 4 x i32> [[WIDE_LOAD]], i32 1)
+; CHECK-TF-ONLYRED-NEXT:    [[TMP9:%.*]] = add nsw <vscale x 4 x i32> [[WIDE_LOAD]], [[TMP8]]
+; CHECK-TF-ONLYRED-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[INDEX]]
+; CHECK-TF-ONLYRED-NEXT:    store <vscale x 4 x i32> [[TMP9]], ptr [[TMP10]], align 4
+; CHECK-TF-ONLYRED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; CHECK-TF-ONLYRED-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-TF-ONLYRED-NEXT:    br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-TF-ONLYRED:       [[MIDDLE_BLOCK]]:
+; CHECK-TF-ONLYRED-NEXT:    [[TMP12:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-TF-ONLYRED-NEXT:    [[TMP13:%.*]] = mul nuw i32 [[TMP12]], 4
+; CHECK-TF-ONLYRED-NEXT:    [[TMP14:%.*]] = sub i32 [[TMP13]], 1
+; CHECK-TF-ONLYRED-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 4 x i32> [[WIDE_LOAD]], i32 [[TMP14]]
+; CHECK-TF-ONLYRED-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-TF-ONLYRED-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-TF-ONLYRED:       [[SCALAR_PH]]:
+; CHECK-TF-ONLYRED-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ [[DOTPRE]], %[[ENTRY]] ]
+; CHECK-TF-ONLYRED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-TF-ONLYRED-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK-TF-ONLYRED:       [[FOR_BODY]]:
+; CHECK-TF-ONLYRED-NEXT:    [[TMP15:%.*]] = phi i32 [ [[TMP16:%.*]], %[[FOR_BODY]] ], [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ]
+; CHECK-TF-ONLYRED-NEXT:    [[I_010:%.*]] = phi i64 [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-TF-ONLYRED-NEXT:    [[ADD]] = add nuw nsw i64 [[I_010]], 1
+; CHECK-TF-ONLYRED-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[ADD]]
+; CHECK-TF-ONLYRED-NEXT:    [[TMP16]] = load i32, ptr [[ARRAYIDX1]], align 4
+; CHECK-TF-ONLYRED-NEXT:    [[ADD2:%.*]] = add nsw i32 [[TMP16]], [[TMP15]]
+; CHECK-TF-ONLYRED-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[I_010]]
+; CHECK-TF-ONLYRED-NEXT:    store i32 [[ADD2]], ptr [[ARRAYIDX3]], align 4
+; CHECK-TF-ONLYRED-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[ADD]], [[N]]
+; CHECK-TF-ONLYRED-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-TF-ONLYRED:       [[FOR_END]]:
+; CHECK-TF-ONLYRED-NEXT:    ret void
+;
+; CHECK-NEOVERSE-V1-LABEL: define void @add_recur(
+; CHECK-NEOVERSE-V1-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEOVERSE-V1-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEOVERSE-V1-NEXT:    [[DOTPRE:%.*]] = load i32, ptr [[SRC]], align 4
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NEOVERSE-V1-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-NEOVERSE-V1-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEOVERSE-V1:       [[VECTOR_PH]]:
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NEOVERSE-V1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP2]]
+; CHECK-NEOVERSE-V1-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP3:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP4:%.*]] = mul nuw i32 [[TMP3]], 4
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP5:%.*]] = sub i32 [[TMP4]], 1
+; CHECK-NEOVERSE-V1-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[DOTPRE]], i32 [[TMP5]]
+; CHECK-NEOVERSE-V1-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-NEOVERSE-V1:       [[VECTOR_BODY]]:
+; CHECK-NEOVERSE-V1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEOVERSE-V1-NEXT:    [[VECTOR_RECUR:%.*]] = phi <vscale x 4 x i32> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ [[WIDE_LOAD:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP6:%.*]] = add nuw nsw i64 [[INDEX]], 1
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP6]]
+; CHECK-NEOVERSE-V1-NEXT:    [[WIDE_LOAD]] = load <vscale x 4 x i32>, ptr [[TMP7]], align 4
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP8:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> [[VECTOR_RECUR]], <vscale x 4 x i32> [[WIDE_LOAD]], i32 1)
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP9:%.*]] = add nsw <vscale x 4 x i32> [[WIDE_LOAD]], [[TMP8]]
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEOVERSE-V1-NEXT:    store <vscale x 4 x i32> [[TMP9]], ptr [[TMP10]], align 4
+; CHECK-NEOVERSE-V1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEOVERSE-V1-NEXT:    br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEOVERSE-V1:       [[MIDDLE_BLOCK]]:
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP12:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP13:%.*]] = mul nuw i32 [[TMP12]], 4
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP14:%.*]] = sub i32 [[TMP13]], 1
+; CHECK-NEOVERSE-V1-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 4 x i32> [[WIDE_LOAD]], i32 [[TMP14]]
+; CHECK-NEOVERSE-V1-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEOVERSE-V1-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-NEOVERSE-V1:       [[SCALAR_PH]]:
+; CHECK-NEOVERSE-V1-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ [[DOTPRE]], %[[ENTRY]] ]
+; CHECK-NEOVERSE-V1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEOVERSE-V1-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK-NEOVERSE-V1:       [[FOR_BODY]]:
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP15:%.*]] = phi i32 [ [[TMP16:%.*]], %[[FOR_BODY]] ], [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ]
+; CHECK-NEOVERSE-V1-NEXT:    [[I_010:%.*]] = phi i64 [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEOVERSE-V1-NEXT:    [[ADD]] = add nuw nsw i64 [[I_010]], 1
+; CHECK-NEOVERSE-V1-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[ADD]]
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP16]] = load i32, ptr [[ARRAYIDX1]], align 4
+; CHECK-NEOVERSE-V1-NEXT:    [[ADD2:%.*]] = add nsw i32 [[TMP16]], [[TMP15]]
+; CHECK-NEOVERSE-V1-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[I_010]]
+; CHECK-NEOVERSE-V1-NEXT:    store i32 [[ADD2]], ptr [[ARRAYIDX3]], align 4
+; CHECK-NEOVERSE-V1-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[ADD]], [[N]]
+; CHECK-NEOVERSE-V1-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEOVERSE-V1:       [[FOR_END]]:
+; CHECK-NEOVERSE-V1-NEXT:    ret void
+;
+
+
+
+
+
+
 
 entry:
   %.pre = load i32, ptr %src, align 4
@@ -281,41 +1046,605 @@ for.end:
 }
 
 define void @interleave(ptr noalias %dst, ptr noalias %src, i64 %n) #0 {
-; CHECK-NOTF-LABEL: @interleave(
-; CHECK-NOTF:       vector.body:
-; CHECK-NOTF:         %[[LOAD:.*]] = load <8 x float>, ptr
-; CHECK-NOTF:         %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NOTF:         %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-
-; CHECK-TF-LABEL: @interleave(
-; CHECK-TF:       vector.body:
-; CHECK-TF:         %[[LOAD:.*]] = load <8 x float>, ptr
-; CHECK-TF:         %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-TF:         %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-
-; CHECK-TF-NORED-LABEL: @interleave(
-; CHECK-TF-NORED:       vector.body:
-; CHECK-TF-NORED:         %[[LOAD:.*]] = load <8 x float>, ptr
-; CHECK-TF-NORED:         %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-TF-NORED:         %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-
-; CHECK-TF-NOREC-LABEL: @interleave(
-; CHECK-TF-NOREC:       vector.body:
-; CHECK-TF-NOREC:         %[[LOAD:.*]] = load <8 x float>, ptr
-; CHECK-TF-NOREC:         %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-TF-NOREC:         %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-
-; CHECK-TF-NOREV-LABEL: @interleave(
-; CHECK-TF-NOREV:       vector.body:
-; CHECK-TF-NOREV:         %[[LOAD:.*]] = load <8 x float>, ptr
-; CHECK-TF-NOREV:         %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-TF-NOREV:         %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-
-; CHECK-NEOVERSE-V1-LABEL: @interleave(
-; CHECK-NEOVERSE-V1:       vector.body:
-; CHECK-NEOVERSE-V1:         %[[LOAD:.*]] = load <8 x float>, ptr
-; CHECK-NEOVERSE-V1:         %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NEOVERSE-V1:         %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NOTF-LABEL: define void @interleave(
+; CHECK-NOTF-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NOTF-NEXT:  [[ENTRY:.*]]:
+; CHECK-NOTF-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; CHECK-NOTF-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NOTF:       [[VECTOR_PH]]:
+; CHECK-NOTF-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
+; CHECK-NOTF-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NOTF-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-NOTF:       [[VECTOR_BODY]]:
+; CHECK-NOTF-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NOTF-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 4
+; CHECK-NOTF-NEXT:    [[TMP1:%.*]] = shl nuw nsw i64 [[INDEX]], 1
+; CHECK-NOTF-NEXT:    [[TMP2:%.*]] = shl nuw nsw i64 [[TMP0]], 1
+; CHECK-NOTF-NEXT:    [[TMP3:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP1]]
+; CHECK-NOTF-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP2]]
+; CHECK-NOTF-NEXT:    [[WIDE_VEC:%.*]] = load <8 x float>, ptr [[TMP3]], align 4
+; CHECK-NOTF-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <8 x float> [[WIDE_VEC]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NOTF-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <8 x float> [[WIDE_VEC]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NOTF-NEXT:    [[WIDE_VEC2:%.*]] = load <8 x float>, ptr [[TMP4]], align 4
+; CHECK-NOTF-NEXT:    [[STRIDED_VEC3:%.*]] = shufflevector <8 x float> [[WIDE_VEC2]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NOTF-NEXT:    [[STRIDED_VEC4:%.*]] = shufflevector <8 x float> [[WIDE_VEC2]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NOTF-NEXT:    [[TMP5:%.*]] = mul nuw nsw i64 [[INDEX]], 3
+; CHECK-NOTF-NEXT:    [[TMP6:%.*]] = mul nuw nsw i64 [[TMP0]], 3
+; CHECK-NOTF-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP5]]
+; CHECK-NOTF-NEXT:    [[TMP8:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP6]]
+; CHECK-NOTF-NEXT:    [[TMP9:%.*]] = shufflevector <4 x float> [[STRIDED_VEC]], <4 x float> [[STRIDED_VEC1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NOTF-NEXT:    [[TMP10:%.*]] = shufflevector <8 x float> [[TMP9]], <8 x float> <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float undef, float undef, float undef, float undef>, <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
+; CHECK-NOTF-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <12 x float> [[TMP10]], <12 x float> poison, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+; CHECK-NOTF-NEXT:    store <12 x float> [[INTERLEAVED_VEC]], ptr [[TMP7]], align 4
+; CHECK-NOTF-NEXT:    [[TMP11:%.*]] = shufflevector <4 x float> [[STRIDED_VEC3]], <4 x float> [[STRIDED_VEC4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NOTF-NEXT:    [[TMP12:%.*]] = shufflevector <8 x float> [[TMP11]], <8 x float> <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float undef, float undef, float undef, float undef>, <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
+; CHECK-NOTF-NEXT:    [[INTERLEAVED_VEC5:%.*]] = shufflevector <12 x float> [[TMP12]], <12 x float> poison, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+; CHECK-NOTF-NEXT:    store <12 x float> [[INTERLEAVED_VEC5]], ptr [[TMP8]], align 4
+; CHECK-NOTF-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-NOTF-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NOTF-NEXT:    br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NOTF:       [[MIDDLE_BLOCK]]:
+; CHECK-NOTF-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NOTF-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-NOTF:       [[SCALAR_PH]]:
+; CHECK-NOTF-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NOTF-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK-NOTF:       [[FOR_BODY]]:
+; CHECK-NOTF-NEXT:    [[I_021:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NOTF-NEXT:    [[MUL:%.*]] = shl nuw nsw i64 [[I_021]], 1
+; CHECK-NOTF-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[MUL]]
+; CHECK-NOTF-NEXT:    [[TMP14:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-NOTF-NEXT:    [[MUL1:%.*]] = mul nuw nsw i64 [[I_021]], 3
+; CHECK-NOTF-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[MUL1]]
+; CHECK-NOTF-NEXT:    store float [[TMP14]], ptr [[ARRAYIDX2]], align 4
+; CHECK-NOTF-NEXT:    [[ADD:%.*]] = or disjoint i64 [[MUL]], 1
+; CHECK-NOTF-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[ADD]]
+; CHECK-NOTF-NEXT:    [[TMP15:%.*]] = load float, ptr [[ARRAYIDX4]], align 4
+; CHECK-NOTF-NEXT:    [[ADD6:%.*]] = add nuw nsw i64 [[MUL1]], 1
+; CHECK-NOTF-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD6]]
+; CHECK-NOTF-NEXT:    store float [[TMP15]], ptr [[ARRAYIDX7]], align 4
+; CHECK-NOTF-NEXT:    [[ADD9:%.*]] = add nuw nsw i64 [[MUL1]], 2
+; CHECK-NOTF-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD9]]
+; CHECK-NOTF-NEXT:    store float 3.000000e+00, ptr [[ARRAYIDX10]], align 4
+; CHECK-NOTF-NEXT:    [[INC]] = add nuw nsw i64 [[I_021]], 1
+; CHECK-NOTF-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]]
+; CHECK-NOTF-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-NOTF:       [[FOR_END]]:
+; CHECK-NOTF-NEXT:    ret void
+;
+; CHECK-TF-ALL-LABEL: define void @interleave(
+; CHECK-TF-ALL-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-ALL-NEXT:  [[ENTRY:.*]]:
+; CHECK-TF-ALL-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; CHECK-TF-ALL-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-TF-ALL:       [[VECTOR_PH]]:
+; CHECK-TF-ALL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
+; CHECK-TF-ALL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-TF-ALL-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-ALL:       [[VECTOR_BODY]]:
+; CHECK-TF-ALL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-ALL-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 4
+; CHECK-TF-ALL-NEXT:    [[TMP1:%.*]] = shl nuw nsw i64 [[INDEX]], 1
+; CHECK-TF-ALL-NEXT:    [[TMP2:%.*]] = shl nuw nsw i64 [[TMP0]], 1
+; CHECK-TF-ALL-NEXT:    [[TMP3:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP1]]
+; CHECK-TF-ALL-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP2]]
+; CHECK-TF-ALL-NEXT:    [[WIDE_VEC:%.*]] = load <8 x float>, ptr [[TMP3]], align 4
+; CHECK-TF-ALL-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <8 x float> [[WIDE_VEC]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-TF-ALL-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <8 x float> [[WIDE_VEC]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-TF-ALL-NEXT:    [[WIDE_VEC2:%.*]] = load <8 x float>, ptr [[TMP4]], align 4
+; CHECK-TF-ALL-NEXT:    [[STRIDED_VEC3:%.*]] = shufflevector <8 x float> [[WIDE_VEC2]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-TF-ALL-NEXT:    [[STRIDED_VEC4:%.*]] = shufflevector <8 x float> [[WIDE_VEC2]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-TF-ALL-NEXT:    [[TMP5:%.*]] = mul nuw nsw i64 [[INDEX]], 3
+; CHECK-TF-ALL-NEXT:    [[TMP6:%.*]] = mul nuw nsw i64 [[TMP0]], 3
+; CHECK-TF-ALL-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP5]]
+; CHECK-TF-ALL-NEXT:    [[TMP8:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP6]]
+; CHECK-TF-ALL-NEXT:    [[TMP9:%.*]] = shufflevector <4 x float> [[STRIDED_VEC]], <4 x float> [[STRIDED_VEC1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-TF-ALL-NEXT:    [[TMP10:%.*]] = shufflevector <8 x float> [[TMP9]], <8 x float> <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float undef, float undef, float undef, float undef>, <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
+; CHECK-TF-ALL-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <12 x float> [[TMP10]], <12 x float> poison, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+; CHECK-TF-ALL-NEXT:    store <12 x float> [[INTERLEAVED_VEC]], ptr [[TMP7]], align 4
+; CHECK-TF-ALL-NEXT:    [[TMP11:%.*]] = shufflevector <4 x float> [[STRIDED_VEC3]], <4 x float> [[STRIDED_VEC4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-TF-ALL-NEXT:    [[TMP12:%.*]] = shufflevector <8 x float> [[TMP11]], <8 x float> <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float undef, float undef, float undef, float undef>, <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
+; CHECK-TF-ALL-NEXT:    [[INTERLEAVED_VEC5:%.*]] = shufflevector <12 x float> [[TMP12]], <12 x float> poison, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+; CHECK-TF-ALL-NEXT:    store <12 x float> [[INTERLEAVED_VEC5]], ptr [[TMP8]], align 4
+; CHECK-TF-ALL-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-TF-ALL-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-TF-ALL-NEXT:    br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-TF-ALL:       [[MIDDLE_BLOCK]]:
+; CHECK-TF-ALL-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-TF-ALL-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-TF-ALL:       [[SCALAR_PH]]:
+; CHECK-TF-ALL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-TF-ALL-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK-TF-ALL:       [[FOR_BODY]]:
+; CHECK-TF-ALL-NEXT:    [[I_021:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-TF-ALL-NEXT:    [[MUL:%.*]] = shl nuw nsw i64 [[I_021]], 1
+; CHECK-TF-ALL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[MUL]]
+; CHECK-TF-ALL-NEXT:    [[TMP14:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-TF-ALL-NEXT:    [[MUL1:%.*]] = mul nuw nsw i64 [[I_021]], 3
+; CHECK-TF-ALL-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[MUL1]]
+; CHECK-TF-ALL-NEXT:    store float [[TMP14]], ptr [[ARRAYIDX2]], align 4
+; CHECK-TF-ALL-NEXT:    [[ADD:%.*]] = or disjoint i64 [[MUL]], 1
+; CHECK-TF-ALL-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[ADD]]
+; CHECK-TF-ALL-NEXT:    [[TMP15:%.*]] = load float, ptr [[ARRAYIDX4]], align 4
+; CHECK-TF-ALL-NEXT:    [[ADD6:%.*]] = add nuw nsw i64 [[MUL1]], 1
+; CHECK-TF-ALL-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD6]]
+; CHECK-TF-ALL-NEXT:    store float [[TMP15]], ptr [[ARRAYIDX7]], align 4
+; CHECK-TF-ALL-NEXT:    [[ADD9:%.*]] = add nuw nsw i64 [[MUL1]], 2
+; CHECK-TF-ALL-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD9]]
+; CHECK-TF-ALL-NEXT:    store float 3.000000e+00, ptr [[ARRAYIDX10]], align 4
+; CHECK-TF-ALL-NEXT:    [[INC]] = add nuw nsw i64 [[I_021]], 1
+; CHECK-TF-ALL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]]
+; CHECK-TF-ALL-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-TF-ALL:       [[FOR_END]]:
+; CHECK-TF-ALL-NEXT:    ret void
+;
+; CHECK-TF-LABEL: define void @interleave(
+; CHECK-TF-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-NEXT:  [[ENTRY:.*]]:
+; CHECK-TF-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; CHECK-TF-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-TF:       [[VECTOR_PH]]:
+; CHECK-TF-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
+; CHECK-TF-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-TF-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-TF:       [[VECTOR_BODY]]:
+; CHECK-TF-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 4
+; CHECK-TF-NEXT:    [[TMP1:%.*]] = shl nuw nsw i64 [[INDEX]], 1
+; CHECK-TF-NEXT:    [[TMP2:%.*]] = shl nuw nsw i64 [[TMP0]], 1
+; CHECK-TF-NEXT:    [[TMP3:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP1]]
+; CHECK-TF-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP2]]
+; CHECK-TF-NEXT:    [[WIDE_VEC:%.*]] = load <8 x float>, ptr [[TMP3]], align 4
+; CHECK-TF-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <8 x float> [[WIDE_VEC]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-TF-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <8 x float> [[WIDE_VEC]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-TF-NEXT:    [[WIDE_VEC2:%.*]] = load <8 x float>, ptr [[TMP4]], align 4
+; CHECK-TF-NEXT:    [[STRIDED_VEC3:%.*]] = shufflevector <8 x float> [[WIDE_VEC2]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-TF-NEXT:    [[STRIDED_VEC4:%.*]] = shufflevector <8 x float> [[WIDE_VEC2]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-TF-NEXT:    [[TMP5:%.*]] = mul nuw nsw i64 [[INDEX]], 3
+; CHECK-TF-NEXT:    [[TMP6:%.*]] = mul nuw nsw i64 [[TMP0]], 3
+; CHECK-TF-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP5]]
+; CHECK-TF-NEXT:    [[TMP8:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP6]]
+; CHECK-TF-NEXT:    [[TMP9:%.*]] = shufflevector <4 x float> [[STRIDED_VEC]], <4 x float> [[STRIDED_VEC1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-TF-NEXT:    [[TMP10:%.*]] = shufflevector <8 x float> [[TMP9]], <8 x float> <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float undef, float undef, float undef, float undef>, <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
+; CHECK-TF-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <12 x float> [[TMP10]], <12 x float> poison, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+; CHECK-TF-NEXT:    store <12 x float> [[INTERLEAVED_VEC]], ptr [[TMP7]], align 4
+; CHECK-TF-NEXT:    [[TMP11:%.*]] = shufflevector <4 x float> [[STRIDED_VEC3]], <4 x float> [[STRIDED_VEC4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-TF-NEXT:    [[TMP12:%.*]] = shufflevector <8 x float> [[TMP11]], <8 x float> <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float undef, float undef, float undef, float undef>, <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
+; CHECK-TF-NEXT:    [[INTERLEAVED_VEC5:%.*]] = shufflevector <12 x float> [[TMP12]], <12 x float> poison, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+; CHECK-TF-NEXT:    store <12 x float> [[INTERLEAVED_VEC5]], ptr [[TMP8]], align 4
+; CHECK-TF-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-TF-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-TF-NEXT:    br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-TF:       [[MIDDLE_BLOCK]]:
+; CHECK-TF-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-TF-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-TF:       [[SCALAR_PH]]:
+; CHECK-TF-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-TF-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK-TF:       [[FOR_BODY]]:
+; CHECK-TF-NEXT:    [[I_021:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-TF-NEXT:    [[MUL:%.*]] = shl nuw nsw i64 [[I_021]], 1
+; CHECK-TF-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[MUL]]
+; CHECK-TF-NEXT:    [[TMP14:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-TF-NEXT:    [[MUL1:%.*]] = mul nuw nsw i64 [[I_021]], 3
+; CHECK-TF-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[MUL1]]
+; CHECK-TF-NEXT:    store float [[TMP14]], ptr [[ARRAYIDX2]], align 4
+; CHECK-TF-NEXT:    [[ADD:%.*]] = or disjoint i64 [[MUL]], 1
+; CHECK-TF-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[ADD]]
+; CHECK-TF-NEXT:    [[TMP15:%.*]] = load float, ptr [[ARRAYIDX4]], align 4
+; CHECK-TF-NEXT:    [[ADD6:%.*]] = add nuw nsw i64 [[MUL1]], 1
+; CHECK-TF-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD6]]
+; CHECK-TF-NEXT:    store float [[TMP15]], ptr [[ARRAYIDX7]], align 4
+; CHECK-TF-NEXT:    [[ADD9:%.*]] = add nuw nsw i64 [[MUL1]], 2
+; CHECK-TF-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD9]]
+; CHECK-TF-NEXT:    store float 3.000000e+00, ptr [[ARRAYIDX10]], align 4
+; CHECK-TF-NEXT:    [[INC]] = add nuw nsw i64 [[I_021]], 1
+; CHECK-TF-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]]
+; CHECK-TF-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-TF:       [[FOR_END]]:
+; CHECK-TF-NEXT:    ret void
+;
+; CHECK-TF-DEFAULT-LABEL: define void @interleave(
+; CHECK-TF-DEFAULT-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-DEFAULT-NEXT:  [[ENTRY:.*]]:
+; CHECK-TF-DEFAULT-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; CHECK-TF-DEFAULT-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-TF-DEFAULT:       [[VECTOR_PH]]:
+; CHECK-TF-DEFAULT-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
+; CHECK-TF-DEFAULT-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-TF-DEFAULT-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-DEFAULT:       [[VECTOR_BODY]]:
+; CHECK-TF-DEFAULT-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-DEFAULT-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 4
+; CHECK-TF-DEFAULT-NEXT:    [[TMP1:%.*]] = shl nuw nsw i64 [[INDEX]], 1
+; CHECK-TF-DEFAULT-NEXT:    [[TMP2:%.*]] = shl nuw nsw i64 [[TMP0]], 1
+; CHECK-TF-DEFAULT-NEXT:    [[TMP3:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP1]]
+; CHECK-TF-DEFAULT-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP2]]
+; CHECK-TF-DEFAULT-NEXT:    [[WIDE_VEC:%.*]] = load <8 x float>, ptr [[TMP3]], align 4
+; CHECK-TF-DEFAULT-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <8 x float> [[WIDE_VEC]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-TF-DEFAULT-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <8 x float> [[WIDE_VEC]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-TF-DEFAULT-NEXT:    [[WIDE_VEC2:%.*]] = load <8 x float>, ptr [[TMP4]], align 4
+; CHECK-TF-DEFAULT-NEXT:    [[STRIDED_VEC3:%.*]] = shufflevector <8 x float> [[WIDE_VEC2]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-TF-DEFAULT-NEXT:    [[STRIDED_VEC4:%.*]] = shufflevector <8 x float> [[WIDE_VEC2]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-TF-DEFAULT-NEXT:    [[TMP5:%.*]] = mul nuw nsw i64 [[INDEX]], 3
+; CHECK-TF-DEFAULT-NEXT:    [[TMP6:%.*]] = mul nuw nsw i64 [[TMP0]], 3
+; CHECK-TF-DEFAULT-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP5]]
+; CHECK-TF-DEFAULT-NEXT:    [[TMP8:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP6]]
+; CHECK-TF-DEFAULT-NEXT:    [[TMP9:%.*]] = shufflevector <4 x float> [[STRIDED_VEC]], <4 x float> [[STRIDED_VEC1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-TF-DEFAULT-NEXT:    [[TMP10:%.*]] = shufflevector <8 x float> [[TMP9]], <8 x float> <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float undef, float undef, float undef, float undef>, <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
+; CHECK-TF-DEFAULT-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <12 x float> [[TMP10]], <12 x float> poison, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+; CHECK-TF-DEFAULT-NEXT:    store <12 x float> [[INTERLEAVED_VEC]], ptr [[TMP7]], align 4
+; CHECK-TF-DEFAULT-NEXT:    [[TMP11:%.*]] = shufflevector <4 x float> [[STRIDED_VEC3]], <4 x float> [[STRIDED_VEC4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-TF-DEFAULT-NEXT:    [[TMP12:%.*]] = shufflevector <8 x float> [[TMP11]], <8 x float> <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float undef, float undef, float undef, float undef>, <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
+; CHECK-TF-DEFAULT-NEXT:    [[INTERLEAVED_VEC5:%.*]] = shufflevector <12 x float> [[TMP12]], <12 x float> poison, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+; CHECK-TF-DEFAULT-NEXT:    store <12 x float> [[INTERLEAVED_VEC5]], ptr [[TMP8]], align 4
+; CHECK-TF-DEFAULT-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-TF-DEFAULT-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-TF-DEFAULT-NEXT:    br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-TF-DEFAULT:       [[MIDDLE_BLOCK]]:
+; CHECK-TF-DEFAULT-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-TF-DEFAULT-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-TF-DEFAULT:       [[SCALAR_PH]]:
+; CHECK-TF-DEFAULT-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-TF-DEFAULT-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK-TF-DEFAULT:       [[FOR_BODY]]:
+; CHECK-TF-DEFAULT-NEXT:    [[I_021:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-TF-DEFAULT-NEXT:    [[MUL:%.*]] = shl nuw nsw i64 [[I_021]], 1
+; CHECK-TF-DEFAULT-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[MUL]]
+; CHECK-TF-DEFAULT-NEXT:    [[TMP14:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-TF-DEFAULT-NEXT:    [[MUL1:%.*]] = mul nuw nsw i64 [[I_021]], 3
+; CHECK-TF-DEFAULT-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[MUL1]]
+; CHECK-TF-DEFAULT-NEXT:    store float [[TMP14]], ptr [[ARRAYIDX2]], align 4
+; CHECK-TF-DEFAULT-NEXT:    [[ADD:%.*]] = or disjoint i64 [[MUL]], 1
+; CHECK-TF-DEFAULT-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[ADD]]
+; CHECK-TF-DEFAULT-NEXT:    [[TMP15:%.*]] = load float, ptr [[ARRAYIDX4]], align 4
+; CHECK-TF-DEFAULT-NEXT:    [[ADD6:%.*]] = add nuw nsw i64 [[MUL1]], 1
+; CHECK-TF-DEFAULT-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD6]]
+; CHECK-TF-DEFAULT-NEXT:    store float [[TMP15]], ptr [[ARRAYIDX7]], align 4
+; CHECK-TF-DEFAULT-NEXT:    [[ADD9:%.*]] = add nuw nsw i64 [[MUL1]], 2
+; CHECK-TF-DEFAULT-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD9]]
+; CHECK-TF-DEFAULT-NEXT:    store float 3.000000e+00, ptr [[ARRAYIDX10]], align 4
+; CHECK-TF-DEFAULT-NEXT:    [[INC]] = add nuw nsw i64 [[I_021]], 1
+; CHECK-TF-DEFAULT-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]]
+; CHECK-TF-DEFAULT-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-TF-DEFAULT:       [[FOR_END]]:
+; CHECK-TF-DEFAULT-NEXT:    ret void
+;
+; CHECK-TF-NORED-LABEL: define void @interleave(
+; CHECK-TF-NORED-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-NORED-NEXT:  [[ENTRY:.*]]:
+; CHECK-TF-NORED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; CHECK-TF-NORED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-TF-NORED:       [[VECTOR_PH]]:
+; CHECK-TF-NORED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
+; CHECK-TF-NORED-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-TF-NORED-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-NORED:       [[VECTOR_BODY]]:
+; CHECK-TF-NORED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NORED-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 4
+; CHECK-TF-NORED-NEXT:    [[TMP1:%.*]] = shl nuw nsw i64 [[INDEX]], 1
+; CHECK-TF-NORED-NEXT:    [[TMP2:%.*]] = shl nuw nsw i64 [[TMP0]], 1
+; CHECK-TF-NORED-NEXT:    [[TMP3:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP1]]
+; CHECK-TF-NORED-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP2]]
+; CHECK-TF-NORED-NEXT:    [[WIDE_VEC:%.*]] = load <8 x float>, ptr [[TMP3]], align 4
+; CHECK-TF-NORED-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <8 x float> [[WIDE_VEC]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-TF-NORED-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <8 x float> [[WIDE_VEC]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-TF-NORED-NEXT:    [[WIDE_VEC2:%.*]] = load <8 x float>, ptr [[TMP4]], align 4
+; CHECK-TF-NORED-NEXT:    [[STRIDED_VEC3:%.*]] = shufflevector <8 x float> [[WIDE_VEC2]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-TF-NORED-NEXT:    [[STRIDED_VEC4:%.*]] = shufflevector <8 x float> [[WIDE_VEC2]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-TF-NORED-NEXT:    [[TMP5:%.*]] = mul nuw nsw i64 [[INDEX]], 3
+; CHECK-TF-NORED-NEXT:    [[TMP6:%.*]] = mul nuw nsw i64 [[TMP0]], 3
+; CHECK-TF-NORED-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP5]]
+; CHECK-TF-NORED-NEXT:    [[TMP8:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP6]]
+; CHECK-TF-NORED-NEXT:    [[TMP9:%.*]] = shufflevector <4 x float> [[STRIDED_VEC]], <4 x float> [[STRIDED_VEC1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-TF-NORED-NEXT:    [[TMP10:%.*]] = shufflevector <8 x float> [[TMP9]], <8 x float> <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float undef, float undef, float undef, float undef>, <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
+; CHECK-TF-NORED-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <12 x float> [[TMP10]], <12 x float> poison, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+; CHECK-TF-NORED-NEXT:    store <12 x float> [[INTERLEAVED_VEC]], ptr [[TMP7]], align 4
+; CHECK-TF-NORED-NEXT:    [[TMP11:%.*]] = shufflevector <4 x float> [[STRIDED_VEC3]], <4 x float> [[STRIDED_VEC4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-TF-NORED-NEXT:    [[TMP12:%.*]] = shufflevector <8 x float> [[TMP11]], <8 x float> <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float undef, float undef, float undef, float undef>, <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
+; CHECK-TF-NORED-NEXT:    [[INTERLEAVED_VEC5:%.*]] = shufflevector <12 x float> [[TMP12]], <12 x float> poison, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+; CHECK-TF-NORED-NEXT:    store <12 x float> [[INTERLEAVED_VEC5]], ptr [[TMP8]], align 4
+; CHECK-TF-NORED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-TF-NORED-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-TF-NORED-NEXT:    br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-TF-NORED:       [[MIDDLE_BLOCK]]:
+; CHECK-TF-NORED-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-TF-NORED-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-TF-NORED:       [[SCALAR_PH]]:
+; CHECK-TF-NORED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-TF-NORED-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK-TF-NORED:       [[FOR_BODY]]:
+; CHECK-TF-NORED-NEXT:    [[I_021:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-TF-NORED-NEXT:    [[MUL:%.*]] = shl nuw nsw i64 [[I_021]], 1
+; CHECK-TF-NORED-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[MUL]]
+; CHECK-TF-NORED-NEXT:    [[TMP14:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-TF-NORED-NEXT:    [[MUL1:%.*]] = mul nuw nsw i64 [[I_021]], 3
+; CHECK-TF-NORED-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[MUL1]]
+; CHECK-TF-NORED-NEXT:    store float [[TMP14]], ptr [[ARRAYIDX2]], align 4
+; CHECK-TF-NORED-NEXT:    [[ADD:%.*]] = or disjoint i64 [[MUL]], 1
+; CHECK-TF-NORED-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[ADD]]
+; CHECK-TF-NORED-NEXT:    [[TMP15:%.*]] = load float, ptr [[ARRAYIDX4]], align 4
+; CHECK-TF-NORED-NEXT:    [[ADD6:%.*]] = add nuw nsw i64 [[MUL1]], 1
+; CHECK-TF-NORED-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD6]]
+; CHECK-TF-NORED-NEXT:    store float [[TMP15]], ptr [[ARRAYIDX7]], align 4
+; CHECK-TF-NORED-NEXT:    [[ADD9:%.*]] = add nuw nsw i64 [[MUL1]], 2
+; CHECK-TF-NORED-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD9]]
+; CHECK-TF-NORED-NEXT:    store float 3.000000e+00, ptr [[ARRAYIDX10]], align 4
+; CHECK-TF-NORED-NEXT:    [[INC]] = add nuw nsw i64 [[I_021]], 1
+; CHECK-TF-NORED-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]]
+; CHECK-TF-NORED-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-TF-NORED:       [[FOR_END]]:
+; CHECK-TF-NORED-NEXT:    ret void
+;
+; CHECK-TF-NOREC-LABEL: define void @interleave(
+; CHECK-TF-NOREC-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-NOREC-NEXT:  [[ENTRY:.*]]:
+; CHECK-TF-NOREC-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; CHECK-TF-NOREC-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-TF-NOREC:       [[VECTOR_PH]]:
+; CHECK-TF-NOREC-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
+; CHECK-TF-NOREC-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-TF-NOREC-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-NOREC:       [[VECTOR_BODY]]:
+; CHECK-TF-NOREC-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NOREC-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 4
+; CHECK-TF-NOREC-NEXT:    [[TMP1:%.*]] = shl nuw nsw i64 [[INDEX]], 1
+; CHECK-TF-NOREC-NEXT:    [[TMP2:%.*]] = shl nuw nsw i64 [[TMP0]], 1
+; CHECK-TF-NOREC-NEXT:    [[TMP3:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP1]]
+; CHECK-TF-NOREC-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP2]]
+; CHECK-TF-NOREC-NEXT:    [[WIDE_VEC:%.*]] = load <8 x float>, ptr [[TMP3]], align 4
+; CHECK-TF-NOREC-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <8 x float> [[WIDE_VEC]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-TF-NOREC-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <8 x float> [[WIDE_VEC]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-TF-NOREC-NEXT:    [[WIDE_VEC2:%.*]] = load <8 x float>, ptr [[TMP4]], align 4
+; CHECK-TF-NOREC-NEXT:    [[STRIDED_VEC3:%.*]] = shufflevector <8 x float> [[WIDE_VEC2]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-TF-NOREC-NEXT:    [[STRIDED_VEC4:%.*]] = shufflevector <8 x float> [[WIDE_VEC2]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-TF-NOREC-NEXT:    [[TMP5:%.*]] = mul nuw nsw i64 [[INDEX]], 3
+; CHECK-TF-NOREC-NEXT:    [[TMP6:%.*]] = mul nuw nsw i64 [[TMP0]], 3
+; CHECK-TF-NOREC-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP5]]
+; CHECK-TF-NOREC-NEXT:    [[TMP8:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP6]]
+; CHECK-TF-NOREC-NEXT:    [[TMP9:%.*]] = shufflevector <4 x float> [[STRIDED_VEC]], <4 x float> [[STRIDED_VEC1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-TF-NOREC-NEXT:    [[TMP10:%.*]] = shufflevector <8 x float> [[TMP9]], <8 x float> <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float undef, float undef, float undef, float undef>, <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
+; CHECK-TF-NOREC-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <12 x float> [[TMP10]], <12 x float> poison, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+; CHECK-TF-NOREC-NEXT:    store <12 x float> [[INTERLEAVED_VEC]], ptr [[TMP7]], align 4
+; CHECK-TF-NOREC-NEXT:    [[TMP11:%.*]] = shufflevector <4 x float> [[STRIDED_VEC3]], <4 x float> [[STRIDED_VEC4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-TF-NOREC-NEXT:    [[TMP12:%.*]] = shufflevector <8 x float> [[TMP11]], <8 x float> <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float undef, float undef, float undef, float undef>, <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
+; CHECK-TF-NOREC-NEXT:    [[INTERLEAVED_VEC5:%.*]] = shufflevector <12 x float> [[TMP12]], <12 x float> poison, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+; CHECK-TF-NOREC-NEXT:    store <12 x float> [[INTERLEAVED_VEC5]], ptr [[TMP8]], align 4
+; CHECK-TF-NOREC-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-TF-NOREC-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-TF-NOREC-NEXT:    br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-TF-NOREC:       [[MIDDLE_BLOCK]]:
+; CHECK-TF-NOREC-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-TF-NOREC-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-TF-NOREC:       [[SCALAR_PH]]:
+; CHECK-TF-NOREC-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-TF-NOREC-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK-TF-NOREC:       [[FOR_BODY]]:
+; CHECK-TF-NOREC-NEXT:    [[I_021:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-TF-NOREC-NEXT:    [[MUL:%.*]] = shl nuw nsw i64 [[I_021]], 1
+; CHECK-TF-NOREC-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[MUL]]
+; CHECK-TF-NOREC-NEXT:    [[TMP14:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-TF-NOREC-NEXT:    [[MUL1:%.*]] = mul nuw nsw i64 [[I_021]], 3
+; CHECK-TF-NOREC-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[MUL1]]
+; CHECK-TF-NOREC-NEXT:    store float [[TMP14]], ptr [[ARRAYIDX2]], align 4
+; CHECK-TF-NOREC-NEXT:    [[ADD:%.*]] = or disjoint i64 [[MUL]], 1
+; CHECK-TF-NOREC-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[ADD]]
+; CHECK-TF-NOREC-NEXT:    [[TMP15:%.*]] = load float, ptr [[ARRAYIDX4]], align 4
+; CHECK-TF-NOREC-NEXT:    [[ADD6:%.*]] = add nuw nsw i64 [[MUL1]], 1
+; CHECK-TF-NOREC-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD6]]
+; CHECK-TF-NOREC-NEXT:    store float [[TMP15]], ptr [[ARRAYIDX7]], align 4
+; CHECK-TF-NOREC-NEXT:    [[ADD9:%.*]] = add nuw nsw i64 [[MUL1]], 2
+; CHECK-TF-NOREC-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD9]]
+; CHECK-TF-NOREC-NEXT:    store float 3.000000e+00, ptr [[ARRAYIDX10]], align 4
+; CHECK-TF-NOREC-NEXT:    [[INC]] = add nuw nsw i64 [[I_021]], 1
+; CHECK-TF-NOREC-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]]
+; CHECK-TF-NOREC-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-TF-NOREC:       [[FOR_END]]:
+; CHECK-TF-NOREC-NEXT:    ret void
+;
+; CHECK-TF-NOREV-LABEL: define void @interleave(
+; CHECK-TF-NOREV-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-NOREV-NEXT:  [[ENTRY:.*]]:
+; CHECK-TF-NOREV-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; CHECK-TF-NOREV-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-TF-NOREV:       [[VECTOR_PH]]:
+; CHECK-TF-NOREV-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
+; CHECK-TF-NOREV-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-TF-NOREV-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-NOREV:       [[VECTOR_BODY]]:
+; CHECK-TF-NOREV-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NOREV-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 4
+; CHECK-TF-NOREV-NEXT:    [[TMP1:%.*]] = shl nuw nsw i64 [[INDEX]], 1
+; CHECK-TF-NOREV-NEXT:    [[TMP2:%.*]] = shl nuw nsw i64 [[TMP0]], 1
+; CHECK-TF-NOREV-NEXT:    [[TMP3:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP1]]
+; CHECK-TF-NOREV-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP2]]
+; CHECK-TF-NOREV-NEXT:    [[WIDE_VEC:%.*]] = load <8 x float>, ptr [[TMP3]], align 4
+; CHECK-TF-NOREV-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <8 x float> [[WIDE_VEC]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-TF-NOREV-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <8 x float> [[WIDE_VEC]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-TF-NOREV-NEXT:    [[WIDE_VEC2:%.*]] = load <8 x float>, ptr [[TMP4]], align 4
+; CHECK-TF-NOREV-NEXT:    [[STRIDED_VEC3:%.*]] = shufflevector <8 x float> [[WIDE_VEC2]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-TF-NOREV-NEXT:    [[STRIDED_VEC4:%.*]] = shufflevector <8 x float> [[WIDE_VEC2]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-TF-NOREV-NEXT:    [[TMP5:%.*]] = mul nuw nsw i64 [[INDEX]], 3
+; CHECK-TF-NOREV-NEXT:    [[TMP6:%.*]] = mul nuw nsw i64 [[TMP0]], 3
+; CHECK-TF-NOREV-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP5]]
+; CHECK-TF-NOREV-NEXT:    [[TMP8:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP6]]
+; CHECK-TF-NOREV-NEXT:    [[TMP9:%.*]] = shufflevector <4 x float> [[STRIDED_VEC]], <4 x float> [[STRIDED_VEC1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-TF-NOREV-NEXT:    [[TMP10:%.*]] = shufflevector <8 x float> [[TMP9]], <8 x float> <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float undef, float undef, float undef, float undef>, <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
+; CHECK-TF-NOREV-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <12 x float> [[TMP10]], <12 x float> poison, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+; CHECK-TF-NOREV-NEXT:    store <12 x float> [[INTERLEAVED_VEC]], ptr [[TMP7]], align 4
+; CHECK-TF-NOREV-NEXT:    [[TMP11:%.*]] = shufflevector <4 x float> [[STRIDED_VEC3]], <4 x float> [[STRIDED_VEC4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-TF-NOREV-NEXT:    [[TMP12:%.*]] = shufflevector <8 x float> [[TMP11]], <8 x float> <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float undef, float undef, float undef, float undef>, <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
+; CHECK-TF-NOREV-NEXT:    [[INTERLEAVED_VEC5:%.*]] = shufflevector <12 x float> [[TMP12]], <12 x float> poison, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+; CHECK-TF-NOREV-NEXT:    store <12 x float> [[INTERLEAVED_VEC5]], ptr [[TMP8]], align 4
+; CHECK-TF-NOREV-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-TF-NOREV-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-TF-NOREV-NEXT:    br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-TF-NOREV:       [[MIDDLE_BLOCK]]:
+; CHECK-TF-NOREV-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-TF-NOREV-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-TF-NOREV:       [[SCALAR_PH]]:
+; CHECK-TF-NOREV-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-TF-NOREV-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK-TF-NOREV:       [[FOR_BODY]]:
+; CHECK-TF-NOREV-NEXT:    [[I_021:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-TF-NOREV-NEXT:    [[MUL:%.*]] = shl nuw nsw i64 [[I_021]], 1
+; CHECK-TF-NOREV-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[MUL]]
+; CHECK-TF-NOREV-NEXT:    [[TMP14:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-TF-NOREV-NEXT:    [[MUL1:%.*]] = mul nuw nsw i64 [[I_021]], 3
+; CHECK-TF-NOREV-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[MUL1]]
+; CHECK-TF-NOREV-NEXT:    store float [[TMP14]], ptr [[ARRAYIDX2]], align 4
+; CHECK-TF-NOREV-NEXT:    [[ADD:%.*]] = or disjoint i64 [[MUL]], 1
+; CHECK-TF-NOREV-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[ADD]]
+; CHECK-TF-NOREV-NEXT:    [[TMP15:%.*]] = load float, ptr [[ARRAYIDX4]], align 4
+; CHECK-TF-NOREV-NEXT:    [[ADD6:%.*]] = add nuw nsw i64 [[MUL1]], 1
+; CHECK-TF-NOREV-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD6]]
+; CHECK-TF-NOREV-NEXT:    store float [[TMP15]], ptr [[ARRAYIDX7]], align 4
+; CHECK-TF-NOREV-NEXT:    [[ADD9:%.*]] = add nuw nsw i64 [[MUL1]], 2
+; CHECK-TF-NOREV-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD9]]
+; CHECK-TF-NOREV-NEXT:    store float 3.000000e+00, ptr [[ARRAYIDX10]], align 4
+; CHECK-TF-NOREV-NEXT:    [[INC]] = add nuw nsw i64 [[I_021]], 1
+; CHECK-TF-NOREV-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]]
+; CHECK-TF-NOREV-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-TF-NOREV:       [[FOR_END]]:
+; CHECK-TF-NOREV-NEXT:    ret void
+;
+; CHECK-TF-ONLYRED-LABEL: define void @interleave(
+; CHECK-TF-ONLYRED-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-ONLYRED-NEXT:  [[ENTRY:.*]]:
+; CHECK-TF-ONLYRED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; CHECK-TF-ONLYRED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-TF-ONLYRED:       [[VECTOR_PH]]:
+; CHECK-TF-ONLYRED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
+; CHECK-TF-ONLYRED-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-TF-ONLYRED-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-ONLYRED:       [[VECTOR_BODY]]:
+; CHECK-TF-ONLYRED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-ONLYRED-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 4
+; CHECK-TF-ONLYRED-NEXT:    [[TMP1:%.*]] = shl nuw nsw i64 [[INDEX]], 1
+; CHECK-TF-ONLYRED-NEXT:    [[TMP2:%.*]] = shl nuw nsw i64 [[TMP0]], 1
+; CHECK-TF-ONLYRED-NEXT:    [[TMP3:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP1]]
+; CHECK-TF-ONLYRED-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP2]]
+; CHECK-TF-ONLYRED-NEXT:    [[WIDE_VEC:%.*]] = load <8 x float>, ptr [[TMP3]], align 4
+; CHECK-TF-ONLYRED-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <8 x float> [[WIDE_VEC]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-TF-ONLYRED-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <8 x float> [[WIDE_VEC]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-TF-ONLYRED-NEXT:    [[WIDE_VEC2:%.*]] = load <8 x float>, ptr [[TMP4]], align 4
+; CHECK-TF-ONLYRED-NEXT:    [[STRIDED_VEC3:%.*]] = shufflevector <8 x float> [[WIDE_VEC2]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-TF-ONLYRED-NEXT:    [[STRIDED_VEC4:%.*]] = shufflevector <8 x float> [[WIDE_VEC2]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-TF-ONLYRED-NEXT:    [[TMP5:%.*]] = mul nuw nsw i64 [[INDEX]], 3
+; CHECK-TF-ONLYRED-NEXT:    [[TMP6:%.*]] = mul nuw nsw i64 [[TMP0]], 3
+; CHECK-TF-ONLYRED-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP5]]
+; CHECK-TF-ONLYRED-NEXT:    [[TMP8:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP6]]
+; CHECK-TF-ONLYRED-NEXT:    [[TMP9:%.*]] = shufflevector <4 x float> [[STRIDED_VEC]], <4 x float> [[STRIDED_VEC1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-TF-ONLYRED-NEXT:    [[TMP10:%.*]] = shufflevector <8 x float> [[TMP9]], <8 x float> <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float undef, float undef, float undef, float undef>, <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
+; CHECK-TF-ONLYRED-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <12 x float> [[TMP10]], <12 x float> poison, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+; CHECK-TF-ONLYRED-NEXT:    store <12 x float> [[INTERLEAVED_VEC]], ptr [[TMP7]], align 4
+; CHECK-TF-ONLYRED-NEXT:    [[TMP11:%.*]] = shufflevector <4 x float> [[STRIDED_VEC3]], <4 x float> [[STRIDED_VEC4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-TF-ONLYRED-NEXT:    [[TMP12:%.*]] = shufflevector <8 x float> [[TMP11]], <8 x float> <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float undef, float undef, float undef, float undef>, <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
+; CHECK-TF-ONLYRED-NEXT:    [[INTERLEAVED_VEC5:%.*]] = shufflevector <12 x float> [[TMP12]], <12 x float> poison, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+; CHECK-TF-ONLYRED-NEXT:    store <12 x float> [[INTERLEAVED_VEC5]], ptr [[TMP8]], align 4
+; CHECK-TF-ONLYRED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-TF-ONLYRED-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-TF-ONLYRED-NEXT:    br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-TF-ONLYRED:       [[MIDDLE_BLOCK]]:
+; CHECK-TF-ONLYRED-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-TF-ONLYRED-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-TF-ONLYRED:       [[SCALAR_PH]]:
+; CHECK-TF-ONLYRED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-TF-ONLYRED-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK-TF-ONLYRED:       [[FOR_BODY]]:
+; CHECK-TF-ONLYRED-NEXT:    [[I_021:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-TF-ONLYRED-NEXT:    [[MUL:%.*]] = shl nuw nsw i64 [[I_021]], 1
+; CHECK-TF-ONLYRED-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[MUL]]
+; CHECK-TF-ONLYRED-NEXT:    [[TMP14:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-TF-ONLYRED-NEXT:    [[MUL1:%.*]] = mul nuw nsw i64 [[I_021]], 3
+; CHECK-TF-ONLYRED-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[MUL1]]
+; CHECK-TF-ONLYRED-NEXT:    store float [[TMP14]], ptr [[ARRAYIDX2]], align 4
+; CHECK-TF-ONLYRED-NEXT:    [[ADD:%.*]] = or disjoint i64 [[MUL]], 1
+; CHECK-TF-ONLYRED-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[ADD]]
+; CHECK-TF-ONLYRED-NEXT:    [[TMP15:%.*]] = load float, ptr [[ARRAYIDX4]], align 4
+; CHECK-TF-ONLYRED-NEXT:    [[ADD6:%.*]] = add nuw nsw i64 [[MUL1]], 1
+; CHECK-TF-ONLYRED-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD6]]
+; CHECK-TF-ONLYRED-NEXT:    store float [[TMP15]], ptr [[ARRAYIDX7]], align 4
+; CHECK-TF-ONLYRED-NEXT:    [[ADD9:%.*]] = add nuw nsw i64 [[MUL1]], 2
+; CHECK-TF-ONLYRED-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD9]]
+; CHECK-TF-ONLYRED-NEXT:    store float 3.000000e+00, ptr [[ARRAYIDX10]], align 4
+; CHECK-TF-ONLYRED-NEXT:    [[INC]] = add nuw nsw i64 [[I_021]], 1
+; CHECK-TF-ONLYRED-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]]
+; CHECK-TF-ONLYRED-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-TF-ONLYRED:       [[FOR_END]]:
+; CHECK-TF-ONLYRED-NEXT:    ret void
+;
+; CHECK-NEOVERSE-V1-LABEL: define void @interleave(
+; CHECK-NEOVERSE-V1-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEOVERSE-V1-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEOVERSE-V1-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; CHECK-NEOVERSE-V1-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEOVERSE-V1:       [[VECTOR_PH]]:
+; CHECK-NEOVERSE-V1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
+; CHECK-NEOVERSE-V1-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEOVERSE-V1-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-NEOVERSE-V1:       [[VECTOR_BODY]]:
+; CHECK-NEOVERSE-V1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 4
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP1:%.*]] = shl nuw nsw i64 [[INDEX]], 1
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP2:%.*]] = shl nuw nsw i64 [[TMP0]], 1
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP3:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP1]]
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP2]]
+; CHECK-NEOVERSE-V1-NEXT:    [[WIDE_VEC:%.*]] = load <8 x float>, ptr [[TMP3]], align 4
+; CHECK-NEOVERSE-V1-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <8 x float> [[WIDE_VEC]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEOVERSE-V1-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <8 x float> [[WIDE_VEC]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEOVERSE-V1-NEXT:    [[WIDE_VEC2:%.*]] = load <8 x float>, ptr [[TMP4]], align 4
+; CHECK-NEOVERSE-V1-NEXT:    [[STRIDED_VEC3:%.*]] = shufflevector <8 x float> [[WIDE_VEC2]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEOVERSE-V1-NEXT:    [[STRIDED_VEC4:%.*]] = shufflevector <8 x float> [[WIDE_VEC2]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP5:%.*]] = mul nuw nsw i64 [[INDEX]], 3
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP6:%.*]] = mul nuw nsw i64 [[TMP0]], 3
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP5]]
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP8:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP6]]
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP9:%.*]] = shufflevector <4 x float> [[STRIDED_VEC]], <4 x float> [[STRIDED_VEC1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP10:%.*]] = shufflevector <8 x float> [[TMP9]], <8 x float> <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float undef, float undef, float undef, float undef>, <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEOVERSE-V1-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <12 x float> [[TMP10]], <12 x float> poison, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+; CHECK-NEOVERSE-V1-NEXT:    store <12 x float> [[INTERLEAVED_VEC]], ptr [[TMP7]], align 4
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP11:%.*]] = shufflevector <4 x float> [[STRIDED_VEC3]], <4 x float> [[STRIDED_VEC4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP12:%.*]] = shufflevector <8 x float> [[TMP11]], <8 x float> <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float undef, float undef, float undef, float undef>, <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEOVERSE-V1-NEXT:    [[INTERLEAVED_VEC5:%.*]] = shufflevector <12 x float> [[TMP12]], <12 x float> poison, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+; CHECK-NEOVERSE-V1-NEXT:    store <12 x float> [[INTERLEAVED_VEC5]], ptr [[TMP8]], align 4
+; CHECK-NEOVERSE-V1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEOVERSE-V1-NEXT:    br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-NEOVERSE-V1:       [[MIDDLE_BLOCK]]:
+; CHECK-NEOVERSE-V1-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEOVERSE-V1-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-NEOVERSE-V1:       [[SCALAR_PH]]:
+; CHECK-NEOVERSE-V1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEOVERSE-V1-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK-NEOVERSE-V1:       [[FOR_BODY]]:
+; CHECK-NEOVERSE-V1-NEXT:    [[I_021:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEOVERSE-V1-NEXT:    [[MUL:%.*]] = shl nuw nsw i64 [[I_021]], 1
+; CHECK-NEOVERSE-V1-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[MUL]]
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP14:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-NEOVERSE-V1-NEXT:    [[MUL1:%.*]] = mul nuw nsw i64 [[I_021]], 3
+; CHECK-NEOVERSE-V1-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[MUL1]]
+; CHECK-NEOVERSE-V1-NEXT:    store float [[TMP14]], ptr [[ARRAYIDX2]], align 4
+; CHECK-NEOVERSE-V1-NEXT:    [[ADD:%.*]] = or disjoint i64 [[MUL]], 1
+; CHECK-NEOVERSE-V1-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[ADD]]
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP15:%.*]] = load float, ptr [[ARRAYIDX4]], align 4
+; CHECK-NEOVERSE-V1-NEXT:    [[ADD6:%.*]] = add nuw nsw i64 [[MUL1]], 1
+; CHECK-NEOVERSE-V1-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD6]]
+; CHECK-NEOVERSE-V1-NEXT:    store float [[TMP15]], ptr [[ARRAYIDX7]], align 4
+; CHECK-NEOVERSE-V1-NEXT:    [[ADD9:%.*]] = add nuw nsw i64 [[MUL1]], 2
+; CHECK-NEOVERSE-V1-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD9]]
+; CHECK-NEOVERSE-V1-NEXT:    store float 3.000000e+00, ptr [[ARRAYIDX10]], align 4
+; CHECK-NEOVERSE-V1-NEXT:    [[INC]] = add nuw nsw i64 [[I_021]], 1
+; CHECK-NEOVERSE-V1-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]]
+; CHECK-NEOVERSE-V1-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NEOVERSE-V1:       [[FOR_END]]:
+; CHECK-NEOVERSE-V1-NEXT:    ret void
+;
+
+
+
+
+
 
 entry:
   br label %for.body
@@ -346,35 +1675,420 @@ for.end:
 }
 
 define void @reverse(ptr noalias %dst, ptr noalias %src) #0 {
-; CHECK-NOTF-LABEL: @reverse(
-; CHECK-NOTF:       vector.body:
-; CHECK-NOTF-NOT:     %{{.*}} = phi <vscale x 4 x i1>
-; CHECK-NOTF:         %[[LOAD:.*]] = load <vscale x 2 x double>, ptr
-; CHECK-NOTF:         %{{.*}} = call <vscale x 2 x double> @llvm.vector.reverse.nxv2f64(<vscale x 2 x double> %{{.*}})
-
-; CHECK-TF-NOREV-LABEL: @reverse(
-; CHECK-TF-NOREV:       vector.body:
-; CHECK-TF-NOREV-NOT:     %{{.*}} = phi <vscale x 4 x i1>
-; CHECK-TF-NOREV:         %[[LOAD:.*]] = load <vscale x 2 x double>, ptr
-; CHECK-TF-NOREV:         %{{.*}} = call <vscale x 2 x double> @llvm.vector.reverse.nxv2f64(<vscale x 2 x double> %{{.*}})
-
-; CHECK-TF-LABEL: @reverse(
-; CHECK-TF:       vector.body:
-; CHECK-TF:         %[[ACTIVE_LANE_MASK:.*]] = phi <vscale x 2 x i1>
-; CHECK-TF:         %[[REVERSE_MASK:.*]] = call <vscale x 2 x i1> @llvm.vector.reverse.nxv2i1(<vscale x 2 x i1> %[[ACTIVE_LANE_MASK]])
-; CHECK-TF:         %[[MASKED_LOAD:.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0({{.*}} <vscale x 2 x i1> %reverse
-
-; CHECK-TF-NORED-LABEL: @reverse(
-; CHECK-TF-NORED:       vector.body:
-; CHECK-TF-NORED:         %[[ACTIVE_LANE_MASK:.*]] = phi <vscale x 2 x i1>
-; CHECK-TF-NORED:         %[[REVERSE_MASK:.*]] = call <vscale x 2 x i1> @llvm.vector.reverse.nxv2i1(<vscale x 2 x i1> %[[ACTIVE_LANE_MASK]])
-; CHECK-TF-NORED:         %[[MASKED_LOAD:.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0({{.*}} <vscale x 2 x i1> %reverse
-
-; CHECK-TF-NOREC-LABEL: @reverse(
-; CHECK-TF-NOREC:       vector.body:
-; CHECK-TF-NOREC:         %[[ACTIVE_LANE_MASK:.*]] = phi <vscale x 2 x i1>
-; CHECK-TF-NOREC:         %[[REVERSE_MASK:.*]] = call <vscale x 2 x i1> @llvm.vector.reverse.nxv2i1(<vscale x 2 x i1> %[[ACTIVE_LANE_MASK]])
-; CHECK-TF-NOREC:         %[[MASKED_LOAD:.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0({{.*}} <vscale x 2 x i1> %reverse
+; CHECK-NOTF-LABEL: define void @reverse(
+; CHECK-NOTF-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]]) #[[ATTR0]] {
+; CHECK-NOTF-NEXT:  [[ENTRY:.*]]:
+; CHECK-NOTF-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NOTF-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NOTF-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
+; CHECK-NOTF-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NOTF:       [[VECTOR_PH]]:
+; CHECK-NOTF-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 1
+; CHECK-NOTF-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; CHECK-NOTF-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
+; CHECK-NOTF-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-NOTF-NEXT:    [[TMP4:%.*]] = sub i64 1023, [[N_VEC]]
+; CHECK-NOTF-NEXT:    [[TMP5:%.*]] = shl i64 [[N_VEC]], 3
+; CHECK-NOTF-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP5]]
+; CHECK-NOTF-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-NOTF:       [[VECTOR_BODY]]:
+; CHECK-NOTF-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NOTF-NEXT:    [[TMP7:%.*]] = sub i64 1023, [[INDEX]]
+; CHECK-NOTF-NEXT:    [[TMP8:%.*]] = shl i64 [[INDEX]], 3
+; CHECK-NOTF-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP8]]
+; CHECK-NOTF-NEXT:    [[TMP9:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[TMP7]]
+; CHECK-NOTF-NEXT:    [[TMP10:%.*]] = sub nuw nsw i64 [[TMP2]], 1
+; CHECK-NOTF-NEXT:    [[TMP11:%.*]] = sub i64 0, [[TMP10]]
+; CHECK-NOTF-NEXT:    [[TMP12:%.*]] = getelementptr inbounds double, ptr [[TMP9]], i64 [[TMP11]]
+; CHECK-NOTF-NEXT:    [[TMP13:%.*]] = sub i64 [[TMP11]], [[TMP2]]
+; CHECK-NOTF-NEXT:    [[TMP14:%.*]] = getelementptr inbounds double, ptr [[TMP9]], i64 [[TMP13]]
+; CHECK-NOTF-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 2 x double>, ptr [[TMP12]], align 8
+; CHECK-NOTF-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 2 x double>, ptr [[TMP14]], align 8
+; CHECK-NOTF-NEXT:    [[TMP15:%.*]] = fadd <vscale x 2 x double> [[WIDE_LOAD]], splat (double 1.000000e+00)
+; CHECK-NOTF-NEXT:    [[TMP16:%.*]] = fadd <vscale x 2 x double> [[WIDE_LOAD1]], splat (double 1.000000e+00)
+; CHECK-NOTF-NEXT:    [[REVERSE:%.*]] = call <vscale x 2 x double> @llvm.vector.reverse.nxv2f64(<vscale x 2 x double> [[TMP15]])
+; CHECK-NOTF-NEXT:    [[REVERSE2:%.*]] = call <vscale x 2 x double> @llvm.vector.reverse.nxv2f64(<vscale x 2 x double> [[TMP16]])
+; CHECK-NOTF-NEXT:    [[TMP17:%.*]] = getelementptr double, ptr [[NEXT_GEP]], i64 [[TMP2]]
+; CHECK-NOTF-NEXT:    store <vscale x 2 x double> [[REVERSE]], ptr [[NEXT_GEP]], align 8
+; CHECK-NOTF-NEXT:    store <vscale x 2 x double> [[REVERSE2]], ptr [[TMP17]], align 8
+; CHECK-NOTF-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-NOTF-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NOTF-NEXT:    br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-NOTF:       [[MIDDLE_BLOCK]]:
+; CHECK-NOTF-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-NOTF-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-NOTF:       [[SCALAR_PH]]:
+; CHECK-NOTF-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP4]], %[[MIDDLE_BLOCK]] ], [ 1023, %[[ENTRY]] ]
+; CHECK-NOTF-NEXT:    [[BC_RESUME_VAL3:%.*]] = phi ptr [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ [[DST]], %[[ENTRY]] ]
+; CHECK-NOTF-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK-NOTF:       [[FOR_BODY]]:
+; CHECK-NOTF-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NOTF-NEXT:    [[INDVARS_PTR:%.*]] = phi ptr [ [[BC_RESUME_VAL3]], %[[SCALAR_PH]] ], [ [[INDVARS_PTR_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NOTF-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[INDVARS_IV]]
+; CHECK-NOTF-NEXT:    [[TMP19:%.*]] = load double, ptr [[ARRAYIDX]], align 8
+; CHECK-NOTF-NEXT:    [[ADD:%.*]] = fadd double [[TMP19]], 1.000000e+00
+; CHECK-NOTF-NEXT:    store double [[ADD]], ptr [[INDVARS_PTR]], align 8
+; CHECK-NOTF-NEXT:    [[INDVARS_IV_NEXT]] = add nsw i64 [[INDVARS_IV]], -1
+; CHECK-NOTF-NEXT:    [[INDVARS_PTR_NEXT]] = getelementptr inbounds double, ptr [[INDVARS_PTR]], i64 1
+; CHECK-NOTF-NEXT:    [[CMP_NOT:%.*]] = icmp eq i64 [[INDVARS_IV]], 0
+; CHECK-NOTF-NEXT:    br i1 [[CMP_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-NOTF:       [[FOR_END]]:
+; CHECK-NOTF-NEXT:    ret void
+;
+; CHECK-TF-ALL-LABEL: define void @reverse(
+; CHECK-TF-ALL-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]]) #[[ATTR0]] {
+; CHECK-TF-ALL-NEXT:  [[ENTRY:.*:]]
+; CHECK-TF-ALL-NEXT:    br label %[[VECTOR_PH:.*]]
+; CHECK-TF-ALL:       [[VECTOR_PH]]:
+; CHECK-TF-ALL-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-ALL-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; CHECK-TF-ALL-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1024)
+; CHECK-TF-ALL-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-ALL:       [[VECTOR_BODY]]:
+; CHECK-TF-ALL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-ALL-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-ALL-NEXT:    [[TMP2:%.*]] = sub i64 1023, [[INDEX]]
+; CHECK-TF-ALL-NEXT:    [[TMP3:%.*]] = shl i64 [[INDEX]], 3
+; CHECK-TF-ALL-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP3]]
+; CHECK-TF-ALL-NEXT:    [[TMP4:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[TMP2]]
+; CHECK-TF-ALL-NEXT:    [[TMP5:%.*]] = sub nuw nsw i64 [[TMP1]], 1
+; CHECK-TF-ALL-NEXT:    [[TMP6:%.*]] = sub i64 0, [[TMP5]]
+; CHECK-TF-ALL-NEXT:    [[TMP7:%.*]] = getelementptr double, ptr [[TMP4]], i64 [[TMP6]]
+; CHECK-TF-ALL-NEXT:    [[REVERSE:%.*]] = call <vscale x 2 x i1> @llvm.vector.reverse.nxv2i1(<vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-ALL-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP7]], <vscale x 2 x i1> [[REVERSE]], <vscale x 2 x double> poison)
+; CHECK-TF-ALL-NEXT:    [[TMP8:%.*]] = fadd <vscale x 2 x double> [[WIDE_MASKED_LOAD]], splat (double 1.000000e+00)
+; CHECK-TF-ALL-NEXT:    [[REVERSE1:%.*]] = call <vscale x 2 x double> @llvm.vector.reverse.nxv2f64(<vscale x 2 x double> [[TMP8]])
+; CHECK-TF-ALL-NEXT:    call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[REVERSE1]], ptr align 8 [[NEXT_GEP]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-ALL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-TF-ALL-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1024)
+; CHECK-TF-ALL-NEXT:    [[TMP9:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-ALL-NEXT:    [[TMP10:%.*]] = xor i1 [[TMP9]], true
+; CHECK-TF-ALL-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-TF-ALL:       [[MIDDLE_BLOCK]]:
+; CHECK-TF-ALL-NEXT:    br label %[[FOR_END:.*]]
+; CHECK-TF-ALL:       [[FOR_END]]:
+; CHECK-TF-ALL-NEXT:    ret void
+;
+; CHECK-TF-LABEL: define void @reverse(
+; CHECK-TF-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]]) #[[ATTR0]] {
+; CHECK-TF-NEXT:  [[ENTRY:.*:]]
+; CHECK-TF-NEXT:    br label %[[VECTOR_PH:.*]]
+; CHECK-TF:       [[VECTOR_PH]]:
+; CHECK-TF-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1024)
+; CHECK-TF-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-TF:       [[VECTOR_BODY]]:
+; CHECK-TF-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NEXT:    [[TMP2:%.*]] = sub i64 1023, [[INDEX]]
+; CHECK-TF-NEXT:    [[TMP3:%.*]] = shl i64 [[INDEX]], 3
+; CHECK-TF-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP3]]
+; CHECK-TF-NEXT:    [[TMP4:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[TMP2]]
+; CHECK-TF-NEXT:    [[TMP5:%.*]] = sub nuw nsw i64 [[TMP1]], 1
+; CHECK-TF-NEXT:    [[TMP6:%.*]] = sub i64 0, [[TMP5]]
+; CHECK-TF-NEXT:    [[TMP7:%.*]] = getelementptr double, ptr [[TMP4]], i64 [[TMP6]]
+; CHECK-TF-NEXT:    [[REVERSE:%.*]] = call <vscale x 2 x i1> @llvm.vector.reverse.nxv2i1(<vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP7]], <vscale x 2 x i1> [[REVERSE]], <vscale x 2 x double> poison)
+; CHECK-TF-NEXT:    [[TMP8:%.*]] = fadd <vscale x 2 x double> [[WIDE_MASKED_LOAD]], splat (double 1.000000e+00)
+; CHECK-TF-NEXT:    [[REVERSE1:%.*]] = call <vscale x 2 x double> @llvm.vector.reverse.nxv2f64(<vscale x 2 x double> [[TMP8]])
+; CHECK-TF-NEXT:    call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[REVERSE1]], ptr align 8 [[NEXT_GEP]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1024)
+; CHECK-TF-NEXT:    [[TMP9:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-NEXT:    [[TMP10:%.*]] = xor i1 [[TMP9]], true
+; CHECK-TF-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-TF:       [[MIDDLE_BLOCK]]:
+; CHECK-TF-NEXT:    br label %[[FOR_END:.*]]
+; CHECK-TF:       [[FOR_END]]:
+; CHECK-TF-NEXT:    ret void
+;
+; CHECK-TF-DEFAULT-LABEL: define void @reverse(
+; CHECK-TF-DEFAULT-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]]) #[[ATTR0]] {
+; CHECK-TF-DEFAULT-NEXT:  [[ENTRY:.*:]]
+; CHECK-TF-DEFAULT-NEXT:    br label %[[VECTOR_PH:.*]]
+; CHECK-TF-DEFAULT:       [[VECTOR_PH]]:
+; CHECK-TF-DEFAULT-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-DEFAULT-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; CHECK-TF-DEFAULT-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1024)
+; CHECK-TF-DEFAULT-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-DEFAULT:       [[VECTOR_BODY]]:
+; CHECK-TF-DEFAULT-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-DEFAULT-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-DEFAULT-NEXT:    [[TMP2:%.*]] = sub i64 1023, [[INDEX]]
+; CHECK-TF-DEFAULT-NEXT:    [[TMP3:%.*]] = shl i64 [[INDEX]], 3
+; CHECK-TF-DEFAULT-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP3]]
+; CHECK-TF-DEFAULT-NEXT:    [[TMP4:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[TMP2]]
+; CHECK-TF-DEFAULT-NEXT:    [[TMP5:%.*]] = sub nuw nsw i64 [[TMP1]], 1
+; CHECK-TF-DEFAULT-NEXT:    [[TMP6:%.*]] = sub i64 0, [[TMP5]]
+; CHECK-TF-DEFAULT-NEXT:    [[TMP7:%.*]] = getelementptr double, ptr [[TMP4]], i64 [[TMP6]]
+; CHECK-TF-DEFAULT-NEXT:    [[REVERSE:%.*]] = call <vscale x 2 x i1> @llvm.vector.reverse.nxv2i1(<vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-DEFAULT-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP7]], <vscale x 2 x i1> [[REVERSE]], <vscale x 2 x double> poison)
+; CHECK-TF-DEFAULT-NEXT:    [[TMP8:%.*]] = fadd <vscale x 2 x double> [[WIDE_MASKED_LOAD]], splat (double 1.000000e+00)
+; CHECK-TF-DEFAULT-NEXT:    [[REVERSE1:%.*]] = call <vscale x 2 x double> @llvm.vector.reverse.nxv2f64(<vscale x 2 x double> [[TMP8]])
+; CHECK-TF-DEFAULT-NEXT:    call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[REVERSE1]], ptr align 8 [[NEXT_GEP]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-DEFAULT-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-TF-DEFAULT-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1024)
+; CHECK-TF-DEFAULT-NEXT:    [[TMP9:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-DEFAULT-NEXT:    [[TMP10:%.*]] = xor i1 [[TMP9]], true
+; CHECK-TF-DEFAULT-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-TF-DEFAULT:       [[MIDDLE_BLOCK]]:
+; CHECK-TF-DEFAULT-NEXT:    br label %[[FOR_END:.*]]
+; CHECK-TF-DEFAULT:       [[FOR_END]]:
+; CHECK-TF-DEFAULT-NEXT:    ret void
+;
+; CHECK-TF-NORED-LABEL: define void @reverse(
+; CHECK-TF-NORED-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]]) #[[ATTR0]] {
+; CHECK-TF-NORED-NEXT:  [[ENTRY:.*:]]
+; CHECK-TF-NORED-NEXT:    br label %[[VECTOR_PH:.*]]
+; CHECK-TF-NORED:       [[VECTOR_PH]]:
+; CHECK-TF-NORED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NORED-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; CHECK-TF-NORED-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1024)
+; CHECK-TF-NORED-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-NORED:       [[VECTOR_BODY]]:
+; CHECK-TF-NORED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NORED-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NORED-NEXT:    [[TMP2:%.*]] = sub i64 1023, [[INDEX]]
+; CHECK-TF-NORED-NEXT:    [[TMP3:%.*]] = shl i64 [[INDEX]], 3
+; CHECK-TF-NORED-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP3]]
+; CHECK-TF-NORED-NEXT:    [[TMP4:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[TMP2]]
+; CHECK-TF-NORED-NEXT:    [[TMP5:%.*]] = sub nuw nsw i64 [[TMP1]], 1
+; CHECK-TF-NORED-NEXT:    [[TMP6:%.*]] = sub i64 0, [[TMP5]]
+; CHECK-TF-NORED-NEXT:    [[TMP7:%.*]] = getelementptr double, ptr [[TMP4]], i64 [[TMP6]]
+; CHECK-TF-NORED-NEXT:    [[REVERSE:%.*]] = call <vscale x 2 x i1> @llvm.vector.reverse.nxv2i1(<vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-NORED-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP7]], <vscale x 2 x i1> [[REVERSE]], <vscale x 2 x double> poison)
+; CHECK-TF-NORED-NEXT:    [[TMP8:%.*]] = fadd <vscale x 2 x double> [[WIDE_MASKED_LOAD]], splat (double 1.000000e+00)
+; CHECK-TF-NORED-NEXT:    [[REVERSE1:%.*]] = call <vscale x 2 x double> @llvm.vector.reverse.nxv2f64(<vscale x 2 x double> [[TMP8]])
+; CHECK-TF-NORED-NEXT:    call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[REVERSE1]], ptr align 8 [[NEXT_GEP]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-NORED-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-TF-NORED-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1024)
+; CHECK-TF-NORED-NEXT:    [[TMP9:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-NORED-NEXT:    [[TMP10:%.*]] = xor i1 [[TMP9]], true
+; CHECK-TF-NORED-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-TF-NORED:       [[MIDDLE_BLOCK]]:
+; CHECK-TF-NORED-NEXT:    br label %[[FOR_END:.*]]
+; CHECK-TF-NORED:       [[FOR_END]]:
+; CHECK-TF-NORED-NEXT:    ret void
+;
+; CHECK-TF-NOREC-LABEL: define void @reverse(
+; CHECK-TF-NOREC-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]]) #[[ATTR0]] {
+; CHECK-TF-NOREC-NEXT:  [[ENTRY:.*:]]
+; CHECK-TF-NOREC-NEXT:    br label %[[VECTOR_PH:.*]]
+; CHECK-TF-NOREC:       [[VECTOR_PH]]:
+; CHECK-TF-NOREC-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NOREC-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; CHECK-TF-NOREC-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1024)
+; CHECK-TF-NOREC-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-NOREC:       [[VECTOR_BODY]]:
+; CHECK-TF-NOREC-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NOREC-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NOREC-NEXT:    [[TMP2:%.*]] = sub i64 1023, [[INDEX]]
+; CHECK-TF-NOREC-NEXT:    [[TMP3:%.*]] = shl i64 [[INDEX]], 3
+; CHECK-TF-NOREC-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP3]]
+; CHECK-TF-NOREC-NEXT:    [[TMP4:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[TMP2]]
+; CHECK-TF-NOREC-NEXT:    [[TMP5:%.*]] = sub nuw nsw i64 [[TMP1]], 1
+; CHECK-TF-NOREC-NEXT:    [[TMP6:%.*]] = sub i64 0, [[TMP5]]
+; CHECK-TF-NOREC-NEXT:    [[TMP7:%.*]] = getelementptr double, ptr [[TMP4]], i64 [[TMP6]]
+; CHECK-TF-NOREC-NEXT:    [[REVERSE:%.*]] = call <vscale x 2 x i1> @llvm.vector.reverse.nxv2i1(<vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-NOREC-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP7]], <vscale x 2 x i1> [[REVERSE]], <vscale x 2 x double> poison)
+; CHECK-TF-NOREC-NEXT:    [[TMP8:%.*]] = fadd <vscale x 2 x double> [[WIDE_MASKED_LOAD]], splat (double 1.000000e+00)
+; CHECK-TF-NOREC-NEXT:    [[REVERSE1:%.*]] = call <vscale x 2 x double> @llvm.vector.reverse.nxv2f64(<vscale x 2 x double> [[TMP8]])
+; CHECK-TF-NOREC-NEXT:    call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[REVERSE1]], ptr align 8 [[NEXT_GEP]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-NOREC-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-TF-NOREC-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1024)
+; CHECK-TF-NOREC-NEXT:    [[TMP9:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-NOREC-NEXT:    [[TMP10:%.*]] = xor i1 [[TMP9]], true
+; CHECK-TF-NOREC-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-TF-NOREC:       [[MIDDLE_BLOCK]]:
+; CHECK-TF-NOREC-NEXT:    br label %[[FOR_END:.*]]
+; CHECK-TF-NOREC:       [[FOR_END]]:
+; CHECK-TF-NOREC-NEXT:    ret void
+;
+; CHECK-TF-NOREV-LABEL: define void @reverse(
+; CHECK-TF-NOREV-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]]) #[[ATTR0]] {
+; CHECK-TF-NOREV-NEXT:  [[ENTRY:.*]]:
+; CHECK-TF-NOREV-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NOREV-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-NOREV-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
+; CHECK-TF-NOREV-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-TF-NOREV:       [[VECTOR_PH]]:
+; CHECK-TF-NOREV-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 1
+; CHECK-TF-NOREV-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; CHECK-TF-NOREV-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
+; CHECK-TF-NOREV-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-TF-NOREV-NEXT:    [[TMP4:%.*]] = sub i64 1023, [[N_VEC]]
+; CHECK-TF-NOREV-NEXT:    [[TMP5:%.*]] = shl i64 [[N_VEC]], 3
+; CHECK-TF-NOREV-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP5]]
+; CHECK-TF-NOREV-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-NOREV:       [[VECTOR_BODY]]:
+; CHECK-TF-NOREV-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NOREV-NEXT:    [[TMP7:%.*]] = sub i64 1023, [[INDEX]]
+; CHECK-TF-NOREV-NEXT:    [[TMP8:%.*]] = shl i64 [[INDEX]], 3
+; CHECK-TF-NOREV-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP8]]
+; CHECK-TF-NOREV-NEXT:    [[TMP9:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[TMP7]]
+; CHECK-TF-NOREV-NEXT:    [[TMP10:%.*]] = sub nuw nsw i64 [[TMP2]], 1
+; CHECK-TF-NOREV-NEXT:    [[TMP11:%.*]] = sub i64 0, [[TMP10]]
+; CHECK-TF-NOREV-NEXT:    [[TMP12:%.*]] = getelementptr inbounds double, ptr [[TMP9]], i64 [[TMP11]]
+; CHECK-TF-NOREV-NEXT:    [[TMP13:%.*]] = sub i64 [[TMP11]], [[TMP2]]
+; CHECK-TF-NOREV-NEXT:    [[TMP14:%.*]] = getelementptr inbounds double, ptr [[TMP9]], i64 [[TMP13]]
+; CHECK-TF-NOREV-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 2 x double>, ptr [[TMP12]], align 8
+; CHECK-TF-NOREV-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 2 x double>, ptr [[TMP14]], align 8
+; CHECK-TF-NOREV-NEXT:    [[TMP15:%.*]] = fadd <vscale x 2 x double> [[WIDE_LOAD]], splat (double 1.000000e+00)
+; CHECK-TF-NOREV-NEXT:    [[TMP16:%.*]] = fadd <vscale x 2 x double> [[WIDE_LOAD1]], splat (double 1.000000e+00)
+; CHECK-TF-NOREV-NEXT:    [[REVERSE:%.*]] = call <vscale x 2 x double> @llvm.vector.reverse.nxv2f64(<vscale x 2 x double> [[TMP15]])
+; CHECK-TF-NOREV-NEXT:    [[REVERSE2:%.*]] = call <vscale x 2 x double> @llvm.vector.reverse.nxv2f64(<vscale x 2 x double> [[TMP16]])
+; CHECK-TF-NOREV-NEXT:    [[TMP17:%.*]] = getelementptr double, ptr [[NEXT_GEP]], i64 [[TMP2]]
+; CHECK-TF-NOREV-NEXT:    store <vscale x 2 x double> [[REVERSE]], ptr [[NEXT_GEP]], align 8
+; CHECK-TF-NOREV-NEXT:    store <vscale x 2 x double> [[REVERSE2]], ptr [[TMP17]], align 8
+; CHECK-TF-NOREV-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-TF-NOREV-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-TF-NOREV-NEXT:    br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-TF-NOREV:       [[MIDDLE_BLOCK]]:
+; CHECK-TF-NOREV-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-TF-NOREV-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-TF-NOREV:       [[SCALAR_PH]]:
+; CHECK-TF-NOREV-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP4]], %[[MIDDLE_BLOCK]] ], [ 1023, %[[ENTRY]] ]
+; CHECK-TF-NOREV-NEXT:    [[BC_RESUME_VAL3:%.*]] = phi ptr [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ [[DST]], %[[ENTRY]] ]
+; CHECK-TF-NOREV-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK-TF-NOREV:       [[FOR_BODY]]:
+; CHECK-TF-NOREV-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-TF-NOREV-NEXT:    [[INDVARS_PTR:%.*]] = phi ptr [ [[BC_RESUME_VAL3]], %[[SCALAR_PH]] ], [ [[INDVARS_PTR_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-TF-NOREV-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[INDVARS_IV]]
+; CHECK-TF-NOREV-NEXT:    [[TMP19:%.*]] = load double, ptr [[ARRAYIDX]], align 8
+; CHECK-TF-NOREV-NEXT:    [[ADD:%.*]] = fadd double [[TMP19]], 1.000000e+00
+; CHECK-TF-NOREV-NEXT:    store double [[ADD]], ptr [[INDVARS_PTR]], align 8
+; CHECK-TF-NOREV-NEXT:    [[INDVARS_IV_NEXT]] = add nsw i64 [[INDVARS_IV]], -1
+; CHECK-TF-NOREV-NEXT:    [[INDVARS_PTR_NEXT]] = getelementptr inbounds double, ptr [[INDVARS_PTR]], i64 1
+; CHECK-TF-NOREV-NEXT:    [[CMP_NOT:%.*]] = icmp eq i64 [[INDVARS_IV]], 0
+; CHECK-TF-NOREV-NEXT:    br i1 [[CMP_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-TF-NOREV:       [[FOR_END]]:
+; CHECK-TF-NOREV-NEXT:    ret void
+;
+; CHECK-TF-ONLYRED-LABEL: define void @reverse(
+; CHECK-TF-ONLYRED-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]]) #[[ATTR0]] {
+; CHECK-TF-ONLYRED-NEXT:  [[ENTRY:.*]]:
+; CHECK-TF-ONLYRED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-ONLYRED-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-ONLYRED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
+; CHECK-TF-ONLYRED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-TF-ONLYRED:       [[VECTOR_PH]]:
+; CHECK-TF-ONLYRED-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 1
+; CHECK-TF-ONLYRED-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; CHECK-TF-ONLYRED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
+; CHECK-TF-ONLYRED-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-TF-ONLYRED-NEXT:    [[TMP4:%.*]] = sub i64 1023, [[N_VEC]]
+; CHECK-TF-ONLYRED-NEXT:    [[TMP5:%.*]] = shl i64 [[N_VEC]], 3
+; CHECK-TF-ONLYRED-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP5]]
+; CHECK-TF-ONLYRED-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-ONLYRED:       [[VECTOR_BODY]]:
+; CHECK-TF-ONLYRED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-ONLYRED-NEXT:    [[TMP7:%.*]] = sub i64 1023, [[INDEX]]
+; CHECK-TF-ONLYRED-NEXT:    [[TMP8:%.*]] = shl i64 [[INDEX]], 3
+; CHECK-TF-ONLYRED-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP8]]
+; CHECK-TF-ONLYRED-NEXT:    [[TMP9:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[TMP7]]
+; CHECK-TF-ONLYRED-NEXT:    [[TMP10:%.*]] = sub nuw nsw i64 [[TMP2]], 1
+; CHECK-TF-ONLYRED-NEXT:    [[TMP11:%.*]] = sub i64 0, [[TMP10]]
+; CHECK-TF-ONLYRED-NEXT:    [[TMP12:%.*]] = getelementptr inbounds double, ptr [[TMP9]], i64 [[TMP11]]
+; CHECK-TF-ONLYRED-NEXT:    [[TMP13:%.*]] = sub i64 [[TMP11]], [[TMP2]]
+; CHECK-TF-ONLYRED-NEXT:    [[TMP14:%.*]] = getelementptr inbounds double, ptr [[TMP9]], i64 [[TMP13]]
+; CHECK-TF-ONLYRED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 2 x double>, ptr [[TMP12]], align 8
+; CHECK-TF-ONLYRED-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 2 x double>, ptr [[TMP14]], align 8
+; CHECK-TF-ONLYRED-NEXT:    [[TMP15:%.*]] = fadd <vscale x 2 x double> [[WIDE_LOAD]], splat (double 1.000000e+00)
+; CHECK-TF-ONLYRED-NEXT:    [[TMP16:%.*]] = fadd <vscale x 2 x double> [[WIDE_LOAD1]], splat (double 1.000000e+00)
+; CHECK-TF-ONLYRED-NEXT:    [[REVERSE:%.*]] = call <vscale x 2 x double> @llvm.vector.reverse.nxv2f64(<vscale x 2 x double> [[TMP15]])
+; CHECK-TF-ONLYRED-NEXT:    [[REVERSE2:%.*]] = call <vscale x 2 x double> @llvm.vector.reverse.nxv2f64(<vscale x 2 x double> [[TMP16]])
+; CHECK-TF-ONLYRED-NEXT:    [[TMP17:%.*]] = getelementptr double, ptr [[NEXT_GEP]], i64 [[TMP2]]
+; CHECK-TF-ONLYRED-NEXT:    store <vscale x 2 x double> [[REVERSE]], ptr [[NEXT_GEP]], align 8
+; CHECK-TF-ONLYRED-NEXT:    store <vscale x 2 x double> [[REVERSE2]], ptr [[TMP17]], align 8
+; CHECK-TF-ONLYRED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-TF-ONLYRED-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-TF-ONLYRED-NEXT:    br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-TF-ONLYRED:       [[MIDDLE_BLOCK]]:
+; CHECK-TF-ONLYRED-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-TF-ONLYRED-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-TF-ONLYRED:       [[SCALAR_PH]]:
+; CHECK-TF-ONLYRED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP4]], %[[MIDDLE_BLOCK]] ], [ 1023, %[[ENTRY]] ]
+; CHECK-TF-ONLYRED-NEXT:    [[BC_RESUME_VAL3:%.*]] = phi ptr [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ [[DST]], %[[ENTRY]] ]
+; CHECK-TF-ONLYRED-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK-TF-ONLYRED:       [[FOR_BODY]]:
+; CHECK-TF-ONLYRED-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-TF-ONLYRED-NEXT:    [[INDVARS_PTR:%.*]] = phi ptr [ [[BC_RESUME_VAL3]], %[[SCALAR_PH]] ], [ [[INDVARS_PTR_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-TF-ONLYRED-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[INDVARS_IV]]
+; CHECK-TF-ONLYRED-NEXT:    [[TMP19:%.*]] = load double, ptr [[ARRAYIDX]], align 8
+; CHECK-TF-ONLYRED-NEXT:    [[ADD:%.*]] = fadd double [[TMP19]], 1.000000e+00
+; CHECK-TF-ONLYRED-NEXT:    store double [[ADD]], ptr [[INDVARS_PTR]], align 8
+; CHECK-TF-ONLYRED-NEXT:    [[INDVARS_IV_NEXT]] = add nsw i64 [[INDVARS_IV]], -1
+; CHECK-TF-ONLYRED-NEXT:    [[INDVARS_PTR_NEXT]] = getelementptr inbounds double, ptr [[INDVARS_PTR]], i64 1
+; CHECK-TF-ONLYRED-NEXT:    [[CMP_NOT:%.*]] = icmp eq i64 [[INDVARS_IV]], 0
+; CHECK-TF-ONLYRED-NEXT:    br i1 [[CMP_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-TF-ONLYRED:       [[FOR_END]]:
+; CHECK-TF-ONLYRED-NEXT:    ret void
+;
+; CHECK-NEOVERSE-V1-LABEL: define void @reverse(
+; CHECK-NEOVERSE-V1-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]]) #[[ATTR0]] {
+; CHECK-NEOVERSE-V1-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NEOVERSE-V1-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
+; CHECK-NEOVERSE-V1-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEOVERSE-V1:       [[VECTOR_PH]]:
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 1
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; CHECK-NEOVERSE-V1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
+; CHECK-NEOVERSE-V1-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP4:%.*]] = sub i64 1023, [[N_VEC]]
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP5:%.*]] = shl i64 [[N_VEC]], 3
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP5]]
+; CHECK-NEOVERSE-V1-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-NEOVERSE-V1:       [[VECTOR_BODY]]:
+; CHECK-NEOVERSE-V1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP7:%.*]] = sub i64 1023, [[INDEX]]
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP8:%.*]] = shl i64 [[INDEX]], 3
+; CHECK-NEOVERSE-V1-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP8]]
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP9:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[TMP7]]
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP10:%.*]] = sub nuw nsw i64 [[TMP2]], 1
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP11:%.*]] = sub i64 0, [[TMP10]]
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP12:%.*]] = getelementptr inbounds double, ptr [[TMP9]], i64 [[TMP11]]
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP13:%.*]] = sub i64 [[TMP11]], [[TMP2]]
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP14:%.*]] = getelementptr inbounds double, ptr [[TMP9]], i64 [[TMP13]]
+; CHECK-NEOVERSE-V1-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 2 x double>, ptr [[TMP12]], align 8
+; CHECK-NEOVERSE-V1-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 2 x double>, ptr [[TMP14]], align 8
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP15:%.*]] = fadd <vscale x 2 x double> [[WIDE_LOAD]], splat (double 1.000000e+00)
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP16:%.*]] = fadd <vscale x 2 x double> [[WIDE_LOAD1]], splat (double 1.000000e+00)
+; CHECK-NEOVERSE-V1-NEXT:    [[REVERSE:%.*]] = call <vscale x 2 x double> @llvm.vector.reverse.nxv2f64(<vscale x 2 x double> [[TMP15]])
+; CHECK-NEOVERSE-V1-NEXT:    [[REVERSE2:%.*]] = call <vscale x 2 x double> @llvm.vector.reverse.nxv2f64(<vscale x 2 x double> [[TMP16]])
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP17:%.*]] = getelementptr double, ptr [[NEXT_GEP]], i64 [[TMP2]]
+; CHECK-NEOVERSE-V1-NEXT:    store <vscale x 2 x double> [[REVERSE]], ptr [[NEXT_GEP]], align 8
+; CHECK-NEOVERSE-V1-NEXT:    store <vscale x 2 x double> [[REVERSE2]], ptr [[TMP17]], align 8
+; CHECK-NEOVERSE-V1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEOVERSE-V1-NEXT:    br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-NEOVERSE-V1:       [[MIDDLE_BLOCK]]:
+; CHECK-NEOVERSE-V1-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-NEOVERSE-V1-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-NEOVERSE-V1:       [[SCALAR_PH]]:
+; CHECK-NEOVERSE-V1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP4]], %[[MIDDLE_BLOCK]] ], [ 1023, %[[ENTRY]] ]
+; CHECK-NEOVERSE-V1-NEXT:    [[BC_RESUME_VAL3:%.*]] = phi ptr [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ [[DST]], %[[ENTRY]] ]
+; CHECK-NEOVERSE-V1-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK-NEOVERSE-V1:       [[FOR_BODY]]:
+; CHECK-NEOVERSE-V1-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEOVERSE-V1-NEXT:    [[INDVARS_PTR:%.*]] = phi ptr [ [[BC_RESUME_VAL3]], %[[SCALAR_PH]] ], [ [[INDVARS_PTR_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEOVERSE-V1-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[INDVARS_IV]]
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP19:%.*]] = load double, ptr [[ARRAYIDX]], align 8
+; CHECK-NEOVERSE-V1-NEXT:    [[ADD:%.*]] = fadd double [[TMP19]], 1.000000e+00
+; CHECK-NEOVERSE-V1-NEXT:    store double [[ADD]], ptr [[INDVARS_PTR]], align 8
+; CHECK-NEOVERSE-V1-NEXT:    [[INDVARS_IV_NEXT]] = add nsw i64 [[INDVARS_IV]], -1
+; CHECK-NEOVERSE-V1-NEXT:    [[INDVARS_PTR_NEXT]] = getelementptr inbounds double, ptr [[INDVARS_PTR]], i64 1
+; CHECK-NEOVERSE-V1-NEXT:    [[CMP_NOT:%.*]] = icmp eq i64 [[INDVARS_IV]], 0
+; CHECK-NEOVERSE-V1-NEXT:    br i1 [[CMP_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-NEOVERSE-V1:       [[FOR_END]]:
+; CHECK-NEOVERSE-V1-NEXT:    ret void
+;
+
+
+
+
 
 entry:
   br label %for.body
@@ -402,3 +2116,98 @@ attributes #0 = { "target-features"="+sve" }
 !2 = !{!"llvm.loop.vectorize.scalable.enable", i1 true}
 !3 = !{!"llvm.loop.interleave.count", i32 1}
 !4 = !{!"llvm.loop.vectorize.enable", i1 true}
+;.
+; CHECK-NOTF: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK-NOTF: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK-NOTF: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK-NOTF: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
+; CHECK-NOTF: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
+; CHECK-NOTF: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]}
+; CHECK-NOTF: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]}
+; CHECK-NOTF: [[LOOP7]] = distinct !{[[LOOP7]], [[META2]], [[META1]]}
+; CHECK-NOTF: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META2]]}
+; CHECK-NOTF: [[LOOP9]] = distinct !{[[LOOP9]], [[META2]], [[META1]]}
+; CHECK-NOTF: [[LOOP10]] = distinct !{[[LOOP10]], [[META1]], [[META2]]}
+; CHECK-NOTF: [[LOOP11]] = distinct !{[[LOOP11]], [[META2]], [[META1]]}
+;.
+; CHECK-TF-ALL: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK-TF-ALL: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK-TF-ALL: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK-TF-ALL: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]], [[META2]]}
+; CHECK-TF-ALL: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
+; CHECK-TF-ALL: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]], [[META2]]}
+; CHECK-TF-ALL: [[LOOP6]] = distinct !{[[LOOP6]], [[META2]], [[META1]]}
+; CHECK-TF-ALL: [[LOOP7]] = distinct !{[[LOOP7]], [[META1]], [[META2]]}
+;.
+; CHECK-TF: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK-TF: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK-TF: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK-TF: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]], [[META2]]}
+; CHECK-TF: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
+; CHECK-TF: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]], [[META2]]}
+; CHECK-TF: [[LOOP6]] = distinct !{[[LOOP6]], [[META2]], [[META1]]}
+; CHECK-TF: [[LOOP7]] = distinct !{[[LOOP7]], [[META1]], [[META2]]}
+;.
+; CHECK-TF-DEFAULT: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK-TF-DEFAULT: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK-TF-DEFAULT: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK-TF-DEFAULT: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]], [[META2]]}
+; CHECK-TF-DEFAULT: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
+; CHECK-TF-DEFAULT: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]], [[META2]]}
+; CHECK-TF-DEFAULT: [[LOOP6]] = distinct !{[[LOOP6]], [[META2]], [[META1]]}
+; CHECK-TF-DEFAULT: [[LOOP7]] = distinct !{[[LOOP7]], [[META1]], [[META2]]}
+;.
+; CHECK-TF-NORED: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK-TF-NORED: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK-TF-NORED: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK-TF-NORED: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]], [[META2]]}
+; CHECK-TF-NORED: [[LOOP4]] = distinct !{[[LOOP4]], [[META2]], [[META1]]}
+; CHECK-TF-NORED: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]], [[META2]]}
+; CHECK-TF-NORED: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]}
+; CHECK-TF-NORED: [[LOOP7]] = distinct !{[[LOOP7]], [[META2]], [[META1]]}
+; CHECK-TF-NORED: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META2]]}
+;.
+; CHECK-TF-NOREC: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK-TF-NOREC: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK-TF-NOREC: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK-TF-NOREC: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]], [[META2]]}
+; CHECK-TF-NOREC: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
+; CHECK-TF-NOREC: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]}
+; CHECK-TF-NOREC: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]}
+; CHECK-TF-NOREC: [[LOOP7]] = distinct !{[[LOOP7]], [[META2]], [[META1]]}
+; CHECK-TF-NOREC: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META2]]}
+;.
+; CHECK-TF-NOREV: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK-TF-NOREV: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK-TF-NOREV: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK-TF-NOREV: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]], [[META2]]}
+; CHECK-TF-NOREV: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
+; CHECK-TF-NOREV: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]], [[META2]]}
+; CHECK-TF-NOREV: [[LOOP6]] = distinct !{[[LOOP6]], [[META2]], [[META1]]}
+; CHECK-TF-NOREV: [[LOOP7]] = distinct !{[[LOOP7]], [[META1]], [[META2]]}
+; CHECK-TF-NOREV: [[LOOP8]] = distinct !{[[LOOP8]], [[META2]], [[META1]]}
+;.
+; CHECK-TF-ONLYRED: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK-TF-ONLYRED: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK-TF-ONLYRED: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK-TF-ONLYRED: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
+; CHECK-TF-ONLYRED: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
+; CHECK-TF-ONLYRED: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]], [[META2]]}
+; CHECK-TF-ONLYRED: [[LOOP6]] = distinct !{[[LOOP6]], [[META2]], [[META1]]}
+; CHECK-TF-ONLYRED: [[LOOP7]] = distinct !{[[LOOP7]], [[META1]], [[META2]]}
+; CHECK-TF-ONLYRED: [[LOOP8]] = distinct !{[[LOOP8]], [[META2]], [[META1]]}
+; CHECK-TF-ONLYRED: [[LOOP9]] = distinct !{[[LOOP9]], [[META1]], [[META2]]}
+; CHECK-TF-ONLYRED: [[LOOP10]] = distinct !{[[LOOP10]], [[META2]], [[META1]]}
+;.
+; CHECK-NEOVERSE-V1: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK-NEOVERSE-V1: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK-NEOVERSE-V1: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK-NEOVERSE-V1: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]], [[META2]]}
+; CHECK-NEOVERSE-V1: [[LOOP4]] = distinct !{[[LOOP4]], [[META2]], [[META1]]}
+; CHECK-NEOVERSE-V1: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]], [[META2]]}
+; CHECK-NEOVERSE-V1: [[LOOP6]] = distinct !{[[LOOP6]], [[META2]], [[META1]]}
+; CHECK-NEOVERSE-V1: [[LOOP7]] = distinct !{[[LOOP7]], [[META1]], [[META2]]}
+; CHECK-NEOVERSE-V1: [[LOOP8]] = distinct !{[[LOOP8]], [[META2]], [[META1]]}
+; CHECK-NEOVERSE-V1: [[LOOP9]] = distinct !{[[LOOP9]], [[META1]], [[META2]]}
+; CHECK-NEOVERSE-V1: [[LOOP10]] = distinct !{[[LOOP10]], [[META2]], [[META1]]}
+;.

>From bdb3c4083f1e3ec69472789ba9dafc6564cb07b6 Mon Sep 17 00:00:00 2001
From: Harry Ramsey <harry.ramsey at arm.com>
Date: Tue, 21 Jul 2026 12:39:11 +0000
Subject: [PATCH 2/2] fixup! [AArch64][ISel] Enable factor-3 scalable
 interleave operations

---
 .../Target/AArch64/AArch64ISelLowering.cpp    | 107 +-
 .../AArch64/AArch64TargetTransformInfo.cpp    |  16 +-
 .../AArch64/sve-vector-deinterleave.ll        |  53 +
 .../CodeGen/AArch64/sve-vector-interleave.ll  |  52 +
 .../AArch64/force-target-instruction-cost.ll  |  58 +-
 .../AArch64/sve-interleaved-accesses.ll       |  40 +-
 .../AArch64/sve-tail-folding-option.ll        | 928 ++++++++++--------
 7 files changed, 779 insertions(+), 475 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 2e6b0f4006de6..143798b4679fb 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -27771,6 +27771,41 @@ static SDValue getNarrowMaskForInterleavedOps(SelectionDAG &DAG, SDLoc &DL,
                      WideMask->getOperand(0));
 }
 
+static bool
+deinterleaveInterleavedValueForSVESt3(SDValue WideValue, SDLoc DL,
+                                      SelectionDAG &DAG,
+                                      SmallVectorImpl<SDValue> &Ops) {
+  constexpr unsigned Factor = 3;
+  EVT WideVT = WideValue.getValueType();
+  if (!WideVT.isScalableVector())
+    return false;
+
+  ElementCount WideEC = WideVT.getVectorElementCount();
+  if (!WideEC.isKnownMultipleOf(Factor))
+    return false;
+
+  EVT SubVecTy =
+      EVT::getVectorVT(*DAG.getContext(), WideVT.getVectorElementType(),
+                       WideEC.divideCoefficientBy(Factor));
+  if (!SubVecTy.isScalableVector() ||
+      SubVecTy.getSizeInBits().getKnownMinValue() != 128 ||
+      !DAG.getTargetLoweringInfo().isTypeLegal(SubVecTy))
+    return false;
+
+  SmallVector<SDValue, 3> SubVecs;
+  for (unsigned I = 0; I != Factor; ++I)
+    SubVecs.push_back(DAG.getNode(
+        ISD::EXTRACT_SUBVECTOR, DL, SubVecTy, WideValue,
+        DAG.getVectorIdxConstant(I * SubVecTy.getVectorMinNumElements(), DL)));
+
+  SmallVector<EVT, 3> ResultVTs(Factor, SubVecTy);
+  SDValue Deinterleaved = DAG.getNode(ISD::VECTOR_DEINTERLEAVE, DL,
+                                      DAG.getVTList(ResultVTs), SubVecs);
+  for (unsigned I = 0; I != Factor; ++I)
+    Ops.push_back(Deinterleaved.getValue(I));
+  return true;
+}
+
 static SDValue
 performInterleavedStoreCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
                                SelectionDAG &DAG) {
@@ -27809,7 +27844,12 @@ performInterleavedStoreCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
       return SDValue();
   } else if (!isSplatVectorInterleaveOps(DAG, DL, WideValue,
                                          ValueInterleaveOps)) {
-    return SDValue();
+    // A vector.interleave3 defined in another basic block is unknown here
+    // because SelectionDAGs are local to a basic block. Deinterleave the wide
+    // value back into legal operands so it can be used by st3.
+    if (!deinterleaveInterleavedValueForSVESt3(WideValue, DL, DAG,
+                                               ValueInterleaveOps))
+      return SDValue();
   }
 
   unsigned NumParts = ValueInterleaveOps.size();
@@ -27831,8 +27871,6 @@ performInterleavedStoreCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
 
   auto *MemN = cast<MemSDNode>(N);
   if (IsScalable) {
-    if (NumParts == 3)
-      return SDValue();
     SDValue Pred;
     if (IsMasked) {
       Pred = getNarrowMaskForInterleavedOps(DAG, DL, Mask, NumParts);
@@ -27843,8 +27881,18 @@ performInterleavedStoreCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
       Pred = DAG.getConstant(1, DL, PredVT);
     }
 
-    const Intrinsic::ID IID =
-        NumParts == 2 ? Intrinsic::aarch64_sve_st2 : Intrinsic::aarch64_sve_st4;
+    Intrinsic::ID IID;
+    switch (NumParts) {
+    case 2:
+      IID = Intrinsic::aarch64_sve_st2;
+      break;
+    case 3:
+      IID = Intrinsic::aarch64_sve_st3;
+      break;
+    case 4:
+      IID = Intrinsic::aarch64_sve_st4;
+      break;
+    }
     SmallVector<SDValue, 8> Ops;
     Ops.append({Chain, DAG.getConstant(IID, DL, MVT::i32)});
     Ops.append(ValueInterleaveOps);
@@ -30870,8 +30918,6 @@ static SDValue performVectorDeinterleaveCombine(
   SDValue Res;
   MemSDNode *MemNode = dyn_cast<MemSDNode>(WideVec);
   if (IsScalable) {
-    if (NumParts == 3)
-      return SDValue();
     SDValue Chain, BasePtr, Pred;
     if (auto *MaskedLoad = dyn_cast<MaskedLoadSDNode>(WideVec)) {
       // Bail out if the masked load has an unexpected number of uses, since we
@@ -30905,8 +30951,18 @@ static SDValue performVectorDeinterleaveCombine(
       BasePtr = Load->getBasePtr();
     }
 
-    const Intrinsic::ID IID = NumParts == 2 ? Intrinsic::aarch64_sve_ld2_sret
-                                            : Intrinsic::aarch64_sve_ld4_sret;
+    Intrinsic::ID IID;
+    switch (NumParts) {
+    case 2:
+      IID = Intrinsic::aarch64_sve_ld2_sret;
+      break;
+    case 3:
+      IID = Intrinsic::aarch64_sve_ld3_sret;
+      break;
+    case 4:
+      IID = Intrinsic::aarch64_sve_ld4_sret;
+      break;
+    }
     SDValue NewLdOps[] = {Chain, DAG.getConstant(IID, DL, MVT::i32), Pred,
                           BasePtr};
     Res = DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, ResVTList,
@@ -34422,6 +34478,7 @@ AArch64TargetLowering::LowerVECTOR_DEINTERLEAVE(SDValue Op,
   if (OpVT.isScalableVector() && Op->getNumOperands() == 3) {
     // aarch64_sve_ld3 only supports packed datatypes.
     EVT PackedVT = getPackedSVEVectorVT(OpVT.getVectorElementCount());
+    bool IsPredicate = OpVT.getVectorElementType() == MVT::i1;
     Align Alignment = DAG.getReducedAlign(PackedVT, /*UseABI=*/false);
     SDValue StackPtr =
         DAG.CreateStackTemporary(PackedVT.getStoreSize() * 3, Alignment);
@@ -34431,7 +34488,12 @@ AArch64TargetLowering::LowerVECTOR_DEINTERLEAVE(SDValue Op,
     for (unsigned I = 0; I < 3; ++I) {
       SDValue Ptr =
           DAG.getMemBasePlusOffset(StackPtr, PackedVT.getStoreSize() * I, DL);
-      SDValue V = getSVESafeBitCast(PackedVT, Op.getOperand(I), DAG);
+      // Predicate vectors cannot be bitcast to packed data vectors. Materialize
+      // their values in Z registers so they can be written to memory.
+      SDValue V =
+          IsPredicate
+              ? DAG.getNode(ISD::ZERO_EXTEND, DL, PackedVT, Op.getOperand(I))
+              : getSVESafeBitCast(PackedVT, Op.getOperand(I), DAG);
       Chains.push_back(
           DAG.getStore(DAG.getEntryNode(), DL, V, Ptr, MachinePointerInfo()));
     }
@@ -34450,9 +34512,12 @@ AArch64TargetLowering::LowerVECTOR_DEINTERLEAVE(SDValue Op,
     SDValue LD3 = DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops);
 
     SmallVector<SDValue, 3> Results;
-    Results.push_back(getSVESafeBitCast(OpVT, LD3.getValue(0), DAG));
-    Results.push_back(getSVESafeBitCast(OpVT, LD3.getValue(1), DAG));
-    Results.push_back(getSVESafeBitCast(OpVT, LD3.getValue(2), DAG));
+    for (unsigned I = 0; I < 3; ++I)
+      Results.push_back(IsPredicate
+                            ? DAG.getSetCC(DL, OpVT, LD3.getValue(I),
+                                           DAG.getConstant(0, DL, PackedVT),
+                                           ISD::SETNE)
+                            : getSVESafeBitCast(OpVT, LD3.getValue(I), DAG));
     return DAG.getMergeValues(Results, DL);
   }
 
@@ -34531,9 +34596,15 @@ SDValue AArch64TargetLowering::LowerVECTOR_INTERLEAVE(SDValue Op,
   if (OpVT.isScalableVector() && Op->getNumOperands() == 3) {
     // aarch64_sve_st3 only supports packed datatypes.
     EVT PackedVT = getPackedSVEVectorVT(OpVT.getVectorElementCount());
+    bool IsPredicate = OpVT.getVectorElementType() == MVT::i1;
     SmallVector<SDValue, 3> InVecs;
-    for (SDValue V : Op->ops())
-      InVecs.push_back(getSVESafeBitCast(PackedVT, V, DAG));
+    for (SDValue V : Op->ops()) {
+      // Predicate vectors cannot be bitcast to packed data vectors. Materialize
+      // their values in Z registers so they can be interleaved by st3.
+      InVecs.push_back(IsPredicate
+                           ? DAG.getNode(ISD::ZERO_EXTEND, DL, PackedVT, V)
+                           : getSVESafeBitCast(PackedVT, V, DAG));
+    }
 
     Align Alignment = DAG.getReducedAlign(PackedVT, /*UseABI=*/false);
     SDValue StackPtr =
@@ -34558,7 +34629,11 @@ SDValue AArch64TargetLowering::LowerVECTOR_INTERLEAVE(SDValue Op,
       SDValue Ptr =
           DAG.getMemBasePlusOffset(StackPtr, PackedVT.getStoreSize() * I, DL);
       SDValue L = DAG.getLoad(PackedVT, DL, Chain, Ptr, MachinePointerInfo());
-      Results.push_back(getSVESafeBitCast(OpVT, L, DAG));
+      Results.push_back(IsPredicate
+                            ? DAG.getSetCC(DL, OpVT, L,
+                                           DAG.getConstant(0, DL, PackedVT),
+                                           ISD::SETNE)
+                            : getSVESafeBitCast(OpVT, L, DAG));
     }
 
     return DAG.getMergeValues(Results, DL);
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index f462ef8b2c295..fdadc43583868 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -5414,12 +5414,16 @@ InstructionCost AArch64TTIImpl::getInterleavedMemoryOpCost(
   if (VecTy->isScalableTy() && !ST->hasSVE())
     return InstructionCost::getInvalid();
 
-  // Scalable VFs will emit vector.[de]interleave intrinsics, and currently we
-  // only have lowering for power-of-2 factors.
-  // TODO: Add lowering for vector.[de]interleave3 intrinsics and support in
-  // InterleavedAccessPass for ld3/st3
-  if (VecTy->isScalableTy() && !isPowerOf2_32(Factor))
-    return InstructionCost::getInvalid();
+  // Scalable VFs emit vector.[de]interleave intrinsics, for which the target
+  // supports factors up to the maximum supported interleave factor.
+  if (VecTy->isScalableTy()) {
+    if (Factor > TLI->getMaxSupportedInterleaveFactor())
+      return InstructionCost::getInvalid();
+
+    if (Factor == 3 &&
+        DL.getTypeSizeInBits(VecTy).getKnownMinValue() != (Factor * 128))
+      return InstructionCost::getInvalid();
+  }
 
   // Vectorization for masked interleaved accesses is only enabled for scalable
   // VF.
diff --git a/llvm/test/CodeGen/AArch64/sve-vector-deinterleave.ll b/llvm/test/CodeGen/AArch64/sve-vector-deinterleave.ll
index f0d4e86752dec..b76c79be0e399 100644
--- a/llvm/test/CodeGen/AArch64/sve-vector-deinterleave.ll
+++ b/llvm/test/CodeGen/AArch64/sve-vector-deinterleave.ll
@@ -712,6 +712,59 @@ define {<vscale x 2 x i1>, <vscale x 2 x i1>} @vector_deinterleave_nxv2i1_nxv4i1
   ret {<vscale x 2 x i1>, <vscale x 2 x i1>} %retval
 }
 
+define {<vscale x 8 x i1>, <vscale x 8 x i1>, <vscale x 8 x i1>} @deinterleave3_nxv16i1_nxv24i1(<vscale x 24 x i1> %vec) {
+; CHECK-LABEL: deinterleave3_nxv16i1_nxv24i1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-3
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x48, 0x1e, 0x22 // sp + 16 + 24 * VG
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    punpklo p1.h, p1.b
+; CHECK-NEXT:    punpkhi p3.h, p0.b
+; CHECK-NEXT:    punpklo p0.h, p0.b
+; CHECK-NEXT:    mov z0.h, p1/z, #1 // =0x1
+; CHECK-NEXT:    mov z1.h, p3/z, #1 // =0x1
+; CHECK-NEXT:    ptrue p2.h
+; CHECK-NEXT:    mov z2.h, p0/z, #1 // =0x1
+; CHECK-NEXT:    str z0, [sp, #2, mul vl]
+; CHECK-NEXT:    str z1, [sp, #1, mul vl]
+; CHECK-NEXT:    str z2, [sp]
+; CHECK-NEXT:    ld3h { z0.h - z2.h }, p2/z, [sp]
+; CHECK-NEXT:    cmpne p0.h, p2/z, z0.h, #0
+; CHECK-NEXT:    cmpne p1.h, p2/z, z1.h, #0
+; CHECK-NEXT:    cmpne p2.h, p2/z, z2.h, #0
+; CHECK-NEXT:    addvl sp, sp, #3
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %retval = call {<vscale x 8 x i1>, <vscale x 8 x i1>, <vscale x 8 x i1>} @llvm.vector.deinterleave3.nxv24i1(<vscale x 24 x i1> %vec)
+  ret {<vscale x 8 x i1>, <vscale x 8 x i1>, <vscale x 8 x i1>} %retval
+}
+
+define {<vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>} @deinterleave3_nxv16i1_nxv48i1(<vscale x 48 x i1> %vec) {
+; CHECK-LABEL: deinterleave3_nxv16i1_nxv48i1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-3
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x48, 0x1e, 0x22 // sp + 16 + 24 * VG
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    mov z0.b, p2/z, #1 // =0x1
+; CHECK-NEXT:    mov z1.b, p1/z, #1 // =0x1
+; CHECK-NEXT:    mov z2.b, p0/z, #1 // =0x1
+; CHECK-NEXT:    ptrue p2.b
+; CHECK-NEXT:    str z0, [sp, #2, mul vl]
+; CHECK-NEXT:    str z1, [sp, #1, mul vl]
+; CHECK-NEXT:    str z2, [sp]
+; CHECK-NEXT:    ld3b { z0.b - z2.b }, p2/z, [sp]
+; CHECK-NEXT:    cmpne p0.b, p2/z, z0.b, #0
+; CHECK-NEXT:    cmpne p1.b, p2/z, z1.b, #0
+; CHECK-NEXT:    cmpne p2.b, p2/z, z2.b, #0
+; CHECK-NEXT:    addvl sp, sp, #3
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %retval = call {<vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>} @llvm.vector.deinterleave3.nxv48i1(<vscale x 48 x i1> %vec)
+  ret {<vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>} %retval
+}
+
 
 ; Split illegal types
 
diff --git a/llvm/test/CodeGen/AArch64/sve-vector-interleave.ll b/llvm/test/CodeGen/AArch64/sve-vector-interleave.ll
index dbffad1b630d8..5f7fd2a3e0a51 100644
--- a/llvm/test/CodeGen/AArch64/sve-vector-interleave.ll
+++ b/llvm/test/CodeGen/AArch64/sve-vector-interleave.ll
@@ -659,6 +659,58 @@ define <vscale x 2 x i1> @interleave2_nxv2i1(<vscale x 1 x i1> %vec0, <vscale x
   ret <vscale x 2 x i1> %retval
 }
 
+define <vscale x 24 x i1> @interleave3_nxv24i1(<vscale x 8 x i1> %vec0, <vscale x 8 x i1> %vec1, <vscale x 8 x i1> %vec2) {
+; CHECK-LABEL: interleave3_nxv24i1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-3
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x48, 0x1e, 0x22 // sp + 16 + 24 * VG
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    mov z2.h, p2/z, #1 // =0x1
+; CHECK-NEXT:    mov z1.h, p1/z, #1 // =0x1
+; CHECK-NEXT:    mov z0.h, p0/z, #1 // =0x1
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    st3h { z0.h - z2.h }, p0, [sp]
+; CHECK-NEXT:    ldr z0, [sp, #1, mul vl]
+; CHECK-NEXT:    ldr z1, [sp]
+; CHECK-NEXT:    ldr z2, [sp, #2, mul vl]
+; CHECK-NEXT:    cmpne p1.h, p0/z, z0.h, #0
+; CHECK-NEXT:    cmpne p2.h, p0/z, z1.h, #0
+; CHECK-NEXT:    cmpne p3.h, p0/z, z2.h, #0
+; CHECK-NEXT:    uzp1 p0.b, p2.b, p1.b
+; CHECK-NEXT:    uzp1 p1.b, p3.b, p0.b
+; CHECK-NEXT:    addvl sp, sp, #3
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %retval = call <vscale x 24 x i1> @llvm.vector.interleave3.nxv24i1(<vscale x 8 x i1> %vec0, <vscale x 8 x i1> %vec1, <vscale x 8 x i1> %vec2)
+  ret <vscale x 24 x i1> %retval
+}
+
+define <vscale x 48 x i1> @interleave3_nxv48i1(<vscale x 16 x i1> %vec0, <vscale x 16 x i1> %vec1, <vscale x 16 x i1> %vec2) {
+; CHECK-LABEL: interleave3_nxv48i1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-3
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x48, 0x1e, 0x22 // sp + 16 + 24 * VG
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    mov z2.b, p2/z, #1 // =0x1
+; CHECK-NEXT:    mov z1.b, p1/z, #1 // =0x1
+; CHECK-NEXT:    mov z0.b, p0/z, #1 // =0x1
+; CHECK-NEXT:    ptrue p2.b
+; CHECK-NEXT:    st3b { z0.b - z2.b }, p2, [sp]
+; CHECK-NEXT:    ldr z0, [sp]
+; CHECK-NEXT:    ldr z1, [sp, #1, mul vl]
+; CHECK-NEXT:    ldr z2, [sp, #2, mul vl]
+; CHECK-NEXT:    cmpne p0.b, p2/z, z0.b, #0
+; CHECK-NEXT:    cmpne p1.b, p2/z, z1.b, #0
+; CHECK-NEXT:    cmpne p2.b, p2/z, z2.b, #0
+; CHECK-NEXT:    addvl sp, sp, #3
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %retval = call <vscale x 48 x i1> @llvm.vector.interleave3.nxv48i1(<vscale x 16 x i1> %vec0, <vscale x 16 x i1> %vec1, <vscale x 16 x i1> %vec2)
+  ret <vscale x 48 x i1> %retval
+}
+
 ; Split illegal type size
 
 define <vscale x 16 x i32> @interleave2_nxv16i32(<vscale x 8 x i32> %vec0, <vscale x 8 x i32> %vec1) {
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
index 07fc18e46c4fe..8fa200573e2b9 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
@@ -439,27 +439,40 @@ define void @interleave_group(ptr %dst) #1 {
 ; COST1-NEXT:  [[ITER_CHECK:.*:]]
 ; COST1-NEXT:    br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
 ; COST1:       [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
-; COST1-NEXT:    br i1 false, label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; COST1-NEXT:    [[TMP24:%.*]] = call i64 @llvm.vscale.i64()
+; COST1-NEXT:    [[TMP25:%.*]] = shl nuw i64 [[TMP24]], 5
+; COST1-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 101, [[TMP25]]
+; COST1-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
 ; COST1:       [[VECTOR_PH]]:
+; COST1-NEXT:    [[TMP26:%.*]] = shl nuw i64 [[TMP24]], 4
+; COST1-NEXT:    [[TMP27:%.*]] = shl nuw i64 [[TMP26]], 1
+; COST1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 101, [[TMP27]]
+; COST1-NEXT:    [[N_VEC:%.*]] = sub i64 101, [[N_MOD_VF]]
 ; COST1-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; COST1:       [[VECTOR_BODY]]:
 ; COST1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COST1-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 16
+; COST1-NEXT:    [[TMP28:%.*]] = add i64 [[TMP26]], 0
+; COST1-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP28]], 1
+; COST1-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], [[TMP5]]
 ; COST1-NEXT:    [[TMP1:%.*]] = mul i64 [[INDEX]], 3
 ; COST1-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP0]], 3
 ; COST1-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP1]]
 ; COST1-NEXT:    [[TMP4:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP2]]
-; COST1-NEXT:    store <48 x i8> zeroinitializer, ptr [[TMP3]], align 1
-; COST1-NEXT:    store <48 x i8> zeroinitializer, ptr [[TMP4]], align 1
-; COST1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
-; COST1-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96
-; COST1-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; COST1-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 48 x i8> @llvm.vector.interleave3.nxv48i8(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer)
+; COST1-NEXT:    store <vscale x 48 x i8> [[INTERLEAVED_VEC]], ptr [[TMP3]], align 1
+; COST1-NEXT:    [[INTERLEAVED_VEC1:%.*]] = call <vscale x 48 x i8> @llvm.vector.interleave3.nxv48i8(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer)
+; COST1-NEXT:    store <vscale x 48 x i8> [[INTERLEAVED_VEC1]], ptr [[TMP4]], align 1
+; COST1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP27]]
+; COST1-NEXT:    [[TMP29:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COST1-NEXT:    br i1 [[TMP29]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
 ; COST1:       [[MIDDLE_BLOCK]]:
-; COST1-NEXT:    br i1 false, [[EXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; COST1-NEXT:    [[CMP_N:%.*]] = icmp eq i64 101, [[N_VEC]]
+; COST1-NEXT:    br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
 ; COST1:       [[VEC_EPILOG_ITER_CHECK]]:
-; COST1-NEXT:    br i1 false, label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3]]
+; COST1-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4
+; COST1-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3]]
 ; COST1:       [[VEC_EPILOG_PH]]:
-; COST1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 96, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; COST1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; COST1-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[BC_RESUME_VAL]], i64 0
 ; COST1-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
 ; COST1-NEXT:    [[INDUCTION:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 1, i64 2, i64 3>
@@ -509,23 +522,32 @@ define void @interleave_group(ptr %dst) #1 {
 ; COST10-NEXT:  [[ITER_CHECK:.*:]]
 ; COST10-NEXT:    br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
 ; COST10:       [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
-; COST10-NEXT:    br i1 false, label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; COST10-NEXT:    [[TMP21:%.*]] = call i64 @llvm.vscale.i64()
+; COST10-NEXT:    [[TMP22:%.*]] = shl nuw i64 [[TMP21]], 4
+; COST10-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 101, [[TMP22]]
+; COST10-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
 ; COST10:       [[VECTOR_PH]]:
+; COST10-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP21]], 4
+; COST10-NEXT:    [[N_MOD_VF:%.*]] = urem i64 101, [[TMP2]]
+; COST10-NEXT:    [[N_VEC:%.*]] = sub i64 101, [[N_MOD_VF]]
 ; COST10-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; COST10:       [[VECTOR_BODY]]:
 ; COST10-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; COST10-NEXT:    [[TMP0:%.*]] = mul i64 [[INDEX]], 3
 ; COST10-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP0]]
-; COST10-NEXT:    store <48 x i8> zeroinitializer, ptr [[TMP1]], align 1
-; COST10-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
-; COST10-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96
-; COST10-NEXT:    br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; COST10-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 48 x i8> @llvm.vector.interleave3.nxv48i8(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer)
+; COST10-NEXT:    store <vscale x 48 x i8> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 1
+; COST10-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; COST10-NEXT:    [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COST10-NEXT:    br i1 [[TMP23]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
 ; COST10:       [[MIDDLE_BLOCK]]:
-; COST10-NEXT:    br i1 false, [[EXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; COST10-NEXT:    [[CMP_N:%.*]] = icmp eq i64 101, [[N_VEC]]
+; COST10-NEXT:    br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
 ; COST10:       [[VEC_EPILOG_ITER_CHECK]]:
-; COST10-NEXT:    br i1 false, label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3]]
+; COST10-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4
+; COST10-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3]]
 ; COST10:       [[VEC_EPILOG_PH]]:
-; COST10-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 96, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; COST10-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; COST10-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[BC_RESUME_VAL]], i64 0
 ; COST10-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
 ; COST10-NEXT:    [[INDUCTION:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 1, i64 2, i64 3>
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll
index 6d65b4e597075..a48218d0efcdc 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll
@@ -1320,8 +1320,6 @@ end:
 ;   dst[i].z = a[i].z << b[i].z;
 ; }
 ;
-; TODO: Support scalable interleave groups once we can also codegen
-; @llvm.[de]interleave3
 %struct.xyz = type { i32, i32, i32 }
 
 define void @interleave_deinterleave_factor3(ptr writeonly noalias %dst, ptr readonly %a, ptr readonly %b) {
@@ -1335,36 +1333,28 @@ define void @interleave_deinterleave_factor3(ptr writeonly noalias %dst, ptr rea
 ; CHECK-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP2]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP3:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP2]], i64 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP3]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[WIDE_GEP:%.*]] = getelementptr inbounds [[STRUCT_XYZ:%.*]], ptr [[A:%.*]], <vscale x 4 x i64> [[VEC_IND]]
-; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[WIDE_GEP]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
-; CHECK-NEXT:    [[WIDE_GEP1:%.*]] = getelementptr inbounds [[STRUCT_XYZ]], ptr [[B:%.*]], <vscale x 4 x i64> [[VEC_IND]]
-; CHECK-NEXT:    [[WIDE_MASKED_GATHER2:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[WIDE_GEP1]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds [[STRUCT_XYZ:%.*]], ptr [[A:%.*]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 12 x i32>, ptr [[TMP3]], align 4
+; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave3.nxv12i32(<vscale x 12 x i32> [[WIDE_VEC]])
+; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0
+; CHECK-NEXT:    [[WIDE_MASKED_GATHER5:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 1
+; CHECK-NEXT:    [[WIDE_MASKED_GATHER10:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 2
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds [[STRUCT_XYZ]], ptr [[B:%.*]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_VEC1:%.*]] = load <vscale x 12 x i32>, ptr [[TMP8]], align 4
+; CHECK-NEXT:    [[STRIDED_VEC2:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave3.nxv12i32(<vscale x 12 x i32> [[WIDE_VEC1]])
+; CHECK-NEXT:    [[WIDE_MASKED_GATHER2:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC2]], 0
+; CHECK-NEXT:    [[WIDE_MASKED_GATHER7:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC2]], 1
+; CHECK-NEXT:    [[WIDE_MASKED_GATHER12:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC2]], 2
 ; CHECK-NEXT:    [[TMP4:%.*]] = add nsw <vscale x 4 x i32> [[WIDE_MASKED_GATHER2]], [[WIDE_MASKED_GATHER]]
-; CHECK-NEXT:    [[WIDE_GEP3:%.*]] = getelementptr inbounds [[STRUCT_XYZ]], ptr [[DST:%.*]], <vscale x 4 x i64> [[VEC_IND]]
-; CHECK-NEXT:    call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP4]], <vscale x 4 x ptr> align 4 [[WIDE_GEP3]], <vscale x 4 x i1> splat (i1 true))
-; CHECK-NEXT:    [[WIDE_GEP4:%.*]] = getelementptr inbounds nuw i8, <vscale x 4 x ptr> [[WIDE_GEP]], i64 4
-; CHECK-NEXT:    [[WIDE_MASKED_GATHER5:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[WIDE_GEP4]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
-; CHECK-NEXT:    [[WIDE_GEP6:%.*]] = getelementptr inbounds nuw i8, <vscale x 4 x ptr> [[WIDE_GEP1]], i64 4
-; CHECK-NEXT:    [[WIDE_MASKED_GATHER7:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[WIDE_GEP6]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds [[STRUCT_XYZ]], ptr [[DST:%.*]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = sub nsw <vscale x 4 x i32> [[WIDE_MASKED_GATHER5]], [[WIDE_MASKED_GATHER7]]
-; CHECK-NEXT:    [[WIDE_GEP8:%.*]] = getelementptr inbounds nuw i8, <vscale x 4 x ptr> [[WIDE_GEP3]], i64 4
-; CHECK-NEXT:    call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP5]], <vscale x 4 x ptr> align 4 [[WIDE_GEP8]], <vscale x 4 x i1> splat (i1 true))
-; CHECK-NEXT:    [[WIDE_GEP9:%.*]] = getelementptr inbounds nuw i8, <vscale x 4 x ptr> [[WIDE_GEP]], i64 8
-; CHECK-NEXT:    [[WIDE_MASKED_GATHER10:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[WIDE_GEP9]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
-; CHECK-NEXT:    [[WIDE_GEP11:%.*]] = getelementptr inbounds nuw i8, <vscale x 4 x ptr> [[WIDE_GEP1]], i64 8
-; CHECK-NEXT:    [[WIDE_MASKED_GATHER12:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[WIDE_GEP11]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
 ; CHECK-NEXT:    [[TMP6:%.*]] = shl <vscale x 4 x i32> [[WIDE_MASKED_GATHER10]], [[WIDE_MASKED_GATHER12]]
-; CHECK-NEXT:    [[WIDE_GEP13:%.*]] = getelementptr inbounds nuw i8, <vscale x 4 x ptr> [[WIDE_GEP3]], i64 8
-; CHECK-NEXT:    call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP6]], <vscale x 4 x ptr> align 4 [[WIDE_GEP13]], <vscale x 4 x i1> splat (i1 true))
+; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 12 x i32> @llvm.vector.interleave3.nxv12i32(<vscale x 4 x i32> [[TMP4]], <vscale x 4 x i32> [[TMP5]], <vscale x 4 x i32> [[TMP6]])
+; CHECK-NEXT:    store <vscale x 12 x i32> [[INTERLEAVED_VEC]], ptr [[TMP12]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
-; CHECK-NEXT:    [[VEC_IND_NEXT]] = add nuw nsw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP33:![0-9]+]]
 ; CHECK:       middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-option.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-option.ll
index 802c0d460eee8..d826987e9fee4 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-option.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-option.ll
@@ -19,27 +19,27 @@ define void @simple_memset(i32 %val, ptr %ptr, i64 %n) #0 {
 ; CHECK-NOTF-LABEL: define void @simple_memset(
 ; CHECK-NOTF-SAME: i32 [[VAL:%.*]], ptr [[PTR:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NOTF-NEXT:  [[ENTRY:.*]]:
-; CHECK-NOTF-NEXT:    [[TMP0:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
-; CHECK-NOTF-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NOTF-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
-; CHECK-NOTF-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
+; CHECK-NOTF-NEXT:    [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
+; CHECK-NOTF-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NOTF-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NOTF-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[UMAX]], [[TMP1]]
 ; CHECK-NOTF-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK-NOTF:       [[VECTOR_PH]]:
-; CHECK-NOTF-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP1]], 2
-; CHECK-NOTF-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP3]]
-; CHECK-NOTF-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
+; CHECK-NOTF-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NOTF-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[UMAX]], [[TMP2]]
+; CHECK-NOTF-NEXT:    [[N_VEC:%.*]] = sub i64 [[UMAX]], [[N_MOD_VF]]
 ; CHECK-NOTF-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[VAL]], i64 0
 ; CHECK-NOTF-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-NOTF-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK-NOTF:       [[VECTOR_BODY]]:
 ; CHECK-NOTF-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NOTF-NEXT:    [[TMP4:%.*]] = getelementptr i32, ptr [[PTR]], i64 [[INDEX1]]
-; CHECK-NOTF-NEXT:    store <vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr [[TMP4]], align 4
-; CHECK-NOTF-NEXT:    [[INDEX_NEXT2]] = add nuw i64 [[INDEX1]], [[TMP3]]
-; CHECK-NOTF-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT2]], [[N_VEC]]
-; CHECK-NOTF-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NOTF-NEXT:    [[TMP3:%.*]] = getelementptr i32, ptr [[PTR]], i64 [[INDEX1]]
+; CHECK-NOTF-NEXT:    store <vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr [[TMP3]], align 4
+; CHECK-NOTF-NEXT:    [[INDEX_NEXT2]] = add nuw i64 [[INDEX1]], [[TMP2]]
+; CHECK-NOTF-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT2]], [[N_VEC]]
+; CHECK-NOTF-NEXT:    br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK-NOTF:       [[MIDDLE_BLOCK]]:
-; CHECK-NOTF-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
+; CHECK-NOTF-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[UMAX]], [[N_VEC]]
 ; CHECK-NOTF-NEXT:    br i1 [[CMP_N]], label %[[WHILE_END_LOOPEXIT:.*]], label %[[SCALAR_PH]]
 ; CHECK-NOTF:       [[SCALAR_PH]]:
 ; CHECK-NOTF-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
@@ -57,25 +57,25 @@ define void @simple_memset(i32 %val, ptr %ptr, i64 %n) #0 {
 ; CHECK-TF-ALL-LABEL: define void @simple_memset(
 ; CHECK-TF-ALL-SAME: i32 [[VAL:%.*]], ptr [[PTR:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-TF-ALL-NEXT:  [[ENTRY:.*:]]
-; CHECK-TF-ALL-NEXT:    [[TMP0:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
+; CHECK-TF-ALL-NEXT:    [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
 ; CHECK-TF-ALL-NEXT:    br label %[[VECTOR_PH:.*]]
 ; CHECK-TF-ALL:       [[VECTOR_PH]]:
-; CHECK-TF-ALL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-TF-ALL-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
-; CHECK-TF-ALL-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[TMP0]])
+; CHECK-TF-ALL-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-ALL-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-ALL-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[UMAX]])
 ; CHECK-TF-ALL-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[VAL]], i64 0
 ; CHECK-TF-ALL-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-TF-ALL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK-TF-ALL:       [[VECTOR_BODY]]:
 ; CHECK-TF-ALL-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-TF-ALL-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-TF-ALL-NEXT:    [[TMP3:%.*]] = getelementptr i32, ptr [[PTR]], i64 [[INDEX1]]
-; CHECK-TF-ALL-NEXT:    call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr align 4 [[TMP3]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
-; CHECK-TF-ALL-NEXT:    [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP2]]
-; CHECK-TF-ALL-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT2]], i64 [[TMP0]])
-; CHECK-TF-ALL-NEXT:    [[TMP4:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
-; CHECK-TF-ALL-NEXT:    [[TMP5:%.*]] = xor i1 [[TMP4]], true
-; CHECK-TF-ALL-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-TF-ALL-NEXT:    [[TMP2:%.*]] = getelementptr i32, ptr [[PTR]], i64 [[INDEX1]]
+; CHECK-TF-ALL-NEXT:    call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr align 4 [[TMP2]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-ALL-NEXT:    [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP1]]
+; CHECK-TF-ALL-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT2]], i64 [[UMAX]])
+; CHECK-TF-ALL-NEXT:    [[TMP3:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-ALL-NEXT:    [[TMP4:%.*]] = xor i1 [[TMP3]], true
+; CHECK-TF-ALL-NEXT:    br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK-TF-ALL:       [[MIDDLE_BLOCK]]:
 ; CHECK-TF-ALL-NEXT:    br label %[[WHILE_END_LOOPEXIT:.*]]
 ; CHECK-TF-ALL:       [[WHILE_END_LOOPEXIT]]:
@@ -84,25 +84,25 @@ define void @simple_memset(i32 %val, ptr %ptr, i64 %n) #0 {
 ; CHECK-TF-LABEL: define void @simple_memset(
 ; CHECK-TF-SAME: i32 [[VAL:%.*]], ptr [[PTR:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-TF-NEXT:  [[ENTRY:.*:]]
-; CHECK-TF-NEXT:    [[TMP0:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
+; CHECK-TF-NEXT:    [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
 ; CHECK-TF-NEXT:    br label %[[VECTOR_PH:.*]]
 ; CHECK-TF:       [[VECTOR_PH]]:
-; CHECK-TF-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-TF-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
-; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[TMP0]])
+; CHECK-TF-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[UMAX]])
 ; CHECK-TF-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[VAL]], i64 0
 ; CHECK-TF-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-TF-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK-TF:       [[VECTOR_BODY]]:
 ; CHECK-TF-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-TF-NEXT:    [[TMP3:%.*]] = getelementptr i32, ptr [[PTR]], i64 [[INDEX1]]
-; CHECK-TF-NEXT:    call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr align 4 [[TMP3]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
-; CHECK-TF-NEXT:    [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP2]]
-; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT2]], i64 [[TMP0]])
-; CHECK-TF-NEXT:    [[TMP4:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
-; CHECK-TF-NEXT:    [[TMP5:%.*]] = xor i1 [[TMP4]], true
-; CHECK-TF-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-TF-NEXT:    [[TMP2:%.*]] = getelementptr i32, ptr [[PTR]], i64 [[INDEX1]]
+; CHECK-TF-NEXT:    call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr align 4 [[TMP2]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-NEXT:    [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP1]]
+; CHECK-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT2]], i64 [[UMAX]])
+; CHECK-TF-NEXT:    [[TMP3:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-NEXT:    [[TMP4:%.*]] = xor i1 [[TMP3]], true
+; CHECK-TF-NEXT:    br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK-TF:       [[MIDDLE_BLOCK]]:
 ; CHECK-TF-NEXT:    br label %[[WHILE_END_LOOPEXIT:.*]]
 ; CHECK-TF:       [[WHILE_END_LOOPEXIT]]:
@@ -111,25 +111,25 @@ define void @simple_memset(i32 %val, ptr %ptr, i64 %n) #0 {
 ; CHECK-TF-DEFAULT-LABEL: define void @simple_memset(
 ; CHECK-TF-DEFAULT-SAME: i32 [[VAL:%.*]], ptr [[PTR:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-TF-DEFAULT-NEXT:  [[ENTRY:.*:]]
-; CHECK-TF-DEFAULT-NEXT:    [[TMP0:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
+; CHECK-TF-DEFAULT-NEXT:    [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
 ; CHECK-TF-DEFAULT-NEXT:    br label %[[VECTOR_PH:.*]]
 ; CHECK-TF-DEFAULT:       [[VECTOR_PH]]:
-; CHECK-TF-DEFAULT-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-TF-DEFAULT-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
-; CHECK-TF-DEFAULT-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[TMP0]])
+; CHECK-TF-DEFAULT-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-DEFAULT-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-DEFAULT-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[UMAX]])
 ; CHECK-TF-DEFAULT-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[VAL]], i64 0
 ; CHECK-TF-DEFAULT-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-TF-DEFAULT-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK-TF-DEFAULT:       [[VECTOR_BODY]]:
 ; CHECK-TF-DEFAULT-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-TF-DEFAULT-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-TF-DEFAULT-NEXT:    [[TMP3:%.*]] = getelementptr i32, ptr [[PTR]], i64 [[INDEX1]]
-; CHECK-TF-DEFAULT-NEXT:    call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr align 4 [[TMP3]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
-; CHECK-TF-DEFAULT-NEXT:    [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP2]]
-; CHECK-TF-DEFAULT-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT2]], i64 [[TMP0]])
-; CHECK-TF-DEFAULT-NEXT:    [[TMP4:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
-; CHECK-TF-DEFAULT-NEXT:    [[TMP5:%.*]] = xor i1 [[TMP4]], true
-; CHECK-TF-DEFAULT-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-TF-DEFAULT-NEXT:    [[TMP2:%.*]] = getelementptr i32, ptr [[PTR]], i64 [[INDEX1]]
+; CHECK-TF-DEFAULT-NEXT:    call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr align 4 [[TMP2]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-DEFAULT-NEXT:    [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP1]]
+; CHECK-TF-DEFAULT-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT2]], i64 [[UMAX]])
+; CHECK-TF-DEFAULT-NEXT:    [[TMP3:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-DEFAULT-NEXT:    [[TMP4:%.*]] = xor i1 [[TMP3]], true
+; CHECK-TF-DEFAULT-NEXT:    br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK-TF-DEFAULT:       [[MIDDLE_BLOCK]]:
 ; CHECK-TF-DEFAULT-NEXT:    br label %[[WHILE_END_LOOPEXIT:.*]]
 ; CHECK-TF-DEFAULT:       [[WHILE_END_LOOPEXIT]]:
@@ -138,25 +138,25 @@ define void @simple_memset(i32 %val, ptr %ptr, i64 %n) #0 {
 ; CHECK-TF-NORED-LABEL: define void @simple_memset(
 ; CHECK-TF-NORED-SAME: i32 [[VAL:%.*]], ptr [[PTR:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-TF-NORED-NEXT:  [[ENTRY:.*:]]
-; CHECK-TF-NORED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
+; CHECK-TF-NORED-NEXT:    [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
 ; CHECK-TF-NORED-NEXT:    br label %[[VECTOR_PH:.*]]
 ; CHECK-TF-NORED:       [[VECTOR_PH]]:
-; CHECK-TF-NORED-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-TF-NORED-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
-; CHECK-TF-NORED-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[TMP0]])
+; CHECK-TF-NORED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NORED-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-NORED-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[UMAX]])
 ; CHECK-TF-NORED-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[VAL]], i64 0
 ; CHECK-TF-NORED-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-TF-NORED-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK-TF-NORED:       [[VECTOR_BODY]]:
 ; CHECK-TF-NORED-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-TF-NORED-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-TF-NORED-NEXT:    [[TMP3:%.*]] = getelementptr i32, ptr [[PTR]], i64 [[INDEX1]]
-; CHECK-TF-NORED-NEXT:    call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr align 4 [[TMP3]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
-; CHECK-TF-NORED-NEXT:    [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP2]]
-; CHECK-TF-NORED-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT2]], i64 [[TMP0]])
-; CHECK-TF-NORED-NEXT:    [[TMP4:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
-; CHECK-TF-NORED-NEXT:    [[TMP5:%.*]] = xor i1 [[TMP4]], true
-; CHECK-TF-NORED-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-TF-NORED-NEXT:    [[TMP2:%.*]] = getelementptr i32, ptr [[PTR]], i64 [[INDEX1]]
+; CHECK-TF-NORED-NEXT:    call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr align 4 [[TMP2]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-NORED-NEXT:    [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP1]]
+; CHECK-TF-NORED-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT2]], i64 [[UMAX]])
+; CHECK-TF-NORED-NEXT:    [[TMP3:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-NORED-NEXT:    [[TMP4:%.*]] = xor i1 [[TMP3]], true
+; CHECK-TF-NORED-NEXT:    br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK-TF-NORED:       [[MIDDLE_BLOCK]]:
 ; CHECK-TF-NORED-NEXT:    br label %[[WHILE_END_LOOPEXIT:.*]]
 ; CHECK-TF-NORED:       [[WHILE_END_LOOPEXIT]]:
@@ -165,25 +165,25 @@ define void @simple_memset(i32 %val, ptr %ptr, i64 %n) #0 {
 ; CHECK-TF-NOREC-LABEL: define void @simple_memset(
 ; CHECK-TF-NOREC-SAME: i32 [[VAL:%.*]], ptr [[PTR:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-TF-NOREC-NEXT:  [[ENTRY:.*:]]
-; CHECK-TF-NOREC-NEXT:    [[TMP0:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
+; CHECK-TF-NOREC-NEXT:    [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
 ; CHECK-TF-NOREC-NEXT:    br label %[[VECTOR_PH:.*]]
 ; CHECK-TF-NOREC:       [[VECTOR_PH]]:
-; CHECK-TF-NOREC-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-TF-NOREC-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
-; CHECK-TF-NOREC-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[TMP0]])
+; CHECK-TF-NOREC-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NOREC-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-NOREC-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[UMAX]])
 ; CHECK-TF-NOREC-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[VAL]], i64 0
 ; CHECK-TF-NOREC-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-TF-NOREC-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK-TF-NOREC:       [[VECTOR_BODY]]:
 ; CHECK-TF-NOREC-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-TF-NOREC-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-TF-NOREC-NEXT:    [[TMP3:%.*]] = getelementptr i32, ptr [[PTR]], i64 [[INDEX1]]
-; CHECK-TF-NOREC-NEXT:    call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr align 4 [[TMP3]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
-; CHECK-TF-NOREC-NEXT:    [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP2]]
-; CHECK-TF-NOREC-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT2]], i64 [[TMP0]])
-; CHECK-TF-NOREC-NEXT:    [[TMP4:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
-; CHECK-TF-NOREC-NEXT:    [[TMP5:%.*]] = xor i1 [[TMP4]], true
-; CHECK-TF-NOREC-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-TF-NOREC-NEXT:    [[TMP2:%.*]] = getelementptr i32, ptr [[PTR]], i64 [[INDEX1]]
+; CHECK-TF-NOREC-NEXT:    call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr align 4 [[TMP2]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-NOREC-NEXT:    [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP1]]
+; CHECK-TF-NOREC-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT2]], i64 [[UMAX]])
+; CHECK-TF-NOREC-NEXT:    [[TMP3:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-NOREC-NEXT:    [[TMP4:%.*]] = xor i1 [[TMP3]], true
+; CHECK-TF-NOREC-NEXT:    br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK-TF-NOREC:       [[MIDDLE_BLOCK]]:
 ; CHECK-TF-NOREC-NEXT:    br label %[[WHILE_END_LOOPEXIT:.*]]
 ; CHECK-TF-NOREC:       [[WHILE_END_LOOPEXIT]]:
@@ -192,25 +192,25 @@ define void @simple_memset(i32 %val, ptr %ptr, i64 %n) #0 {
 ; CHECK-TF-NOREV-LABEL: define void @simple_memset(
 ; CHECK-TF-NOREV-SAME: i32 [[VAL:%.*]], ptr [[PTR:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-TF-NOREV-NEXT:  [[ENTRY:.*:]]
-; CHECK-TF-NOREV-NEXT:    [[TMP0:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
+; CHECK-TF-NOREV-NEXT:    [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
 ; CHECK-TF-NOREV-NEXT:    br label %[[VECTOR_PH:.*]]
 ; CHECK-TF-NOREV:       [[VECTOR_PH]]:
-; CHECK-TF-NOREV-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-TF-NOREV-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
-; CHECK-TF-NOREV-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[TMP0]])
+; CHECK-TF-NOREV-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NOREV-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-NOREV-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[UMAX]])
 ; CHECK-TF-NOREV-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[VAL]], i64 0
 ; CHECK-TF-NOREV-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-TF-NOREV-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK-TF-NOREV:       [[VECTOR_BODY]]:
 ; CHECK-TF-NOREV-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-TF-NOREV-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-TF-NOREV-NEXT:    [[TMP3:%.*]] = getelementptr i32, ptr [[PTR]], i64 [[INDEX1]]
-; CHECK-TF-NOREV-NEXT:    call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr align 4 [[TMP3]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
-; CHECK-TF-NOREV-NEXT:    [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP2]]
-; CHECK-TF-NOREV-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT2]], i64 [[TMP0]])
-; CHECK-TF-NOREV-NEXT:    [[TMP4:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
-; CHECK-TF-NOREV-NEXT:    [[TMP5:%.*]] = xor i1 [[TMP4]], true
-; CHECK-TF-NOREV-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-TF-NOREV-NEXT:    [[TMP2:%.*]] = getelementptr i32, ptr [[PTR]], i64 [[INDEX1]]
+; CHECK-TF-NOREV-NEXT:    call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr align 4 [[TMP2]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-NOREV-NEXT:    [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP1]]
+; CHECK-TF-NOREV-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT2]], i64 [[UMAX]])
+; CHECK-TF-NOREV-NEXT:    [[TMP3:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-NOREV-NEXT:    [[TMP4:%.*]] = xor i1 [[TMP3]], true
+; CHECK-TF-NOREV-NEXT:    br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK-TF-NOREV:       [[MIDDLE_BLOCK]]:
 ; CHECK-TF-NOREV-NEXT:    br label %[[WHILE_END_LOOPEXIT:.*]]
 ; CHECK-TF-NOREV:       [[WHILE_END_LOOPEXIT]]:
@@ -219,27 +219,27 @@ define void @simple_memset(i32 %val, ptr %ptr, i64 %n) #0 {
 ; CHECK-TF-ONLYRED-LABEL: define void @simple_memset(
 ; CHECK-TF-ONLYRED-SAME: i32 [[VAL:%.*]], ptr [[PTR:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-TF-ONLYRED-NEXT:  [[ENTRY:.*]]:
-; CHECK-TF-ONLYRED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
-; CHECK-TF-ONLYRED-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-TF-ONLYRED-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
-; CHECK-TF-ONLYRED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
+; CHECK-TF-ONLYRED-NEXT:    [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
+; CHECK-TF-ONLYRED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-ONLYRED-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-ONLYRED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[UMAX]], [[TMP1]]
 ; CHECK-TF-ONLYRED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK-TF-ONLYRED:       [[VECTOR_PH]]:
-; CHECK-TF-ONLYRED-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP1]], 2
-; CHECK-TF-ONLYRED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP3]]
-; CHECK-TF-ONLYRED-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
+; CHECK-TF-ONLYRED-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-ONLYRED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[UMAX]], [[TMP2]]
+; CHECK-TF-ONLYRED-NEXT:    [[N_VEC:%.*]] = sub i64 [[UMAX]], [[N_MOD_VF]]
 ; CHECK-TF-ONLYRED-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[VAL]], i64 0
 ; CHECK-TF-ONLYRED-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-TF-ONLYRED-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK-TF-ONLYRED:       [[VECTOR_BODY]]:
 ; CHECK-TF-ONLYRED-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-TF-ONLYRED-NEXT:    [[TMP4:%.*]] = getelementptr i32, ptr [[PTR]], i64 [[INDEX1]]
-; CHECK-TF-ONLYRED-NEXT:    store <vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr [[TMP4]], align 4
-; CHECK-TF-ONLYRED-NEXT:    [[INDEX_NEXT2]] = add nuw i64 [[INDEX1]], [[TMP3]]
-; CHECK-TF-ONLYRED-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT2]], [[N_VEC]]
-; CHECK-TF-ONLYRED-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-TF-ONLYRED-NEXT:    [[TMP3:%.*]] = getelementptr i32, ptr [[PTR]], i64 [[INDEX1]]
+; CHECK-TF-ONLYRED-NEXT:    store <vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr [[TMP3]], align 4
+; CHECK-TF-ONLYRED-NEXT:    [[INDEX_NEXT2]] = add nuw i64 [[INDEX1]], [[TMP2]]
+; CHECK-TF-ONLYRED-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT2]], [[N_VEC]]
+; CHECK-TF-ONLYRED-NEXT:    br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK-TF-ONLYRED:       [[MIDDLE_BLOCK]]:
-; CHECK-TF-ONLYRED-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
+; CHECK-TF-ONLYRED-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[UMAX]], [[N_VEC]]
 ; CHECK-TF-ONLYRED-NEXT:    br i1 [[CMP_N]], label %[[WHILE_END_LOOPEXIT:.*]], label %[[SCALAR_PH]]
 ; CHECK-TF-ONLYRED:       [[SCALAR_PH]]:
 ; CHECK-TF-ONLYRED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
@@ -257,25 +257,25 @@ define void @simple_memset(i32 %val, ptr %ptr, i64 %n) #0 {
 ; CHECK-NEOVERSE-V1-LABEL: define void @simple_memset(
 ; CHECK-NEOVERSE-V1-SAME: i32 [[VAL:%.*]], ptr [[PTR:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEOVERSE-V1-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEOVERSE-V1-NEXT:    [[TMP0:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
+; CHECK-NEOVERSE-V1-NEXT:    [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
 ; CHECK-NEOVERSE-V1-NEXT:    br label %[[VECTOR_PH:.*]]
 ; CHECK-NEOVERSE-V1:       [[VECTOR_PH]]:
-; CHECK-NEOVERSE-V1-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEOVERSE-V1-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
-; CHECK-NEOVERSE-V1-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[TMP0]])
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NEOVERSE-V1-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[UMAX]])
 ; CHECK-NEOVERSE-V1-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[VAL]], i64 0
 ; CHECK-NEOVERSE-V1-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-NEOVERSE-V1-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK-NEOVERSE-V1:       [[VECTOR_BODY]]:
 ; CHECK-NEOVERSE-V1-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEOVERSE-V1-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEOVERSE-V1-NEXT:    [[TMP3:%.*]] = getelementptr i32, ptr [[PTR]], i64 [[INDEX1]]
-; CHECK-NEOVERSE-V1-NEXT:    call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr align 4 [[TMP3]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
-; CHECK-NEOVERSE-V1-NEXT:    [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP2]]
-; CHECK-NEOVERSE-V1-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT2]], i64 [[TMP0]])
-; CHECK-NEOVERSE-V1-NEXT:    [[TMP4:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
-; CHECK-NEOVERSE-V1-NEXT:    [[TMP5:%.*]] = xor i1 [[TMP4]], true
-; CHECK-NEOVERSE-V1-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP2:%.*]] = getelementptr i32, ptr [[PTR]], i64 [[INDEX1]]
+; CHECK-NEOVERSE-V1-NEXT:    call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr align 4 [[TMP2]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-NEOVERSE-V1-NEXT:    [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP1]]
+; CHECK-NEOVERSE-V1-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT2]], i64 [[UMAX]])
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP3:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP4:%.*]] = xor i1 [[TMP3]], true
+; CHECK-NEOVERSE-V1-NEXT:    br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK-NEOVERSE-V1:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEOVERSE-V1-NEXT:    br label %[[WHILE_END_LOOPEXIT:.*]]
 ; CHECK-NEOVERSE-V1:       [[WHILE_END_LOOPEXIT]]:
@@ -1049,40 +1049,44 @@ define void @interleave(ptr noalias %dst, ptr noalias %src, i64 %n) #0 {
 ; CHECK-NOTF-LABEL: define void @interleave(
 ; CHECK-NOTF-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; CHECK-NOTF-NEXT:  [[ENTRY:.*]]:
-; CHECK-NOTF-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; CHECK-NOTF-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NOTF-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-NOTF-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; CHECK-NOTF-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK-NOTF:       [[VECTOR_PH]]:
-; CHECK-NOTF-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
+; CHECK-NOTF-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NOTF-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; CHECK-NOTF-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-NOTF-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; CHECK-NOTF-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK-NOTF:       [[VECTOR_BODY]]:
 ; CHECK-NOTF-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NOTF-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 4
-; CHECK-NOTF-NEXT:    [[TMP1:%.*]] = shl nuw nsw i64 [[INDEX]], 1
-; CHECK-NOTF-NEXT:    [[TMP2:%.*]] = shl nuw nsw i64 [[TMP0]], 1
-; CHECK-NOTF-NEXT:    [[TMP3:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP1]]
-; CHECK-NOTF-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP2]]
-; CHECK-NOTF-NEXT:    [[WIDE_VEC:%.*]] = load <8 x float>, ptr [[TMP3]], align 4
-; CHECK-NOTF-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <8 x float> [[WIDE_VEC]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NOTF-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <8 x float> [[WIDE_VEC]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; CHECK-NOTF-NEXT:    [[WIDE_VEC2:%.*]] = load <8 x float>, ptr [[TMP4]], align 4
-; CHECK-NOTF-NEXT:    [[STRIDED_VEC3:%.*]] = shufflevector <8 x float> [[WIDE_VEC2]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NOTF-NEXT:    [[STRIDED_VEC4:%.*]] = shufflevector <8 x float> [[WIDE_VEC2]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; CHECK-NOTF-NEXT:    [[TMP5:%.*]] = mul nuw nsw i64 [[INDEX]], 3
-; CHECK-NOTF-NEXT:    [[TMP6:%.*]] = mul nuw nsw i64 [[TMP0]], 3
-; CHECK-NOTF-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP5]]
-; CHECK-NOTF-NEXT:    [[TMP8:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP6]]
-; CHECK-NOTF-NEXT:    [[TMP9:%.*]] = shufflevector <4 x float> [[STRIDED_VEC]], <4 x float> [[STRIDED_VEC1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NOTF-NEXT:    [[TMP10:%.*]] = shufflevector <8 x float> [[TMP9]], <8 x float> <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float undef, float undef, float undef, float undef>, <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-; CHECK-NOTF-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <12 x float> [[TMP10]], <12 x float> poison, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
-; CHECK-NOTF-NEXT:    store <12 x float> [[INTERLEAVED_VEC]], ptr [[TMP7]], align 4
-; CHECK-NOTF-NEXT:    [[TMP11:%.*]] = shufflevector <4 x float> [[STRIDED_VEC3]], <4 x float> [[STRIDED_VEC4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NOTF-NEXT:    [[TMP12:%.*]] = shufflevector <8 x float> [[TMP11]], <8 x float> <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float undef, float undef, float undef, float undef>, <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-; CHECK-NOTF-NEXT:    [[INTERLEAVED_VEC5:%.*]] = shufflevector <12 x float> [[TMP12]], <12 x float> poison, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
-; CHECK-NOTF-NEXT:    store <12 x float> [[INTERLEAVED_VEC5]], ptr [[TMP8]], align 4
-; CHECK-NOTF-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
-; CHECK-NOTF-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NOTF-NEXT:    br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NOTF-NEXT:    [[TMP4:%.*]] = add i64 [[TMP2]], 0
+; CHECK-NOTF-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 1
+; CHECK-NOTF-NEXT:    [[TMP6:%.*]] = add i64 [[INDEX]], [[TMP5]]
+; CHECK-NOTF-NEXT:    [[TMP7:%.*]] = shl nuw nsw i64 [[INDEX]], 1
+; CHECK-NOTF-NEXT:    [[TMP8:%.*]] = shl nuw nsw i64 [[TMP6]], 1
+; CHECK-NOTF-NEXT:    [[TMP9:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP7]]
+; CHECK-NOTF-NEXT:    [[TMP10:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP8]]
+; CHECK-NOTF-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x float>, ptr [[TMP9]], align 4
+; CHECK-NOTF-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC]])
+; CHECK-NOTF-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 0
+; CHECK-NOTF-NEXT:    [[TMP12:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 1
+; CHECK-NOTF-NEXT:    [[WIDE_VEC1:%.*]] = load <vscale x 8 x float>, ptr [[TMP10]], align 4
+; CHECK-NOTF-NEXT:    [[STRIDED_VEC2:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC1]])
+; CHECK-NOTF-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC2]], 0
+; CHECK-NOTF-NEXT:    [[TMP14:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC2]], 1
+; CHECK-NOTF-NEXT:    [[TMP15:%.*]] = mul nuw nsw i64 [[INDEX]], 3
+; CHECK-NOTF-NEXT:    [[TMP16:%.*]] = mul nuw nsw i64 [[TMP6]], 3
+; CHECK-NOTF-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP15]]
+; CHECK-NOTF-NEXT:    [[TMP18:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP16]]
+; CHECK-NOTF-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 12 x float> @llvm.vector.interleave3.nxv12f32(<vscale x 4 x float> [[TMP11]], <vscale x 4 x float> [[TMP12]], <vscale x 4 x float> splat (float 3.000000e+00))
+; CHECK-NOTF-NEXT:    store <vscale x 12 x float> [[INTERLEAVED_VEC]], ptr [[TMP17]], align 4
+; CHECK-NOTF-NEXT:    [[INTERLEAVED_VEC3:%.*]] = call <vscale x 12 x float> @llvm.vector.interleave3.nxv12f32(<vscale x 4 x float> [[TMP13]], <vscale x 4 x float> [[TMP14]], <vscale x 4 x float> splat (float 3.000000e+00))
+; CHECK-NOTF-NEXT:    store <vscale x 12 x float> [[INTERLEAVED_VEC3]], ptr [[TMP18]], align 4
+; CHECK-NOTF-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-NOTF-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NOTF-NEXT:    br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; CHECK-NOTF:       [[MIDDLE_BLOCK]]:
 ; CHECK-NOTF-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; CHECK-NOTF-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
@@ -1093,16 +1097,16 @@ define void @interleave(ptr noalias %dst, ptr noalias %src, i64 %n) #0 {
 ; CHECK-NOTF-NEXT:    [[I_021:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
 ; CHECK-NOTF-NEXT:    [[MUL:%.*]] = shl nuw nsw i64 [[I_021]], 1
 ; CHECK-NOTF-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[MUL]]
-; CHECK-NOTF-NEXT:    [[TMP14:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-NOTF-NEXT:    [[TMP20:%.*]] = load float, ptr [[ARRAYIDX]], align 4
 ; CHECK-NOTF-NEXT:    [[MUL1:%.*]] = mul nuw nsw i64 [[I_021]], 3
 ; CHECK-NOTF-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[MUL1]]
-; CHECK-NOTF-NEXT:    store float [[TMP14]], ptr [[ARRAYIDX2]], align 4
+; CHECK-NOTF-NEXT:    store float [[TMP20]], ptr [[ARRAYIDX2]], align 4
 ; CHECK-NOTF-NEXT:    [[ADD:%.*]] = or disjoint i64 [[MUL]], 1
 ; CHECK-NOTF-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[ADD]]
-; CHECK-NOTF-NEXT:    [[TMP15:%.*]] = load float, ptr [[ARRAYIDX4]], align 4
+; CHECK-NOTF-NEXT:    [[TMP21:%.*]] = load float, ptr [[ARRAYIDX4]], align 4
 ; CHECK-NOTF-NEXT:    [[ADD6:%.*]] = add nuw nsw i64 [[MUL1]], 1
 ; CHECK-NOTF-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD6]]
-; CHECK-NOTF-NEXT:    store float [[TMP15]], ptr [[ARRAYIDX7]], align 4
+; CHECK-NOTF-NEXT:    store float [[TMP21]], ptr [[ARRAYIDX7]], align 4
 ; CHECK-NOTF-NEXT:    [[ADD9:%.*]] = add nuw nsw i64 [[MUL1]], 2
 ; CHECK-NOTF-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD9]]
 ; CHECK-NOTF-NEXT:    store float 3.000000e+00, ptr [[ARRAYIDX10]], align 4
@@ -1115,40 +1119,44 @@ define void @interleave(ptr noalias %dst, ptr noalias %src, i64 %n) #0 {
 ; CHECK-TF-ALL-LABEL: define void @interleave(
 ; CHECK-TF-ALL-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; CHECK-TF-ALL-NEXT:  [[ENTRY:.*]]:
-; CHECK-TF-ALL-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; CHECK-TF-ALL-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-ALL-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-TF-ALL-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; CHECK-TF-ALL-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK-TF-ALL:       [[VECTOR_PH]]:
-; CHECK-TF-ALL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
+; CHECK-TF-ALL-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-ALL-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; CHECK-TF-ALL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-TF-ALL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; CHECK-TF-ALL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK-TF-ALL:       [[VECTOR_BODY]]:
 ; CHECK-TF-ALL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-TF-ALL-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 4
-; CHECK-TF-ALL-NEXT:    [[TMP1:%.*]] = shl nuw nsw i64 [[INDEX]], 1
-; CHECK-TF-ALL-NEXT:    [[TMP2:%.*]] = shl nuw nsw i64 [[TMP0]], 1
-; CHECK-TF-ALL-NEXT:    [[TMP3:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP1]]
-; CHECK-TF-ALL-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP2]]
-; CHECK-TF-ALL-NEXT:    [[WIDE_VEC:%.*]] = load <8 x float>, ptr [[TMP3]], align 4
-; CHECK-TF-ALL-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <8 x float> [[WIDE_VEC]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-TF-ALL-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <8 x float> [[WIDE_VEC]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; CHECK-TF-ALL-NEXT:    [[WIDE_VEC2:%.*]] = load <8 x float>, ptr [[TMP4]], align 4
-; CHECK-TF-ALL-NEXT:    [[STRIDED_VEC3:%.*]] = shufflevector <8 x float> [[WIDE_VEC2]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-TF-ALL-NEXT:    [[STRIDED_VEC4:%.*]] = shufflevector <8 x float> [[WIDE_VEC2]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; CHECK-TF-ALL-NEXT:    [[TMP5:%.*]] = mul nuw nsw i64 [[INDEX]], 3
-; CHECK-TF-ALL-NEXT:    [[TMP6:%.*]] = mul nuw nsw i64 [[TMP0]], 3
-; CHECK-TF-ALL-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP5]]
-; CHECK-TF-ALL-NEXT:    [[TMP8:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP6]]
-; CHECK-TF-ALL-NEXT:    [[TMP9:%.*]] = shufflevector <4 x float> [[STRIDED_VEC]], <4 x float> [[STRIDED_VEC1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-TF-ALL-NEXT:    [[TMP10:%.*]] = shufflevector <8 x float> [[TMP9]], <8 x float> <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float undef, float undef, float undef, float undef>, <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-; CHECK-TF-ALL-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <12 x float> [[TMP10]], <12 x float> poison, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
-; CHECK-TF-ALL-NEXT:    store <12 x float> [[INTERLEAVED_VEC]], ptr [[TMP7]], align 4
-; CHECK-TF-ALL-NEXT:    [[TMP11:%.*]] = shufflevector <4 x float> [[STRIDED_VEC3]], <4 x float> [[STRIDED_VEC4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-TF-ALL-NEXT:    [[TMP12:%.*]] = shufflevector <8 x float> [[TMP11]], <8 x float> <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float undef, float undef, float undef, float undef>, <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-; CHECK-TF-ALL-NEXT:    [[INTERLEAVED_VEC5:%.*]] = shufflevector <12 x float> [[TMP12]], <12 x float> poison, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
-; CHECK-TF-ALL-NEXT:    store <12 x float> [[INTERLEAVED_VEC5]], ptr [[TMP8]], align 4
-; CHECK-TF-ALL-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
-; CHECK-TF-ALL-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-TF-ALL-NEXT:    br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-TF-ALL-NEXT:    [[TMP4:%.*]] = add i64 [[TMP2]], 0
+; CHECK-TF-ALL-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 1
+; CHECK-TF-ALL-NEXT:    [[TMP6:%.*]] = add i64 [[INDEX]], [[TMP5]]
+; CHECK-TF-ALL-NEXT:    [[TMP7:%.*]] = shl nuw nsw i64 [[INDEX]], 1
+; CHECK-TF-ALL-NEXT:    [[TMP8:%.*]] = shl nuw nsw i64 [[TMP6]], 1
+; CHECK-TF-ALL-NEXT:    [[TMP9:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP7]]
+; CHECK-TF-ALL-NEXT:    [[TMP10:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP8]]
+; CHECK-TF-ALL-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x float>, ptr [[TMP9]], align 4
+; CHECK-TF-ALL-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC]])
+; CHECK-TF-ALL-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 0
+; CHECK-TF-ALL-NEXT:    [[TMP12:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 1
+; CHECK-TF-ALL-NEXT:    [[WIDE_VEC1:%.*]] = load <vscale x 8 x float>, ptr [[TMP10]], align 4
+; CHECK-TF-ALL-NEXT:    [[STRIDED_VEC2:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC1]])
+; CHECK-TF-ALL-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC2]], 0
+; CHECK-TF-ALL-NEXT:    [[TMP14:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC2]], 1
+; CHECK-TF-ALL-NEXT:    [[TMP15:%.*]] = mul nuw nsw i64 [[INDEX]], 3
+; CHECK-TF-ALL-NEXT:    [[TMP16:%.*]] = mul nuw nsw i64 [[TMP6]], 3
+; CHECK-TF-ALL-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP15]]
+; CHECK-TF-ALL-NEXT:    [[TMP18:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP16]]
+; CHECK-TF-ALL-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 12 x float> @llvm.vector.interleave3.nxv12f32(<vscale x 4 x float> [[TMP11]], <vscale x 4 x float> [[TMP12]], <vscale x 4 x float> splat (float 3.000000e+00))
+; CHECK-TF-ALL-NEXT:    store <vscale x 12 x float> [[INTERLEAVED_VEC]], ptr [[TMP17]], align 4
+; CHECK-TF-ALL-NEXT:    [[INTERLEAVED_VEC3:%.*]] = call <vscale x 12 x float> @llvm.vector.interleave3.nxv12f32(<vscale x 4 x float> [[TMP13]], <vscale x 4 x float> [[TMP14]], <vscale x 4 x float> splat (float 3.000000e+00))
+; CHECK-TF-ALL-NEXT:    store <vscale x 12 x float> [[INTERLEAVED_VEC3]], ptr [[TMP18]], align 4
+; CHECK-TF-ALL-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-TF-ALL-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-TF-ALL-NEXT:    br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK-TF-ALL:       [[MIDDLE_BLOCK]]:
 ; CHECK-TF-ALL-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; CHECK-TF-ALL-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
@@ -1159,16 +1167,16 @@ define void @interleave(ptr noalias %dst, ptr noalias %src, i64 %n) #0 {
 ; CHECK-TF-ALL-NEXT:    [[I_021:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
 ; CHECK-TF-ALL-NEXT:    [[MUL:%.*]] = shl nuw nsw i64 [[I_021]], 1
 ; CHECK-TF-ALL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[MUL]]
-; CHECK-TF-ALL-NEXT:    [[TMP14:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-TF-ALL-NEXT:    [[TMP20:%.*]] = load float, ptr [[ARRAYIDX]], align 4
 ; CHECK-TF-ALL-NEXT:    [[MUL1:%.*]] = mul nuw nsw i64 [[I_021]], 3
 ; CHECK-TF-ALL-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[MUL1]]
-; CHECK-TF-ALL-NEXT:    store float [[TMP14]], ptr [[ARRAYIDX2]], align 4
+; CHECK-TF-ALL-NEXT:    store float [[TMP20]], ptr [[ARRAYIDX2]], align 4
 ; CHECK-TF-ALL-NEXT:    [[ADD:%.*]] = or disjoint i64 [[MUL]], 1
 ; CHECK-TF-ALL-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[ADD]]
-; CHECK-TF-ALL-NEXT:    [[TMP15:%.*]] = load float, ptr [[ARRAYIDX4]], align 4
+; CHECK-TF-ALL-NEXT:    [[TMP21:%.*]] = load float, ptr [[ARRAYIDX4]], align 4
 ; CHECK-TF-ALL-NEXT:    [[ADD6:%.*]] = add nuw nsw i64 [[MUL1]], 1
 ; CHECK-TF-ALL-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD6]]
-; CHECK-TF-ALL-NEXT:    store float [[TMP15]], ptr [[ARRAYIDX7]], align 4
+; CHECK-TF-ALL-NEXT:    store float [[TMP21]], ptr [[ARRAYIDX7]], align 4
 ; CHECK-TF-ALL-NEXT:    [[ADD9:%.*]] = add nuw nsw i64 [[MUL1]], 2
 ; CHECK-TF-ALL-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD9]]
 ; CHECK-TF-ALL-NEXT:    store float 3.000000e+00, ptr [[ARRAYIDX10]], align 4
@@ -1181,40 +1189,44 @@ define void @interleave(ptr noalias %dst, ptr noalias %src, i64 %n) #0 {
 ; CHECK-TF-LABEL: define void @interleave(
 ; CHECK-TF-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; CHECK-TF-NEXT:  [[ENTRY:.*]]:
-; CHECK-TF-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; CHECK-TF-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-TF-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; CHECK-TF-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK-TF:       [[VECTOR_PH]]:
-; CHECK-TF-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
+; CHECK-TF-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; CHECK-TF-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-TF-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; CHECK-TF-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK-TF:       [[VECTOR_BODY]]:
 ; CHECK-TF-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-TF-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 4
-; CHECK-TF-NEXT:    [[TMP1:%.*]] = shl nuw nsw i64 [[INDEX]], 1
-; CHECK-TF-NEXT:    [[TMP2:%.*]] = shl nuw nsw i64 [[TMP0]], 1
-; CHECK-TF-NEXT:    [[TMP3:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP1]]
-; CHECK-TF-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP2]]
-; CHECK-TF-NEXT:    [[WIDE_VEC:%.*]] = load <8 x float>, ptr [[TMP3]], align 4
-; CHECK-TF-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <8 x float> [[WIDE_VEC]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-TF-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <8 x float> [[WIDE_VEC]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; CHECK-TF-NEXT:    [[WIDE_VEC2:%.*]] = load <8 x float>, ptr [[TMP4]], align 4
-; CHECK-TF-NEXT:    [[STRIDED_VEC3:%.*]] = shufflevector <8 x float> [[WIDE_VEC2]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-TF-NEXT:    [[STRIDED_VEC4:%.*]] = shufflevector <8 x float> [[WIDE_VEC2]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; CHECK-TF-NEXT:    [[TMP5:%.*]] = mul nuw nsw i64 [[INDEX]], 3
-; CHECK-TF-NEXT:    [[TMP6:%.*]] = mul nuw nsw i64 [[TMP0]], 3
-; CHECK-TF-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP5]]
-; CHECK-TF-NEXT:    [[TMP8:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP6]]
-; CHECK-TF-NEXT:    [[TMP9:%.*]] = shufflevector <4 x float> [[STRIDED_VEC]], <4 x float> [[STRIDED_VEC1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-TF-NEXT:    [[TMP10:%.*]] = shufflevector <8 x float> [[TMP9]], <8 x float> <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float undef, float undef, float undef, float undef>, <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-; CHECK-TF-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <12 x float> [[TMP10]], <12 x float> poison, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
-; CHECK-TF-NEXT:    store <12 x float> [[INTERLEAVED_VEC]], ptr [[TMP7]], align 4
-; CHECK-TF-NEXT:    [[TMP11:%.*]] = shufflevector <4 x float> [[STRIDED_VEC3]], <4 x float> [[STRIDED_VEC4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-TF-NEXT:    [[TMP12:%.*]] = shufflevector <8 x float> [[TMP11]], <8 x float> <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float undef, float undef, float undef, float undef>, <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-; CHECK-TF-NEXT:    [[INTERLEAVED_VEC5:%.*]] = shufflevector <12 x float> [[TMP12]], <12 x float> poison, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
-; CHECK-TF-NEXT:    store <12 x float> [[INTERLEAVED_VEC5]], ptr [[TMP8]], align 4
-; CHECK-TF-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
-; CHECK-TF-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-TF-NEXT:    br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-TF-NEXT:    [[TMP4:%.*]] = add i64 [[TMP2]], 0
+; CHECK-TF-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 1
+; CHECK-TF-NEXT:    [[TMP6:%.*]] = add i64 [[INDEX]], [[TMP5]]
+; CHECK-TF-NEXT:    [[TMP7:%.*]] = shl nuw nsw i64 [[INDEX]], 1
+; CHECK-TF-NEXT:    [[TMP8:%.*]] = shl nuw nsw i64 [[TMP6]], 1
+; CHECK-TF-NEXT:    [[TMP9:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP7]]
+; CHECK-TF-NEXT:    [[TMP10:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP8]]
+; CHECK-TF-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x float>, ptr [[TMP9]], align 4
+; CHECK-TF-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC]])
+; CHECK-TF-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 0
+; CHECK-TF-NEXT:    [[TMP12:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 1
+; CHECK-TF-NEXT:    [[WIDE_VEC1:%.*]] = load <vscale x 8 x float>, ptr [[TMP10]], align 4
+; CHECK-TF-NEXT:    [[STRIDED_VEC2:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC1]])
+; CHECK-TF-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC2]], 0
+; CHECK-TF-NEXT:    [[TMP14:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC2]], 1
+; CHECK-TF-NEXT:    [[TMP15:%.*]] = mul nuw nsw i64 [[INDEX]], 3
+; CHECK-TF-NEXT:    [[TMP16:%.*]] = mul nuw nsw i64 [[TMP6]], 3
+; CHECK-TF-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP15]]
+; CHECK-TF-NEXT:    [[TMP18:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP16]]
+; CHECK-TF-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 12 x float> @llvm.vector.interleave3.nxv12f32(<vscale x 4 x float> [[TMP11]], <vscale x 4 x float> [[TMP12]], <vscale x 4 x float> splat (float 3.000000e+00))
+; CHECK-TF-NEXT:    store <vscale x 12 x float> [[INTERLEAVED_VEC]], ptr [[TMP17]], align 4
+; CHECK-TF-NEXT:    [[INTERLEAVED_VEC3:%.*]] = call <vscale x 12 x float> @llvm.vector.interleave3.nxv12f32(<vscale x 4 x float> [[TMP13]], <vscale x 4 x float> [[TMP14]], <vscale x 4 x float> splat (float 3.000000e+00))
+; CHECK-TF-NEXT:    store <vscale x 12 x float> [[INTERLEAVED_VEC3]], ptr [[TMP18]], align 4
+; CHECK-TF-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-TF-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-TF-NEXT:    br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK-TF:       [[MIDDLE_BLOCK]]:
 ; CHECK-TF-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; CHECK-TF-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
@@ -1225,16 +1237,16 @@ define void @interleave(ptr noalias %dst, ptr noalias %src, i64 %n) #0 {
 ; CHECK-TF-NEXT:    [[I_021:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
 ; CHECK-TF-NEXT:    [[MUL:%.*]] = shl nuw nsw i64 [[I_021]], 1
 ; CHECK-TF-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[MUL]]
-; CHECK-TF-NEXT:    [[TMP14:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-TF-NEXT:    [[TMP20:%.*]] = load float, ptr [[ARRAYIDX]], align 4
 ; CHECK-TF-NEXT:    [[MUL1:%.*]] = mul nuw nsw i64 [[I_021]], 3
 ; CHECK-TF-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[MUL1]]
-; CHECK-TF-NEXT:    store float [[TMP14]], ptr [[ARRAYIDX2]], align 4
+; CHECK-TF-NEXT:    store float [[TMP20]], ptr [[ARRAYIDX2]], align 4
 ; CHECK-TF-NEXT:    [[ADD:%.*]] = or disjoint i64 [[MUL]], 1
 ; CHECK-TF-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[ADD]]
-; CHECK-TF-NEXT:    [[TMP15:%.*]] = load float, ptr [[ARRAYIDX4]], align 4
+; CHECK-TF-NEXT:    [[TMP21:%.*]] = load float, ptr [[ARRAYIDX4]], align 4
 ; CHECK-TF-NEXT:    [[ADD6:%.*]] = add nuw nsw i64 [[MUL1]], 1
 ; CHECK-TF-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD6]]
-; CHECK-TF-NEXT:    store float [[TMP15]], ptr [[ARRAYIDX7]], align 4
+; CHECK-TF-NEXT:    store float [[TMP21]], ptr [[ARRAYIDX7]], align 4
 ; CHECK-TF-NEXT:    [[ADD9:%.*]] = add nuw nsw i64 [[MUL1]], 2
 ; CHECK-TF-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD9]]
 ; CHECK-TF-NEXT:    store float 3.000000e+00, ptr [[ARRAYIDX10]], align 4
@@ -1246,107 +1258,149 @@ define void @interleave(ptr noalias %dst, ptr noalias %src, i64 %n) #0 {
 ;
 ; CHECK-TF-DEFAULT-LABEL: define void @interleave(
 ; CHECK-TF-DEFAULT-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-TF-DEFAULT-NEXT:  [[ENTRY:.*]]:
-; CHECK-TF-DEFAULT-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
-; CHECK-TF-DEFAULT-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-TF-DEFAULT-NEXT:  [[ITER_CHECK:.*]]:
+; CHECK-TF-DEFAULT-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-DEFAULT-NEXT:    [[TMP25:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-DEFAULT-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP25]]
+; CHECK-TF-DEFAULT-NEXT:    [[TMP29:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-DEFAULT-NEXT:    [[TMP30:%.*]] = shl nuw i64 [[TMP29]], 2
+; CHECK-TF-DEFAULT-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK-TF-DEFAULT:       [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-TF-DEFAULT-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-TF-DEFAULT-NEXT:    [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-TF-DEFAULT-NEXT:    br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK-TF-DEFAULT:       [[VECTOR_PH]]:
-; CHECK-TF-DEFAULT-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
+; CHECK-TF-DEFAULT-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-DEFAULT-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; CHECK-TF-DEFAULT-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-TF-DEFAULT-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; CHECK-TF-DEFAULT-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK-TF-DEFAULT:       [[VECTOR_BODY]]:
 ; CHECK-TF-DEFAULT-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-TF-DEFAULT-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 4
-; CHECK-TF-DEFAULT-NEXT:    [[TMP1:%.*]] = shl nuw nsw i64 [[INDEX]], 1
-; CHECK-TF-DEFAULT-NEXT:    [[TMP2:%.*]] = shl nuw nsw i64 [[TMP0]], 1
-; CHECK-TF-DEFAULT-NEXT:    [[TMP3:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP1]]
-; CHECK-TF-DEFAULT-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP2]]
-; CHECK-TF-DEFAULT-NEXT:    [[WIDE_VEC:%.*]] = load <8 x float>, ptr [[TMP3]], align 4
-; CHECK-TF-DEFAULT-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <8 x float> [[WIDE_VEC]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-TF-DEFAULT-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <8 x float> [[WIDE_VEC]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; CHECK-TF-DEFAULT-NEXT:    [[WIDE_VEC2:%.*]] = load <8 x float>, ptr [[TMP4]], align 4
-; CHECK-TF-DEFAULT-NEXT:    [[STRIDED_VEC3:%.*]] = shufflevector <8 x float> [[WIDE_VEC2]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-TF-DEFAULT-NEXT:    [[STRIDED_VEC4:%.*]] = shufflevector <8 x float> [[WIDE_VEC2]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; CHECK-TF-DEFAULT-NEXT:    [[TMP5:%.*]] = mul nuw nsw i64 [[INDEX]], 3
-; CHECK-TF-DEFAULT-NEXT:    [[TMP6:%.*]] = mul nuw nsw i64 [[TMP0]], 3
-; CHECK-TF-DEFAULT-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP5]]
-; CHECK-TF-DEFAULT-NEXT:    [[TMP8:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP6]]
-; CHECK-TF-DEFAULT-NEXT:    [[TMP9:%.*]] = shufflevector <4 x float> [[STRIDED_VEC]], <4 x float> [[STRIDED_VEC1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-TF-DEFAULT-NEXT:    [[TMP10:%.*]] = shufflevector <8 x float> [[TMP9]], <8 x float> <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float undef, float undef, float undef, float undef>, <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-; CHECK-TF-DEFAULT-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <12 x float> [[TMP10]], <12 x float> poison, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
-; CHECK-TF-DEFAULT-NEXT:    store <12 x float> [[INTERLEAVED_VEC]], ptr [[TMP7]], align 4
-; CHECK-TF-DEFAULT-NEXT:    [[TMP11:%.*]] = shufflevector <4 x float> [[STRIDED_VEC3]], <4 x float> [[STRIDED_VEC4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-TF-DEFAULT-NEXT:    [[TMP12:%.*]] = shufflevector <8 x float> [[TMP11]], <8 x float> <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float undef, float undef, float undef, float undef>, <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-; CHECK-TF-DEFAULT-NEXT:    [[INTERLEAVED_VEC5:%.*]] = shufflevector <12 x float> [[TMP12]], <12 x float> poison, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
-; CHECK-TF-DEFAULT-NEXT:    store <12 x float> [[INTERLEAVED_VEC5]], ptr [[TMP8]], align 4
-; CHECK-TF-DEFAULT-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
-; CHECK-TF-DEFAULT-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-TF-DEFAULT-NEXT:    br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-TF-DEFAULT-NEXT:    [[TMP4:%.*]] = add i64 [[TMP2]], 0
+; CHECK-TF-DEFAULT-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 1
+; CHECK-TF-DEFAULT-NEXT:    [[TMP6:%.*]] = add i64 [[INDEX]], [[TMP5]]
+; CHECK-TF-DEFAULT-NEXT:    [[TMP7:%.*]] = shl nuw nsw i64 [[INDEX]], 1
+; CHECK-TF-DEFAULT-NEXT:    [[TMP8:%.*]] = shl nuw nsw i64 [[TMP6]], 1
+; CHECK-TF-DEFAULT-NEXT:    [[TMP9:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP7]]
+; CHECK-TF-DEFAULT-NEXT:    [[TMP10:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP8]]
+; CHECK-TF-DEFAULT-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x float>, ptr [[TMP9]], align 4
+; CHECK-TF-DEFAULT-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC]])
+; CHECK-TF-DEFAULT-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 0
+; CHECK-TF-DEFAULT-NEXT:    [[TMP12:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 1
+; CHECK-TF-DEFAULT-NEXT:    [[WIDE_VEC2:%.*]] = load <vscale x 8 x float>, ptr [[TMP10]], align 4
+; CHECK-TF-DEFAULT-NEXT:    [[STRIDED_VEC3:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC2]])
+; CHECK-TF-DEFAULT-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC3]], 0
+; CHECK-TF-DEFAULT-NEXT:    [[TMP14:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC3]], 1
+; CHECK-TF-DEFAULT-NEXT:    [[TMP15:%.*]] = mul nuw nsw i64 [[INDEX]], 3
+; CHECK-TF-DEFAULT-NEXT:    [[TMP16:%.*]] = mul nuw nsw i64 [[TMP6]], 3
+; CHECK-TF-DEFAULT-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP15]]
+; CHECK-TF-DEFAULT-NEXT:    [[TMP18:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP16]]
+; CHECK-TF-DEFAULT-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 12 x float> @llvm.vector.interleave3.nxv12f32(<vscale x 4 x float> [[TMP11]], <vscale x 4 x float> [[TMP12]], <vscale x 4 x float> splat (float 3.000000e+00))
+; CHECK-TF-DEFAULT-NEXT:    store <vscale x 12 x float> [[INTERLEAVED_VEC]], ptr [[TMP17]], align 4
+; CHECK-TF-DEFAULT-NEXT:    [[INTERLEAVED_VEC4:%.*]] = call <vscale x 12 x float> @llvm.vector.interleave3.nxv12f32(<vscale x 4 x float> [[TMP13]], <vscale x 4 x float> [[TMP14]], <vscale x 4 x float> splat (float 3.000000e+00))
+; CHECK-TF-DEFAULT-NEXT:    store <vscale x 12 x float> [[INTERLEAVED_VEC4]], ptr [[TMP18]], align 4
+; CHECK-TF-DEFAULT-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-TF-DEFAULT-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-TF-DEFAULT-NEXT:    br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK-TF-DEFAULT:       [[MIDDLE_BLOCK]]:
 ; CHECK-TF-DEFAULT-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-TF-DEFAULT-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
-; CHECK-TF-DEFAULT:       [[SCALAR_PH]]:
-; CHECK-TF-DEFAULT-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-TF-DEFAULT-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK-TF-DEFAULT:       [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-TF-DEFAULT-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP30]]
+; CHECK-TF-DEFAULT-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF6:![0-9]+]]
+; CHECK-TF-DEFAULT:       [[VEC_EPILOG_PH]]:
+; CHECK-TF-DEFAULT-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-TF-DEFAULT-NEXT:    [[TMP31:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-DEFAULT-NEXT:    [[TMP24:%.*]] = shl nuw i64 [[TMP31]], 2
+; CHECK-TF-DEFAULT-NEXT:    [[N_MOD_VF5:%.*]] = urem i64 [[N]], [[TMP24]]
+; CHECK-TF-DEFAULT-NEXT:    [[N_VEC6:%.*]] = sub i64 [[N]], [[N_MOD_VF5]]
+; CHECK-TF-DEFAULT-NEXT:    br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK-TF-DEFAULT:       [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-TF-DEFAULT-NEXT:    [[INDEX7:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT12:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-TF-DEFAULT-NEXT:    [[TMP20:%.*]] = shl nuw nsw i64 [[INDEX7]], 1
+; CHECK-TF-DEFAULT-NEXT:    [[TMP21:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP20]]
+; CHECK-TF-DEFAULT-NEXT:    [[WIDE_VEC8:%.*]] = load <vscale x 8 x float>, ptr [[TMP21]], align 4
+; CHECK-TF-DEFAULT-NEXT:    [[STRIDED_VEC9:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC8]])
+; CHECK-TF-DEFAULT-NEXT:    [[TMP32:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC9]], 0
+; CHECK-TF-DEFAULT-NEXT:    [[TMP33:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC9]], 1
+; CHECK-TF-DEFAULT-NEXT:    [[TMP22:%.*]] = mul nuw nsw i64 [[INDEX7]], 3
+; CHECK-TF-DEFAULT-NEXT:    [[TMP23:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP22]]
+; CHECK-TF-DEFAULT-NEXT:    [[INTERLEAVED_VEC10:%.*]] = call <vscale x 12 x float> @llvm.vector.interleave3.nxv12f32(<vscale x 4 x float> [[TMP32]], <vscale x 4 x float> [[TMP33]], <vscale x 4 x float> splat (float 3.000000e+00))
+; CHECK-TF-DEFAULT-NEXT:    store <vscale x 12 x float> [[INTERLEAVED_VEC10]], ptr [[TMP23]], align 4
+; CHECK-TF-DEFAULT-NEXT:    [[INDEX_NEXT12]] = add nuw i64 [[INDEX7]], [[TMP24]]
+; CHECK-TF-DEFAULT-NEXT:    [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT12]], [[N_VEC6]]
+; CHECK-TF-DEFAULT-NEXT:    br i1 [[TMP26]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-TF-DEFAULT:       [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-TF-DEFAULT-NEXT:    [[CMP_N13:%.*]] = icmp eq i64 [[N]], [[N_VEC6]]
+; CHECK-TF-DEFAULT-NEXT:    br i1 [[CMP_N13]], label %[[FOR_END]], label %[[VEC_EPILOG_SCALAR_PH]]
+; CHECK-TF-DEFAULT:       [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-TF-DEFAULT-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC6]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
 ; CHECK-TF-DEFAULT-NEXT:    br label %[[FOR_BODY:.*]]
 ; CHECK-TF-DEFAULT:       [[FOR_BODY]]:
-; CHECK-TF-DEFAULT-NEXT:    [[I_021:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-TF-DEFAULT-NEXT:    [[I_021:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ]
 ; CHECK-TF-DEFAULT-NEXT:    [[MUL:%.*]] = shl nuw nsw i64 [[I_021]], 1
 ; CHECK-TF-DEFAULT-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[MUL]]
-; CHECK-TF-DEFAULT-NEXT:    [[TMP14:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-TF-DEFAULT-NEXT:    [[TMP27:%.*]] = load float, ptr [[ARRAYIDX]], align 4
 ; CHECK-TF-DEFAULT-NEXT:    [[MUL1:%.*]] = mul nuw nsw i64 [[I_021]], 3
 ; CHECK-TF-DEFAULT-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[MUL1]]
-; CHECK-TF-DEFAULT-NEXT:    store float [[TMP14]], ptr [[ARRAYIDX2]], align 4
+; CHECK-TF-DEFAULT-NEXT:    store float [[TMP27]], ptr [[ARRAYIDX2]], align 4
 ; CHECK-TF-DEFAULT-NEXT:    [[ADD:%.*]] = or disjoint i64 [[MUL]], 1
 ; CHECK-TF-DEFAULT-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[ADD]]
-; CHECK-TF-DEFAULT-NEXT:    [[TMP15:%.*]] = load float, ptr [[ARRAYIDX4]], align 4
+; CHECK-TF-DEFAULT-NEXT:    [[TMP28:%.*]] = load float, ptr [[ARRAYIDX4]], align 4
 ; CHECK-TF-DEFAULT-NEXT:    [[ADD6:%.*]] = add nuw nsw i64 [[MUL1]], 1
 ; CHECK-TF-DEFAULT-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD6]]
-; CHECK-TF-DEFAULT-NEXT:    store float [[TMP15]], ptr [[ARRAYIDX7]], align 4
+; CHECK-TF-DEFAULT-NEXT:    store float [[TMP28]], ptr [[ARRAYIDX7]], align 4
 ; CHECK-TF-DEFAULT-NEXT:    [[ADD9:%.*]] = add nuw nsw i64 [[MUL1]], 2
 ; CHECK-TF-DEFAULT-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD9]]
 ; CHECK-TF-DEFAULT-NEXT:    store float 3.000000e+00, ptr [[ARRAYIDX10]], align 4
 ; CHECK-TF-DEFAULT-NEXT:    [[INC]] = add nuw nsw i64 [[I_021]], 1
 ; CHECK-TF-DEFAULT-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]]
-; CHECK-TF-DEFAULT-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-TF-DEFAULT-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; CHECK-TF-DEFAULT:       [[FOR_END]]:
 ; CHECK-TF-DEFAULT-NEXT:    ret void
 ;
 ; CHECK-TF-NORED-LABEL: define void @interleave(
 ; CHECK-TF-NORED-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; CHECK-TF-NORED-NEXT:  [[ENTRY:.*]]:
-; CHECK-TF-NORED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; CHECK-TF-NORED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NORED-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-TF-NORED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; CHECK-TF-NORED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK-TF-NORED:       [[VECTOR_PH]]:
-; CHECK-TF-NORED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
+; CHECK-TF-NORED-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-NORED-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; CHECK-TF-NORED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-TF-NORED-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; CHECK-TF-NORED-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK-TF-NORED:       [[VECTOR_BODY]]:
 ; CHECK-TF-NORED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-TF-NORED-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 4
-; CHECK-TF-NORED-NEXT:    [[TMP1:%.*]] = shl nuw nsw i64 [[INDEX]], 1
-; CHECK-TF-NORED-NEXT:    [[TMP2:%.*]] = shl nuw nsw i64 [[TMP0]], 1
-; CHECK-TF-NORED-NEXT:    [[TMP3:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP1]]
-; CHECK-TF-NORED-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP2]]
-; CHECK-TF-NORED-NEXT:    [[WIDE_VEC:%.*]] = load <8 x float>, ptr [[TMP3]], align 4
-; CHECK-TF-NORED-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <8 x float> [[WIDE_VEC]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-TF-NORED-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <8 x float> [[WIDE_VEC]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; CHECK-TF-NORED-NEXT:    [[WIDE_VEC2:%.*]] = load <8 x float>, ptr [[TMP4]], align 4
-; CHECK-TF-NORED-NEXT:    [[STRIDED_VEC3:%.*]] = shufflevector <8 x float> [[WIDE_VEC2]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-TF-NORED-NEXT:    [[STRIDED_VEC4:%.*]] = shufflevector <8 x float> [[WIDE_VEC2]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; CHECK-TF-NORED-NEXT:    [[TMP5:%.*]] = mul nuw nsw i64 [[INDEX]], 3
-; CHECK-TF-NORED-NEXT:    [[TMP6:%.*]] = mul nuw nsw i64 [[TMP0]], 3
-; CHECK-TF-NORED-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP5]]
-; CHECK-TF-NORED-NEXT:    [[TMP8:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP6]]
-; CHECK-TF-NORED-NEXT:    [[TMP9:%.*]] = shufflevector <4 x float> [[STRIDED_VEC]], <4 x float> [[STRIDED_VEC1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-TF-NORED-NEXT:    [[TMP10:%.*]] = shufflevector <8 x float> [[TMP9]], <8 x float> <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float undef, float undef, float undef, float undef>, <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-; CHECK-TF-NORED-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <12 x float> [[TMP10]], <12 x float> poison, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
-; CHECK-TF-NORED-NEXT:    store <12 x float> [[INTERLEAVED_VEC]], ptr [[TMP7]], align 4
-; CHECK-TF-NORED-NEXT:    [[TMP11:%.*]] = shufflevector <4 x float> [[STRIDED_VEC3]], <4 x float> [[STRIDED_VEC4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-TF-NORED-NEXT:    [[TMP12:%.*]] = shufflevector <8 x float> [[TMP11]], <8 x float> <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float undef, float undef, float undef, float undef>, <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-; CHECK-TF-NORED-NEXT:    [[INTERLEAVED_VEC5:%.*]] = shufflevector <12 x float> [[TMP12]], <12 x float> poison, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
-; CHECK-TF-NORED-NEXT:    store <12 x float> [[INTERLEAVED_VEC5]], ptr [[TMP8]], align 4
-; CHECK-TF-NORED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
-; CHECK-TF-NORED-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-TF-NORED-NEXT:    br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-TF-NORED-NEXT:    [[TMP4:%.*]] = add i64 [[TMP2]], 0
+; CHECK-TF-NORED-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 1
+; CHECK-TF-NORED-NEXT:    [[TMP6:%.*]] = add i64 [[INDEX]], [[TMP5]]
+; CHECK-TF-NORED-NEXT:    [[TMP7:%.*]] = shl nuw nsw i64 [[INDEX]], 1
+; CHECK-TF-NORED-NEXT:    [[TMP8:%.*]] = shl nuw nsw i64 [[TMP6]], 1
+; CHECK-TF-NORED-NEXT:    [[TMP9:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP7]]
+; CHECK-TF-NORED-NEXT:    [[TMP10:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP8]]
+; CHECK-TF-NORED-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x float>, ptr [[TMP9]], align 4
+; CHECK-TF-NORED-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC]])
+; CHECK-TF-NORED-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 0
+; CHECK-TF-NORED-NEXT:    [[TMP12:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 1
+; CHECK-TF-NORED-NEXT:    [[WIDE_VEC1:%.*]] = load <vscale x 8 x float>, ptr [[TMP10]], align 4
+; CHECK-TF-NORED-NEXT:    [[STRIDED_VEC2:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC1]])
+; CHECK-TF-NORED-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC2]], 0
+; CHECK-TF-NORED-NEXT:    [[TMP14:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC2]], 1
+; CHECK-TF-NORED-NEXT:    [[TMP15:%.*]] = mul nuw nsw i64 [[INDEX]], 3
+; CHECK-TF-NORED-NEXT:    [[TMP16:%.*]] = mul nuw nsw i64 [[TMP6]], 3
+; CHECK-TF-NORED-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP15]]
+; CHECK-TF-NORED-NEXT:    [[TMP18:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP16]]
+; CHECK-TF-NORED-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 12 x float> @llvm.vector.interleave3.nxv12f32(<vscale x 4 x float> [[TMP11]], <vscale x 4 x float> [[TMP12]], <vscale x 4 x float> splat (float 3.000000e+00))
+; CHECK-TF-NORED-NEXT:    store <vscale x 12 x float> [[INTERLEAVED_VEC]], ptr [[TMP17]], align 4
+; CHECK-TF-NORED-NEXT:    [[INTERLEAVED_VEC3:%.*]] = call <vscale x 12 x float> @llvm.vector.interleave3.nxv12f32(<vscale x 4 x float> [[TMP13]], <vscale x 4 x float> [[TMP14]], <vscale x 4 x float> splat (float 3.000000e+00))
+; CHECK-TF-NORED-NEXT:    store <vscale x 12 x float> [[INTERLEAVED_VEC3]], ptr [[TMP18]], align 4
+; CHECK-TF-NORED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-TF-NORED-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-TF-NORED-NEXT:    br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK-TF-NORED:       [[MIDDLE_BLOCK]]:
 ; CHECK-TF-NORED-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; CHECK-TF-NORED-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
@@ -1357,16 +1411,16 @@ define void @interleave(ptr noalias %dst, ptr noalias %src, i64 %n) #0 {
 ; CHECK-TF-NORED-NEXT:    [[I_021:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
 ; CHECK-TF-NORED-NEXT:    [[MUL:%.*]] = shl nuw nsw i64 [[I_021]], 1
 ; CHECK-TF-NORED-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[MUL]]
-; CHECK-TF-NORED-NEXT:    [[TMP14:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-TF-NORED-NEXT:    [[TMP20:%.*]] = load float, ptr [[ARRAYIDX]], align 4
 ; CHECK-TF-NORED-NEXT:    [[MUL1:%.*]] = mul nuw nsw i64 [[I_021]], 3
 ; CHECK-TF-NORED-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[MUL1]]
-; CHECK-TF-NORED-NEXT:    store float [[TMP14]], ptr [[ARRAYIDX2]], align 4
+; CHECK-TF-NORED-NEXT:    store float [[TMP20]], ptr [[ARRAYIDX2]], align 4
 ; CHECK-TF-NORED-NEXT:    [[ADD:%.*]] = or disjoint i64 [[MUL]], 1
 ; CHECK-TF-NORED-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[ADD]]
-; CHECK-TF-NORED-NEXT:    [[TMP15:%.*]] = load float, ptr [[ARRAYIDX4]], align 4
+; CHECK-TF-NORED-NEXT:    [[TMP21:%.*]] = load float, ptr [[ARRAYIDX4]], align 4
 ; CHECK-TF-NORED-NEXT:    [[ADD6:%.*]] = add nuw nsw i64 [[MUL1]], 1
 ; CHECK-TF-NORED-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD6]]
-; CHECK-TF-NORED-NEXT:    store float [[TMP15]], ptr [[ARRAYIDX7]], align 4
+; CHECK-TF-NORED-NEXT:    store float [[TMP21]], ptr [[ARRAYIDX7]], align 4
 ; CHECK-TF-NORED-NEXT:    [[ADD9:%.*]] = add nuw nsw i64 [[MUL1]], 2
 ; CHECK-TF-NORED-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD9]]
 ; CHECK-TF-NORED-NEXT:    store float 3.000000e+00, ptr [[ARRAYIDX10]], align 4
@@ -1379,40 +1433,44 @@ define void @interleave(ptr noalias %dst, ptr noalias %src, i64 %n) #0 {
 ; CHECK-TF-NOREC-LABEL: define void @interleave(
 ; CHECK-TF-NOREC-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; CHECK-TF-NOREC-NEXT:  [[ENTRY:.*]]:
-; CHECK-TF-NOREC-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; CHECK-TF-NOREC-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NOREC-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-TF-NOREC-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; CHECK-TF-NOREC-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK-TF-NOREC:       [[VECTOR_PH]]:
-; CHECK-TF-NOREC-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
+; CHECK-TF-NOREC-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-NOREC-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; CHECK-TF-NOREC-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-TF-NOREC-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; CHECK-TF-NOREC-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK-TF-NOREC:       [[VECTOR_BODY]]:
 ; CHECK-TF-NOREC-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-TF-NOREC-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 4
-; CHECK-TF-NOREC-NEXT:    [[TMP1:%.*]] = shl nuw nsw i64 [[INDEX]], 1
-; CHECK-TF-NOREC-NEXT:    [[TMP2:%.*]] = shl nuw nsw i64 [[TMP0]], 1
-; CHECK-TF-NOREC-NEXT:    [[TMP3:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP1]]
-; CHECK-TF-NOREC-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP2]]
-; CHECK-TF-NOREC-NEXT:    [[WIDE_VEC:%.*]] = load <8 x float>, ptr [[TMP3]], align 4
-; CHECK-TF-NOREC-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <8 x float> [[WIDE_VEC]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-TF-NOREC-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <8 x float> [[WIDE_VEC]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; CHECK-TF-NOREC-NEXT:    [[WIDE_VEC2:%.*]] = load <8 x float>, ptr [[TMP4]], align 4
-; CHECK-TF-NOREC-NEXT:    [[STRIDED_VEC3:%.*]] = shufflevector <8 x float> [[WIDE_VEC2]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-TF-NOREC-NEXT:    [[STRIDED_VEC4:%.*]] = shufflevector <8 x float> [[WIDE_VEC2]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; CHECK-TF-NOREC-NEXT:    [[TMP5:%.*]] = mul nuw nsw i64 [[INDEX]], 3
-; CHECK-TF-NOREC-NEXT:    [[TMP6:%.*]] = mul nuw nsw i64 [[TMP0]], 3
-; CHECK-TF-NOREC-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP5]]
-; CHECK-TF-NOREC-NEXT:    [[TMP8:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP6]]
-; CHECK-TF-NOREC-NEXT:    [[TMP9:%.*]] = shufflevector <4 x float> [[STRIDED_VEC]], <4 x float> [[STRIDED_VEC1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-TF-NOREC-NEXT:    [[TMP10:%.*]] = shufflevector <8 x float> [[TMP9]], <8 x float> <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float undef, float undef, float undef, float undef>, <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-; CHECK-TF-NOREC-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <12 x float> [[TMP10]], <12 x float> poison, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
-; CHECK-TF-NOREC-NEXT:    store <12 x float> [[INTERLEAVED_VEC]], ptr [[TMP7]], align 4
-; CHECK-TF-NOREC-NEXT:    [[TMP11:%.*]] = shufflevector <4 x float> [[STRIDED_VEC3]], <4 x float> [[STRIDED_VEC4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-TF-NOREC-NEXT:    [[TMP12:%.*]] = shufflevector <8 x float> [[TMP11]], <8 x float> <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float undef, float undef, float undef, float undef>, <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-; CHECK-TF-NOREC-NEXT:    [[INTERLEAVED_VEC5:%.*]] = shufflevector <12 x float> [[TMP12]], <12 x float> poison, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
-; CHECK-TF-NOREC-NEXT:    store <12 x float> [[INTERLEAVED_VEC5]], ptr [[TMP8]], align 4
-; CHECK-TF-NOREC-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
-; CHECK-TF-NOREC-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-TF-NOREC-NEXT:    br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-TF-NOREC-NEXT:    [[TMP4:%.*]] = add i64 [[TMP2]], 0
+; CHECK-TF-NOREC-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 1
+; CHECK-TF-NOREC-NEXT:    [[TMP6:%.*]] = add i64 [[INDEX]], [[TMP5]]
+; CHECK-TF-NOREC-NEXT:    [[TMP7:%.*]] = shl nuw nsw i64 [[INDEX]], 1
+; CHECK-TF-NOREC-NEXT:    [[TMP8:%.*]] = shl nuw nsw i64 [[TMP6]], 1
+; CHECK-TF-NOREC-NEXT:    [[TMP9:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP7]]
+; CHECK-TF-NOREC-NEXT:    [[TMP10:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP8]]
+; CHECK-TF-NOREC-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x float>, ptr [[TMP9]], align 4
+; CHECK-TF-NOREC-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC]])
+; CHECK-TF-NOREC-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 0
+; CHECK-TF-NOREC-NEXT:    [[TMP12:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 1
+; CHECK-TF-NOREC-NEXT:    [[WIDE_VEC1:%.*]] = load <vscale x 8 x float>, ptr [[TMP10]], align 4
+; CHECK-TF-NOREC-NEXT:    [[STRIDED_VEC2:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC1]])
+; CHECK-TF-NOREC-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC2]], 0
+; CHECK-TF-NOREC-NEXT:    [[TMP14:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC2]], 1
+; CHECK-TF-NOREC-NEXT:    [[TMP15:%.*]] = mul nuw nsw i64 [[INDEX]], 3
+; CHECK-TF-NOREC-NEXT:    [[TMP16:%.*]] = mul nuw nsw i64 [[TMP6]], 3
+; CHECK-TF-NOREC-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP15]]
+; CHECK-TF-NOREC-NEXT:    [[TMP18:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP16]]
+; CHECK-TF-NOREC-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 12 x float> @llvm.vector.interleave3.nxv12f32(<vscale x 4 x float> [[TMP11]], <vscale x 4 x float> [[TMP12]], <vscale x 4 x float> splat (float 3.000000e+00))
+; CHECK-TF-NOREC-NEXT:    store <vscale x 12 x float> [[INTERLEAVED_VEC]], ptr [[TMP17]], align 4
+; CHECK-TF-NOREC-NEXT:    [[INTERLEAVED_VEC3:%.*]] = call <vscale x 12 x float> @llvm.vector.interleave3.nxv12f32(<vscale x 4 x float> [[TMP13]], <vscale x 4 x float> [[TMP14]], <vscale x 4 x float> splat (float 3.000000e+00))
+; CHECK-TF-NOREC-NEXT:    store <vscale x 12 x float> [[INTERLEAVED_VEC3]], ptr [[TMP18]], align 4
+; CHECK-TF-NOREC-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-TF-NOREC-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-TF-NOREC-NEXT:    br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK-TF-NOREC:       [[MIDDLE_BLOCK]]:
 ; CHECK-TF-NOREC-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; CHECK-TF-NOREC-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
@@ -1423,16 +1481,16 @@ define void @interleave(ptr noalias %dst, ptr noalias %src, i64 %n) #0 {
 ; CHECK-TF-NOREC-NEXT:    [[I_021:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
 ; CHECK-TF-NOREC-NEXT:    [[MUL:%.*]] = shl nuw nsw i64 [[I_021]], 1
 ; CHECK-TF-NOREC-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[MUL]]
-; CHECK-TF-NOREC-NEXT:    [[TMP14:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-TF-NOREC-NEXT:    [[TMP20:%.*]] = load float, ptr [[ARRAYIDX]], align 4
 ; CHECK-TF-NOREC-NEXT:    [[MUL1:%.*]] = mul nuw nsw i64 [[I_021]], 3
 ; CHECK-TF-NOREC-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[MUL1]]
-; CHECK-TF-NOREC-NEXT:    store float [[TMP14]], ptr [[ARRAYIDX2]], align 4
+; CHECK-TF-NOREC-NEXT:    store float [[TMP20]], ptr [[ARRAYIDX2]], align 4
 ; CHECK-TF-NOREC-NEXT:    [[ADD:%.*]] = or disjoint i64 [[MUL]], 1
 ; CHECK-TF-NOREC-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[ADD]]
-; CHECK-TF-NOREC-NEXT:    [[TMP15:%.*]] = load float, ptr [[ARRAYIDX4]], align 4
+; CHECK-TF-NOREC-NEXT:    [[TMP21:%.*]] = load float, ptr [[ARRAYIDX4]], align 4
 ; CHECK-TF-NOREC-NEXT:    [[ADD6:%.*]] = add nuw nsw i64 [[MUL1]], 1
 ; CHECK-TF-NOREC-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD6]]
-; CHECK-TF-NOREC-NEXT:    store float [[TMP15]], ptr [[ARRAYIDX7]], align 4
+; CHECK-TF-NOREC-NEXT:    store float [[TMP21]], ptr [[ARRAYIDX7]], align 4
 ; CHECK-TF-NOREC-NEXT:    [[ADD9:%.*]] = add nuw nsw i64 [[MUL1]], 2
 ; CHECK-TF-NOREC-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD9]]
 ; CHECK-TF-NOREC-NEXT:    store float 3.000000e+00, ptr [[ARRAYIDX10]], align 4
@@ -1445,40 +1503,44 @@ define void @interleave(ptr noalias %dst, ptr noalias %src, i64 %n) #0 {
 ; CHECK-TF-NOREV-LABEL: define void @interleave(
 ; CHECK-TF-NOREV-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; CHECK-TF-NOREV-NEXT:  [[ENTRY:.*]]:
-; CHECK-TF-NOREV-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; CHECK-TF-NOREV-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NOREV-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-TF-NOREV-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; CHECK-TF-NOREV-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK-TF-NOREV:       [[VECTOR_PH]]:
-; CHECK-TF-NOREV-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
+; CHECK-TF-NOREV-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-NOREV-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; CHECK-TF-NOREV-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-TF-NOREV-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; CHECK-TF-NOREV-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK-TF-NOREV:       [[VECTOR_BODY]]:
 ; CHECK-TF-NOREV-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-TF-NOREV-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 4
-; CHECK-TF-NOREV-NEXT:    [[TMP1:%.*]] = shl nuw nsw i64 [[INDEX]], 1
-; CHECK-TF-NOREV-NEXT:    [[TMP2:%.*]] = shl nuw nsw i64 [[TMP0]], 1
-; CHECK-TF-NOREV-NEXT:    [[TMP3:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP1]]
-; CHECK-TF-NOREV-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP2]]
-; CHECK-TF-NOREV-NEXT:    [[WIDE_VEC:%.*]] = load <8 x float>, ptr [[TMP3]], align 4
-; CHECK-TF-NOREV-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <8 x float> [[WIDE_VEC]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-TF-NOREV-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <8 x float> [[WIDE_VEC]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; CHECK-TF-NOREV-NEXT:    [[WIDE_VEC2:%.*]] = load <8 x float>, ptr [[TMP4]], align 4
-; CHECK-TF-NOREV-NEXT:    [[STRIDED_VEC3:%.*]] = shufflevector <8 x float> [[WIDE_VEC2]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-TF-NOREV-NEXT:    [[STRIDED_VEC4:%.*]] = shufflevector <8 x float> [[WIDE_VEC2]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; CHECK-TF-NOREV-NEXT:    [[TMP5:%.*]] = mul nuw nsw i64 [[INDEX]], 3
-; CHECK-TF-NOREV-NEXT:    [[TMP6:%.*]] = mul nuw nsw i64 [[TMP0]], 3
-; CHECK-TF-NOREV-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP5]]
-; CHECK-TF-NOREV-NEXT:    [[TMP8:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP6]]
-; CHECK-TF-NOREV-NEXT:    [[TMP9:%.*]] = shufflevector <4 x float> [[STRIDED_VEC]], <4 x float> [[STRIDED_VEC1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-TF-NOREV-NEXT:    [[TMP10:%.*]] = shufflevector <8 x float> [[TMP9]], <8 x float> <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float undef, float undef, float undef, float undef>, <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-; CHECK-TF-NOREV-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <12 x float> [[TMP10]], <12 x float> poison, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
-; CHECK-TF-NOREV-NEXT:    store <12 x float> [[INTERLEAVED_VEC]], ptr [[TMP7]], align 4
-; CHECK-TF-NOREV-NEXT:    [[TMP11:%.*]] = shufflevector <4 x float> [[STRIDED_VEC3]], <4 x float> [[STRIDED_VEC4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-TF-NOREV-NEXT:    [[TMP12:%.*]] = shufflevector <8 x float> [[TMP11]], <8 x float> <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float undef, float undef, float undef, float undef>, <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-; CHECK-TF-NOREV-NEXT:    [[INTERLEAVED_VEC5:%.*]] = shufflevector <12 x float> [[TMP12]], <12 x float> poison, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
-; CHECK-TF-NOREV-NEXT:    store <12 x float> [[INTERLEAVED_VEC5]], ptr [[TMP8]], align 4
-; CHECK-TF-NOREV-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
-; CHECK-TF-NOREV-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-TF-NOREV-NEXT:    br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-TF-NOREV-NEXT:    [[TMP4:%.*]] = add i64 [[TMP2]], 0
+; CHECK-TF-NOREV-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 1
+; CHECK-TF-NOREV-NEXT:    [[TMP6:%.*]] = add i64 [[INDEX]], [[TMP5]]
+; CHECK-TF-NOREV-NEXT:    [[TMP7:%.*]] = shl nuw nsw i64 [[INDEX]], 1
+; CHECK-TF-NOREV-NEXT:    [[TMP8:%.*]] = shl nuw nsw i64 [[TMP6]], 1
+; CHECK-TF-NOREV-NEXT:    [[TMP9:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP7]]
+; CHECK-TF-NOREV-NEXT:    [[TMP10:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP8]]
+; CHECK-TF-NOREV-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x float>, ptr [[TMP9]], align 4
+; CHECK-TF-NOREV-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC]])
+; CHECK-TF-NOREV-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 0
+; CHECK-TF-NOREV-NEXT:    [[TMP12:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 1
+; CHECK-TF-NOREV-NEXT:    [[WIDE_VEC1:%.*]] = load <vscale x 8 x float>, ptr [[TMP10]], align 4
+; CHECK-TF-NOREV-NEXT:    [[STRIDED_VEC2:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC1]])
+; CHECK-TF-NOREV-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC2]], 0
+; CHECK-TF-NOREV-NEXT:    [[TMP14:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC2]], 1
+; CHECK-TF-NOREV-NEXT:    [[TMP15:%.*]] = mul nuw nsw i64 [[INDEX]], 3
+; CHECK-TF-NOREV-NEXT:    [[TMP16:%.*]] = mul nuw nsw i64 [[TMP6]], 3
+; CHECK-TF-NOREV-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP15]]
+; CHECK-TF-NOREV-NEXT:    [[TMP18:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP16]]
+; CHECK-TF-NOREV-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 12 x float> @llvm.vector.interleave3.nxv12f32(<vscale x 4 x float> [[TMP11]], <vscale x 4 x float> [[TMP12]], <vscale x 4 x float> splat (float 3.000000e+00))
+; CHECK-TF-NOREV-NEXT:    store <vscale x 12 x float> [[INTERLEAVED_VEC]], ptr [[TMP17]], align 4
+; CHECK-TF-NOREV-NEXT:    [[INTERLEAVED_VEC3:%.*]] = call <vscale x 12 x float> @llvm.vector.interleave3.nxv12f32(<vscale x 4 x float> [[TMP13]], <vscale x 4 x float> [[TMP14]], <vscale x 4 x float> splat (float 3.000000e+00))
+; CHECK-TF-NOREV-NEXT:    store <vscale x 12 x float> [[INTERLEAVED_VEC3]], ptr [[TMP18]], align 4
+; CHECK-TF-NOREV-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-TF-NOREV-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-TF-NOREV-NEXT:    br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK-TF-NOREV:       [[MIDDLE_BLOCK]]:
 ; CHECK-TF-NOREV-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; CHECK-TF-NOREV-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
@@ -1489,16 +1551,16 @@ define void @interleave(ptr noalias %dst, ptr noalias %src, i64 %n) #0 {
 ; CHECK-TF-NOREV-NEXT:    [[I_021:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
 ; CHECK-TF-NOREV-NEXT:    [[MUL:%.*]] = shl nuw nsw i64 [[I_021]], 1
 ; CHECK-TF-NOREV-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[MUL]]
-; CHECK-TF-NOREV-NEXT:    [[TMP14:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-TF-NOREV-NEXT:    [[TMP20:%.*]] = load float, ptr [[ARRAYIDX]], align 4
 ; CHECK-TF-NOREV-NEXT:    [[MUL1:%.*]] = mul nuw nsw i64 [[I_021]], 3
 ; CHECK-TF-NOREV-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[MUL1]]
-; CHECK-TF-NOREV-NEXT:    store float [[TMP14]], ptr [[ARRAYIDX2]], align 4
+; CHECK-TF-NOREV-NEXT:    store float [[TMP20]], ptr [[ARRAYIDX2]], align 4
 ; CHECK-TF-NOREV-NEXT:    [[ADD:%.*]] = or disjoint i64 [[MUL]], 1
 ; CHECK-TF-NOREV-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[ADD]]
-; CHECK-TF-NOREV-NEXT:    [[TMP15:%.*]] = load float, ptr [[ARRAYIDX4]], align 4
+; CHECK-TF-NOREV-NEXT:    [[TMP21:%.*]] = load float, ptr [[ARRAYIDX4]], align 4
 ; CHECK-TF-NOREV-NEXT:    [[ADD6:%.*]] = add nuw nsw i64 [[MUL1]], 1
 ; CHECK-TF-NOREV-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD6]]
-; CHECK-TF-NOREV-NEXT:    store float [[TMP15]], ptr [[ARRAYIDX7]], align 4
+; CHECK-TF-NOREV-NEXT:    store float [[TMP21]], ptr [[ARRAYIDX7]], align 4
 ; CHECK-TF-NOREV-NEXT:    [[ADD9:%.*]] = add nuw nsw i64 [[MUL1]], 2
 ; CHECK-TF-NOREV-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD9]]
 ; CHECK-TF-NOREV-NEXT:    store float 3.000000e+00, ptr [[ARRAYIDX10]], align 4
@@ -1511,40 +1573,44 @@ define void @interleave(ptr noalias %dst, ptr noalias %src, i64 %n) #0 {
 ; CHECK-TF-ONLYRED-LABEL: define void @interleave(
 ; CHECK-TF-ONLYRED-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; CHECK-TF-ONLYRED-NEXT:  [[ENTRY:.*]]:
-; CHECK-TF-ONLYRED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; CHECK-TF-ONLYRED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-ONLYRED-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-TF-ONLYRED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; CHECK-TF-ONLYRED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK-TF-ONLYRED:       [[VECTOR_PH]]:
-; CHECK-TF-ONLYRED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
+; CHECK-TF-ONLYRED-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-ONLYRED-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; CHECK-TF-ONLYRED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-TF-ONLYRED-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; CHECK-TF-ONLYRED-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK-TF-ONLYRED:       [[VECTOR_BODY]]:
 ; CHECK-TF-ONLYRED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-TF-ONLYRED-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 4
-; CHECK-TF-ONLYRED-NEXT:    [[TMP1:%.*]] = shl nuw nsw i64 [[INDEX]], 1
-; CHECK-TF-ONLYRED-NEXT:    [[TMP2:%.*]] = shl nuw nsw i64 [[TMP0]], 1
-; CHECK-TF-ONLYRED-NEXT:    [[TMP3:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP1]]
-; CHECK-TF-ONLYRED-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP2]]
-; CHECK-TF-ONLYRED-NEXT:    [[WIDE_VEC:%.*]] = load <8 x float>, ptr [[TMP3]], align 4
-; CHECK-TF-ONLYRED-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <8 x float> [[WIDE_VEC]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-TF-ONLYRED-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <8 x float> [[WIDE_VEC]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; CHECK-TF-ONLYRED-NEXT:    [[WIDE_VEC2:%.*]] = load <8 x float>, ptr [[TMP4]], align 4
-; CHECK-TF-ONLYRED-NEXT:    [[STRIDED_VEC3:%.*]] = shufflevector <8 x float> [[WIDE_VEC2]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-TF-ONLYRED-NEXT:    [[STRIDED_VEC4:%.*]] = shufflevector <8 x float> [[WIDE_VEC2]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; CHECK-TF-ONLYRED-NEXT:    [[TMP5:%.*]] = mul nuw nsw i64 [[INDEX]], 3
-; CHECK-TF-ONLYRED-NEXT:    [[TMP6:%.*]] = mul nuw nsw i64 [[TMP0]], 3
-; CHECK-TF-ONLYRED-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP5]]
-; CHECK-TF-ONLYRED-NEXT:    [[TMP8:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP6]]
-; CHECK-TF-ONLYRED-NEXT:    [[TMP9:%.*]] = shufflevector <4 x float> [[STRIDED_VEC]], <4 x float> [[STRIDED_VEC1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-TF-ONLYRED-NEXT:    [[TMP10:%.*]] = shufflevector <8 x float> [[TMP9]], <8 x float> <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float undef, float undef, float undef, float undef>, <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-; CHECK-TF-ONLYRED-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <12 x float> [[TMP10]], <12 x float> poison, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
-; CHECK-TF-ONLYRED-NEXT:    store <12 x float> [[INTERLEAVED_VEC]], ptr [[TMP7]], align 4
-; CHECK-TF-ONLYRED-NEXT:    [[TMP11:%.*]] = shufflevector <4 x float> [[STRIDED_VEC3]], <4 x float> [[STRIDED_VEC4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-TF-ONLYRED-NEXT:    [[TMP12:%.*]] = shufflevector <8 x float> [[TMP11]], <8 x float> <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float undef, float undef, float undef, float undef>, <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-; CHECK-TF-ONLYRED-NEXT:    [[INTERLEAVED_VEC5:%.*]] = shufflevector <12 x float> [[TMP12]], <12 x float> poison, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
-; CHECK-TF-ONLYRED-NEXT:    store <12 x float> [[INTERLEAVED_VEC5]], ptr [[TMP8]], align 4
-; CHECK-TF-ONLYRED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
-; CHECK-TF-ONLYRED-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-TF-ONLYRED-NEXT:    br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-TF-ONLYRED-NEXT:    [[TMP4:%.*]] = add i64 [[TMP2]], 0
+; CHECK-TF-ONLYRED-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 1
+; CHECK-TF-ONLYRED-NEXT:    [[TMP6:%.*]] = add i64 [[INDEX]], [[TMP5]]
+; CHECK-TF-ONLYRED-NEXT:    [[TMP7:%.*]] = shl nuw nsw i64 [[INDEX]], 1
+; CHECK-TF-ONLYRED-NEXT:    [[TMP8:%.*]] = shl nuw nsw i64 [[TMP6]], 1
+; CHECK-TF-ONLYRED-NEXT:    [[TMP9:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP7]]
+; CHECK-TF-ONLYRED-NEXT:    [[TMP10:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP8]]
+; CHECK-TF-ONLYRED-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x float>, ptr [[TMP9]], align 4
+; CHECK-TF-ONLYRED-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC]])
+; CHECK-TF-ONLYRED-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 0
+; CHECK-TF-ONLYRED-NEXT:    [[TMP12:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 1
+; CHECK-TF-ONLYRED-NEXT:    [[WIDE_VEC1:%.*]] = load <vscale x 8 x float>, ptr [[TMP10]], align 4
+; CHECK-TF-ONLYRED-NEXT:    [[STRIDED_VEC2:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC1]])
+; CHECK-TF-ONLYRED-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC2]], 0
+; CHECK-TF-ONLYRED-NEXT:    [[TMP14:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC2]], 1
+; CHECK-TF-ONLYRED-NEXT:    [[TMP15:%.*]] = mul nuw nsw i64 [[INDEX]], 3
+; CHECK-TF-ONLYRED-NEXT:    [[TMP16:%.*]] = mul nuw nsw i64 [[TMP6]], 3
+; CHECK-TF-ONLYRED-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP15]]
+; CHECK-TF-ONLYRED-NEXT:    [[TMP18:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP16]]
+; CHECK-TF-ONLYRED-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 12 x float> @llvm.vector.interleave3.nxv12f32(<vscale x 4 x float> [[TMP11]], <vscale x 4 x float> [[TMP12]], <vscale x 4 x float> splat (float 3.000000e+00))
+; CHECK-TF-ONLYRED-NEXT:    store <vscale x 12 x float> [[INTERLEAVED_VEC]], ptr [[TMP17]], align 4
+; CHECK-TF-ONLYRED-NEXT:    [[INTERLEAVED_VEC3:%.*]] = call <vscale x 12 x float> @llvm.vector.interleave3.nxv12f32(<vscale x 4 x float> [[TMP13]], <vscale x 4 x float> [[TMP14]], <vscale x 4 x float> splat (float 3.000000e+00))
+; CHECK-TF-ONLYRED-NEXT:    store <vscale x 12 x float> [[INTERLEAVED_VEC3]], ptr [[TMP18]], align 4
+; CHECK-TF-ONLYRED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-TF-ONLYRED-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-TF-ONLYRED-NEXT:    br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
 ; CHECK-TF-ONLYRED:       [[MIDDLE_BLOCK]]:
 ; CHECK-TF-ONLYRED-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; CHECK-TF-ONLYRED-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
@@ -1555,16 +1621,16 @@ define void @interleave(ptr noalias %dst, ptr noalias %src, i64 %n) #0 {
 ; CHECK-TF-ONLYRED-NEXT:    [[I_021:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
 ; CHECK-TF-ONLYRED-NEXT:    [[MUL:%.*]] = shl nuw nsw i64 [[I_021]], 1
 ; CHECK-TF-ONLYRED-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[MUL]]
-; CHECK-TF-ONLYRED-NEXT:    [[TMP14:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-TF-ONLYRED-NEXT:    [[TMP20:%.*]] = load float, ptr [[ARRAYIDX]], align 4
 ; CHECK-TF-ONLYRED-NEXT:    [[MUL1:%.*]] = mul nuw nsw i64 [[I_021]], 3
 ; CHECK-TF-ONLYRED-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[MUL1]]
-; CHECK-TF-ONLYRED-NEXT:    store float [[TMP14]], ptr [[ARRAYIDX2]], align 4
+; CHECK-TF-ONLYRED-NEXT:    store float [[TMP20]], ptr [[ARRAYIDX2]], align 4
 ; CHECK-TF-ONLYRED-NEXT:    [[ADD:%.*]] = or disjoint i64 [[MUL]], 1
 ; CHECK-TF-ONLYRED-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[ADD]]
-; CHECK-TF-ONLYRED-NEXT:    [[TMP15:%.*]] = load float, ptr [[ARRAYIDX4]], align 4
+; CHECK-TF-ONLYRED-NEXT:    [[TMP21:%.*]] = load float, ptr [[ARRAYIDX4]], align 4
 ; CHECK-TF-ONLYRED-NEXT:    [[ADD6:%.*]] = add nuw nsw i64 [[MUL1]], 1
 ; CHECK-TF-ONLYRED-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD6]]
-; CHECK-TF-ONLYRED-NEXT:    store float [[TMP15]], ptr [[ARRAYIDX7]], align 4
+; CHECK-TF-ONLYRED-NEXT:    store float [[TMP21]], ptr [[ARRAYIDX7]], align 4
 ; CHECK-TF-ONLYRED-NEXT:    [[ADD9:%.*]] = add nuw nsw i64 [[MUL1]], 2
 ; CHECK-TF-ONLYRED-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD9]]
 ; CHECK-TF-ONLYRED-NEXT:    store float 3.000000e+00, ptr [[ARRAYIDX10]], align 4
@@ -1576,67 +1642,105 @@ define void @interleave(ptr noalias %dst, ptr noalias %src, i64 %n) #0 {
 ;
 ; CHECK-NEOVERSE-V1-LABEL: define void @interleave(
 ; CHECK-NEOVERSE-V1-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-NEOVERSE-V1-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEOVERSE-V1-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
-; CHECK-NEOVERSE-V1-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEOVERSE-V1-NEXT:  [[ITER_CHECK:.*]]:
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP25:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NEOVERSE-V1-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP25]]
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP29:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP30:%.*]] = shl nuw i64 [[TMP29]], 2
+; CHECK-NEOVERSE-V1-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK-NEOVERSE-V1:       [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-NEOVERSE-V1-NEXT:    [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-NEOVERSE-V1-NEXT:    br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK-NEOVERSE-V1:       [[VECTOR_PH]]:
-; CHECK-NEOVERSE-V1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; CHECK-NEOVERSE-V1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-NEOVERSE-V1-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; CHECK-NEOVERSE-V1-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK-NEOVERSE-V1:       [[VECTOR_BODY]]:
 ; CHECK-NEOVERSE-V1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEOVERSE-V1-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 4
-; CHECK-NEOVERSE-V1-NEXT:    [[TMP1:%.*]] = shl nuw nsw i64 [[INDEX]], 1
-; CHECK-NEOVERSE-V1-NEXT:    [[TMP2:%.*]] = shl nuw nsw i64 [[TMP0]], 1
-; CHECK-NEOVERSE-V1-NEXT:    [[TMP3:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP1]]
-; CHECK-NEOVERSE-V1-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP2]]
-; CHECK-NEOVERSE-V1-NEXT:    [[WIDE_VEC:%.*]] = load <8 x float>, ptr [[TMP3]], align 4
-; CHECK-NEOVERSE-V1-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <8 x float> [[WIDE_VEC]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NEOVERSE-V1-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <8 x float> [[WIDE_VEC]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; CHECK-NEOVERSE-V1-NEXT:    [[WIDE_VEC2:%.*]] = load <8 x float>, ptr [[TMP4]], align 4
-; CHECK-NEOVERSE-V1-NEXT:    [[STRIDED_VEC3:%.*]] = shufflevector <8 x float> [[WIDE_VEC2]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NEOVERSE-V1-NEXT:    [[STRIDED_VEC4:%.*]] = shufflevector <8 x float> [[WIDE_VEC2]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; CHECK-NEOVERSE-V1-NEXT:    [[TMP5:%.*]] = mul nuw nsw i64 [[INDEX]], 3
-; CHECK-NEOVERSE-V1-NEXT:    [[TMP6:%.*]] = mul nuw nsw i64 [[TMP0]], 3
-; CHECK-NEOVERSE-V1-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP5]]
-; CHECK-NEOVERSE-V1-NEXT:    [[TMP8:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP6]]
-; CHECK-NEOVERSE-V1-NEXT:    [[TMP9:%.*]] = shufflevector <4 x float> [[STRIDED_VEC]], <4 x float> [[STRIDED_VEC1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEOVERSE-V1-NEXT:    [[TMP10:%.*]] = shufflevector <8 x float> [[TMP9]], <8 x float> <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float undef, float undef, float undef, float undef>, <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-; CHECK-NEOVERSE-V1-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <12 x float> [[TMP10]], <12 x float> poison, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
-; CHECK-NEOVERSE-V1-NEXT:    store <12 x float> [[INTERLEAVED_VEC]], ptr [[TMP7]], align 4
-; CHECK-NEOVERSE-V1-NEXT:    [[TMP11:%.*]] = shufflevector <4 x float> [[STRIDED_VEC3]], <4 x float> [[STRIDED_VEC4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEOVERSE-V1-NEXT:    [[TMP12:%.*]] = shufflevector <8 x float> [[TMP11]], <8 x float> <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float undef, float undef, float undef, float undef>, <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-; CHECK-NEOVERSE-V1-NEXT:    [[INTERLEAVED_VEC5:%.*]] = shufflevector <12 x float> [[TMP12]], <12 x float> poison, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
-; CHECK-NEOVERSE-V1-NEXT:    store <12 x float> [[INTERLEAVED_VEC5]], ptr [[TMP8]], align 4
-; CHECK-NEOVERSE-V1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
-; CHECK-NEOVERSE-V1-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEOVERSE-V1-NEXT:    br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP4:%.*]] = add i64 [[TMP2]], 0
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 1
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP6:%.*]] = add i64 [[INDEX]], [[TMP5]]
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP7:%.*]] = shl nuw nsw i64 [[INDEX]], 1
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP8:%.*]] = shl nuw nsw i64 [[TMP6]], 1
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP9:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP7]]
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP10:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP8]]
+; CHECK-NEOVERSE-V1-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x float>, ptr [[TMP9]], align 4
+; CHECK-NEOVERSE-V1-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC]])
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 0
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP12:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 1
+; CHECK-NEOVERSE-V1-NEXT:    [[WIDE_VEC2:%.*]] = load <vscale x 8 x float>, ptr [[TMP10]], align 4
+; CHECK-NEOVERSE-V1-NEXT:    [[STRIDED_VEC3:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC2]])
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC3]], 0
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP14:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC3]], 1
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP15:%.*]] = mul nuw nsw i64 [[INDEX]], 3
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP16:%.*]] = mul nuw nsw i64 [[TMP6]], 3
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP15]]
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP18:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP16]]
+; CHECK-NEOVERSE-V1-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 12 x float> @llvm.vector.interleave3.nxv12f32(<vscale x 4 x float> [[TMP11]], <vscale x 4 x float> [[TMP12]], <vscale x 4 x float> splat (float 3.000000e+00))
+; CHECK-NEOVERSE-V1-NEXT:    store <vscale x 12 x float> [[INTERLEAVED_VEC]], ptr [[TMP17]], align 4
+; CHECK-NEOVERSE-V1-NEXT:    [[INTERLEAVED_VEC4:%.*]] = call <vscale x 12 x float> @llvm.vector.interleave3.nxv12f32(<vscale x 4 x float> [[TMP13]], <vscale x 4 x float> [[TMP14]], <vscale x 4 x float> splat (float 3.000000e+00))
+; CHECK-NEOVERSE-V1-NEXT:    store <vscale x 12 x float> [[INTERLEAVED_VEC4]], ptr [[TMP18]], align 4
+; CHECK-NEOVERSE-V1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEOVERSE-V1-NEXT:    br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
 ; CHECK-NEOVERSE-V1:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEOVERSE-V1-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEOVERSE-V1-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
-; CHECK-NEOVERSE-V1:       [[SCALAR_PH]]:
-; CHECK-NEOVERSE-V1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEOVERSE-V1-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK-NEOVERSE-V1:       [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-NEOVERSE-V1-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP30]]
+; CHECK-NEOVERSE-V1-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF8:![0-9]+]]
+; CHECK-NEOVERSE-V1:       [[VEC_EPILOG_PH]]:
+; CHECK-NEOVERSE-V1-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP31:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP24:%.*]] = shl nuw i64 [[TMP31]], 2
+; CHECK-NEOVERSE-V1-NEXT:    [[N_MOD_VF5:%.*]] = urem i64 [[N]], [[TMP24]]
+; CHECK-NEOVERSE-V1-NEXT:    [[N_VEC6:%.*]] = sub i64 [[N]], [[N_MOD_VF5]]
+; CHECK-NEOVERSE-V1-NEXT:    br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK-NEOVERSE-V1:       [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-NEOVERSE-V1-NEXT:    [[INDEX7:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT12:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP20:%.*]] = shl nuw nsw i64 [[INDEX7]], 1
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP21:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP20]]
+; CHECK-NEOVERSE-V1-NEXT:    [[WIDE_VEC8:%.*]] = load <vscale x 8 x float>, ptr [[TMP21]], align 4
+; CHECK-NEOVERSE-V1-NEXT:    [[STRIDED_VEC9:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC8]])
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP32:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC9]], 0
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP33:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC9]], 1
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP22:%.*]] = mul nuw nsw i64 [[INDEX7]], 3
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP23:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP22]]
+; CHECK-NEOVERSE-V1-NEXT:    [[INTERLEAVED_VEC10:%.*]] = call <vscale x 12 x float> @llvm.vector.interleave3.nxv12f32(<vscale x 4 x float> [[TMP32]], <vscale x 4 x float> [[TMP33]], <vscale x 4 x float> splat (float 3.000000e+00))
+; CHECK-NEOVERSE-V1-NEXT:    store <vscale x 12 x float> [[INTERLEAVED_VEC10]], ptr [[TMP23]], align 4
+; CHECK-NEOVERSE-V1-NEXT:    [[INDEX_NEXT12]] = add nuw i64 [[INDEX7]], [[TMP24]]
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT12]], [[N_VEC6]]
+; CHECK-NEOVERSE-V1-NEXT:    br i1 [[TMP26]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-NEOVERSE-V1:       [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-NEOVERSE-V1-NEXT:    [[CMP_N13:%.*]] = icmp eq i64 [[N]], [[N_VEC6]]
+; CHECK-NEOVERSE-V1-NEXT:    br i1 [[CMP_N13]], label %[[FOR_END]], label %[[VEC_EPILOG_SCALAR_PH]]
+; CHECK-NEOVERSE-V1:       [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-NEOVERSE-V1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC6]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
 ; CHECK-NEOVERSE-V1-NEXT:    br label %[[FOR_BODY:.*]]
 ; CHECK-NEOVERSE-V1:       [[FOR_BODY]]:
-; CHECK-NEOVERSE-V1-NEXT:    [[I_021:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEOVERSE-V1-NEXT:    [[I_021:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ]
 ; CHECK-NEOVERSE-V1-NEXT:    [[MUL:%.*]] = shl nuw nsw i64 [[I_021]], 1
 ; CHECK-NEOVERSE-V1-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[MUL]]
-; CHECK-NEOVERSE-V1-NEXT:    [[TMP14:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP27:%.*]] = load float, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEOVERSE-V1-NEXT:    [[MUL1:%.*]] = mul nuw nsw i64 [[I_021]], 3
 ; CHECK-NEOVERSE-V1-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[MUL1]]
-; CHECK-NEOVERSE-V1-NEXT:    store float [[TMP14]], ptr [[ARRAYIDX2]], align 4
+; CHECK-NEOVERSE-V1-NEXT:    store float [[TMP27]], ptr [[ARRAYIDX2]], align 4
 ; CHECK-NEOVERSE-V1-NEXT:    [[ADD:%.*]] = or disjoint i64 [[MUL]], 1
 ; CHECK-NEOVERSE-V1-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[ADD]]
-; CHECK-NEOVERSE-V1-NEXT:    [[TMP15:%.*]] = load float, ptr [[ARRAYIDX4]], align 4
+; CHECK-NEOVERSE-V1-NEXT:    [[TMP28:%.*]] = load float, ptr [[ARRAYIDX4]], align 4
 ; CHECK-NEOVERSE-V1-NEXT:    [[ADD6:%.*]] = add nuw nsw i64 [[MUL1]], 1
 ; CHECK-NEOVERSE-V1-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD6]]
-; CHECK-NEOVERSE-V1-NEXT:    store float [[TMP15]], ptr [[ARRAYIDX7]], align 4
+; CHECK-NEOVERSE-V1-NEXT:    store float [[TMP28]], ptr [[ARRAYIDX7]], align 4
 ; CHECK-NEOVERSE-V1-NEXT:    [[ADD9:%.*]] = add nuw nsw i64 [[MUL1]], 2
 ; CHECK-NEOVERSE-V1-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD9]]
 ; CHECK-NEOVERSE-V1-NEXT:    store float 3.000000e+00, ptr [[ARRAYIDX10]], align 4
 ; CHECK-NEOVERSE-V1-NEXT:    [[INC]] = add nuw nsw i64 [[I_021]], 1
 ; CHECK-NEOVERSE-V1-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]]
-; CHECK-NEOVERSE-V1-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NEOVERSE-V1-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
 ; CHECK-NEOVERSE-V1:       [[FOR_END]]:
 ; CHECK-NEOVERSE-V1-NEXT:    ret void
 ;
@@ -1831,7 +1935,7 @@ define void @reverse(ptr noalias %dst, ptr noalias %src) #0 {
 ; CHECK-TF-DEFAULT-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1024)
 ; CHECK-TF-DEFAULT-NEXT:    [[TMP9:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
 ; CHECK-TF-DEFAULT-NEXT:    [[TMP10:%.*]] = xor i1 [[TMP9]], true
-; CHECK-TF-DEFAULT-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-TF-DEFAULT-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
 ; CHECK-TF-DEFAULT:       [[MIDDLE_BLOCK]]:
 ; CHECK-TF-DEFAULT-NEXT:    br label %[[FOR_END:.*]]
 ; CHECK-TF-DEFAULT:       [[FOR_END]]:
@@ -2063,7 +2167,7 @@ define void @reverse(ptr noalias %dst, ptr noalias %src) #0 {
 ; CHECK-NEOVERSE-V1-NEXT:    store <vscale x 2 x double> [[REVERSE2]], ptr [[TMP17]], align 8
 ; CHECK-NEOVERSE-V1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-NEOVERSE-V1-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEOVERSE-V1-NEXT:    br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-NEOVERSE-V1-NEXT:    br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
 ; CHECK-NEOVERSE-V1:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEOVERSE-V1-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
 ; CHECK-NEOVERSE-V1-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
@@ -2081,7 +2185,7 @@ define void @reverse(ptr noalias %dst, ptr noalias %src) #0 {
 ; CHECK-NEOVERSE-V1-NEXT:    [[INDVARS_IV_NEXT]] = add nsw i64 [[INDVARS_IV]], -1
 ; CHECK-NEOVERSE-V1-NEXT:    [[INDVARS_PTR_NEXT]] = getelementptr inbounds double, ptr [[INDVARS_PTR]], i64 1
 ; CHECK-NEOVERSE-V1-NEXT:    [[CMP_NOT:%.*]] = icmp eq i64 [[INDVARS_IV]], 0
-; CHECK-NEOVERSE-V1-NEXT:    br i1 [[CMP_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-NEOVERSE-V1-NEXT:    br i1 [[CMP_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
 ; CHECK-NEOVERSE-V1:       [[FOR_END]]:
 ; CHECK-NEOVERSE-V1-NEXT:    ret void
 ;
@@ -2154,8 +2258,10 @@ attributes #0 = { "target-features"="+sve" }
 ; CHECK-TF-DEFAULT: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]], [[META2]]}
 ; CHECK-TF-DEFAULT: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
 ; CHECK-TF-DEFAULT: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]], [[META2]]}
-; CHECK-TF-DEFAULT: [[LOOP6]] = distinct !{[[LOOP6]], [[META2]], [[META1]]}
+; CHECK-TF-DEFAULT: [[PROF6]] = !{!"branch_weights", i32 8, i32 8}
 ; CHECK-TF-DEFAULT: [[LOOP7]] = distinct !{[[LOOP7]], [[META1]], [[META2]]}
+; CHECK-TF-DEFAULT: [[LOOP8]] = distinct !{[[LOOP8]], [[META2]], [[META1]]}
+; CHECK-TF-DEFAULT: [[LOOP9]] = distinct !{[[LOOP9]], [[META1]], [[META2]]}
 ;.
 ; CHECK-TF-NORED: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
 ; CHECK-TF-NORED: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
@@ -2207,7 +2313,9 @@ attributes #0 = { "target-features"="+sve" }
 ; CHECK-NEOVERSE-V1: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]], [[META2]]}
 ; CHECK-NEOVERSE-V1: [[LOOP6]] = distinct !{[[LOOP6]], [[META2]], [[META1]]}
 ; CHECK-NEOVERSE-V1: [[LOOP7]] = distinct !{[[LOOP7]], [[META1]], [[META2]]}
-; CHECK-NEOVERSE-V1: [[LOOP8]] = distinct !{[[LOOP8]], [[META2]], [[META1]]}
+; CHECK-NEOVERSE-V1: [[PROF8]] = !{!"branch_weights", i32 8, i32 8}
 ; CHECK-NEOVERSE-V1: [[LOOP9]] = distinct !{[[LOOP9]], [[META1]], [[META2]]}
 ; CHECK-NEOVERSE-V1: [[LOOP10]] = distinct !{[[LOOP10]], [[META2]], [[META1]]}
+; CHECK-NEOVERSE-V1: [[LOOP11]] = distinct !{[[LOOP11]], [[META1]], [[META2]]}
+; CHECK-NEOVERSE-V1: [[LOOP12]] = distinct !{[[LOOP12]], [[META2]], [[META1]]}
 ;.

