[llvm] [AArch64][SVE] Allow scalable factor-3 interleaved accesses (PR #200424)
Harry Ramsey via llvm-commits
llvm-commits at lists.llvm.org
Thu Jul 16 05:36:07 PDT 2026
https://github.com/Harry-Ramsey updated https://github.com/llvm/llvm-project/pull/200424
>From 486039c535558405f6283bafefa1fbfb6a4dde8e Mon Sep 17 00:00:00 2001
From: Harry Ramsey <harry.ramsey at arm.com>
Date: Tue, 14 Jul 2026 22:48:50 +0000
Subject: [PATCH] [AArch64][ISel] Enable factor-3 scalable interleave
operations
Allow scalable interleaved memory operations with factor 3 in the
AArch64 TTI cost model.
SelectionDAG can lower explicit scalable vector.interleave3 and
vector.deinterleave3 patterns for legal packed SVE vector types. For
vector.interleave3 values declared in seperate blocks must be
deinterleaved first before being used.
---
.../Target/AArch64/AArch64ISelLowering.cpp | 86 +-
.../AArch64/AArch64TargetTransformInfo.cpp | 16 +-
.../scalable_masked_deinterleaved_loads.ll | 11 +
.../scalable_masked_interleaved_stores.ll | 68 +
.../AArch64/force-target-instruction-cost.ll | 58 +-
.../AArch64/sve-interleaved-accesses.ll | 40 +-
.../AArch64/sve-tail-folding-option.ll | 2465 +++++++++++++++--
7 files changed, 2412 insertions(+), 332 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 6c27fef9e519d..5addd72c9e886 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -27622,6 +27622,41 @@ static SDValue getNarrowMaskForInterleavedOps(SelectionDAG &DAG, SDLoc &DL,
WideMask->getOperand(0));
}
+static bool
+deinterleaveInterleavedValueForSVESt3(SDValue WideValue, SDLoc DL,
+ SelectionDAG &DAG,
+ SmallVectorImpl<SDValue> &Ops) {
+ constexpr unsigned Factor = 3;
+ EVT WideVT = WideValue.getValueType();
+ if (!WideVT.isScalableVector())
+ return false;
+
+ ElementCount WideEC = WideVT.getVectorElementCount();
+ if (!WideEC.isKnownMultipleOf(Factor))
+ return false;
+
+ EVT SubVecTy =
+ EVT::getVectorVT(*DAG.getContext(), WideVT.getVectorElementType(),
+ WideEC.divideCoefficientBy(Factor));
+ if (!SubVecTy.isScalableVector() ||
+ SubVecTy.getSizeInBits().getKnownMinValue() != 128 ||
+ !DAG.getTargetLoweringInfo().isTypeLegal(SubVecTy))
+ return false;
+
+ SmallVector<SDValue, 3> SubVecs;
+ for (unsigned I = 0; I != Factor; ++I)
+ SubVecs.push_back(DAG.getNode(
+ ISD::EXTRACT_SUBVECTOR, DL, SubVecTy, WideValue,
+ DAG.getVectorIdxConstant(I * SubVecTy.getVectorMinNumElements(), DL)));
+
+ SmallVector<EVT, 3> ResultVTs(Factor, SubVecTy);
+ SDValue Deinterleaved = DAG.getNode(ISD::VECTOR_DEINTERLEAVE, DL,
+ DAG.getVTList(ResultVTs), SubVecs);
+ for (unsigned I = 0; I != Factor; ++I)
+ Ops.push_back(Deinterleaved.getValue(I));
+ return true;
+}
+
static SDValue performInterleavedMaskedStoreCombine(
SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG) {
if (!DCI.isBeforeLegalize())
@@ -27646,11 +27681,16 @@ static SDValue performInterleavedMaskedStoreCombine(
return SDValue();
} else if (!isSplatVectorInterleaveOps(DAG, DL, WideValue,
ValueInterleaveOps)) {
- return SDValue();
+ // A vector.interleave3 defined in another basic block is unknown here
+ // because SelectionDAGs are local to a basic block. Deinterleave the wide
+ // value back into legal operands so it can be used by st3.
+ if (!deinterleaveInterleavedValueForSVESt3(WideValue, DL, DAG,
+ ValueInterleaveOps))
+ return SDValue();
}
unsigned NumParts = ValueInterleaveOps.size();
- if (NumParts != 2 && NumParts != 4)
+ if (NumParts != 2 && NumParts != 3 && NumParts != 4)
return SDValue();
// At the moment we're unlikely to see a fixed-width vector interleave as
@@ -27666,8 +27706,18 @@ static SDValue performInterleavedMaskedStoreCombine(
if (!NarrowMask)
return SDValue();
- const Intrinsic::ID IID =
- NumParts == 2 ? Intrinsic::aarch64_sve_st2 : Intrinsic::aarch64_sve_st4;
+ Intrinsic::ID IID;
+ switch (NumParts) {
+ case 2:
+ IID = Intrinsic::aarch64_sve_st2;
+ break;
+ case 3:
+ IID = Intrinsic::aarch64_sve_st3;
+ break;
+ case 4:
+ IID = Intrinsic::aarch64_sve_st4;
+ break;
+ }
SmallVector<SDValue, 8> NewStOps;
NewStOps.append({MST->getChain(), DAG.getConstant(IID, DL, MVT::i32)});
NewStOps.append(ValueInterleaveOps);
@@ -30555,7 +30605,7 @@ static SDValue performVectorDeinterleaveCombine(
return SDValue();
unsigned NumParts = N->getNumOperands();
- if (NumParts != 2 && NumParts != 4)
+ if (NumParts != 2 && NumParts != 3 && NumParts != 4)
return SDValue();
EVT SubVecTy = N->getValueType(0);
@@ -30601,19 +30651,37 @@ static SDValue performVectorDeinterleaveCombine(
if (!NarrowMask)
return SDValue();
- const Intrinsic::ID IID = NumParts == 2 ? Intrinsic::aarch64_sve_ld2_sret
- : Intrinsic::aarch64_sve_ld4_sret;
+ Intrinsic::ID IID;
+ switch (NumParts) {
+ case 2:
+ IID = Intrinsic::aarch64_sve_ld2_sret;
+ break;
+ case 3:
+ IID = Intrinsic::aarch64_sve_ld3_sret;
+ break;
+ case 4:
+ IID = Intrinsic::aarch64_sve_ld4_sret;
+ break;
+ }
SDValue NewLdOps[] = {MaskedLoad->getChain(),
DAG.getConstant(IID, DL, MVT::i32), NarrowMask,
MaskedLoad->getBasePtr()};
SDValue Res;
- if (NumParts == 2)
+ switch (NumParts) {
+ case 2:
Res = DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL,
{SubVecTy, SubVecTy, MVT::Other}, NewLdOps);
- else
+ break;
+ case 3:
+ Res = DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL,
+ {SubVecTy, SubVecTy, SubVecTy, MVT::Other}, NewLdOps);
+ break;
+ case 4:
Res = DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL,
{SubVecTy, SubVecTy, SubVecTy, SubVecTy, MVT::Other},
NewLdOps);
+ break;
+ }
// We can now generate a structured load!
SmallVector<SDValue, 4> ResOps(NumParts);
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 9d448b4a8681a..985a11ccfc232 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -5414,12 +5414,16 @@ InstructionCost AArch64TTIImpl::getInterleavedMemoryOpCost(
if (VecTy->isScalableTy() && !ST->hasSVE())
return InstructionCost::getInvalid();
- // Scalable VFs will emit vector.[de]interleave intrinsics, and currently we
- // only have lowering for power-of-2 factors.
- // TODO: Add lowering for vector.[de]interleave3 intrinsics and support in
- // InterleavedAccessPass for ld3/st3
- if (VecTy->isScalableTy() && !isPowerOf2_32(Factor))
- return InstructionCost::getInvalid();
+ // Scalable VFs emit vector.[de]interleave intrinsics, for which the target
+ // supports factors up to the maximum supported interleave factor.
+ if (VecTy->isScalableTy()) {
+ if (Factor > TLI->getMaxSupportedInterleaveFactor())
+ return InstructionCost::getInvalid();
+
+ if (Factor == 3 &&
+ DL.getTypeSizeInBits(VecTy).getKnownMinValue() < Factor * 128)
+ return InstructionCost::getInvalid();
+ }
// Vectorization for masked interleaved accesses is only enabled for scalable
// VF.
diff --git a/llvm/test/CodeGen/AArch64/scalable_masked_deinterleaved_loads.ll b/llvm/test/CodeGen/AArch64/scalable_masked_deinterleaved_loads.ll
index ee2d482d9ffb5..031509d5ff6c3 100644
--- a/llvm/test/CodeGen/AArch64/scalable_masked_deinterleaved_loads.ll
+++ b/llvm/test/CodeGen/AArch64/scalable_masked_deinterleaved_loads.ll
@@ -45,6 +45,17 @@ define { <vscale x 2 x double>, <vscale x 2 x double> } @foo_ld2_nxv2f64(<vscale
ret { <vscale x 2 x double>, <vscale x 2 x double> } %deinterleaved.vec
}
+define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @foo_ld3_nxv16i8(<vscale x 16 x i1> %mask, ptr %p) {
+; CHECK-LABEL: foo_ld3_nxv16i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ld3b { z0.b - z2.b }, p0/z, [x0]
+; CHECK-NEXT: ret
+ %interleaved.mask = call <vscale x 48 x i1> @llvm.vector.interleave3.nxv48i1(<vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask)
+ %wide.masked.vec = call <vscale x 48 x i8> @llvm.masked.load.nxv48i8(ptr %p, i32 1, <vscale x 48 x i1> %interleaved.mask, <vscale x 48 x i8> poison)
+ %deinterleaved.vec = call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.vector.deinterleave3.nxv48i8(<vscale x 48 x i8> %wide.masked.vec)
+ ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %deinterleaved.vec
+}
+
define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @foo_ld4_nxv16i8(<vscale x 16 x i1> %mask, ptr %p) {
; CHECK-LABEL: foo_ld4_nxv16i8:
; CHECK: // %bb.0:
diff --git a/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll b/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll
index 87833306571c2..2f9152c764bf2 100644
--- a/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll
+++ b/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll
@@ -53,6 +53,46 @@ define void @foo_st2_nxv2i64(<vscale x 2 x i1> %mask, <vscale x 2 x i64> %val1,
ret void
}
+define void @foo_st3_nxv16i8(<vscale x 16 x i1> %mask, <vscale x 16 x i8> %val1, <vscale x 16 x i8> %val2, <vscale x 16 x i8> %val3, ptr %p) {
+; CHECK-LABEL: foo_st3_nxv16i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2 def $z0_z1_z2
+; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2 def $z0_z1_z2
+; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2 def $z0_z1_z2
+; CHECK-NEXT: st3b { z0.b - z2.b }, p0, [x0]
+; CHECK-NEXT: ret
+ %interleaved.mask = call <vscale x 48 x i1> @llvm.vector.interleave3.nxv48i1(<vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask)
+ %interleaved.value = call <vscale x 48 x i8> @llvm.vector.interleave3.nxv48i8(<vscale x 16 x i8> %val1, <vscale x 16 x i8> %val2, <vscale x 16 x i8> %val3)
+ call void @llvm.masked.store.nxv48i8.p0(<vscale x 48 x i8> %interleaved.value, ptr %p, i32 1, <vscale x 48 x i1> %interleaved.mask)
+ ret void
+}
+
+; This test is specific to interleaves of factor 3.
+; SelectionDAG combines are local to a basic blocks. Keep the interleave in the
+; predecessor so the store sees an opaque, target-illegal nxv48i8 value.
+; The DAG should split the vectors back into three legal SVE vectors before type
+; legalisation.
+define void @interleave3_cross_block(<vscale x 16 x i1> %mask,
+; CHECK-LABEL: interleave3_cross_block:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2 def $z0_z1_z2
+; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2 def $z0_z1_z2
+; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2 def $z0_z1_z2
+; CHECK-NEXT: st3b { z0.b - z2.b }, p0, [x0]
+; CHECK-NEXT: ret
+ <vscale x 16 x i8> %a,
+ <vscale x 16 x i8> %b,
+ <vscale x 16 x i8> %c, ptr %p) {
+entry:
+ %wide.value = call <vscale x 48 x i8> @llvm.vector.interleave3.nxv48i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i8> %c)
+ br label %store
+
+store:
+ %wide.mask = call <vscale x 48 x i1> @llvm.vector.interleave3.nxv48i1(<vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask)
+ call void @llvm.masked.store.nxv48i8.p0(<vscale x 48 x i8> %wide.value, ptr %p, i32 1, <vscale x 48 x i1> %wide.mask)
+ ret void
+}
+
define void @foo_st4_nxv16i8(<vscale x 16 x i1> %mask, <vscale x 16 x i8> %val1, <vscale x 16 x i8> %val2, <vscale x 16 x i8> %val3, <vscale x 16 x i8> %val4, ptr %p) {
; CHECK-LABEL: foo_st4_nxv16i8:
; CHECK: // %bb.0:
@@ -295,6 +335,19 @@ define void @foo_st2_nxv16i8_zeroinitializer(<vscale x 16 x i1> %mask, ptr %p) {
ret void
}
+define void @foo_st3_nxv16i8_zeroinitializer(<vscale x 16 x i1> %mask, <vscale x 16 x i8> %val1, <vscale x 16 x i8> %val2, <vscale x 16 x i8> %val3, ptr %p) {
+; CHECK-LABEL: foo_st3_nxv16i8_zeroinitializer:
+; CHECK: // %bb.0:
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: mov z1.d, z0.d
+; CHECK-NEXT: mov z2.d, z0.d
+; CHECK-NEXT: st3b { z0.b - z2.b }, p0, [x0]
+; CHECK-NEXT: ret
+ %interleaved.mask = call <vscale x 48 x i1> @llvm.vector.interleave3.nxv48i1(<vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask)
+ call void @llvm.masked.store.nxv48i8.p0(<vscale x 48 x i8> zeroinitializer, ptr %p, i32 1, <vscale x 48 x i1> %interleaved.mask)
+ ret void
+}
+
define void @foo_st4_nxv16i8_zeroinitializer(<vscale x 16 x i1> %mask, ptr %p) {
; CHECK-LABEL: foo_st4_nxv16i8_zeroinitializer:
; CHECK: // %bb.0:
@@ -323,6 +376,21 @@ define void @foo_st2_nxv16i8_splat(<vscale x 16 x i1> %mask, ptr %p) {
ret void
}
+define void @foo_st3_nxv16i8_splat(<vscale x 16 x i1> %mask, ptr %p) {
+; CHECK-LABEL: foo_st3_nxv16i8_splat:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov z0.b, #1 // =0x1
+; CHECK-NEXT: mov z1.d, z0.d
+; CHECK-NEXT: mov z2.d, z0.d
+; CHECK-NEXT: st3b { z0.b - z2.b }, p0, [x0]
+; CHECK-NEXT: ret
+ %base = insertelement <vscale x 48 x i8> poison, i8 1, i32 0
+ %interleaved.value = shufflevector <vscale x 48 x i8> %base, <vscale x 48 x i8> poison, <vscale x 48 x i32> zeroinitializer
+ %interleaved.mask = call <vscale x 48 x i1> @llvm.vector.interleave3.nxv48i1(<vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask)
+ call void @llvm.masked.store.nxv48i8.p0( <vscale x 48 x i8> %interleaved.value, ptr %p, i32 1, <vscale x 48 x i1> %interleaved.mask)
+ ret void
+}
+
define void @foo_st4_nxv16i8_splat(<vscale x 16 x i1> %mask, ptr %p) {
; CHECK-LABEL: foo_st4_nxv16i8_splat:
; CHECK: // %bb.0:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
index 07fc18e46c4fe..8fa200573e2b9 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
@@ -439,27 +439,40 @@ define void @interleave_group(ptr %dst) #1 {
; COST1-NEXT: [[ITER_CHECK:.*:]]
; COST1-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
; COST1: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
-; COST1-NEXT: br i1 false, label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; COST1-NEXT: [[TMP24:%.*]] = call i64 @llvm.vscale.i64()
+; COST1-NEXT: [[TMP25:%.*]] = shl nuw i64 [[TMP24]], 5
+; COST1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 101, [[TMP25]]
+; COST1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
; COST1: [[VECTOR_PH]]:
+; COST1-NEXT: [[TMP26:%.*]] = shl nuw i64 [[TMP24]], 4
+; COST1-NEXT: [[TMP27:%.*]] = shl nuw i64 [[TMP26]], 1
+; COST1-NEXT: [[N_MOD_VF:%.*]] = urem i64 101, [[TMP27]]
+; COST1-NEXT: [[N_VEC:%.*]] = sub i64 101, [[N_MOD_VF]]
; COST1-NEXT: br label %[[VECTOR_BODY:.*]]
; COST1: [[VECTOR_BODY]]:
; COST1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COST1-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 16
+; COST1-NEXT: [[TMP28:%.*]] = add i64 [[TMP26]], 0
+; COST1-NEXT: [[TMP5:%.*]] = mul i64 [[TMP28]], 1
+; COST1-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], [[TMP5]]
; COST1-NEXT: [[TMP1:%.*]] = mul i64 [[INDEX]], 3
; COST1-NEXT: [[TMP2:%.*]] = mul i64 [[TMP0]], 3
; COST1-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP1]]
; COST1-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP2]]
-; COST1-NEXT: store <48 x i8> zeroinitializer, ptr [[TMP3]], align 1
-; COST1-NEXT: store <48 x i8> zeroinitializer, ptr [[TMP4]], align 1
-; COST1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
-; COST1-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96
-; COST1-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; COST1-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 48 x i8> @llvm.vector.interleave3.nxv48i8(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer)
+; COST1-NEXT: store <vscale x 48 x i8> [[INTERLEAVED_VEC]], ptr [[TMP3]], align 1
+; COST1-NEXT: [[INTERLEAVED_VEC1:%.*]] = call <vscale x 48 x i8> @llvm.vector.interleave3.nxv48i8(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer)
+; COST1-NEXT: store <vscale x 48 x i8> [[INTERLEAVED_VEC1]], ptr [[TMP4]], align 1
+; COST1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP27]]
+; COST1-NEXT: [[TMP29:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COST1-NEXT: br i1 [[TMP29]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
; COST1: [[MIDDLE_BLOCK]]:
-; COST1-NEXT: br i1 false, [[EXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; COST1-NEXT: [[CMP_N:%.*]] = icmp eq i64 101, [[N_VEC]]
+; COST1-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; COST1: [[VEC_EPILOG_ITER_CHECK]]:
-; COST1-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3]]
+; COST1-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4
+; COST1-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3]]
; COST1: [[VEC_EPILOG_PH]]:
-; COST1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 96, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; COST1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; COST1-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[BC_RESUME_VAL]], i64 0
; COST1-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COST1-NEXT: [[INDUCTION:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 1, i64 2, i64 3>
@@ -509,23 +522,32 @@ define void @interleave_group(ptr %dst) #1 {
; COST10-NEXT: [[ITER_CHECK:.*:]]
; COST10-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
; COST10: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
-; COST10-NEXT: br i1 false, label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; COST10-NEXT: [[TMP21:%.*]] = call i64 @llvm.vscale.i64()
+; COST10-NEXT: [[TMP22:%.*]] = shl nuw i64 [[TMP21]], 4
+; COST10-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 101, [[TMP22]]
+; COST10-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
; COST10: [[VECTOR_PH]]:
+; COST10-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP21]], 4
+; COST10-NEXT: [[N_MOD_VF:%.*]] = urem i64 101, [[TMP2]]
+; COST10-NEXT: [[N_VEC:%.*]] = sub i64 101, [[N_MOD_VF]]
; COST10-NEXT: br label %[[VECTOR_BODY:.*]]
; COST10: [[VECTOR_BODY]]:
; COST10-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COST10-NEXT: [[TMP0:%.*]] = mul i64 [[INDEX]], 3
; COST10-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP0]]
-; COST10-NEXT: store <48 x i8> zeroinitializer, ptr [[TMP1]], align 1
-; COST10-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
-; COST10-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96
-; COST10-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; COST10-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 48 x i8> @llvm.vector.interleave3.nxv48i8(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer)
+; COST10-NEXT: store <vscale x 48 x i8> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 1
+; COST10-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; COST10-NEXT: [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COST10-NEXT: br i1 [[TMP23]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
; COST10: [[MIDDLE_BLOCK]]:
-; COST10-NEXT: br i1 false, [[EXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; COST10-NEXT: [[CMP_N:%.*]] = icmp eq i64 101, [[N_VEC]]
+; COST10-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; COST10: [[VEC_EPILOG_ITER_CHECK]]:
-; COST10-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3]]
+; COST10-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4
+; COST10-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3]]
; COST10: [[VEC_EPILOG_PH]]:
-; COST10-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 96, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; COST10-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; COST10-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[BC_RESUME_VAL]], i64 0
; COST10-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COST10-NEXT: [[INDUCTION:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 1, i64 2, i64 3>
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll
index 6d65b4e597075..a48218d0efcdc 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll
@@ -1320,8 +1320,6 @@ end:
; dst[i].z = a[i].z << b[i].z;
; }
;
-; TODO: Support scalable interleave groups once we can also codegen
-; @llvm.[de]interleave3
%struct.xyz = type { i32, i32, i32 }
define void @interleave_deinterleave_factor3(ptr writeonly noalias %dst, ptr readonly %a, ptr readonly %b) {
@@ -1335,36 +1333,28 @@ define void @interleave_deinterleave_factor3(ptr writeonly noalias %dst, ptr rea
; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP2]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-NEXT: [[TMP3:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP2]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP3]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[WIDE_GEP:%.*]] = getelementptr inbounds [[STRUCT_XYZ:%.*]], ptr [[A:%.*]], <vscale x 4 x i64> [[VEC_IND]]
-; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[WIDE_GEP]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
-; CHECK-NEXT: [[WIDE_GEP1:%.*]] = getelementptr inbounds [[STRUCT_XYZ]], ptr [[B:%.*]], <vscale x 4 x i64> [[VEC_IND]]
-; CHECK-NEXT: [[WIDE_MASKED_GATHER2:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[WIDE_GEP1]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds [[STRUCT_XYZ:%.*]], ptr [[A:%.*]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <vscale x 12 x i32>, ptr [[TMP3]], align 4
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave3.nxv12i32(<vscale x 12 x i32> [[WIDE_VEC]])
+; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0
+; CHECK-NEXT: [[WIDE_MASKED_GATHER5:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 1
+; CHECK-NEXT: [[WIDE_MASKED_GATHER10:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 2
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds [[STRUCT_XYZ]], ptr [[B:%.*]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_VEC1:%.*]] = load <vscale x 12 x i32>, ptr [[TMP8]], align 4
+; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave3.nxv12i32(<vscale x 12 x i32> [[WIDE_VEC1]])
+; CHECK-NEXT: [[WIDE_MASKED_GATHER2:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC2]], 0
+; CHECK-NEXT: [[WIDE_MASKED_GATHER7:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC2]], 1
+; CHECK-NEXT: [[WIDE_MASKED_GATHER12:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC2]], 2
; CHECK-NEXT: [[TMP4:%.*]] = add nsw <vscale x 4 x i32> [[WIDE_MASKED_GATHER2]], [[WIDE_MASKED_GATHER]]
-; CHECK-NEXT: [[WIDE_GEP3:%.*]] = getelementptr inbounds [[STRUCT_XYZ]], ptr [[DST:%.*]], <vscale x 4 x i64> [[VEC_IND]]
-; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP4]], <vscale x 4 x ptr> align 4 [[WIDE_GEP3]], <vscale x 4 x i1> splat (i1 true))
-; CHECK-NEXT: [[WIDE_GEP4:%.*]] = getelementptr inbounds nuw i8, <vscale x 4 x ptr> [[WIDE_GEP]], i64 4
-; CHECK-NEXT: [[WIDE_MASKED_GATHER5:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[WIDE_GEP4]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
-; CHECK-NEXT: [[WIDE_GEP6:%.*]] = getelementptr inbounds nuw i8, <vscale x 4 x ptr> [[WIDE_GEP1]], i64 4
-; CHECK-NEXT: [[WIDE_MASKED_GATHER7:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[WIDE_GEP6]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds [[STRUCT_XYZ]], ptr [[DST:%.*]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP5:%.*]] = sub nsw <vscale x 4 x i32> [[WIDE_MASKED_GATHER5]], [[WIDE_MASKED_GATHER7]]
-; CHECK-NEXT: [[WIDE_GEP8:%.*]] = getelementptr inbounds nuw i8, <vscale x 4 x ptr> [[WIDE_GEP3]], i64 4
-; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP5]], <vscale x 4 x ptr> align 4 [[WIDE_GEP8]], <vscale x 4 x i1> splat (i1 true))
-; CHECK-NEXT: [[WIDE_GEP9:%.*]] = getelementptr inbounds nuw i8, <vscale x 4 x ptr> [[WIDE_GEP]], i64 8
-; CHECK-NEXT: [[WIDE_MASKED_GATHER10:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[WIDE_GEP9]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
-; CHECK-NEXT: [[WIDE_GEP11:%.*]] = getelementptr inbounds nuw i8, <vscale x 4 x ptr> [[WIDE_GEP1]], i64 8
-; CHECK-NEXT: [[WIDE_MASKED_GATHER12:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[WIDE_GEP11]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
; CHECK-NEXT: [[TMP6:%.*]] = shl <vscale x 4 x i32> [[WIDE_MASKED_GATHER10]], [[WIDE_MASKED_GATHER12]]
-; CHECK-NEXT: [[WIDE_GEP13:%.*]] = getelementptr inbounds nuw i8, <vscale x 4 x ptr> [[WIDE_GEP3]], i64 8
-; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP6]], <vscale x 4 x ptr> align 4 [[WIDE_GEP13]], <vscale x 4 x i1> splat (i1 true))
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 12 x i32> @llvm.vector.interleave3.nxv12i32(<vscale x 4 x i32> [[TMP4]], <vscale x 4 x i32> [[TMP5]], <vscale x 4 x i32> [[TMP6]])
+; CHECK-NEXT: store <vscale x 12 x i32> [[INTERLEAVED_VEC]], ptr [[TMP12]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP33:![0-9]+]]
; CHECK: middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-option.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-option.ll
index fb17b9a80d09b..d826987e9fee4 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-option.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-option.ll
@@ -1,9 +1,10 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding-insn-threshold=0 -sve-tail-folding=disabled -S | FileCheck %s -check-prefix=CHECK-NOTF
; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding-insn-threshold=0 -sve-tail-folding=default -S | FileCheck %s -check-prefix=CHECK-NOTF
; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding-insn-threshold=0 -S | FileCheck %s -check-prefix=CHECK-NOTF
-; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding-insn-threshold=0 -sve-tail-folding=all -S | FileCheck %s -check-prefix=CHECK-TF
+; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding-insn-threshold=0 -sve-tail-folding=all -S | FileCheck %s -check-prefix=CHECK-TF-ALL
; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding-insn-threshold=0 -sve-tail-folding=simple+reductions+recurrences+reverse -S | FileCheck %s -check-prefix=CHECK-TF
-; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding-insn-threshold=0 -S -mcpu=neoverse-v1 -sve-tail-folding=default+reductions+recurrences+reverse | FileCheck %s -check-prefix=CHECK-TF
+; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding-insn-threshold=0 -S -mcpu=neoverse-v1 -sve-tail-folding=default+reductions+recurrences+reverse | FileCheck %s -check-prefix=CHECK-TF-DEFAULT
; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding-insn-threshold=0 -sve-tail-folding=all+noreductions -S | FileCheck %s -check-prefix=CHECK-TF-NORED
; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding-insn-threshold=0 -sve-tail-folding=all+norecurrences -S | FileCheck %s -check-prefix=CHECK-TF-NOREC
; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding-insn-threshold=0 -sve-tail-folding=all+noreverse -S | FileCheck %s -check-prefix=CHECK-TF-NOREV
@@ -15,61 +16,277 @@
target triple = "aarch64-unknown-linux-gnu"
define void @simple_memset(i32 %val, ptr %ptr, i64 %n) #0 {
-; CHECK-NOTF-LABEL: @simple_memset(
-; CHECK-NOTF: vector.ph:
-; CHECK-NOTF: %[[INSERT:.*]] = insertelement <vscale x 4 x i32> poison, i32 %val, i64 0
-; CHECK-NOTF: %[[SPLAT:.*]] = shufflevector <vscale x 4 x i32> %[[INSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NOTF: vector.body:
-; CHECK-NOTF-NOT: %{{.*}} = phi <vscale x 4 x i1>
-; CHECK-NOTF: store <vscale x 4 x i32> %[[SPLAT]], ptr
-
-; CHECK-TF-NORED-LABEL: @simple_memset(
-; CHECK-TF-NORED: vector.ph:
-; CHECK-TF-NORED: %[[INSERT:.*]] = insertelement <vscale x 4 x i32> poison, i32 %val, i64 0
-; CHECK-TF-NORED: %[[SPLAT:.*]] = shufflevector <vscale x 4 x i32> %[[INSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-TF-NORED: vector.body:
-; CHECK-TF-NORED: %[[ACTIVE_LANE_MASK:.*]] = phi <vscale x 4 x i1>
-; CHECK-TF-NORED: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> %[[SPLAT]], {{.*}} %[[ACTIVE_LANE_MASK]]
-
-; CHECK-TF-NOREC-LABEL: @simple_memset(
-; CHECK-TF-NOREC: vector.ph:
-; CHECK-TF-NOREC: %[[INSERT:.*]] = insertelement <vscale x 4 x i32> poison, i32 %val, i64 0
-; CHECK-TF-NOREC: %[[SPLAT:.*]] = shufflevector <vscale x 4 x i32> %[[INSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-TF-NOREC: vector.body:
-; CHECK-TF-NOREC: %[[ACTIVE_LANE_MASK:.*]] = phi <vscale x 4 x i1>
-; CHECK-TF-NOREC: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> %[[SPLAT]], {{.*}} %[[ACTIVE_LANE_MASK]]
-
-; CHECK-TF-NOREV-LABEL: @simple_memset(
-; CHECK-TF-NOREV: vector.ph:
-; CHECK-TF-NOREV: %[[INSERT:.*]] = insertelement <vscale x 4 x i32> poison, i32 %val, i64 0
-; CHECK-TF-NOREV: %[[SPLAT:.*]] = shufflevector <vscale x 4 x i32> %[[INSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-TF-NOREV: vector.body:
-; CHECK-TF-NOREV: %[[ACTIVE_LANE_MASK:.*]] = phi <vscale x 4 x i1>
-; CHECK-TF-NOREV: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> %[[SPLAT]], {{.*}} %[[ACTIVE_LANE_MASK]]
-
-; CHECK-TF-LABEL: @simple_memset(
-; CHECK-TF: vector.ph:
-; CHECK-TF: %[[INSERT:.*]] = insertelement <vscale x 4 x i32> poison, i32 %val, i64 0
-; CHECK-TF: %[[SPLAT:.*]] = shufflevector <vscale x 4 x i32> %[[INSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-TF: vector.body:
-; CHECK-TF: %[[ACTIVE_LANE_MASK:.*]] = phi <vscale x 4 x i1>
-; CHECK-TF: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> %[[SPLAT]], {{.*}} %[[ACTIVE_LANE_MASK]]
-
-; CHECK-TF-ONLYRED-LABEL: @simple_memset(
-; CHECK-TF-ONLYRED: vector.ph:
-; CHECK-TF-ONLYRED: %[[INSERT:.*]] = insertelement <vscale x 4 x i32> poison, i32 %val, i64 0
-; CHECK-TF-ONLYRED: %[[SPLAT:.*]] = shufflevector <vscale x 4 x i32> %[[INSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-TF-ONLYRED: vector.body:
-; CHECK-TF-ONLYRED-NOT: %{{.*}} = phi <vscale x 4 x i1>
-; CHECK-TF-ONLYRED: store <vscale x 4 x i32> %[[SPLAT]], ptr
-
-; CHECK-NEOVERSE-V1-LABEL: @simple_memset(
-; CHECK-NEOVERSE-V1: vector.ph:
-; CHECK-NEOVERSE-V1: %[[INSERT:.*]] = insertelement <vscale x 4 x i32> poison, i32 %val, i64 0
-; CHECK-NEOVERSE-V1: %[[SPLAT:.*]] = shufflevector <vscale x 4 x i32> %[[INSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEOVERSE-V1: vector.body:
-; CHECK-NEOVERSE-V1: %[[ACTIVE_LANE_MASK:.*]] = phi <vscale x 4 x i1>
-; CHECK-NEOVERSE-V1: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> %[[SPLAT]], {{.*}} %[[ACTIVE_LANE_MASK]]
+; CHECK-NOTF-LABEL: define void @simple_memset(
+; CHECK-NOTF-SAME: i32 [[VAL:%.*]], ptr [[PTR:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NOTF-NEXT: [[ENTRY:.*]]:
+; CHECK-NOTF-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
+; CHECK-NOTF-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NOTF-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NOTF-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[UMAX]], [[TMP1]]
+; CHECK-NOTF-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NOTF: [[VECTOR_PH]]:
+; CHECK-NOTF-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NOTF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[UMAX]], [[TMP2]]
+; CHECK-NOTF-NEXT: [[N_VEC:%.*]] = sub i64 [[UMAX]], [[N_MOD_VF]]
+; CHECK-NOTF-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[VAL]], i64 0
+; CHECK-NOTF-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NOTF-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-NOTF: [[VECTOR_BODY]]:
+; CHECK-NOTF-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NOTF-NEXT: [[TMP3:%.*]] = getelementptr i32, ptr [[PTR]], i64 [[INDEX1]]
+; CHECK-NOTF-NEXT: store <vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr [[TMP3]], align 4
+; CHECK-NOTF-NEXT: [[INDEX_NEXT2]] = add nuw i64 [[INDEX1]], [[TMP2]]
+; CHECK-NOTF-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT2]], [[N_VEC]]
+; CHECK-NOTF-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NOTF: [[MIDDLE_BLOCK]]:
+; CHECK-NOTF-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[UMAX]], [[N_VEC]]
+; CHECK-NOTF-NEXT: br i1 [[CMP_N]], label %[[WHILE_END_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NOTF: [[SCALAR_PH]]:
+; CHECK-NOTF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NOTF-NEXT: br label %[[WHILE_BODY:.*]]
+; CHECK-NOTF: [[WHILE_BODY]]:
+; CHECK-NOTF-NEXT: [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], %[[WHILE_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NOTF-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[PTR]], i64 [[INDEX]]
+; CHECK-NOTF-NEXT: store i32 [[VAL]], ptr [[GEP]], align 4
+; CHECK-NOTF-NEXT: [[INDEX_NEXT]] = add nsw i64 [[INDEX]], 1
+; CHECK-NOTF-NEXT: [[CMP10:%.*]] = icmp ult i64 [[INDEX_NEXT]], [[N]]
+; CHECK-NOTF-NEXT: br i1 [[CMP10]], label %[[WHILE_BODY]], label %[[WHILE_END_LOOPEXIT]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NOTF: [[WHILE_END_LOOPEXIT]]:
+; CHECK-NOTF-NEXT: ret void
+;
+; CHECK-TF-ALL-LABEL: define void @simple_memset(
+; CHECK-TF-ALL-SAME: i32 [[VAL:%.*]], ptr [[PTR:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-TF-ALL-NEXT: [[ENTRY:.*:]]
+; CHECK-TF-ALL-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
+; CHECK-TF-ALL-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-TF-ALL: [[VECTOR_PH]]:
+; CHECK-TF-ALL-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-ALL-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-ALL-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[UMAX]])
+; CHECK-TF-ALL-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[VAL]], i64 0
+; CHECK-TF-ALL-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-TF-ALL-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-ALL: [[VECTOR_BODY]]:
+; CHECK-TF-ALL-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-ALL-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-ALL-NEXT: [[TMP2:%.*]] = getelementptr i32, ptr [[PTR]], i64 [[INDEX1]]
+; CHECK-TF-ALL-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr align 4 [[TMP2]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-ALL-NEXT: [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP1]]
+; CHECK-TF-ALL-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT2]], i64 [[UMAX]])
+; CHECK-TF-ALL-NEXT: [[TMP3:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-ALL-NEXT: [[TMP4:%.*]] = xor i1 [[TMP3]], true
+; CHECK-TF-ALL-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-TF-ALL: [[MIDDLE_BLOCK]]:
+; CHECK-TF-ALL-NEXT: br label %[[WHILE_END_LOOPEXIT:.*]]
+; CHECK-TF-ALL: [[WHILE_END_LOOPEXIT]]:
+; CHECK-TF-ALL-NEXT: ret void
+;
+; CHECK-TF-LABEL: define void @simple_memset(
+; CHECK-TF-SAME: i32 [[VAL:%.*]], ptr [[PTR:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-TF-NEXT: [[ENTRY:.*:]]
+; CHECK-TF-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
+; CHECK-TF-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-TF: [[VECTOR_PH]]:
+; CHECK-TF-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[UMAX]])
+; CHECK-TF-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[VAL]], i64 0
+; CHECK-TF-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-TF-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF: [[VECTOR_BODY]]:
+; CHECK-TF-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NEXT: [[TMP2:%.*]] = getelementptr i32, ptr [[PTR]], i64 [[INDEX1]]
+; CHECK-TF-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr align 4 [[TMP2]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-NEXT: [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP1]]
+; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT2]], i64 [[UMAX]])
+; CHECK-TF-NEXT: [[TMP3:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-NEXT: [[TMP4:%.*]] = xor i1 [[TMP3]], true
+; CHECK-TF-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-TF: [[MIDDLE_BLOCK]]:
+; CHECK-TF-NEXT: br label %[[WHILE_END_LOOPEXIT:.*]]
+; CHECK-TF: [[WHILE_END_LOOPEXIT]]:
+; CHECK-TF-NEXT: ret void
+;
+; CHECK-TF-DEFAULT-LABEL: define void @simple_memset(
+; CHECK-TF-DEFAULT-SAME: i32 [[VAL:%.*]], ptr [[PTR:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-TF-DEFAULT-NEXT: [[ENTRY:.*:]]
+; CHECK-TF-DEFAULT-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
+; CHECK-TF-DEFAULT-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-TF-DEFAULT: [[VECTOR_PH]]:
+; CHECK-TF-DEFAULT-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-DEFAULT-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-DEFAULT-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[UMAX]])
+; CHECK-TF-DEFAULT-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[VAL]], i64 0
+; CHECK-TF-DEFAULT-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-TF-DEFAULT-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-DEFAULT: [[VECTOR_BODY]]:
+; CHECK-TF-DEFAULT-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-DEFAULT-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-DEFAULT-NEXT: [[TMP2:%.*]] = getelementptr i32, ptr [[PTR]], i64 [[INDEX1]]
+; CHECK-TF-DEFAULT-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr align 4 [[TMP2]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-DEFAULT-NEXT: [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP1]]
+; CHECK-TF-DEFAULT-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT2]], i64 [[UMAX]])
+; CHECK-TF-DEFAULT-NEXT: [[TMP3:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-DEFAULT-NEXT: [[TMP4:%.*]] = xor i1 [[TMP3]], true
+; CHECK-TF-DEFAULT-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-TF-DEFAULT: [[MIDDLE_BLOCK]]:
+; CHECK-TF-DEFAULT-NEXT: br label %[[WHILE_END_LOOPEXIT:.*]]
+; CHECK-TF-DEFAULT: [[WHILE_END_LOOPEXIT]]:
+; CHECK-TF-DEFAULT-NEXT: ret void
+;
+; CHECK-TF-NORED-LABEL: define void @simple_memset(
+; CHECK-TF-NORED-SAME: i32 [[VAL:%.*]], ptr [[PTR:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-TF-NORED-NEXT: [[ENTRY:.*:]]
+; CHECK-TF-NORED-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
+; CHECK-TF-NORED-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-TF-NORED: [[VECTOR_PH]]:
+; CHECK-TF-NORED-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NORED-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-NORED-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[UMAX]])
+; CHECK-TF-NORED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[VAL]], i64 0
+; CHECK-TF-NORED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-TF-NORED-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-NORED: [[VECTOR_BODY]]:
+; CHECK-TF-NORED-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NORED-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NORED-NEXT: [[TMP2:%.*]] = getelementptr i32, ptr [[PTR]], i64 [[INDEX1]]
+; CHECK-TF-NORED-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr align 4 [[TMP2]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-NORED-NEXT: [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP1]]
+; CHECK-TF-NORED-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT2]], i64 [[UMAX]])
+; CHECK-TF-NORED-NEXT: [[TMP3:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-NORED-NEXT: [[TMP4:%.*]] = xor i1 [[TMP3]], true
+; CHECK-TF-NORED-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-TF-NORED: [[MIDDLE_BLOCK]]:
+; CHECK-TF-NORED-NEXT: br label %[[WHILE_END_LOOPEXIT:.*]]
+; CHECK-TF-NORED: [[WHILE_END_LOOPEXIT]]:
+; CHECK-TF-NORED-NEXT: ret void
+;
+; CHECK-TF-NOREC-LABEL: define void @simple_memset(
+; CHECK-TF-NOREC-SAME: i32 [[VAL:%.*]], ptr [[PTR:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-TF-NOREC-NEXT: [[ENTRY:.*:]]
+; CHECK-TF-NOREC-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
+; CHECK-TF-NOREC-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-TF-NOREC: [[VECTOR_PH]]:
+; CHECK-TF-NOREC-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NOREC-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-NOREC-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[UMAX]])
+; CHECK-TF-NOREC-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[VAL]], i64 0
+; CHECK-TF-NOREC-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-TF-NOREC-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-NOREC: [[VECTOR_BODY]]:
+; CHECK-TF-NOREC-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NOREC-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NOREC-NEXT: [[TMP2:%.*]] = getelementptr i32, ptr [[PTR]], i64 [[INDEX1]]
+; CHECK-TF-NOREC-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr align 4 [[TMP2]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-NOREC-NEXT: [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP1]]
+; CHECK-TF-NOREC-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT2]], i64 [[UMAX]])
+; CHECK-TF-NOREC-NEXT: [[TMP3:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-NOREC-NEXT: [[TMP4:%.*]] = xor i1 [[TMP3]], true
+; CHECK-TF-NOREC-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-TF-NOREC: [[MIDDLE_BLOCK]]:
+; CHECK-TF-NOREC-NEXT: br label %[[WHILE_END_LOOPEXIT:.*]]
+; CHECK-TF-NOREC: [[WHILE_END_LOOPEXIT]]:
+; CHECK-TF-NOREC-NEXT: ret void
+;
+; CHECK-TF-NOREV-LABEL: define void @simple_memset(
+; CHECK-TF-NOREV-SAME: i32 [[VAL:%.*]], ptr [[PTR:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-TF-NOREV-NEXT: [[ENTRY:.*:]]
+; CHECK-TF-NOREV-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
+; CHECK-TF-NOREV-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-TF-NOREV: [[VECTOR_PH]]:
+; CHECK-TF-NOREV-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NOREV-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-NOREV-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[UMAX]])
+; CHECK-TF-NOREV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[VAL]], i64 0
+; CHECK-TF-NOREV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-TF-NOREV-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-NOREV: [[VECTOR_BODY]]:
+; CHECK-TF-NOREV-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NOREV-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NOREV-NEXT: [[TMP2:%.*]] = getelementptr i32, ptr [[PTR]], i64 [[INDEX1]]
+; CHECK-TF-NOREV-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr align 4 [[TMP2]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-NOREV-NEXT: [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP1]]
+; CHECK-TF-NOREV-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT2]], i64 [[UMAX]])
+; CHECK-TF-NOREV-NEXT: [[TMP3:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-NOREV-NEXT: [[TMP4:%.*]] = xor i1 [[TMP3]], true
+; CHECK-TF-NOREV-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-TF-NOREV: [[MIDDLE_BLOCK]]:
+; CHECK-TF-NOREV-NEXT: br label %[[WHILE_END_LOOPEXIT:.*]]
+; CHECK-TF-NOREV: [[WHILE_END_LOOPEXIT]]:
+; CHECK-TF-NOREV-NEXT: ret void
+;
+; CHECK-TF-ONLYRED-LABEL: define void @simple_memset(
+; CHECK-TF-ONLYRED-SAME: i32 [[VAL:%.*]], ptr [[PTR:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-TF-ONLYRED-NEXT: [[ENTRY:.*]]:
+; CHECK-TF-ONLYRED-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
+; CHECK-TF-ONLYRED-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-ONLYRED-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-ONLYRED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[UMAX]], [[TMP1]]
+; CHECK-TF-ONLYRED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-TF-ONLYRED: [[VECTOR_PH]]:
+; CHECK-TF-ONLYRED-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-ONLYRED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[UMAX]], [[TMP2]]
+; CHECK-TF-ONLYRED-NEXT: [[N_VEC:%.*]] = sub i64 [[UMAX]], [[N_MOD_VF]]
+; CHECK-TF-ONLYRED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[VAL]], i64 0
+; CHECK-TF-ONLYRED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-TF-ONLYRED-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-ONLYRED: [[VECTOR_BODY]]:
+; CHECK-TF-ONLYRED-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-ONLYRED-NEXT: [[TMP3:%.*]] = getelementptr i32, ptr [[PTR]], i64 [[INDEX1]]
+; CHECK-TF-ONLYRED-NEXT: store <vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr [[TMP3]], align 4
+; CHECK-TF-ONLYRED-NEXT: [[INDEX_NEXT2]] = add nuw i64 [[INDEX1]], [[TMP2]]
+; CHECK-TF-ONLYRED-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT2]], [[N_VEC]]
+; CHECK-TF-ONLYRED-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-TF-ONLYRED: [[MIDDLE_BLOCK]]:
+; CHECK-TF-ONLYRED-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[UMAX]], [[N_VEC]]
+; CHECK-TF-ONLYRED-NEXT: br i1 [[CMP_N]], label %[[WHILE_END_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-TF-ONLYRED: [[SCALAR_PH]]:
+; CHECK-TF-ONLYRED-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-TF-ONLYRED-NEXT: br label %[[WHILE_BODY:.*]]
+; CHECK-TF-ONLYRED: [[WHILE_BODY]]:
+; CHECK-TF-ONLYRED-NEXT: [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], %[[WHILE_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-TF-ONLYRED-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[PTR]], i64 [[INDEX]]
+; CHECK-TF-ONLYRED-NEXT: store i32 [[VAL]], ptr [[GEP]], align 4
+; CHECK-TF-ONLYRED-NEXT: [[INDEX_NEXT]] = add nsw i64 [[INDEX]], 1
+; CHECK-TF-ONLYRED-NEXT: [[CMP10:%.*]] = icmp ult i64 [[INDEX_NEXT]], [[N]]
+; CHECK-TF-ONLYRED-NEXT: br i1 [[CMP10]], label %[[WHILE_BODY]], label %[[WHILE_END_LOOPEXIT]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-TF-ONLYRED: [[WHILE_END_LOOPEXIT]]:
+; CHECK-TF-ONLYRED-NEXT: ret void
+;
+; CHECK-NEOVERSE-V1-LABEL: define void @simple_memset(
+; CHECK-NEOVERSE-V1-SAME: i32 [[VAL:%.*]], ptr [[PTR:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEOVERSE-V1-NEXT: [[ENTRY:.*:]]
+; CHECK-NEOVERSE-V1-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
+; CHECK-NEOVERSE-V1-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-NEOVERSE-V1: [[VECTOR_PH]]:
+; CHECK-NEOVERSE-V1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEOVERSE-V1-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NEOVERSE-V1-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[UMAX]])
+; CHECK-NEOVERSE-V1-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[VAL]], i64 0
+; CHECK-NEOVERSE-V1-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEOVERSE-V1-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-NEOVERSE-V1: [[VECTOR_BODY]]:
+; CHECK-NEOVERSE-V1-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEOVERSE-V1-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEOVERSE-V1-NEXT: [[TMP2:%.*]] = getelementptr i32, ptr [[PTR]], i64 [[INDEX1]]
+; CHECK-NEOVERSE-V1-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr align 4 [[TMP2]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-NEOVERSE-V1-NEXT: [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP1]]
+; CHECK-NEOVERSE-V1-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT2]], i64 [[UMAX]])
+; CHECK-NEOVERSE-V1-NEXT: [[TMP3:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-NEOVERSE-V1-NEXT: [[TMP4:%.*]] = xor i1 [[TMP3]], true
+; CHECK-NEOVERSE-V1-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEOVERSE-V1: [[MIDDLE_BLOCK]]:
+; CHECK-NEOVERSE-V1-NEXT: br label %[[WHILE_END_LOOPEXIT:.*]]
+; CHECK-NEOVERSE-V1: [[WHILE_END_LOOPEXIT]]:
+; CHECK-NEOVERSE-V1-NEXT: ret void
+;
+
+
+
+
+
+
entry:
br label %while.body
@@ -87,69 +304,306 @@ while.end.loopexit:
}
define float @fadd_red_fast(ptr noalias nocapture readonly %a, i64 %n) #0 {
-; CHECK-NOTF-LABEL: @fadd_red_fast
-; CHECK-NOTF: vector.body:
-; CHECK-NOTF-NOT: %{{.*}} = phi <vscale x 4 x i1>
-; CHECK-NOTF: %[[LOAD:.*]] = load <vscale x 4 x float>
-; CHECK-NOTF: %[[ADD:.*]] = fadd fast <vscale x 4 x float> %[[LOAD]]
-; CHECK-NOTF: middle.block:
-; CHECK-NOTF-NEXT: call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> %[[ADD]])
-
-; CHECK-TF-NORED-LABEL: @fadd_red_fast
-; CHECK-TF-NORED: vector.body:
-; CHECK-TF-NORED-NOT: %{{.*}} = phi <vscale x 4 x i1>
-; CHECK-TF-NORED: %[[LOAD:.*]] = load <vscale x 4 x float>
-; CHECK-TF-NORED: %[[ADD:.*]] = fadd fast <vscale x 4 x float> %[[LOAD]]
-; CHECK-TF-NORED: middle.block:
-; CHECK-TF-NORED-NEXT: call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> %[[ADD]])
-
-; CHECK-TF-NOREC-LABEL: @fadd_red_fast
-; CHECK-TF-NOREC: vector.body:
-; CHECK-TF-NOREC: %[[ACTIVE_LANE_MASK:.*]] = phi <vscale x 4 x i1>
-; CHECK-TF-NOREC: %[[VEC_PHI:.*]] = phi <vscale x 4 x float>
-; CHECK-TF-NOREC: %[[LOAD:.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0({{.*}} %[[ACTIVE_LANE_MASK]]
-; CHECK-TF-NOREC: %[[ADD:.*]] = fadd fast <vscale x 4 x float> %[[LOAD]]
-; CHECK-TF-NOREC: %[[SEL:.*]] = select fast <vscale x 4 x i1> %[[ACTIVE_LANE_MASK]], <vscale x 4 x float> %[[ADD]], <vscale x 4 x float> %[[VEC_PHI]]
-; CHECK-TF-NOREC: middle.block:
-; CHECK-TF-NOREC-NEXT: call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> %[[SEL]])
-
-; CHECK-TF-NOREV-LABEL: @fadd_red_fast
-; CHECK-TF-NOREV: vector.body:
-; CHECK-TF-NOREV: %[[ACTIVE_LANE_MASK:.*]] = phi <vscale x 4 x i1>
-; CHECK-TF-NOREV: %[[VEC_PHI:.*]] = phi <vscale x 4 x float>
-; CHECK-TF-NOREV: %[[LOAD:.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0({{.*}} %[[ACTIVE_LANE_MASK]]
-; CHECK-TF-NOREV: %[[ADD:.*]] = fadd fast <vscale x 4 x float> %[[LOAD]]
-; CHECK-TF-NOREV: %[[SEL:.*]] = select fast <vscale x 4 x i1> %[[ACTIVE_LANE_MASK]], <vscale x 4 x float> %[[ADD]], <vscale x 4 x float> %[[VEC_PHI]]
-; CHECK-TF-NOREV: middle.block:
-; CHECK-TF-NOREV-NEXT: call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> %[[SEL]])
-
-; CHECK-TF-LABEL: @fadd_red_fast
-; CHECK-TF: vector.body:
-; CHECK-TF: %[[ACTIVE_LANE_MASK:.*]] = phi <vscale x 4 x i1>
-; CHECK-TF: %[[VEC_PHI:.*]] = phi <vscale x 4 x float>
-; CHECK-TF: %[[LOAD:.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0({{.*}} %[[ACTIVE_LANE_MASK]]
-; CHECK-TF: %[[ADD:.*]] = fadd fast <vscale x 4 x float> %[[LOAD]]
-; CHECK-TF: %[[SEL:.*]] = select fast <vscale x 4 x i1> %[[ACTIVE_LANE_MASK]], <vscale x 4 x float> %[[ADD]], <vscale x 4 x float> %[[VEC_PHI]]
-; CHECK-TF: middle.block:
-; CHECK-TF-NEXT: call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> %[[SEL]])
-
-; CHECK-TF-ONLYRED-LABEL: @fadd_red_fast
-; CHECK-TF-ONLYRED: vector.body:
-; CHECK-TF-ONLYRED: %[[ACTIVE_LANE_MASK:.*]] = phi <vscale x 4 x i1>
-; CHECK-TF-ONLYRED: %[[VEC_PHI:.*]] = phi <vscale x 4 x float>
-; CHECK-TF-ONLYRED: %[[LOAD:.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0({{.*}} %[[ACTIVE_LANE_MASK]]
-; CHECK-TF-ONLYRED: %[[ADD:.*]] = fadd fast <vscale x 4 x float> %[[LOAD]]
-; CHECK-TF-ONLYRED: %[[SEL:.*]] = select fast <vscale x 4 x i1> %[[ACTIVE_LANE_MASK]], <vscale x 4 x float> %[[ADD]], <vscale x 4 x float> %[[VEC_PHI]]
-; CHECK-TF-ONLYRED: middle.block:
-; CHECK-TF-ONLYRED-NEXT: call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> %[[SEL]])
-
-; CHECK-NEOVERSE-V1-LABEL: @fadd_red_fast
-; CHECK-NEOVERSE-V1: vector.body:
-; CHECK-NEOVERSE-V1-NOT: %{{.*}} = phi <vscale x 4 x i1>
-; CHECK-NEOVERSE-V1: %[[LOAD:.*]] = load <vscale x 4 x float>
-; CHECK-NEOVERSE-V1: %[[ADD:.*]] = fadd fast <vscale x 4 x float> %[[LOAD]]
-; CHECK-NEOVERSE-V1: middle.block:
-; CHECK-NEOVERSE-V1-NEXT: call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> %[[ADD]])
+; CHECK-NOTF-LABEL: define float @fadd_red_fast(
+; CHECK-NOTF-SAME: ptr noalias readonly captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NOTF-NEXT: [[ENTRY:.*]]:
+; CHECK-NOTF-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NOTF-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NOTF-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-NOTF-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NOTF: [[VECTOR_PH]]:
+; CHECK-NOTF-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NOTF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP2]]
+; CHECK-NOTF-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NOTF-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-NOTF: [[VECTOR_BODY]]:
+; CHECK-NOTF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NOTF-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NOTF-NEXT: [[TMP3:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-NOTF-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP3]], align 4
+; CHECK-NOTF-NEXT: [[TMP4]] = fadd fast <vscale x 4 x float> [[WIDE_LOAD]], [[VEC_PHI]]
+; CHECK-NOTF-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; CHECK-NOTF-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NOTF-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NOTF: [[MIDDLE_BLOCK]]:
+; CHECK-NOTF-NEXT: [[TMP6:%.*]] = call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> [[TMP4]])
+; CHECK-NOTF-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NOTF-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-NOTF: [[SCALAR_PH]]:
+; CHECK-NOTF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NOTF-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; CHECK-NOTF-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-NOTF: [[FOR_BODY]]:
+; CHECK-NOTF-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NOTF-NEXT: [[SUM_07:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[ADD:%.*]], %[[FOR_BODY]] ]
+; CHECK-NOTF-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
+; CHECK-NOTF-NEXT: [[TMP7:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-NOTF-NEXT: [[ADD]] = fadd fast float [[TMP7]], [[SUM_07]]
+; CHECK-NOTF-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NOTF-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NOTF-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NOTF: [[FOR_END]]:
+; CHECK-NOTF-NEXT: [[ADD_LCSSA:%.*]] = phi float [ [[ADD]], %[[FOR_BODY]] ], [ [[TMP6]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NOTF-NEXT: ret float [[ADD_LCSSA]]
+;
+; CHECK-TF-ALL-LABEL: define float @fadd_red_fast(
+; CHECK-TF-ALL-SAME: ptr noalias readonly captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-ALL-NEXT: [[ENTRY:.*:]]
+; CHECK-TF-ALL-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-TF-ALL: [[VECTOR_PH]]:
+; CHECK-TF-ALL-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-ALL-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-ALL-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
+; CHECK-TF-ALL-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-ALL: [[VECTOR_BODY]]:
+; CHECK-TF-ALL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-ALL-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-ALL-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-ALL-NEXT: [[TMP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-TF-ALL-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0(ptr align 4 [[TMP2]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> poison)
+; CHECK-TF-ALL-NEXT: [[TMP3:%.*]] = fadd fast <vscale x 4 x float> [[WIDE_MASKED_LOAD]], [[VEC_PHI]]
+; CHECK-TF-ALL-NEXT: [[TMP4]] = select fast <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> [[TMP3]], <vscale x 4 x float> [[VEC_PHI]]
+; CHECK-TF-ALL-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-TF-ALL-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-TF-ALL-NEXT: [[TMP5:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-ALL-NEXT: [[TMP6:%.*]] = xor i1 [[TMP5]], true
+; CHECK-TF-ALL-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-TF-ALL: [[MIDDLE_BLOCK]]:
+; CHECK-TF-ALL-NEXT: [[TMP7:%.*]] = call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> [[TMP4]])
+; CHECK-TF-ALL-NEXT: br label %[[FOR_END:.*]]
+; CHECK-TF-ALL: [[FOR_END]]:
+; CHECK-TF-ALL-NEXT: ret float [[TMP7]]
+;
+; CHECK-TF-LABEL: define float @fadd_red_fast(
+; CHECK-TF-SAME: ptr noalias readonly captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-NEXT: [[ENTRY:.*:]]
+; CHECK-TF-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-TF: [[VECTOR_PH]]:
+; CHECK-TF-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
+; CHECK-TF-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF: [[VECTOR_BODY]]:
+; CHECK-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NEXT: [[TMP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-TF-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0(ptr align 4 [[TMP2]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> poison)
+; CHECK-TF-NEXT: [[TMP3:%.*]] = fadd fast <vscale x 4 x float> [[WIDE_MASKED_LOAD]], [[VEC_PHI]]
+; CHECK-TF-NEXT: [[TMP4]] = select fast <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> [[TMP3]], <vscale x 4 x float> [[VEC_PHI]]
+; CHECK-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-TF-NEXT: [[TMP5:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-NEXT: [[TMP6:%.*]] = xor i1 [[TMP5]], true
+; CHECK-TF-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-TF: [[MIDDLE_BLOCK]]:
+; CHECK-TF-NEXT: [[TMP7:%.*]] = call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> [[TMP4]])
+; CHECK-TF-NEXT: br label %[[FOR_END:.*]]
+; CHECK-TF: [[FOR_END]]:
+; CHECK-TF-NEXT: ret float [[TMP7]]
+;
+; CHECK-TF-DEFAULT-LABEL: define float @fadd_red_fast(
+; CHECK-TF-DEFAULT-SAME: ptr noalias readonly captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-DEFAULT-NEXT: [[ENTRY:.*:]]
+; CHECK-TF-DEFAULT-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-TF-DEFAULT: [[VECTOR_PH]]:
+; CHECK-TF-DEFAULT-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-DEFAULT-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-DEFAULT-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
+; CHECK-TF-DEFAULT-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-DEFAULT: [[VECTOR_BODY]]:
+; CHECK-TF-DEFAULT-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-DEFAULT-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-DEFAULT-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-DEFAULT-NEXT: [[TMP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-TF-DEFAULT-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0(ptr align 4 [[TMP2]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> poison)
+; CHECK-TF-DEFAULT-NEXT: [[TMP3:%.*]] = fadd fast <vscale x 4 x float> [[WIDE_MASKED_LOAD]], [[VEC_PHI]]
+; CHECK-TF-DEFAULT-NEXT: [[TMP4]] = select fast <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> [[TMP3]], <vscale x 4 x float> [[VEC_PHI]]
+; CHECK-TF-DEFAULT-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-TF-DEFAULT-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-TF-DEFAULT-NEXT: [[TMP5:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-DEFAULT-NEXT: [[TMP6:%.*]] = xor i1 [[TMP5]], true
+; CHECK-TF-DEFAULT-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-TF-DEFAULT: [[MIDDLE_BLOCK]]:
+; CHECK-TF-DEFAULT-NEXT: [[TMP7:%.*]] = call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> [[TMP4]])
+; CHECK-TF-DEFAULT-NEXT: br label %[[FOR_END:.*]]
+; CHECK-TF-DEFAULT: [[FOR_END]]:
+; CHECK-TF-DEFAULT-NEXT: ret float [[TMP7]]
+;
+; CHECK-TF-NORED-LABEL: define float @fadd_red_fast(
+; CHECK-TF-NORED-SAME: ptr noalias readonly captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-NORED-NEXT: [[ENTRY:.*]]:
+; CHECK-TF-NORED-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NORED-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-NORED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-TF-NORED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-TF-NORED: [[VECTOR_PH]]:
+; CHECK-TF-NORED-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-NORED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP2]]
+; CHECK-TF-NORED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-TF-NORED-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-NORED: [[VECTOR_BODY]]:
+; CHECK-TF-NORED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NORED-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NORED-NEXT: [[TMP3:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-TF-NORED-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP3]], align 4
+; CHECK-TF-NORED-NEXT: [[TMP4]] = fadd fast <vscale x 4 x float> [[WIDE_LOAD]], [[VEC_PHI]]
+; CHECK-TF-NORED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; CHECK-TF-NORED-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-TF-NORED-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-TF-NORED: [[MIDDLE_BLOCK]]:
+; CHECK-TF-NORED-NEXT: [[TMP6:%.*]] = call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> [[TMP4]])
+; CHECK-TF-NORED-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-TF-NORED-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-TF-NORED: [[SCALAR_PH]]:
+; CHECK-TF-NORED-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-TF-NORED-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; CHECK-TF-NORED-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-TF-NORED: [[FOR_BODY]]:
+; CHECK-TF-NORED-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-TF-NORED-NEXT: [[SUM_07:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[ADD:%.*]], %[[FOR_BODY]] ]
+; CHECK-TF-NORED-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
+; CHECK-TF-NORED-NEXT: [[TMP7:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-TF-NORED-NEXT: [[ADD]] = fadd fast float [[TMP7]], [[SUM_07]]
+; CHECK-TF-NORED-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-TF-NORED-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-TF-NORED-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-TF-NORED: [[FOR_END]]:
+; CHECK-TF-NORED-NEXT: [[ADD_LCSSA:%.*]] = phi float [ [[ADD]], %[[FOR_BODY]] ], [ [[TMP6]], %[[MIDDLE_BLOCK]] ]
+; CHECK-TF-NORED-NEXT: ret float [[ADD_LCSSA]]
+;
+; CHECK-TF-NOREC-LABEL: define float @fadd_red_fast(
+; CHECK-TF-NOREC-SAME: ptr noalias readonly captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-NOREC-NEXT: [[ENTRY:.*:]]
+; CHECK-TF-NOREC-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-TF-NOREC: [[VECTOR_PH]]:
+; CHECK-TF-NOREC-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NOREC-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-NOREC-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
+; CHECK-TF-NOREC-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-NOREC: [[VECTOR_BODY]]:
+; CHECK-TF-NOREC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NOREC-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NOREC-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NOREC-NEXT: [[TMP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-TF-NOREC-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0(ptr align 4 [[TMP2]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> poison)
+; CHECK-TF-NOREC-NEXT: [[TMP3:%.*]] = fadd fast <vscale x 4 x float> [[WIDE_MASKED_LOAD]], [[VEC_PHI]]
+; CHECK-TF-NOREC-NEXT: [[TMP4]] = select fast <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> [[TMP3]], <vscale x 4 x float> [[VEC_PHI]]
+; CHECK-TF-NOREC-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-TF-NOREC-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-TF-NOREC-NEXT: [[TMP5:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-NOREC-NEXT: [[TMP6:%.*]] = xor i1 [[TMP5]], true
+; CHECK-TF-NOREC-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-TF-NOREC: [[MIDDLE_BLOCK]]:
+; CHECK-TF-NOREC-NEXT: [[TMP7:%.*]] = call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> [[TMP4]])
+; CHECK-TF-NOREC-NEXT: br label %[[FOR_END:.*]]
+; CHECK-TF-NOREC: [[FOR_END]]:
+; CHECK-TF-NOREC-NEXT: ret float [[TMP7]]
+;
+; CHECK-TF-NOREV-LABEL: define float @fadd_red_fast(
+; CHECK-TF-NOREV-SAME: ptr noalias readonly captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-NOREV-NEXT: [[ENTRY:.*:]]
+; CHECK-TF-NOREV-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-TF-NOREV: [[VECTOR_PH]]:
+; CHECK-TF-NOREV-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NOREV-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-NOREV-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
+; CHECK-TF-NOREV-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-NOREV: [[VECTOR_BODY]]:
+; CHECK-TF-NOREV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NOREV-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NOREV-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NOREV-NEXT: [[TMP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-TF-NOREV-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0(ptr align 4 [[TMP2]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> poison)
+; CHECK-TF-NOREV-NEXT: [[TMP3:%.*]] = fadd fast <vscale x 4 x float> [[WIDE_MASKED_LOAD]], [[VEC_PHI]]
+; CHECK-TF-NOREV-NEXT: [[TMP4]] = select fast <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> [[TMP3]], <vscale x 4 x float> [[VEC_PHI]]
+; CHECK-TF-NOREV-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-TF-NOREV-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-TF-NOREV-NEXT: [[TMP5:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-NOREV-NEXT: [[TMP6:%.*]] = xor i1 [[TMP5]], true
+; CHECK-TF-NOREV-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-TF-NOREV: [[MIDDLE_BLOCK]]:
+; CHECK-TF-NOREV-NEXT: [[TMP7:%.*]] = call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> [[TMP4]])
+; CHECK-TF-NOREV-NEXT: br label %[[FOR_END:.*]]
+; CHECK-TF-NOREV: [[FOR_END]]:
+; CHECK-TF-NOREV-NEXT: ret float [[TMP7]]
+;
+; CHECK-TF-ONLYRED-LABEL: define float @fadd_red_fast(
+; CHECK-TF-ONLYRED-SAME: ptr noalias readonly captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-ONLYRED-NEXT: [[ENTRY:.*:]]
+; CHECK-TF-ONLYRED-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-TF-ONLYRED: [[VECTOR_PH]]:
+; CHECK-TF-ONLYRED-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-ONLYRED-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-ONLYRED-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
+; CHECK-TF-ONLYRED-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-ONLYRED: [[VECTOR_BODY]]:
+; CHECK-TF-ONLYRED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-ONLYRED-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-ONLYRED-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-ONLYRED-NEXT: [[TMP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-TF-ONLYRED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0(ptr align 4 [[TMP2]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> poison)
+; CHECK-TF-ONLYRED-NEXT: [[TMP3:%.*]] = fadd fast <vscale x 4 x float> [[WIDE_MASKED_LOAD]], [[VEC_PHI]]
+; CHECK-TF-ONLYRED-NEXT: [[TMP4]] = select fast <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> [[TMP3]], <vscale x 4 x float> [[VEC_PHI]]
+; CHECK-TF-ONLYRED-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-TF-ONLYRED-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-TF-ONLYRED-NEXT: [[TMP5:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-ONLYRED-NEXT: [[TMP6:%.*]] = xor i1 [[TMP5]], true
+; CHECK-TF-ONLYRED-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-TF-ONLYRED: [[MIDDLE_BLOCK]]:
+; CHECK-TF-ONLYRED-NEXT: [[TMP7:%.*]] = call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> [[TMP4]])
+; CHECK-TF-ONLYRED-NEXT: br label %[[FOR_END:.*]]
+; CHECK-TF-ONLYRED: [[FOR_END]]:
+; CHECK-TF-ONLYRED-NEXT: ret float [[TMP7]]
+;
+; CHECK-NEOVERSE-V1-LABEL: define float @fadd_red_fast(
+; CHECK-NEOVERSE-V1-SAME: ptr noalias readonly captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEOVERSE-V1-NEXT: [[ENTRY:.*]]:
+; CHECK-NEOVERSE-V1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEOVERSE-V1-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NEOVERSE-V1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-NEOVERSE-V1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEOVERSE-V1: [[VECTOR_PH]]:
+; CHECK-NEOVERSE-V1-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NEOVERSE-V1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP2]]
+; CHECK-NEOVERSE-V1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEOVERSE-V1-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-NEOVERSE-V1: [[VECTOR_BODY]]:
+; CHECK-NEOVERSE-V1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEOVERSE-V1-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEOVERSE-V1-NEXT: [[TMP3:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEOVERSE-V1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP3]], align 4
+; CHECK-NEOVERSE-V1-NEXT: [[TMP4]] = fadd fast <vscale x 4 x float> [[WIDE_LOAD]], [[VEC_PHI]]
+; CHECK-NEOVERSE-V1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; CHECK-NEOVERSE-V1-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEOVERSE-V1-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEOVERSE-V1: [[MIDDLE_BLOCK]]:
+; CHECK-NEOVERSE-V1-NEXT: [[TMP6:%.*]] = call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> [[TMP4]])
+; CHECK-NEOVERSE-V1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEOVERSE-V1-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-NEOVERSE-V1: [[SCALAR_PH]]:
+; CHECK-NEOVERSE-V1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEOVERSE-V1-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; CHECK-NEOVERSE-V1-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-NEOVERSE-V1: [[FOR_BODY]]:
+; CHECK-NEOVERSE-V1-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEOVERSE-V1-NEXT: [[SUM_07:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[ADD:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEOVERSE-V1-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
+; CHECK-NEOVERSE-V1-NEXT: [[TMP7:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-NEOVERSE-V1-NEXT: [[ADD]] = fadd fast float [[TMP7]], [[SUM_07]]
+; CHECK-NEOVERSE-V1-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEOVERSE-V1-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEOVERSE-V1-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEOVERSE-V1: [[FOR_END]]:
+; CHECK-NEOVERSE-V1-NEXT: [[ADD_LCSSA:%.*]] = phi float [ [[ADD]], %[[FOR_BODY]] ], [ [[TMP6]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEOVERSE-V1-NEXT: ret float [[ADD_LCSSA]]
+;
+
+
+
+
+
+
entry:
br label %for.body
@@ -169,96 +623,407 @@ for.end:
}
define void @add_recur(ptr noalias %dst, ptr noalias %src, i64 %n) #0 {
-; CHECK-NOTF-LABEL: @add_recur
-; CHECK-NOTF: entry:
-; CHECK-NOTF: %[[PRE:.*]] = load i32, ptr %src, align 4
-; CHECK-NOTF: vector.ph:
-; CHECK-NOTF: %[[RECUR_INIT:.*]] = insertelement <vscale x 4 x i32> poison, i32 %[[PRE]]
-; CHECK-NOTF: vector.body:
-; CHECK-NOTF-NOT: %{{.*}} = phi <vscale x 4 x i1>
-; CHECK-NOTF: %[[VECTOR_RECUR:.*]] = phi <vscale x 4 x i32> [ %[[RECUR_INIT]], %vector.ph ], [ %[[LOAD:.*]], %vector.body ]
-; CHECK-NOTF: %[[LOAD]] = load <vscale x 4 x i32>
-; CHECK-NOTF: %[[SPLICE:.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> %[[VECTOR_RECUR]], <vscale x 4 x i32> %[[LOAD]], i32 1)
-; CHECK-NOTF: %[[ADD:.*]] = add nsw <vscale x 4 x i32> %[[LOAD]], %[[SPLICE]]
-; CHECK-NOTF: store <vscale x 4 x i32> %[[ADD]]
-
-; CHECK-TF-NORED-LABEL: @add_recur
-; CHECK-TF-NORED: entry:
-; CHECK-TF-NORED: %[[PRE:.*]] = load i32, ptr %src, align 4
-; CHECK-TF-NORED: vector.ph:
-; CHECK-TF-NORED: %[[RECUR_INIT:.*]] = insertelement <vscale x 4 x i32> poison, i32 %[[PRE]]
-; CHECK-TF-NORED: vector.body:
-; CHECK-TF-NORED: %[[ACTIVE_LANE_MASK:.*]] = phi <vscale x 4 x i1>
-; CHECK-TF-NORED: %[[VECTOR_RECUR:.*]] = phi <vscale x 4 x i32> [ %[[RECUR_INIT]], %vector.ph ], [ %[[LOAD:.*]], %vector.body ]
-; CHECK-TF-NORED: %[[LOAD]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0({{.*}} %[[ACTIVE_LANE_MASK]]
-; CHECK-TF-NORED: %[[SPLICE:.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> %[[VECTOR_RECUR]], <vscale x 4 x i32> %[[LOAD]], i32 1)
-; CHECK-TF-NORED: %[[ADD:.*]] = add nsw <vscale x 4 x i32> %[[LOAD]], %[[SPLICE]]
-; CHECK-TF-NORED: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> %[[ADD]], {{.*}} <vscale x 4 x i1> %[[ACTIVE_LANE_MASK]])
-
-; CHECK-TF-NOREC-LABEL: @add_recur
-; CHECK-TF-NOREC: entry:
-; CHECK-TF-NOREC: %[[PRE:.*]] = load i32, ptr %src, align 4
-; CHECK-TF-NOREC: vector.ph:
-; CHECK-TF-NOREC: %[[RECUR_INIT:.*]] = insertelement <vscale x 4 x i32> poison, i32 %[[PRE]]
-; CHECK-TF-NOREC: vector.body:
-; CHECK-TF-NOREC-NOT: %{{.*}} = phi <vscale x 4 x i1>
-; CHECK-TF-NOREC: %[[VECTOR_RECUR:.*]] = phi <vscale x 4 x i32> [ %[[RECUR_INIT]], %vector.ph ], [ %[[LOAD:.*]], %vector.body ]
-; CHECK-TF-NOREC: %[[LOAD]] = load <vscale x 4 x i32>
-; CHECK-TF-NOREC: %[[SPLICE:.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> %[[VECTOR_RECUR]], <vscale x 4 x i32> %[[LOAD]], i32 1)
-; CHECK-TF-NOREC: %[[ADD:.*]] = add nsw <vscale x 4 x i32> %[[LOAD]], %[[SPLICE]]
-; CHECK-TF-NOREC: store <vscale x 4 x i32> %[[ADD]]
-
-; CHECK-TF-NOREV-LABEL: @add_recur
-; CHECK-TF-NOREV: entry:
-; CHECK-TF-NOREV: %[[PRE:.*]] = load i32, ptr %src, align 4
-; CHECK-TF-NOREV: vector.ph:
-; CHECK-TF-NOREV: %[[RECUR_INIT:.*]] = insertelement <vscale x 4 x i32> poison, i32 %[[PRE]]
-; CHECK-TF-NOREV: vector.body:
-; CHECK-TF-NOREV: %[[ACTIVE_LANE_MASK:.*]] = phi <vscale x 4 x i1>
-; CHECK-TF-NOREV: %[[VECTOR_RECUR:.*]] = phi <vscale x 4 x i32> [ %[[RECUR_INIT]], %vector.ph ], [ %[[LOAD:.*]], %vector.body ]
-; CHECK-TF-NOREV: %[[LOAD]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0({{.*}} %[[ACTIVE_LANE_MASK]]
-; CHECK-TF-NOREV: %[[SPLICE:.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> %[[VECTOR_RECUR]], <vscale x 4 x i32> %[[LOAD]], i32 1)
-; CHECK-TF-NOREV: %[[ADD:.*]] = add nsw <vscale x 4 x i32> %[[LOAD]], %[[SPLICE]]
-; CHECK-TF-NOREV: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> %[[ADD]], {{.*}} <vscale x 4 x i1> %[[ACTIVE_LANE_MASK]])
-
-; CHECK-TF-LABEL: @add_recur
-; CHECK-TF: entry:
-; CHECK-TF: %[[PRE:.*]] = load i32, ptr %src, align 4
-; CHECK-TF: vector.ph:
-; CHECK-TF: %[[RECUR_INIT:.*]] = insertelement <vscale x 4 x i32> poison, i32 %[[PRE]]
-; CHECK-TF: vector.body:
-; CHECK-TF: %[[ACTIVE_LANE_MASK:.*]] = phi <vscale x 4 x i1>
-; CHECK-TF: %[[VECTOR_RECUR:.*]] = phi <vscale x 4 x i32> [ %[[RECUR_INIT]], %vector.ph ], [ %[[LOAD:.*]], %vector.body ]
-; CHECK-TF: %[[LOAD]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0({{.*}} %[[ACTIVE_LANE_MASK]]
-; CHECK-TF: %[[SPLICE:.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> %[[VECTOR_RECUR]], <vscale x 4 x i32> %[[LOAD]], i32 1)
-; CHECK-TF: %[[ADD:.*]] = add nsw <vscale x 4 x i32> %[[LOAD]], %[[SPLICE]]
-; CHECK-TF: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> %[[ADD]], {{.*}} <vscale x 4 x i1> %[[ACTIVE_LANE_MASK]])
-
-; CHECK-TF-ONLYRED-LABEL: @add_recur
-; CHECK-TF-ONLYRED: entry:
-; CHECK-TF-ONLYRED: %[[PRE:.*]] = load i32, ptr %src, align 4
-; CHECK-TF-ONLYRED: vector.ph:
-; CHECK-TF-ONLYRED: %[[RECUR_INIT:.*]] = insertelement <vscale x 4 x i32> poison, i32 %[[PRE]]
-; CHECK-TF-ONLYRED: vector.body:
-; CHECK-TF-ONLYRED-NOT: %{{.*}} = phi <vscale x 4 x i1>
-; CHECK-TF-ONLYRED: %[[VECTOR_RECUR:.*]] = phi <vscale x 4 x i32> [ %[[RECUR_INIT]], %vector.ph ], [ %[[LOAD:.*]], %vector.body ]
-; CHECK-TF-ONLYRED: %[[LOAD]] = load <vscale x 4 x i32>
-; CHECK-TF-ONLYRED: %[[SPLICE:.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> %[[VECTOR_RECUR]], <vscale x 4 x i32> %[[LOAD]], i32 1)
-; CHECK-TF-ONLYRED: %[[ADD:.*]] = add nsw <vscale x 4 x i32> %[[LOAD]], %[[SPLICE]]
-; CHECK-TF-ONLYRED: store <vscale x 4 x i32> %[[ADD]]
-
-; CHECK-NEOVERSE-V1-LABEL: @add_recur
-; CHECK-NEOVERSE-V1: entry:
-; CHECK-NEOVERSE-V1: %[[PRE:.*]] = load i32, ptr %src, align 4
-; CHECK-NEOVERSE-V1: vector.ph:
-; CHECK-NEOVERSE-V1: %[[RECUR_INIT:.*]] = insertelement <vscale x 4 x i32> poison, i32 %[[PRE]]
-; CHECK-NEOVERSE-V1: vector.body:
-; CHECK-NEOVERSE-V1-NOT: %{{.*}} = phi <vscale x 4 x i1>
-; CHECK-NEOVERSE-V1: %[[VECTOR_RECUR:.*]] = phi <vscale x 4 x i32> [ %[[RECUR_INIT]], %vector.ph ], [ %[[LOAD:.*]], %vector.body ]
-; CHECK-NEOVERSE-V1: %[[LOAD]] = load <vscale x 4 x i32>
-; CHECK-NEOVERSE-V1: %[[SPLICE:.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> %[[VECTOR_RECUR]], <vscale x 4 x i32> %[[LOAD]], i32 1)
-; CHECK-NEOVERSE-V1: %[[ADD:.*]] = add nsw <vscale x 4 x i32> %[[LOAD]], %[[SPLICE]]
-; CHECK-NEOVERSE-V1: store <vscale x 4 x i32> %[[ADD]]
+; CHECK-NOTF-LABEL: define void @add_recur(
+; CHECK-NOTF-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NOTF-NEXT: [[ENTRY:.*]]:
+; CHECK-NOTF-NEXT: [[DOTPRE:%.*]] = load i32, ptr [[SRC]], align 4
+; CHECK-NOTF-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NOTF-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NOTF-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-NOTF-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NOTF: [[VECTOR_PH]]:
+; CHECK-NOTF-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NOTF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP2]]
+; CHECK-NOTF-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NOTF-NEXT: [[TMP3:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NOTF-NEXT: [[TMP4:%.*]] = mul nuw i32 [[TMP3]], 4
+; CHECK-NOTF-NEXT: [[TMP5:%.*]] = sub i32 [[TMP4]], 1
+; CHECK-NOTF-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[DOTPRE]], i32 [[TMP5]]
+; CHECK-NOTF-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-NOTF: [[VECTOR_BODY]]:
+; CHECK-NOTF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NOTF-NEXT: [[VECTOR_RECUR:%.*]] = phi <vscale x 4 x i32> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ [[WIDE_LOAD:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NOTF-NEXT: [[TMP6:%.*]] = add nuw nsw i64 [[INDEX]], 1
+; CHECK-NOTF-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP6]]
+; CHECK-NOTF-NEXT: [[WIDE_LOAD]] = load <vscale x 4 x i32>, ptr [[TMP7]], align 4
+; CHECK-NOTF-NEXT: [[TMP8:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> [[VECTOR_RECUR]], <vscale x 4 x i32> [[WIDE_LOAD]], i32 1)
+; CHECK-NOTF-NEXT: [[TMP9:%.*]] = add nsw <vscale x 4 x i32> [[WIDE_LOAD]], [[TMP8]]
+; CHECK-NOTF-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NOTF-NEXT: store <vscale x 4 x i32> [[TMP9]], ptr [[TMP10]], align 4
+; CHECK-NOTF-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; CHECK-NOTF-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NOTF-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NOTF: [[MIDDLE_BLOCK]]:
+; CHECK-NOTF-NEXT: [[TMP12:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NOTF-NEXT: [[TMP13:%.*]] = mul nuw i32 [[TMP12]], 4
+; CHECK-NOTF-NEXT: [[TMP14:%.*]] = sub i32 [[TMP13]], 1
+; CHECK-NOTF-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 4 x i32> [[WIDE_LOAD]], i32 [[TMP14]]
+; CHECK-NOTF-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NOTF-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-NOTF: [[SCALAR_PH]]:
+; CHECK-NOTF-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ [[DOTPRE]], %[[ENTRY]] ]
+; CHECK-NOTF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NOTF-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-NOTF: [[FOR_BODY]]:
+; CHECK-NOTF-NEXT: [[TMP15:%.*]] = phi i32 [ [[TMP16:%.*]], %[[FOR_BODY]] ], [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ]
+; CHECK-NOTF-NEXT: [[I_010:%.*]] = phi i64 [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NOTF-NEXT: [[ADD]] = add nuw nsw i64 [[I_010]], 1
+; CHECK-NOTF-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[ADD]]
+; CHECK-NOTF-NEXT: [[TMP16]] = load i32, ptr [[ARRAYIDX1]], align 4
+; CHECK-NOTF-NEXT: [[ADD2:%.*]] = add nsw i32 [[TMP16]], [[TMP15]]
+; CHECK-NOTF-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[I_010]]
+; CHECK-NOTF-NEXT: store i32 [[ADD2]], ptr [[ARRAYIDX3]], align 4
+; CHECK-NOTF-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[ADD]], [[N]]
+; CHECK-NOTF-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-NOTF: [[FOR_END]]:
+; CHECK-NOTF-NEXT: ret void
+;
+; CHECK-TF-ALL-LABEL: define void @add_recur(
+; CHECK-TF-ALL-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-ALL-NEXT: [[ENTRY:.*:]]
+; CHECK-TF-ALL-NEXT: [[DOTPRE:%.*]] = load i32, ptr [[SRC]], align 4
+; CHECK-TF-ALL-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-TF-ALL: [[VECTOR_PH]]:
+; CHECK-TF-ALL-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-ALL-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-ALL-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
+; CHECK-TF-ALL-NEXT: [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-TF-ALL-NEXT: [[TMP3:%.*]] = mul nuw i32 [[TMP2]], 4
+; CHECK-TF-ALL-NEXT: [[TMP4:%.*]] = sub i32 [[TMP3]], 1
+; CHECK-TF-ALL-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[DOTPRE]], i32 [[TMP4]]
+; CHECK-TF-ALL-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-ALL: [[VECTOR_BODY]]:
+; CHECK-TF-ALL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-ALL-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-ALL-NEXT: [[VECTOR_RECUR:%.*]] = phi <vscale x 4 x i32> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ [[WIDE_MASKED_LOAD:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-ALL-NEXT: [[TMP5:%.*]] = add nuw nsw i64 [[INDEX]], 1
+; CHECK-TF-ALL-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP5]]
+; CHECK-TF-ALL-NEXT: [[WIDE_MASKED_LOAD]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 4 [[TMP6]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison)
+; CHECK-TF-ALL-NEXT: [[TMP7:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> [[VECTOR_RECUR]], <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], i32 1)
+; CHECK-TF-ALL-NEXT: [[TMP8:%.*]] = add nsw <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], [[TMP7]]
+; CHECK-TF-ALL-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[INDEX]]
+; CHECK-TF-ALL-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP8]], ptr align 4 [[TMP9]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-ALL-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-TF-ALL-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-TF-ALL-NEXT: [[TMP10:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-ALL-NEXT: [[TMP11:%.*]] = xor i1 [[TMP10]], true
+; CHECK-TF-ALL-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-TF-ALL: [[MIDDLE_BLOCK]]:
+; CHECK-TF-ALL-NEXT: br label %[[FOR_END:.*]]
+; CHECK-TF-ALL: [[FOR_END]]:
+; CHECK-TF-ALL-NEXT: ret void
+;
+; CHECK-TF-LABEL: define void @add_recur(
+; CHECK-TF-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-NEXT: [[ENTRY:.*:]]
+; CHECK-TF-NEXT: [[DOTPRE:%.*]] = load i32, ptr [[SRC]], align 4
+; CHECK-TF-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-TF: [[VECTOR_PH]]:
+; CHECK-TF-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
+; CHECK-TF-NEXT: [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-TF-NEXT: [[TMP3:%.*]] = mul nuw i32 [[TMP2]], 4
+; CHECK-TF-NEXT: [[TMP4:%.*]] = sub i32 [[TMP3]], 1
+; CHECK-TF-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[DOTPRE]], i32 [[TMP4]]
+; CHECK-TF-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF: [[VECTOR_BODY]]:
+; CHECK-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NEXT: [[VECTOR_RECUR:%.*]] = phi <vscale x 4 x i32> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ [[WIDE_MASKED_LOAD:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NEXT: [[TMP5:%.*]] = add nuw nsw i64 [[INDEX]], 1
+; CHECK-TF-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP5]]
+; CHECK-TF-NEXT: [[WIDE_MASKED_LOAD]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 4 [[TMP6]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison)
+; CHECK-TF-NEXT: [[TMP7:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> [[VECTOR_RECUR]], <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], i32 1)
+; CHECK-TF-NEXT: [[TMP8:%.*]] = add nsw <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], [[TMP7]]
+; CHECK-TF-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[INDEX]]
+; CHECK-TF-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP8]], ptr align 4 [[TMP9]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-TF-NEXT: [[TMP10:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-NEXT: [[TMP11:%.*]] = xor i1 [[TMP10]], true
+; CHECK-TF-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-TF: [[MIDDLE_BLOCK]]:
+; CHECK-TF-NEXT: br label %[[FOR_END:.*]]
+; CHECK-TF: [[FOR_END]]:
+; CHECK-TF-NEXT: ret void
+;
+; CHECK-TF-DEFAULT-LABEL: define void @add_recur(
+; CHECK-TF-DEFAULT-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-DEFAULT-NEXT: [[ENTRY:.*:]]
+; CHECK-TF-DEFAULT-NEXT: [[DOTPRE:%.*]] = load i32, ptr [[SRC]], align 4
+; CHECK-TF-DEFAULT-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-TF-DEFAULT: [[VECTOR_PH]]:
+; CHECK-TF-DEFAULT-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-DEFAULT-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-DEFAULT-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
+; CHECK-TF-DEFAULT-NEXT: [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-TF-DEFAULT-NEXT: [[TMP3:%.*]] = mul nuw i32 [[TMP2]], 4
+; CHECK-TF-DEFAULT-NEXT: [[TMP4:%.*]] = sub i32 [[TMP3]], 1
+; CHECK-TF-DEFAULT-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[DOTPRE]], i32 [[TMP4]]
+; CHECK-TF-DEFAULT-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-DEFAULT: [[VECTOR_BODY]]:
+; CHECK-TF-DEFAULT-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-DEFAULT-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-DEFAULT-NEXT: [[VECTOR_RECUR:%.*]] = phi <vscale x 4 x i32> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ [[WIDE_MASKED_LOAD:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-DEFAULT-NEXT: [[TMP5:%.*]] = add nuw nsw i64 [[INDEX]], 1
+; CHECK-TF-DEFAULT-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP5]]
+; CHECK-TF-DEFAULT-NEXT: [[WIDE_MASKED_LOAD]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 4 [[TMP6]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison)
+; CHECK-TF-DEFAULT-NEXT: [[TMP7:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> [[VECTOR_RECUR]], <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], i32 1)
+; CHECK-TF-DEFAULT-NEXT: [[TMP8:%.*]] = add nsw <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], [[TMP7]]
+; CHECK-TF-DEFAULT-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[INDEX]]
+; CHECK-TF-DEFAULT-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP8]], ptr align 4 [[TMP9]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-DEFAULT-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-TF-DEFAULT-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-TF-DEFAULT-NEXT: [[TMP10:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-DEFAULT-NEXT: [[TMP11:%.*]] = xor i1 [[TMP10]], true
+; CHECK-TF-DEFAULT-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-TF-DEFAULT: [[MIDDLE_BLOCK]]:
+; CHECK-TF-DEFAULT-NEXT: br label %[[FOR_END:.*]]
+; CHECK-TF-DEFAULT: [[FOR_END]]:
+; CHECK-TF-DEFAULT-NEXT: ret void
+;
+; CHECK-TF-NORED-LABEL: define void @add_recur(
+; CHECK-TF-NORED-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-NORED-NEXT: [[ENTRY:.*:]]
+; CHECK-TF-NORED-NEXT: [[DOTPRE:%.*]] = load i32, ptr [[SRC]], align 4
+; CHECK-TF-NORED-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-TF-NORED: [[VECTOR_PH]]:
+; CHECK-TF-NORED-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NORED-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-NORED-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
+; CHECK-TF-NORED-NEXT: [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-TF-NORED-NEXT: [[TMP3:%.*]] = mul nuw i32 [[TMP2]], 4
+; CHECK-TF-NORED-NEXT: [[TMP4:%.*]] = sub i32 [[TMP3]], 1
+; CHECK-TF-NORED-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[DOTPRE]], i32 [[TMP4]]
+; CHECK-TF-NORED-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-NORED: [[VECTOR_BODY]]:
+; CHECK-TF-NORED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NORED-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NORED-NEXT: [[VECTOR_RECUR:%.*]] = phi <vscale x 4 x i32> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ [[WIDE_MASKED_LOAD:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NORED-NEXT: [[TMP5:%.*]] = add nuw nsw i64 [[INDEX]], 1
+; CHECK-TF-NORED-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP5]]
+; CHECK-TF-NORED-NEXT: [[WIDE_MASKED_LOAD]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 4 [[TMP6]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison)
+; CHECK-TF-NORED-NEXT: [[TMP7:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> [[VECTOR_RECUR]], <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], i32 1)
+; CHECK-TF-NORED-NEXT: [[TMP8:%.*]] = add nsw <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], [[TMP7]]
+; CHECK-TF-NORED-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[INDEX]]
+; CHECK-TF-NORED-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP8]], ptr align 4 [[TMP9]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-NORED-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-TF-NORED-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-TF-NORED-NEXT: [[TMP10:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-NORED-NEXT: [[TMP11:%.*]] = xor i1 [[TMP10]], true
+; CHECK-TF-NORED-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-TF-NORED: [[MIDDLE_BLOCK]]:
+; CHECK-TF-NORED-NEXT: br label %[[FOR_END:.*]]
+; CHECK-TF-NORED: [[FOR_END]]:
+; CHECK-TF-NORED-NEXT: ret void
+;
+; CHECK-TF-NOREC-LABEL: define void @add_recur(
+; CHECK-TF-NOREC-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-NOREC-NEXT: [[ENTRY:.*]]:
+; CHECK-TF-NOREC-NEXT: [[DOTPRE:%.*]] = load i32, ptr [[SRC]], align 4
+; CHECK-TF-NOREC-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NOREC-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-NOREC-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-TF-NOREC-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-TF-NOREC: [[VECTOR_PH]]:
+; CHECK-TF-NOREC-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-NOREC-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP2]]
+; CHECK-TF-NOREC-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-TF-NOREC-NEXT: [[TMP3:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-TF-NOREC-NEXT: [[TMP4:%.*]] = mul nuw i32 [[TMP3]], 4
+; CHECK-TF-NOREC-NEXT: [[TMP5:%.*]] = sub i32 [[TMP4]], 1
+; CHECK-TF-NOREC-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[DOTPRE]], i32 [[TMP5]]
+; CHECK-TF-NOREC-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-NOREC: [[VECTOR_BODY]]:
+; CHECK-TF-NOREC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NOREC-NEXT: [[VECTOR_RECUR:%.*]] = phi <vscale x 4 x i32> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ [[WIDE_LOAD:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NOREC-NEXT: [[TMP6:%.*]] = add nuw nsw i64 [[INDEX]], 1
+; CHECK-TF-NOREC-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP6]]
+; CHECK-TF-NOREC-NEXT: [[WIDE_LOAD]] = load <vscale x 4 x i32>, ptr [[TMP7]], align 4
+; CHECK-TF-NOREC-NEXT: [[TMP8:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> [[VECTOR_RECUR]], <vscale x 4 x i32> [[WIDE_LOAD]], i32 1)
+; CHECK-TF-NOREC-NEXT: [[TMP9:%.*]] = add nsw <vscale x 4 x i32> [[WIDE_LOAD]], [[TMP8]]
+; CHECK-TF-NOREC-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[INDEX]]
+; CHECK-TF-NOREC-NEXT: store <vscale x 4 x i32> [[TMP9]], ptr [[TMP10]], align 4
+; CHECK-TF-NOREC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; CHECK-TF-NOREC-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-TF-NOREC-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-TF-NOREC: [[MIDDLE_BLOCK]]:
+; CHECK-TF-NOREC-NEXT: [[TMP12:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-TF-NOREC-NEXT: [[TMP13:%.*]] = mul nuw i32 [[TMP12]], 4
+; CHECK-TF-NOREC-NEXT: [[TMP14:%.*]] = sub i32 [[TMP13]], 1
+; CHECK-TF-NOREC-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 4 x i32> [[WIDE_LOAD]], i32 [[TMP14]]
+; CHECK-TF-NOREC-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-TF-NOREC-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-TF-NOREC: [[SCALAR_PH]]:
+; CHECK-TF-NOREC-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ [[DOTPRE]], %[[ENTRY]] ]
+; CHECK-TF-NOREC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-TF-NOREC-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-TF-NOREC: [[FOR_BODY]]:
+; CHECK-TF-NOREC-NEXT: [[TMP15:%.*]] = phi i32 [ [[TMP16:%.*]], %[[FOR_BODY]] ], [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ]
+; CHECK-TF-NOREC-NEXT: [[I_010:%.*]] = phi i64 [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-TF-NOREC-NEXT: [[ADD]] = add nuw nsw i64 [[I_010]], 1
+; CHECK-TF-NOREC-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[ADD]]
+; CHECK-TF-NOREC-NEXT: [[TMP16]] = load i32, ptr [[ARRAYIDX1]], align 4
+; CHECK-TF-NOREC-NEXT: [[ADD2:%.*]] = add nsw i32 [[TMP16]], [[TMP15]]
+; CHECK-TF-NOREC-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[I_010]]
+; CHECK-TF-NOREC-NEXT: store i32 [[ADD2]], ptr [[ARRAYIDX3]], align 4
+; CHECK-TF-NOREC-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[ADD]], [[N]]
+; CHECK-TF-NOREC-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-TF-NOREC: [[FOR_END]]:
+; CHECK-TF-NOREC-NEXT: ret void
+;
+; CHECK-TF-NOREV-LABEL: define void @add_recur(
+; CHECK-TF-NOREV-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-NOREV-NEXT: [[ENTRY:.*:]]
+; CHECK-TF-NOREV-NEXT: [[DOTPRE:%.*]] = load i32, ptr [[SRC]], align 4
+; CHECK-TF-NOREV-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-TF-NOREV: [[VECTOR_PH]]:
+; CHECK-TF-NOREV-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NOREV-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-NOREV-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
+; CHECK-TF-NOREV-NEXT: [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-TF-NOREV-NEXT: [[TMP3:%.*]] = mul nuw i32 [[TMP2]], 4
+; CHECK-TF-NOREV-NEXT: [[TMP4:%.*]] = sub i32 [[TMP3]], 1
+; CHECK-TF-NOREV-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[DOTPRE]], i32 [[TMP4]]
+; CHECK-TF-NOREV-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-NOREV: [[VECTOR_BODY]]:
+; CHECK-TF-NOREV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NOREV-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NOREV-NEXT: [[VECTOR_RECUR:%.*]] = phi <vscale x 4 x i32> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ [[WIDE_MASKED_LOAD:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NOREV-NEXT: [[TMP5:%.*]] = add nuw nsw i64 [[INDEX]], 1
+; CHECK-TF-NOREV-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP5]]
+; CHECK-TF-NOREV-NEXT: [[WIDE_MASKED_LOAD]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 4 [[TMP6]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison)
+; CHECK-TF-NOREV-NEXT: [[TMP7:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> [[VECTOR_RECUR]], <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], i32 1)
+; CHECK-TF-NOREV-NEXT: [[TMP8:%.*]] = add nsw <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], [[TMP7]]
+; CHECK-TF-NOREV-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[INDEX]]
+; CHECK-TF-NOREV-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP8]], ptr align 4 [[TMP9]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-NOREV-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-TF-NOREV-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-TF-NOREV-NEXT: [[TMP10:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-NOREV-NEXT: [[TMP11:%.*]] = xor i1 [[TMP10]], true
+; CHECK-TF-NOREV-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-TF-NOREV: [[MIDDLE_BLOCK]]:
+; CHECK-TF-NOREV-NEXT: br label %[[FOR_END:.*]]
+; CHECK-TF-NOREV: [[FOR_END]]:
+; CHECK-TF-NOREV-NEXT: ret void
+;
+; CHECK-TF-ONLYRED-LABEL: define void @add_recur(
+; CHECK-TF-ONLYRED-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-ONLYRED-NEXT: [[ENTRY:.*]]:
+; CHECK-TF-ONLYRED-NEXT: [[DOTPRE:%.*]] = load i32, ptr [[SRC]], align 4
+; CHECK-TF-ONLYRED-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-ONLYRED-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-ONLYRED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-TF-ONLYRED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-TF-ONLYRED: [[VECTOR_PH]]:
+; CHECK-TF-ONLYRED-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-ONLYRED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP2]]
+; CHECK-TF-ONLYRED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-TF-ONLYRED-NEXT: [[TMP3:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-TF-ONLYRED-NEXT: [[TMP4:%.*]] = mul nuw i32 [[TMP3]], 4
+; CHECK-TF-ONLYRED-NEXT: [[TMP5:%.*]] = sub i32 [[TMP4]], 1
+; CHECK-TF-ONLYRED-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[DOTPRE]], i32 [[TMP5]]
+; CHECK-TF-ONLYRED-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-ONLYRED: [[VECTOR_BODY]]:
+; CHECK-TF-ONLYRED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-ONLYRED-NEXT: [[VECTOR_RECUR:%.*]] = phi <vscale x 4 x i32> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ [[WIDE_LOAD:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-ONLYRED-NEXT: [[TMP6:%.*]] = add nuw nsw i64 [[INDEX]], 1
+; CHECK-TF-ONLYRED-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP6]]
+; CHECK-TF-ONLYRED-NEXT: [[WIDE_LOAD]] = load <vscale x 4 x i32>, ptr [[TMP7]], align 4
+; CHECK-TF-ONLYRED-NEXT: [[TMP8:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> [[VECTOR_RECUR]], <vscale x 4 x i32> [[WIDE_LOAD]], i32 1)
+; CHECK-TF-ONLYRED-NEXT: [[TMP9:%.*]] = add nsw <vscale x 4 x i32> [[WIDE_LOAD]], [[TMP8]]
+; CHECK-TF-ONLYRED-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[INDEX]]
+; CHECK-TF-ONLYRED-NEXT: store <vscale x 4 x i32> [[TMP9]], ptr [[TMP10]], align 4
+; CHECK-TF-ONLYRED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; CHECK-TF-ONLYRED-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-TF-ONLYRED-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-TF-ONLYRED: [[MIDDLE_BLOCK]]:
+; CHECK-TF-ONLYRED-NEXT: [[TMP12:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-TF-ONLYRED-NEXT: [[TMP13:%.*]] = mul nuw i32 [[TMP12]], 4
+; CHECK-TF-ONLYRED-NEXT: [[TMP14:%.*]] = sub i32 [[TMP13]], 1
+; CHECK-TF-ONLYRED-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 4 x i32> [[WIDE_LOAD]], i32 [[TMP14]]
+; CHECK-TF-ONLYRED-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-TF-ONLYRED-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-TF-ONLYRED: [[SCALAR_PH]]:
+; CHECK-TF-ONLYRED-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ [[DOTPRE]], %[[ENTRY]] ]
+; CHECK-TF-ONLYRED-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-TF-ONLYRED-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-TF-ONLYRED: [[FOR_BODY]]:
+; CHECK-TF-ONLYRED-NEXT: [[TMP15:%.*]] = phi i32 [ [[TMP16:%.*]], %[[FOR_BODY]] ], [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ]
+; CHECK-TF-ONLYRED-NEXT: [[I_010:%.*]] = phi i64 [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-TF-ONLYRED-NEXT: [[ADD]] = add nuw nsw i64 [[I_010]], 1
+; CHECK-TF-ONLYRED-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[ADD]]
+; CHECK-TF-ONLYRED-NEXT: [[TMP16]] = load i32, ptr [[ARRAYIDX1]], align 4
+; CHECK-TF-ONLYRED-NEXT: [[ADD2:%.*]] = add nsw i32 [[TMP16]], [[TMP15]]
+; CHECK-TF-ONLYRED-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[I_010]]
+; CHECK-TF-ONLYRED-NEXT: store i32 [[ADD2]], ptr [[ARRAYIDX3]], align 4
+; CHECK-TF-ONLYRED-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[ADD]], [[N]]
+; CHECK-TF-ONLYRED-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-TF-ONLYRED: [[FOR_END]]:
+; CHECK-TF-ONLYRED-NEXT: ret void
+;
+; CHECK-NEOVERSE-V1-LABEL: define void @add_recur(
+; CHECK-NEOVERSE-V1-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEOVERSE-V1-NEXT: [[ENTRY:.*]]:
+; CHECK-NEOVERSE-V1-NEXT: [[DOTPRE:%.*]] = load i32, ptr [[SRC]], align 4
+; CHECK-NEOVERSE-V1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEOVERSE-V1-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NEOVERSE-V1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-NEOVERSE-V1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEOVERSE-V1: [[VECTOR_PH]]:
+; CHECK-NEOVERSE-V1-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NEOVERSE-V1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP2]]
+; CHECK-NEOVERSE-V1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEOVERSE-V1-NEXT: [[TMP3:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEOVERSE-V1-NEXT: [[TMP4:%.*]] = mul nuw i32 [[TMP3]], 4
+; CHECK-NEOVERSE-V1-NEXT: [[TMP5:%.*]] = sub i32 [[TMP4]], 1
+; CHECK-NEOVERSE-V1-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[DOTPRE]], i32 [[TMP5]]
+; CHECK-NEOVERSE-V1-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-NEOVERSE-V1: [[VECTOR_BODY]]:
+; CHECK-NEOVERSE-V1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEOVERSE-V1-NEXT: [[VECTOR_RECUR:%.*]] = phi <vscale x 4 x i32> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ [[WIDE_LOAD:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEOVERSE-V1-NEXT: [[TMP6:%.*]] = add nuw nsw i64 [[INDEX]], 1
+; CHECK-NEOVERSE-V1-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP6]]
+; CHECK-NEOVERSE-V1-NEXT: [[WIDE_LOAD]] = load <vscale x 4 x i32>, ptr [[TMP7]], align 4
+; CHECK-NEOVERSE-V1-NEXT: [[TMP8:%.*]] = call <vscale x 4 x i32> @llvm.vector.splice.right.nxv4i32(<vscale x 4 x i32> [[VECTOR_RECUR]], <vscale x 4 x i32> [[WIDE_LOAD]], i32 1)
+; CHECK-NEOVERSE-V1-NEXT: [[TMP9:%.*]] = add nsw <vscale x 4 x i32> [[WIDE_LOAD]], [[TMP8]]
+; CHECK-NEOVERSE-V1-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEOVERSE-V1-NEXT: store <vscale x 4 x i32> [[TMP9]], ptr [[TMP10]], align 4
+; CHECK-NEOVERSE-V1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; CHECK-NEOVERSE-V1-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEOVERSE-V1-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEOVERSE-V1: [[MIDDLE_BLOCK]]:
+; CHECK-NEOVERSE-V1-NEXT: [[TMP12:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEOVERSE-V1-NEXT: [[TMP13:%.*]] = mul nuw i32 [[TMP12]], 4
+; CHECK-NEOVERSE-V1-NEXT: [[TMP14:%.*]] = sub i32 [[TMP13]], 1
+; CHECK-NEOVERSE-V1-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 4 x i32> [[WIDE_LOAD]], i32 [[TMP14]]
+; CHECK-NEOVERSE-V1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEOVERSE-V1-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-NEOVERSE-V1: [[SCALAR_PH]]:
+; CHECK-NEOVERSE-V1-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ [[DOTPRE]], %[[ENTRY]] ]
+; CHECK-NEOVERSE-V1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEOVERSE-V1-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-NEOVERSE-V1: [[FOR_BODY]]:
+; CHECK-NEOVERSE-V1-NEXT: [[TMP15:%.*]] = phi i32 [ [[TMP16:%.*]], %[[FOR_BODY]] ], [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ]
+; CHECK-NEOVERSE-V1-NEXT: [[I_010:%.*]] = phi i64 [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEOVERSE-V1-NEXT: [[ADD]] = add nuw nsw i64 [[I_010]], 1
+; CHECK-NEOVERSE-V1-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[ADD]]
+; CHECK-NEOVERSE-V1-NEXT: [[TMP16]] = load i32, ptr [[ARRAYIDX1]], align 4
+; CHECK-NEOVERSE-V1-NEXT: [[ADD2:%.*]] = add nsw i32 [[TMP16]], [[TMP15]]
+; CHECK-NEOVERSE-V1-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[I_010]]
+; CHECK-NEOVERSE-V1-NEXT: store i32 [[ADD2]], ptr [[ARRAYIDX3]], align 4
+; CHECK-NEOVERSE-V1-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[ADD]], [[N]]
+; CHECK-NEOVERSE-V1-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEOVERSE-V1: [[FOR_END]]:
+; CHECK-NEOVERSE-V1-NEXT: ret void
+;
+
+
+
+
+
+
entry:
%.pre = load i32, ptr %src, align 4
@@ -281,41 +1046,709 @@ for.end:
}
define void @interleave(ptr noalias %dst, ptr noalias %src, i64 %n) #0 {
-; CHECK-NOTF-LABEL: @interleave(
-; CHECK-NOTF: vector.body:
-; CHECK-NOTF: %[[LOAD:.*]] = load <8 x float>, ptr
-; CHECK-NOTF: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NOTF: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-
-; CHECK-TF-LABEL: @interleave(
-; CHECK-TF: vector.body:
-; CHECK-TF: %[[LOAD:.*]] = load <8 x float>, ptr
-; CHECK-TF: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-TF: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-
-; CHECK-TF-NORED-LABEL: @interleave(
-; CHECK-TF-NORED: vector.body:
-; CHECK-TF-NORED: %[[LOAD:.*]] = load <8 x float>, ptr
-; CHECK-TF-NORED: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-TF-NORED: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-
-; CHECK-TF-NOREC-LABEL: @interleave(
-; CHECK-TF-NOREC: vector.body:
-; CHECK-TF-NOREC: %[[LOAD:.*]] = load <8 x float>, ptr
-; CHECK-TF-NOREC: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-TF-NOREC: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-
-; CHECK-TF-NOREV-LABEL: @interleave(
-; CHECK-TF-NOREV: vector.body:
-; CHECK-TF-NOREV: %[[LOAD:.*]] = load <8 x float>, ptr
-; CHECK-TF-NOREV: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-TF-NOREV: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-
-; CHECK-NEOVERSE-V1-LABEL: @interleave(
-; CHECK-NEOVERSE-V1: vector.body:
-; CHECK-NEOVERSE-V1: %[[LOAD:.*]] = load <8 x float>, ptr
-; CHECK-NEOVERSE-V1: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NEOVERSE-V1: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NOTF-LABEL: define void @interleave(
+; CHECK-NOTF-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NOTF-NEXT: [[ENTRY:.*]]:
+; CHECK-NOTF-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NOTF-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-NOTF-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-NOTF-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NOTF: [[VECTOR_PH]]:
+; CHECK-NOTF-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NOTF-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; CHECK-NOTF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
+; CHECK-NOTF-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NOTF-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-NOTF: [[VECTOR_BODY]]:
+; CHECK-NOTF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NOTF-NEXT: [[TMP4:%.*]] = add i64 [[TMP2]], 0
+; CHECK-NOTF-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 1
+; CHECK-NOTF-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], [[TMP5]]
+; CHECK-NOTF-NEXT: [[TMP7:%.*]] = shl nuw nsw i64 [[INDEX]], 1
+; CHECK-NOTF-NEXT: [[TMP8:%.*]] = shl nuw nsw i64 [[TMP6]], 1
+; CHECK-NOTF-NEXT: [[TMP9:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP7]]
+; CHECK-NOTF-NEXT: [[TMP10:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP8]]
+; CHECK-NOTF-NEXT: [[WIDE_VEC:%.*]] = load <vscale x 8 x float>, ptr [[TMP9]], align 4
+; CHECK-NOTF-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC]])
+; CHECK-NOTF-NEXT: [[TMP11:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 0
+; CHECK-NOTF-NEXT: [[TMP12:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 1
+; CHECK-NOTF-NEXT: [[WIDE_VEC1:%.*]] = load <vscale x 8 x float>, ptr [[TMP10]], align 4
+; CHECK-NOTF-NEXT: [[STRIDED_VEC2:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC1]])
+; CHECK-NOTF-NEXT: [[TMP13:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC2]], 0
+; CHECK-NOTF-NEXT: [[TMP14:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC2]], 1
+; CHECK-NOTF-NEXT: [[TMP15:%.*]] = mul nuw nsw i64 [[INDEX]], 3
+; CHECK-NOTF-NEXT: [[TMP16:%.*]] = mul nuw nsw i64 [[TMP6]], 3
+; CHECK-NOTF-NEXT: [[TMP17:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP15]]
+; CHECK-NOTF-NEXT: [[TMP18:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP16]]
+; CHECK-NOTF-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 12 x float> @llvm.vector.interleave3.nxv12f32(<vscale x 4 x float> [[TMP11]], <vscale x 4 x float> [[TMP12]], <vscale x 4 x float> splat (float 3.000000e+00))
+; CHECK-NOTF-NEXT: store <vscale x 12 x float> [[INTERLEAVED_VEC]], ptr [[TMP17]], align 4
+; CHECK-NOTF-NEXT: [[INTERLEAVED_VEC3:%.*]] = call <vscale x 12 x float> @llvm.vector.interleave3.nxv12f32(<vscale x 4 x float> [[TMP13]], <vscale x 4 x float> [[TMP14]], <vscale x 4 x float> splat (float 3.000000e+00))
+; CHECK-NOTF-NEXT: store <vscale x 12 x float> [[INTERLEAVED_VEC3]], ptr [[TMP18]], align 4
+; CHECK-NOTF-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-NOTF-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NOTF-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NOTF: [[MIDDLE_BLOCK]]:
+; CHECK-NOTF-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NOTF-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-NOTF: [[SCALAR_PH]]:
+; CHECK-NOTF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NOTF-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-NOTF: [[FOR_BODY]]:
+; CHECK-NOTF-NEXT: [[I_021:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NOTF-NEXT: [[MUL:%.*]] = shl nuw nsw i64 [[I_021]], 1
+; CHECK-NOTF-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[MUL]]
+; CHECK-NOTF-NEXT: [[TMP20:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-NOTF-NEXT: [[MUL1:%.*]] = mul nuw nsw i64 [[I_021]], 3
+; CHECK-NOTF-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[MUL1]]
+; CHECK-NOTF-NEXT: store float [[TMP20]], ptr [[ARRAYIDX2]], align 4
+; CHECK-NOTF-NEXT: [[ADD:%.*]] = or disjoint i64 [[MUL]], 1
+; CHECK-NOTF-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[ADD]]
+; CHECK-NOTF-NEXT: [[TMP21:%.*]] = load float, ptr [[ARRAYIDX4]], align 4
+; CHECK-NOTF-NEXT: [[ADD6:%.*]] = add nuw nsw i64 [[MUL1]], 1
+; CHECK-NOTF-NEXT: [[ARRAYIDX7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD6]]
+; CHECK-NOTF-NEXT: store float [[TMP21]], ptr [[ARRAYIDX7]], align 4
+; CHECK-NOTF-NEXT: [[ADD9:%.*]] = add nuw nsw i64 [[MUL1]], 2
+; CHECK-NOTF-NEXT: [[ARRAYIDX10:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD9]]
+; CHECK-NOTF-NEXT: store float 3.000000e+00, ptr [[ARRAYIDX10]], align 4
+; CHECK-NOTF-NEXT: [[INC]] = add nuw nsw i64 [[I_021]], 1
+; CHECK-NOTF-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]]
+; CHECK-NOTF-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-NOTF: [[FOR_END]]:
+; CHECK-NOTF-NEXT: ret void
+;
+; CHECK-TF-ALL-LABEL: define void @interleave(
+; CHECK-TF-ALL-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-ALL-NEXT: [[ENTRY:.*]]:
+; CHECK-TF-ALL-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-ALL-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-TF-ALL-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-TF-ALL-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-TF-ALL: [[VECTOR_PH]]:
+; CHECK-TF-ALL-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-ALL-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; CHECK-TF-ALL-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
+; CHECK-TF-ALL-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-TF-ALL-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-ALL: [[VECTOR_BODY]]:
+; CHECK-TF-ALL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-ALL-NEXT: [[TMP4:%.*]] = add i64 [[TMP2]], 0
+; CHECK-TF-ALL-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 1
+; CHECK-TF-ALL-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], [[TMP5]]
+; CHECK-TF-ALL-NEXT: [[TMP7:%.*]] = shl nuw nsw i64 [[INDEX]], 1
+; CHECK-TF-ALL-NEXT: [[TMP8:%.*]] = shl nuw nsw i64 [[TMP6]], 1
+; CHECK-TF-ALL-NEXT: [[TMP9:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP7]]
+; CHECK-TF-ALL-NEXT: [[TMP10:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP8]]
+; CHECK-TF-ALL-NEXT: [[WIDE_VEC:%.*]] = load <vscale x 8 x float>, ptr [[TMP9]], align 4
+; CHECK-TF-ALL-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC]])
+; CHECK-TF-ALL-NEXT: [[TMP11:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 0
+; CHECK-TF-ALL-NEXT: [[TMP12:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 1
+; CHECK-TF-ALL-NEXT: [[WIDE_VEC1:%.*]] = load <vscale x 8 x float>, ptr [[TMP10]], align 4
+; CHECK-TF-ALL-NEXT: [[STRIDED_VEC2:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC1]])
+; CHECK-TF-ALL-NEXT: [[TMP13:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC2]], 0
+; CHECK-TF-ALL-NEXT: [[TMP14:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC2]], 1
+; CHECK-TF-ALL-NEXT: [[TMP15:%.*]] = mul nuw nsw i64 [[INDEX]], 3
+; CHECK-TF-ALL-NEXT: [[TMP16:%.*]] = mul nuw nsw i64 [[TMP6]], 3
+; CHECK-TF-ALL-NEXT: [[TMP17:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP15]]
+; CHECK-TF-ALL-NEXT: [[TMP18:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP16]]
+; CHECK-TF-ALL-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 12 x float> @llvm.vector.interleave3.nxv12f32(<vscale x 4 x float> [[TMP11]], <vscale x 4 x float> [[TMP12]], <vscale x 4 x float> splat (float 3.000000e+00))
+; CHECK-TF-ALL-NEXT: store <vscale x 12 x float> [[INTERLEAVED_VEC]], ptr [[TMP17]], align 4
+; CHECK-TF-ALL-NEXT: [[INTERLEAVED_VEC3:%.*]] = call <vscale x 12 x float> @llvm.vector.interleave3.nxv12f32(<vscale x 4 x float> [[TMP13]], <vscale x 4 x float> [[TMP14]], <vscale x 4 x float> splat (float 3.000000e+00))
+; CHECK-TF-ALL-NEXT: store <vscale x 12 x float> [[INTERLEAVED_VEC3]], ptr [[TMP18]], align 4
+; CHECK-TF-ALL-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-TF-ALL-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-TF-ALL-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-TF-ALL: [[MIDDLE_BLOCK]]:
+; CHECK-TF-ALL-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-TF-ALL-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-TF-ALL: [[SCALAR_PH]]:
+; CHECK-TF-ALL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-TF-ALL-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-TF-ALL: [[FOR_BODY]]:
+; CHECK-TF-ALL-NEXT: [[I_021:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-TF-ALL-NEXT: [[MUL:%.*]] = shl nuw nsw i64 [[I_021]], 1
+; CHECK-TF-ALL-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[MUL]]
+; CHECK-TF-ALL-NEXT: [[TMP20:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-TF-ALL-NEXT: [[MUL1:%.*]] = mul nuw nsw i64 [[I_021]], 3
+; CHECK-TF-ALL-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[MUL1]]
+; CHECK-TF-ALL-NEXT: store float [[TMP20]], ptr [[ARRAYIDX2]], align 4
+; CHECK-TF-ALL-NEXT: [[ADD:%.*]] = or disjoint i64 [[MUL]], 1
+; CHECK-TF-ALL-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[ADD]]
+; CHECK-TF-ALL-NEXT: [[TMP21:%.*]] = load float, ptr [[ARRAYIDX4]], align 4
+; CHECK-TF-ALL-NEXT: [[ADD6:%.*]] = add nuw nsw i64 [[MUL1]], 1
+; CHECK-TF-ALL-NEXT: [[ARRAYIDX7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD6]]
+; CHECK-TF-ALL-NEXT: store float [[TMP21]], ptr [[ARRAYIDX7]], align 4
+; CHECK-TF-ALL-NEXT: [[ADD9:%.*]] = add nuw nsw i64 [[MUL1]], 2
+; CHECK-TF-ALL-NEXT: [[ARRAYIDX10:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD9]]
+; CHECK-TF-ALL-NEXT: store float 3.000000e+00, ptr [[ARRAYIDX10]], align 4
+; CHECK-TF-ALL-NEXT: [[INC]] = add nuw nsw i64 [[I_021]], 1
+; CHECK-TF-ALL-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]]
+; CHECK-TF-ALL-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-TF-ALL: [[FOR_END]]:
+; CHECK-TF-ALL-NEXT: ret void
+;
+; CHECK-TF-LABEL: define void @interleave(
+; CHECK-TF-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-NEXT: [[ENTRY:.*]]:
+; CHECK-TF-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-TF-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-TF-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-TF: [[VECTOR_PH]]:
+; CHECK-TF-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; CHECK-TF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
+; CHECK-TF-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-TF-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF: [[VECTOR_BODY]]:
+; CHECK-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NEXT: [[TMP4:%.*]] = add i64 [[TMP2]], 0
+; CHECK-TF-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 1
+; CHECK-TF-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], [[TMP5]]
+; CHECK-TF-NEXT: [[TMP7:%.*]] = shl nuw nsw i64 [[INDEX]], 1
+; CHECK-TF-NEXT: [[TMP8:%.*]] = shl nuw nsw i64 [[TMP6]], 1
+; CHECK-TF-NEXT: [[TMP9:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP7]]
+; CHECK-TF-NEXT: [[TMP10:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP8]]
+; CHECK-TF-NEXT: [[WIDE_VEC:%.*]] = load <vscale x 8 x float>, ptr [[TMP9]], align 4
+; CHECK-TF-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC]])
+; CHECK-TF-NEXT: [[TMP11:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 0
+; CHECK-TF-NEXT: [[TMP12:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 1
+; CHECK-TF-NEXT: [[WIDE_VEC1:%.*]] = load <vscale x 8 x float>, ptr [[TMP10]], align 4
+; CHECK-TF-NEXT: [[STRIDED_VEC2:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC1]])
+; CHECK-TF-NEXT: [[TMP13:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC2]], 0
+; CHECK-TF-NEXT: [[TMP14:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC2]], 1
+; CHECK-TF-NEXT: [[TMP15:%.*]] = mul nuw nsw i64 [[INDEX]], 3
+; CHECK-TF-NEXT: [[TMP16:%.*]] = mul nuw nsw i64 [[TMP6]], 3
+; CHECK-TF-NEXT: [[TMP17:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP15]]
+; CHECK-TF-NEXT: [[TMP18:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP16]]
+; CHECK-TF-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 12 x float> @llvm.vector.interleave3.nxv12f32(<vscale x 4 x float> [[TMP11]], <vscale x 4 x float> [[TMP12]], <vscale x 4 x float> splat (float 3.000000e+00))
+; CHECK-TF-NEXT: store <vscale x 12 x float> [[INTERLEAVED_VEC]], ptr [[TMP17]], align 4
+; CHECK-TF-NEXT: [[INTERLEAVED_VEC3:%.*]] = call <vscale x 12 x float> @llvm.vector.interleave3.nxv12f32(<vscale x 4 x float> [[TMP13]], <vscale x 4 x float> [[TMP14]], <vscale x 4 x float> splat (float 3.000000e+00))
+; CHECK-TF-NEXT: store <vscale x 12 x float> [[INTERLEAVED_VEC3]], ptr [[TMP18]], align 4
+; CHECK-TF-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-TF-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-TF-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-TF: [[MIDDLE_BLOCK]]:
+; CHECK-TF-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-TF-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-TF: [[SCALAR_PH]]:
+; CHECK-TF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-TF-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-TF: [[FOR_BODY]]:
+; CHECK-TF-NEXT: [[I_021:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-TF-NEXT: [[MUL:%.*]] = shl nuw nsw i64 [[I_021]], 1
+; CHECK-TF-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[MUL]]
+; CHECK-TF-NEXT: [[TMP20:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-TF-NEXT: [[MUL1:%.*]] = mul nuw nsw i64 [[I_021]], 3
+; CHECK-TF-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[MUL1]]
+; CHECK-TF-NEXT: store float [[TMP20]], ptr [[ARRAYIDX2]], align 4
+; CHECK-TF-NEXT: [[ADD:%.*]] = or disjoint i64 [[MUL]], 1
+; CHECK-TF-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[ADD]]
+; CHECK-TF-NEXT: [[TMP21:%.*]] = load float, ptr [[ARRAYIDX4]], align 4
+; CHECK-TF-NEXT: [[ADD6:%.*]] = add nuw nsw i64 [[MUL1]], 1
+; CHECK-TF-NEXT: [[ARRAYIDX7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD6]]
+; CHECK-TF-NEXT: store float [[TMP21]], ptr [[ARRAYIDX7]], align 4
+; CHECK-TF-NEXT: [[ADD9:%.*]] = add nuw nsw i64 [[MUL1]], 2
+; CHECK-TF-NEXT: [[ARRAYIDX10:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD9]]
+; CHECK-TF-NEXT: store float 3.000000e+00, ptr [[ARRAYIDX10]], align 4
+; CHECK-TF-NEXT: [[INC]] = add nuw nsw i64 [[I_021]], 1
+; CHECK-TF-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]]
+; CHECK-TF-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-TF: [[FOR_END]]:
+; CHECK-TF-NEXT: ret void
+;
+; CHECK-TF-DEFAULT-LABEL: define void @interleave(
+; CHECK-TF-DEFAULT-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-DEFAULT-NEXT: [[ITER_CHECK:.*]]:
+; CHECK-TF-DEFAULT-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-DEFAULT-NEXT: [[TMP25:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-DEFAULT-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP25]]
+; CHECK-TF-DEFAULT-NEXT: [[TMP29:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-DEFAULT-NEXT: [[TMP30:%.*]] = shl nuw i64 [[TMP29]], 2
+; CHECK-TF-DEFAULT-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK-TF-DEFAULT: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-TF-DEFAULT-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-TF-DEFAULT-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-TF-DEFAULT-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-TF-DEFAULT: [[VECTOR_PH]]:
+; CHECK-TF-DEFAULT-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-DEFAULT-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; CHECK-TF-DEFAULT-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
+; CHECK-TF-DEFAULT-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-TF-DEFAULT-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-DEFAULT: [[VECTOR_BODY]]:
+; CHECK-TF-DEFAULT-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-DEFAULT-NEXT: [[TMP4:%.*]] = add i64 [[TMP2]], 0
+; CHECK-TF-DEFAULT-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 1
+; CHECK-TF-DEFAULT-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], [[TMP5]]
+; CHECK-TF-DEFAULT-NEXT: [[TMP7:%.*]] = shl nuw nsw i64 [[INDEX]], 1
+; CHECK-TF-DEFAULT-NEXT: [[TMP8:%.*]] = shl nuw nsw i64 [[TMP6]], 1
+; CHECK-TF-DEFAULT-NEXT: [[TMP9:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP7]]
+; CHECK-TF-DEFAULT-NEXT: [[TMP10:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP8]]
+; CHECK-TF-DEFAULT-NEXT: [[WIDE_VEC:%.*]] = load <vscale x 8 x float>, ptr [[TMP9]], align 4
+; CHECK-TF-DEFAULT-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC]])
+; CHECK-TF-DEFAULT-NEXT: [[TMP11:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 0
+; CHECK-TF-DEFAULT-NEXT: [[TMP12:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 1
+; CHECK-TF-DEFAULT-NEXT: [[WIDE_VEC2:%.*]] = load <vscale x 8 x float>, ptr [[TMP10]], align 4
+; CHECK-TF-DEFAULT-NEXT: [[STRIDED_VEC3:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC2]])
+; CHECK-TF-DEFAULT-NEXT: [[TMP13:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC3]], 0
+; CHECK-TF-DEFAULT-NEXT: [[TMP14:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC3]], 1
+; CHECK-TF-DEFAULT-NEXT: [[TMP15:%.*]] = mul nuw nsw i64 [[INDEX]], 3
+; CHECK-TF-DEFAULT-NEXT: [[TMP16:%.*]] = mul nuw nsw i64 [[TMP6]], 3
+; CHECK-TF-DEFAULT-NEXT: [[TMP17:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP15]]
+; CHECK-TF-DEFAULT-NEXT: [[TMP18:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP16]]
+; CHECK-TF-DEFAULT-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 12 x float> @llvm.vector.interleave3.nxv12f32(<vscale x 4 x float> [[TMP11]], <vscale x 4 x float> [[TMP12]], <vscale x 4 x float> splat (float 3.000000e+00))
+; CHECK-TF-DEFAULT-NEXT: store <vscale x 12 x float> [[INTERLEAVED_VEC]], ptr [[TMP17]], align 4
+; CHECK-TF-DEFAULT-NEXT: [[INTERLEAVED_VEC4:%.*]] = call <vscale x 12 x float> @llvm.vector.interleave3.nxv12f32(<vscale x 4 x float> [[TMP13]], <vscale x 4 x float> [[TMP14]], <vscale x 4 x float> splat (float 3.000000e+00))
+; CHECK-TF-DEFAULT-NEXT: store <vscale x 12 x float> [[INTERLEAVED_VEC4]], ptr [[TMP18]], align 4
+; CHECK-TF-DEFAULT-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-TF-DEFAULT-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-TF-DEFAULT-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-TF-DEFAULT: [[MIDDLE_BLOCK]]:
+; CHECK-TF-DEFAULT-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-TF-DEFAULT-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK-TF-DEFAULT: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-TF-DEFAULT-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP30]]
+; CHECK-TF-DEFAULT-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF6:![0-9]+]]
+; CHECK-TF-DEFAULT: [[VEC_EPILOG_PH]]:
+; CHECK-TF-DEFAULT-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-TF-DEFAULT-NEXT: [[TMP31:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-DEFAULT-NEXT: [[TMP24:%.*]] = shl nuw i64 [[TMP31]], 2
+; CHECK-TF-DEFAULT-NEXT: [[N_MOD_VF5:%.*]] = urem i64 [[N]], [[TMP24]]
+; CHECK-TF-DEFAULT-NEXT: [[N_VEC6:%.*]] = sub i64 [[N]], [[N_MOD_VF5]]
+; CHECK-TF-DEFAULT-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK-TF-DEFAULT: [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-TF-DEFAULT-NEXT: [[INDEX7:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT12:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-TF-DEFAULT-NEXT: [[TMP20:%.*]] = shl nuw nsw i64 [[INDEX7]], 1
+; CHECK-TF-DEFAULT-NEXT: [[TMP21:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP20]]
+; CHECK-TF-DEFAULT-NEXT: [[WIDE_VEC8:%.*]] = load <vscale x 8 x float>, ptr [[TMP21]], align 4
+; CHECK-TF-DEFAULT-NEXT: [[STRIDED_VEC9:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC8]])
+; CHECK-TF-DEFAULT-NEXT: [[TMP32:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC9]], 0
+; CHECK-TF-DEFAULT-NEXT: [[TMP33:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC9]], 1
+; CHECK-TF-DEFAULT-NEXT: [[TMP22:%.*]] = mul nuw nsw i64 [[INDEX7]], 3
+; CHECK-TF-DEFAULT-NEXT: [[TMP23:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP22]]
+; CHECK-TF-DEFAULT-NEXT: [[INTERLEAVED_VEC10:%.*]] = call <vscale x 12 x float> @llvm.vector.interleave3.nxv12f32(<vscale x 4 x float> [[TMP32]], <vscale x 4 x float> [[TMP33]], <vscale x 4 x float> splat (float 3.000000e+00))
+; CHECK-TF-DEFAULT-NEXT: store <vscale x 12 x float> [[INTERLEAVED_VEC10]], ptr [[TMP23]], align 4
+; CHECK-TF-DEFAULT-NEXT: [[INDEX_NEXT12]] = add nuw i64 [[INDEX7]], [[TMP24]]
+; CHECK-TF-DEFAULT-NEXT: [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT12]], [[N_VEC6]]
+; CHECK-TF-DEFAULT-NEXT: br i1 [[TMP26]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-TF-DEFAULT: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-TF-DEFAULT-NEXT: [[CMP_N13:%.*]] = icmp eq i64 [[N]], [[N_VEC6]]
+; CHECK-TF-DEFAULT-NEXT: br i1 [[CMP_N13]], label %[[FOR_END]], label %[[VEC_EPILOG_SCALAR_PH]]
+; CHECK-TF-DEFAULT: [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-TF-DEFAULT-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC6]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; CHECK-TF-DEFAULT-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-TF-DEFAULT: [[FOR_BODY]]:
+; CHECK-TF-DEFAULT-NEXT: [[I_021:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; CHECK-TF-DEFAULT-NEXT: [[MUL:%.*]] = shl nuw nsw i64 [[I_021]], 1
+; CHECK-TF-DEFAULT-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[MUL]]
+; CHECK-TF-DEFAULT-NEXT: [[TMP27:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-TF-DEFAULT-NEXT: [[MUL1:%.*]] = mul nuw nsw i64 [[I_021]], 3
+; CHECK-TF-DEFAULT-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[MUL1]]
+; CHECK-TF-DEFAULT-NEXT: store float [[TMP27]], ptr [[ARRAYIDX2]], align 4
+; CHECK-TF-DEFAULT-NEXT: [[ADD:%.*]] = or disjoint i64 [[MUL]], 1
+; CHECK-TF-DEFAULT-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[ADD]]
+; CHECK-TF-DEFAULT-NEXT: [[TMP28:%.*]] = load float, ptr [[ARRAYIDX4]], align 4
+; CHECK-TF-DEFAULT-NEXT: [[ADD6:%.*]] = add nuw nsw i64 [[MUL1]], 1
+; CHECK-TF-DEFAULT-NEXT: [[ARRAYIDX7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD6]]
+; CHECK-TF-DEFAULT-NEXT: store float [[TMP28]], ptr [[ARRAYIDX7]], align 4
+; CHECK-TF-DEFAULT-NEXT: [[ADD9:%.*]] = add nuw nsw i64 [[MUL1]], 2
+; CHECK-TF-DEFAULT-NEXT: [[ARRAYIDX10:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD9]]
+; CHECK-TF-DEFAULT-NEXT: store float 3.000000e+00, ptr [[ARRAYIDX10]], align 4
+; CHECK-TF-DEFAULT-NEXT: [[INC]] = add nuw nsw i64 [[I_021]], 1
+; CHECK-TF-DEFAULT-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]]
+; CHECK-TF-DEFAULT-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-TF-DEFAULT: [[FOR_END]]:
+; CHECK-TF-DEFAULT-NEXT: ret void
+;
+; CHECK-TF-NORED-LABEL: define void @interleave(
+; CHECK-TF-NORED-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-NORED-NEXT: [[ENTRY:.*]]:
+; CHECK-TF-NORED-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NORED-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-TF-NORED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-TF-NORED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-TF-NORED: [[VECTOR_PH]]:
+; CHECK-TF-NORED-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-NORED-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; CHECK-TF-NORED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
+; CHECK-TF-NORED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-TF-NORED-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-NORED: [[VECTOR_BODY]]:
+; CHECK-TF-NORED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NORED-NEXT: [[TMP4:%.*]] = add i64 [[TMP2]], 0
+; CHECK-TF-NORED-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 1
+; CHECK-TF-NORED-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], [[TMP5]]
+; CHECK-TF-NORED-NEXT: [[TMP7:%.*]] = shl nuw nsw i64 [[INDEX]], 1
+; CHECK-TF-NORED-NEXT: [[TMP8:%.*]] = shl nuw nsw i64 [[TMP6]], 1
+; CHECK-TF-NORED-NEXT: [[TMP9:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP7]]
+; CHECK-TF-NORED-NEXT: [[TMP10:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP8]]
+; CHECK-TF-NORED-NEXT: [[WIDE_VEC:%.*]] = load <vscale x 8 x float>, ptr [[TMP9]], align 4
+; CHECK-TF-NORED-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC]])
+; CHECK-TF-NORED-NEXT: [[TMP11:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 0
+; CHECK-TF-NORED-NEXT: [[TMP12:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 1
+; CHECK-TF-NORED-NEXT: [[WIDE_VEC1:%.*]] = load <vscale x 8 x float>, ptr [[TMP10]], align 4
+; CHECK-TF-NORED-NEXT: [[STRIDED_VEC2:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC1]])
+; CHECK-TF-NORED-NEXT: [[TMP13:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC2]], 0
+; CHECK-TF-NORED-NEXT: [[TMP14:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC2]], 1
+; CHECK-TF-NORED-NEXT: [[TMP15:%.*]] = mul nuw nsw i64 [[INDEX]], 3
+; CHECK-TF-NORED-NEXT: [[TMP16:%.*]] = mul nuw nsw i64 [[TMP6]], 3
+; CHECK-TF-NORED-NEXT: [[TMP17:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP15]]
+; CHECK-TF-NORED-NEXT: [[TMP18:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP16]]
+; CHECK-TF-NORED-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 12 x float> @llvm.vector.interleave3.nxv12f32(<vscale x 4 x float> [[TMP11]], <vscale x 4 x float> [[TMP12]], <vscale x 4 x float> splat (float 3.000000e+00))
+; CHECK-TF-NORED-NEXT: store <vscale x 12 x float> [[INTERLEAVED_VEC]], ptr [[TMP17]], align 4
+; CHECK-TF-NORED-NEXT: [[INTERLEAVED_VEC3:%.*]] = call <vscale x 12 x float> @llvm.vector.interleave3.nxv12f32(<vscale x 4 x float> [[TMP13]], <vscale x 4 x float> [[TMP14]], <vscale x 4 x float> splat (float 3.000000e+00))
+; CHECK-TF-NORED-NEXT: store <vscale x 12 x float> [[INTERLEAVED_VEC3]], ptr [[TMP18]], align 4
+; CHECK-TF-NORED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-TF-NORED-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-TF-NORED-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-TF-NORED: [[MIDDLE_BLOCK]]:
+; CHECK-TF-NORED-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-TF-NORED-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-TF-NORED: [[SCALAR_PH]]:
+; CHECK-TF-NORED-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-TF-NORED-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-TF-NORED: [[FOR_BODY]]:
+; CHECK-TF-NORED-NEXT: [[I_021:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-TF-NORED-NEXT: [[MUL:%.*]] = shl nuw nsw i64 [[I_021]], 1
+; CHECK-TF-NORED-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[MUL]]
+; CHECK-TF-NORED-NEXT: [[TMP20:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-TF-NORED-NEXT: [[MUL1:%.*]] = mul nuw nsw i64 [[I_021]], 3
+; CHECK-TF-NORED-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[MUL1]]
+; CHECK-TF-NORED-NEXT: store float [[TMP20]], ptr [[ARRAYIDX2]], align 4
+; CHECK-TF-NORED-NEXT: [[ADD:%.*]] = or disjoint i64 [[MUL]], 1
+; CHECK-TF-NORED-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[ADD]]
+; CHECK-TF-NORED-NEXT: [[TMP21:%.*]] = load float, ptr [[ARRAYIDX4]], align 4
+; CHECK-TF-NORED-NEXT: [[ADD6:%.*]] = add nuw nsw i64 [[MUL1]], 1
+; CHECK-TF-NORED-NEXT: [[ARRAYIDX7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD6]]
+; CHECK-TF-NORED-NEXT: store float [[TMP21]], ptr [[ARRAYIDX7]], align 4
+; CHECK-TF-NORED-NEXT: [[ADD9:%.*]] = add nuw nsw i64 [[MUL1]], 2
+; CHECK-TF-NORED-NEXT: [[ARRAYIDX10:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD9]]
+; CHECK-TF-NORED-NEXT: store float 3.000000e+00, ptr [[ARRAYIDX10]], align 4
+; CHECK-TF-NORED-NEXT: [[INC]] = add nuw nsw i64 [[I_021]], 1
+; CHECK-TF-NORED-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]]
+; CHECK-TF-NORED-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-TF-NORED: [[FOR_END]]:
+; CHECK-TF-NORED-NEXT: ret void
+;
+; CHECK-TF-NOREC-LABEL: define void @interleave(
+; CHECK-TF-NOREC-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-NOREC-NEXT: [[ENTRY:.*]]:
+; CHECK-TF-NOREC-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NOREC-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-TF-NOREC-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-TF-NOREC-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-TF-NOREC: [[VECTOR_PH]]:
+; CHECK-TF-NOREC-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-NOREC-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; CHECK-TF-NOREC-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
+; CHECK-TF-NOREC-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-TF-NOREC-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-NOREC: [[VECTOR_BODY]]:
+; CHECK-TF-NOREC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NOREC-NEXT: [[TMP4:%.*]] = add i64 [[TMP2]], 0
+; CHECK-TF-NOREC-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 1
+; CHECK-TF-NOREC-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], [[TMP5]]
+; CHECK-TF-NOREC-NEXT: [[TMP7:%.*]] = shl nuw nsw i64 [[INDEX]], 1
+; CHECK-TF-NOREC-NEXT: [[TMP8:%.*]] = shl nuw nsw i64 [[TMP6]], 1
+; CHECK-TF-NOREC-NEXT: [[TMP9:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP7]]
+; CHECK-TF-NOREC-NEXT: [[TMP10:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP8]]
+; CHECK-TF-NOREC-NEXT: [[WIDE_VEC:%.*]] = load <vscale x 8 x float>, ptr [[TMP9]], align 4
+; CHECK-TF-NOREC-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC]])
+; CHECK-TF-NOREC-NEXT: [[TMP11:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 0
+; CHECK-TF-NOREC-NEXT: [[TMP12:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 1
+; CHECK-TF-NOREC-NEXT: [[WIDE_VEC1:%.*]] = load <vscale x 8 x float>, ptr [[TMP10]], align 4
+; CHECK-TF-NOREC-NEXT: [[STRIDED_VEC2:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC1]])
+; CHECK-TF-NOREC-NEXT: [[TMP13:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC2]], 0
+; CHECK-TF-NOREC-NEXT: [[TMP14:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC2]], 1
+; CHECK-TF-NOREC-NEXT: [[TMP15:%.*]] = mul nuw nsw i64 [[INDEX]], 3
+; CHECK-TF-NOREC-NEXT: [[TMP16:%.*]] = mul nuw nsw i64 [[TMP6]], 3
+; CHECK-TF-NOREC-NEXT: [[TMP17:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP15]]
+; CHECK-TF-NOREC-NEXT: [[TMP18:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP16]]
+; CHECK-TF-NOREC-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 12 x float> @llvm.vector.interleave3.nxv12f32(<vscale x 4 x float> [[TMP11]], <vscale x 4 x float> [[TMP12]], <vscale x 4 x float> splat (float 3.000000e+00))
+; CHECK-TF-NOREC-NEXT: store <vscale x 12 x float> [[INTERLEAVED_VEC]], ptr [[TMP17]], align 4
+; CHECK-TF-NOREC-NEXT: [[INTERLEAVED_VEC3:%.*]] = call <vscale x 12 x float> @llvm.vector.interleave3.nxv12f32(<vscale x 4 x float> [[TMP13]], <vscale x 4 x float> [[TMP14]], <vscale x 4 x float> splat (float 3.000000e+00))
+; CHECK-TF-NOREC-NEXT: store <vscale x 12 x float> [[INTERLEAVED_VEC3]], ptr [[TMP18]], align 4
+; CHECK-TF-NOREC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-TF-NOREC-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-TF-NOREC-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-TF-NOREC: [[MIDDLE_BLOCK]]:
+; CHECK-TF-NOREC-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-TF-NOREC-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-TF-NOREC: [[SCALAR_PH]]:
+; CHECK-TF-NOREC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-TF-NOREC-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-TF-NOREC: [[FOR_BODY]]:
+; CHECK-TF-NOREC-NEXT: [[I_021:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-TF-NOREC-NEXT: [[MUL:%.*]] = shl nuw nsw i64 [[I_021]], 1
+; CHECK-TF-NOREC-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[MUL]]
+; CHECK-TF-NOREC-NEXT: [[TMP20:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-TF-NOREC-NEXT: [[MUL1:%.*]] = mul nuw nsw i64 [[I_021]], 3
+; CHECK-TF-NOREC-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[MUL1]]
+; CHECK-TF-NOREC-NEXT: store float [[TMP20]], ptr [[ARRAYIDX2]], align 4
+; CHECK-TF-NOREC-NEXT: [[ADD:%.*]] = or disjoint i64 [[MUL]], 1
+; CHECK-TF-NOREC-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[ADD]]
+; CHECK-TF-NOREC-NEXT: [[TMP21:%.*]] = load float, ptr [[ARRAYIDX4]], align 4
+; CHECK-TF-NOREC-NEXT: [[ADD6:%.*]] = add nuw nsw i64 [[MUL1]], 1
+; CHECK-TF-NOREC-NEXT: [[ARRAYIDX7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD6]]
+; CHECK-TF-NOREC-NEXT: store float [[TMP21]], ptr [[ARRAYIDX7]], align 4
+; CHECK-TF-NOREC-NEXT: [[ADD9:%.*]] = add nuw nsw i64 [[MUL1]], 2
+; CHECK-TF-NOREC-NEXT: [[ARRAYIDX10:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD9]]
+; CHECK-TF-NOREC-NEXT: store float 3.000000e+00, ptr [[ARRAYIDX10]], align 4
+; CHECK-TF-NOREC-NEXT: [[INC]] = add nuw nsw i64 [[I_021]], 1
+; CHECK-TF-NOREC-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]]
+; CHECK-TF-NOREC-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-TF-NOREC: [[FOR_END]]:
+; CHECK-TF-NOREC-NEXT: ret void
+;
+; CHECK-TF-NOREV-LABEL: define void @interleave(
+; CHECK-TF-NOREV-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-NOREV-NEXT: [[ENTRY:.*]]:
+; CHECK-TF-NOREV-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NOREV-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-TF-NOREV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-TF-NOREV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-TF-NOREV: [[VECTOR_PH]]:
+; CHECK-TF-NOREV-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-NOREV-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; CHECK-TF-NOREV-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
+; CHECK-TF-NOREV-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-TF-NOREV-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-NOREV: [[VECTOR_BODY]]:
+; CHECK-TF-NOREV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NOREV-NEXT: [[TMP4:%.*]] = add i64 [[TMP2]], 0
+; CHECK-TF-NOREV-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 1
+; CHECK-TF-NOREV-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], [[TMP5]]
+; CHECK-TF-NOREV-NEXT: [[TMP7:%.*]] = shl nuw nsw i64 [[INDEX]], 1
+; CHECK-TF-NOREV-NEXT: [[TMP8:%.*]] = shl nuw nsw i64 [[TMP6]], 1
+; CHECK-TF-NOREV-NEXT: [[TMP9:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP7]]
+; CHECK-TF-NOREV-NEXT: [[TMP10:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP8]]
+; CHECK-TF-NOREV-NEXT: [[WIDE_VEC:%.*]] = load <vscale x 8 x float>, ptr [[TMP9]], align 4
+; CHECK-TF-NOREV-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC]])
+; CHECK-TF-NOREV-NEXT: [[TMP11:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 0
+; CHECK-TF-NOREV-NEXT: [[TMP12:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 1
+; CHECK-TF-NOREV-NEXT: [[WIDE_VEC1:%.*]] = load <vscale x 8 x float>, ptr [[TMP10]], align 4
+; CHECK-TF-NOREV-NEXT: [[STRIDED_VEC2:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC1]])
+; CHECK-TF-NOREV-NEXT: [[TMP13:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC2]], 0
+; CHECK-TF-NOREV-NEXT: [[TMP14:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC2]], 1
+; CHECK-TF-NOREV-NEXT: [[TMP15:%.*]] = mul nuw nsw i64 [[INDEX]], 3
+; CHECK-TF-NOREV-NEXT: [[TMP16:%.*]] = mul nuw nsw i64 [[TMP6]], 3
+; CHECK-TF-NOREV-NEXT: [[TMP17:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP15]]
+; CHECK-TF-NOREV-NEXT: [[TMP18:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP16]]
+; CHECK-TF-NOREV-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 12 x float> @llvm.vector.interleave3.nxv12f32(<vscale x 4 x float> [[TMP11]], <vscale x 4 x float> [[TMP12]], <vscale x 4 x float> splat (float 3.000000e+00))
+; CHECK-TF-NOREV-NEXT: store <vscale x 12 x float> [[INTERLEAVED_VEC]], ptr [[TMP17]], align 4
+; CHECK-TF-NOREV-NEXT: [[INTERLEAVED_VEC3:%.*]] = call <vscale x 12 x float> @llvm.vector.interleave3.nxv12f32(<vscale x 4 x float> [[TMP13]], <vscale x 4 x float> [[TMP14]], <vscale x 4 x float> splat (float 3.000000e+00))
+; CHECK-TF-NOREV-NEXT: store <vscale x 12 x float> [[INTERLEAVED_VEC3]], ptr [[TMP18]], align 4
+; CHECK-TF-NOREV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-TF-NOREV-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-TF-NOREV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-TF-NOREV: [[MIDDLE_BLOCK]]:
+; CHECK-TF-NOREV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-TF-NOREV-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-TF-NOREV: [[SCALAR_PH]]:
+; CHECK-TF-NOREV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-TF-NOREV-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-TF-NOREV: [[FOR_BODY]]:
+; CHECK-TF-NOREV-NEXT: [[I_021:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-TF-NOREV-NEXT: [[MUL:%.*]] = shl nuw nsw i64 [[I_021]], 1
+; CHECK-TF-NOREV-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[MUL]]
+; CHECK-TF-NOREV-NEXT: [[TMP20:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-TF-NOREV-NEXT: [[MUL1:%.*]] = mul nuw nsw i64 [[I_021]], 3
+; CHECK-TF-NOREV-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[MUL1]]
+; CHECK-TF-NOREV-NEXT: store float [[TMP20]], ptr [[ARRAYIDX2]], align 4
+; CHECK-TF-NOREV-NEXT: [[ADD:%.*]] = or disjoint i64 [[MUL]], 1
+; CHECK-TF-NOREV-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[ADD]]
+; CHECK-TF-NOREV-NEXT: [[TMP21:%.*]] = load float, ptr [[ARRAYIDX4]], align 4
+; CHECK-TF-NOREV-NEXT: [[ADD6:%.*]] = add nuw nsw i64 [[MUL1]], 1
+; CHECK-TF-NOREV-NEXT: [[ARRAYIDX7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD6]]
+; CHECK-TF-NOREV-NEXT: store float [[TMP21]], ptr [[ARRAYIDX7]], align 4
+; CHECK-TF-NOREV-NEXT: [[ADD9:%.*]] = add nuw nsw i64 [[MUL1]], 2
+; CHECK-TF-NOREV-NEXT: [[ARRAYIDX10:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD9]]
+; CHECK-TF-NOREV-NEXT: store float 3.000000e+00, ptr [[ARRAYIDX10]], align 4
+; CHECK-TF-NOREV-NEXT: [[INC]] = add nuw nsw i64 [[I_021]], 1
+; CHECK-TF-NOREV-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]]
+; CHECK-TF-NOREV-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-TF-NOREV: [[FOR_END]]:
+; CHECK-TF-NOREV-NEXT: ret void
+;
+; CHECK-TF-ONLYRED-LABEL: define void @interleave(
+; CHECK-TF-ONLYRED-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TF-ONLYRED-NEXT: [[ENTRY:.*]]:
+; CHECK-TF-ONLYRED-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-ONLYRED-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-TF-ONLYRED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-TF-ONLYRED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-TF-ONLYRED: [[VECTOR_PH]]:
+; CHECK-TF-ONLYRED-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-ONLYRED-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; CHECK-TF-ONLYRED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
+; CHECK-TF-ONLYRED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-TF-ONLYRED-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-ONLYRED: [[VECTOR_BODY]]:
+; CHECK-TF-ONLYRED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-ONLYRED-NEXT: [[TMP4:%.*]] = add i64 [[TMP2]], 0
+; CHECK-TF-ONLYRED-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 1
+; CHECK-TF-ONLYRED-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], [[TMP5]]
+; CHECK-TF-ONLYRED-NEXT: [[TMP7:%.*]] = shl nuw nsw i64 [[INDEX]], 1
+; CHECK-TF-ONLYRED-NEXT: [[TMP8:%.*]] = shl nuw nsw i64 [[TMP6]], 1
+; CHECK-TF-ONLYRED-NEXT: [[TMP9:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP7]]
+; CHECK-TF-ONLYRED-NEXT: [[TMP10:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP8]]
+; CHECK-TF-ONLYRED-NEXT: [[WIDE_VEC:%.*]] = load <vscale x 8 x float>, ptr [[TMP9]], align 4
+; CHECK-TF-ONLYRED-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC]])
+; CHECK-TF-ONLYRED-NEXT: [[TMP11:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 0
+; CHECK-TF-ONLYRED-NEXT: [[TMP12:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 1
+; CHECK-TF-ONLYRED-NEXT: [[WIDE_VEC1:%.*]] = load <vscale x 8 x float>, ptr [[TMP10]], align 4
+; CHECK-TF-ONLYRED-NEXT: [[STRIDED_VEC2:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC1]])
+; CHECK-TF-ONLYRED-NEXT: [[TMP13:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC2]], 0
+; CHECK-TF-ONLYRED-NEXT: [[TMP14:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC2]], 1
+; CHECK-TF-ONLYRED-NEXT: [[TMP15:%.*]] = mul nuw nsw i64 [[INDEX]], 3
+; CHECK-TF-ONLYRED-NEXT: [[TMP16:%.*]] = mul nuw nsw i64 [[TMP6]], 3
+; CHECK-TF-ONLYRED-NEXT: [[TMP17:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP15]]
+; CHECK-TF-ONLYRED-NEXT: [[TMP18:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP16]]
+; CHECK-TF-ONLYRED-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 12 x float> @llvm.vector.interleave3.nxv12f32(<vscale x 4 x float> [[TMP11]], <vscale x 4 x float> [[TMP12]], <vscale x 4 x float> splat (float 3.000000e+00))
+; CHECK-TF-ONLYRED-NEXT: store <vscale x 12 x float> [[INTERLEAVED_VEC]], ptr [[TMP17]], align 4
+; CHECK-TF-ONLYRED-NEXT: [[INTERLEAVED_VEC3:%.*]] = call <vscale x 12 x float> @llvm.vector.interleave3.nxv12f32(<vscale x 4 x float> [[TMP13]], <vscale x 4 x float> [[TMP14]], <vscale x 4 x float> splat (float 3.000000e+00))
+; CHECK-TF-ONLYRED-NEXT: store <vscale x 12 x float> [[INTERLEAVED_VEC3]], ptr [[TMP18]], align 4
+; CHECK-TF-ONLYRED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-TF-ONLYRED-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-TF-ONLYRED-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-TF-ONLYRED: [[MIDDLE_BLOCK]]:
+; CHECK-TF-ONLYRED-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-TF-ONLYRED-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-TF-ONLYRED: [[SCALAR_PH]]:
+; CHECK-TF-ONLYRED-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-TF-ONLYRED-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-TF-ONLYRED: [[FOR_BODY]]:
+; CHECK-TF-ONLYRED-NEXT: [[I_021:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-TF-ONLYRED-NEXT: [[MUL:%.*]] = shl nuw nsw i64 [[I_021]], 1
+; CHECK-TF-ONLYRED-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[MUL]]
+; CHECK-TF-ONLYRED-NEXT: [[TMP20:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-TF-ONLYRED-NEXT: [[MUL1:%.*]] = mul nuw nsw i64 [[I_021]], 3
+; CHECK-TF-ONLYRED-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[MUL1]]
+; CHECK-TF-ONLYRED-NEXT: store float [[TMP20]], ptr [[ARRAYIDX2]], align 4
+; CHECK-TF-ONLYRED-NEXT: [[ADD:%.*]] = or disjoint i64 [[MUL]], 1
+; CHECK-TF-ONLYRED-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[ADD]]
+; CHECK-TF-ONLYRED-NEXT: [[TMP21:%.*]] = load float, ptr [[ARRAYIDX4]], align 4
+; CHECK-TF-ONLYRED-NEXT: [[ADD6:%.*]] = add nuw nsw i64 [[MUL1]], 1
+; CHECK-TF-ONLYRED-NEXT: [[ARRAYIDX7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD6]]
+; CHECK-TF-ONLYRED-NEXT: store float [[TMP21]], ptr [[ARRAYIDX7]], align 4
+; CHECK-TF-ONLYRED-NEXT: [[ADD9:%.*]] = add nuw nsw i64 [[MUL1]], 2
+; CHECK-TF-ONLYRED-NEXT: [[ARRAYIDX10:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD9]]
+; CHECK-TF-ONLYRED-NEXT: store float 3.000000e+00, ptr [[ARRAYIDX10]], align 4
+; CHECK-TF-ONLYRED-NEXT: [[INC]] = add nuw nsw i64 [[I_021]], 1
+; CHECK-TF-ONLYRED-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]]
+; CHECK-TF-ONLYRED-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-TF-ONLYRED: [[FOR_END]]:
+; CHECK-TF-ONLYRED-NEXT: ret void
+;
+; CHECK-NEOVERSE-V1-LABEL: define void @interleave(
+; CHECK-NEOVERSE-V1-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEOVERSE-V1-NEXT: [[ITER_CHECK:.*]]:
+; CHECK-NEOVERSE-V1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEOVERSE-V1-NEXT: [[TMP25:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NEOVERSE-V1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP25]]
+; CHECK-NEOVERSE-V1-NEXT: [[TMP29:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEOVERSE-V1-NEXT: [[TMP30:%.*]] = shl nuw i64 [[TMP29]], 2
+; CHECK-NEOVERSE-V1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK-NEOVERSE-V1: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-NEOVERSE-V1-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-NEOVERSE-V1-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-NEOVERSE-V1-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEOVERSE-V1: [[VECTOR_PH]]:
+; CHECK-NEOVERSE-V1-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NEOVERSE-V1-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; CHECK-NEOVERSE-V1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
+; CHECK-NEOVERSE-V1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEOVERSE-V1-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-NEOVERSE-V1: [[VECTOR_BODY]]:
+; CHECK-NEOVERSE-V1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEOVERSE-V1-NEXT: [[TMP4:%.*]] = add i64 [[TMP2]], 0
+; CHECK-NEOVERSE-V1-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 1
+; CHECK-NEOVERSE-V1-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], [[TMP5]]
+; CHECK-NEOVERSE-V1-NEXT: [[TMP7:%.*]] = shl nuw nsw i64 [[INDEX]], 1
+; CHECK-NEOVERSE-V1-NEXT: [[TMP8:%.*]] = shl nuw nsw i64 [[TMP6]], 1
+; CHECK-NEOVERSE-V1-NEXT: [[TMP9:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP7]]
+; CHECK-NEOVERSE-V1-NEXT: [[TMP10:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP8]]
+; CHECK-NEOVERSE-V1-NEXT: [[WIDE_VEC:%.*]] = load <vscale x 8 x float>, ptr [[TMP9]], align 4
+; CHECK-NEOVERSE-V1-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC]])
+; CHECK-NEOVERSE-V1-NEXT: [[TMP11:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 0
+; CHECK-NEOVERSE-V1-NEXT: [[TMP12:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 1
+; CHECK-NEOVERSE-V1-NEXT: [[WIDE_VEC2:%.*]] = load <vscale x 8 x float>, ptr [[TMP10]], align 4
+; CHECK-NEOVERSE-V1-NEXT: [[STRIDED_VEC3:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC2]])
+; CHECK-NEOVERSE-V1-NEXT: [[TMP13:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC3]], 0
+; CHECK-NEOVERSE-V1-NEXT: [[TMP14:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC3]], 1
+; CHECK-NEOVERSE-V1-NEXT: [[TMP15:%.*]] = mul nuw nsw i64 [[INDEX]], 3
+; CHECK-NEOVERSE-V1-NEXT: [[TMP16:%.*]] = mul nuw nsw i64 [[TMP6]], 3
+; CHECK-NEOVERSE-V1-NEXT: [[TMP17:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP15]]
+; CHECK-NEOVERSE-V1-NEXT: [[TMP18:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP16]]
+; CHECK-NEOVERSE-V1-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 12 x float> @llvm.vector.interleave3.nxv12f32(<vscale x 4 x float> [[TMP11]], <vscale x 4 x float> [[TMP12]], <vscale x 4 x float> splat (float 3.000000e+00))
+; CHECK-NEOVERSE-V1-NEXT: store <vscale x 12 x float> [[INTERLEAVED_VEC]], ptr [[TMP17]], align 4
+; CHECK-NEOVERSE-V1-NEXT: [[INTERLEAVED_VEC4:%.*]] = call <vscale x 12 x float> @llvm.vector.interleave3.nxv12f32(<vscale x 4 x float> [[TMP13]], <vscale x 4 x float> [[TMP14]], <vscale x 4 x float> splat (float 3.000000e+00))
+; CHECK-NEOVERSE-V1-NEXT: store <vscale x 12 x float> [[INTERLEAVED_VEC4]], ptr [[TMP18]], align 4
+; CHECK-NEOVERSE-V1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-NEOVERSE-V1-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEOVERSE-V1-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-NEOVERSE-V1: [[MIDDLE_BLOCK]]:
+; CHECK-NEOVERSE-V1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEOVERSE-V1-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK-NEOVERSE-V1: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-NEOVERSE-V1-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP30]]
+; CHECK-NEOVERSE-V1-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF8:![0-9]+]]
+; CHECK-NEOVERSE-V1: [[VEC_EPILOG_PH]]:
+; CHECK-NEOVERSE-V1-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEOVERSE-V1-NEXT: [[TMP31:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEOVERSE-V1-NEXT: [[TMP24:%.*]] = shl nuw i64 [[TMP31]], 2
+; CHECK-NEOVERSE-V1-NEXT: [[N_MOD_VF5:%.*]] = urem i64 [[N]], [[TMP24]]
+; CHECK-NEOVERSE-V1-NEXT: [[N_VEC6:%.*]] = sub i64 [[N]], [[N_MOD_VF5]]
+; CHECK-NEOVERSE-V1-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK-NEOVERSE-V1: [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-NEOVERSE-V1-NEXT: [[INDEX7:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT12:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEOVERSE-V1-NEXT: [[TMP20:%.*]] = shl nuw nsw i64 [[INDEX7]], 1
+; CHECK-NEOVERSE-V1-NEXT: [[TMP21:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP20]]
+; CHECK-NEOVERSE-V1-NEXT: [[WIDE_VEC8:%.*]] = load <vscale x 8 x float>, ptr [[TMP21]], align 4
+; CHECK-NEOVERSE-V1-NEXT: [[STRIDED_VEC9:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC8]])
+; CHECK-NEOVERSE-V1-NEXT: [[TMP32:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC9]], 0
+; CHECK-NEOVERSE-V1-NEXT: [[TMP33:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC9]], 1
+; CHECK-NEOVERSE-V1-NEXT: [[TMP22:%.*]] = mul nuw nsw i64 [[INDEX7]], 3
+; CHECK-NEOVERSE-V1-NEXT: [[TMP23:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP22]]
+; CHECK-NEOVERSE-V1-NEXT: [[INTERLEAVED_VEC10:%.*]] = call <vscale x 12 x float> @llvm.vector.interleave3.nxv12f32(<vscale x 4 x float> [[TMP32]], <vscale x 4 x float> [[TMP33]], <vscale x 4 x float> splat (float 3.000000e+00))
+; CHECK-NEOVERSE-V1-NEXT: store <vscale x 12 x float> [[INTERLEAVED_VEC10]], ptr [[TMP23]], align 4
+; CHECK-NEOVERSE-V1-NEXT: [[INDEX_NEXT12]] = add nuw i64 [[INDEX7]], [[TMP24]]
+; CHECK-NEOVERSE-V1-NEXT: [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT12]], [[N_VEC6]]
+; CHECK-NEOVERSE-V1-NEXT: br i1 [[TMP26]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-NEOVERSE-V1: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-NEOVERSE-V1-NEXT: [[CMP_N13:%.*]] = icmp eq i64 [[N]], [[N_VEC6]]
+; CHECK-NEOVERSE-V1-NEXT: br i1 [[CMP_N13]], label %[[FOR_END]], label %[[VEC_EPILOG_SCALAR_PH]]
+; CHECK-NEOVERSE-V1: [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-NEOVERSE-V1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC6]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; CHECK-NEOVERSE-V1-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-NEOVERSE-V1: [[FOR_BODY]]:
+; CHECK-NEOVERSE-V1-NEXT: [[I_021:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; CHECK-NEOVERSE-V1-NEXT: [[MUL:%.*]] = shl nuw nsw i64 [[I_021]], 1
+; CHECK-NEOVERSE-V1-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[MUL]]
+; CHECK-NEOVERSE-V1-NEXT: [[TMP27:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-NEOVERSE-V1-NEXT: [[MUL1:%.*]] = mul nuw nsw i64 [[I_021]], 3
+; CHECK-NEOVERSE-V1-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[MUL1]]
+; CHECK-NEOVERSE-V1-NEXT: store float [[TMP27]], ptr [[ARRAYIDX2]], align 4
+; CHECK-NEOVERSE-V1-NEXT: [[ADD:%.*]] = or disjoint i64 [[MUL]], 1
+; CHECK-NEOVERSE-V1-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[ADD]]
+; CHECK-NEOVERSE-V1-NEXT: [[TMP28:%.*]] = load float, ptr [[ARRAYIDX4]], align 4
+; CHECK-NEOVERSE-V1-NEXT: [[ADD6:%.*]] = add nuw nsw i64 [[MUL1]], 1
+; CHECK-NEOVERSE-V1-NEXT: [[ARRAYIDX7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD6]]
+; CHECK-NEOVERSE-V1-NEXT: store float [[TMP28]], ptr [[ARRAYIDX7]], align 4
+; CHECK-NEOVERSE-V1-NEXT: [[ADD9:%.*]] = add nuw nsw i64 [[MUL1]], 2
+; CHECK-NEOVERSE-V1-NEXT: [[ARRAYIDX10:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[ADD9]]
+; CHECK-NEOVERSE-V1-NEXT: store float 3.000000e+00, ptr [[ARRAYIDX10]], align 4
+; CHECK-NEOVERSE-V1-NEXT: [[INC]] = add nuw nsw i64 [[I_021]], 1
+; CHECK-NEOVERSE-V1-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]]
+; CHECK-NEOVERSE-V1-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-NEOVERSE-V1: [[FOR_END]]:
+; CHECK-NEOVERSE-V1-NEXT: ret void
+;
+
+
+
+
+
entry:
br label %for.body
@@ -346,35 +1779,420 @@ for.end:
}
define void @reverse(ptr noalias %dst, ptr noalias %src) #0 {
-; CHECK-NOTF-LABEL: @reverse(
-; CHECK-NOTF: vector.body:
-; CHECK-NOTF-NOT: %{{.*}} = phi <vscale x 4 x i1>
-; CHECK-NOTF: %[[LOAD:.*]] = load <vscale x 2 x double>, ptr
-; CHECK-NOTF: %{{.*}} = call <vscale x 2 x double> @llvm.vector.reverse.nxv2f64(<vscale x 2 x double> %{{.*}})
-
-; CHECK-TF-NOREV-LABEL: @reverse(
-; CHECK-TF-NOREV: vector.body:
-; CHECK-TF-NOREV-NOT: %{{.*}} = phi <vscale x 4 x i1>
-; CHECK-TF-NOREV: %[[LOAD:.*]] = load <vscale x 2 x double>, ptr
-; CHECK-TF-NOREV: %{{.*}} = call <vscale x 2 x double> @llvm.vector.reverse.nxv2f64(<vscale x 2 x double> %{{.*}})
-
-; CHECK-TF-LABEL: @reverse(
-; CHECK-TF: vector.body:
-; CHECK-TF: %[[ACTIVE_LANE_MASK:.*]] = phi <vscale x 2 x i1>
-; CHECK-TF: %[[REVERSE_MASK:.*]] = call <vscale x 2 x i1> @llvm.vector.reverse.nxv2i1(<vscale x 2 x i1> %[[ACTIVE_LANE_MASK]])
-; CHECK-TF: %[[MASKED_LOAD:.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0({{.*}} <vscale x 2 x i1> %reverse
-
-; CHECK-TF-NORED-LABEL: @reverse(
-; CHECK-TF-NORED: vector.body:
-; CHECK-TF-NORED: %[[ACTIVE_LANE_MASK:.*]] = phi <vscale x 2 x i1>
-; CHECK-TF-NORED: %[[REVERSE_MASK:.*]] = call <vscale x 2 x i1> @llvm.vector.reverse.nxv2i1(<vscale x 2 x i1> %[[ACTIVE_LANE_MASK]])
-; CHECK-TF-NORED: %[[MASKED_LOAD:.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0({{.*}} <vscale x 2 x i1> %reverse
-
-; CHECK-TF-NOREC-LABEL: @reverse(
-; CHECK-TF-NOREC: vector.body:
-; CHECK-TF-NOREC: %[[ACTIVE_LANE_MASK:.*]] = phi <vscale x 2 x i1>
-; CHECK-TF-NOREC: %[[REVERSE_MASK:.*]] = call <vscale x 2 x i1> @llvm.vector.reverse.nxv2i1(<vscale x 2 x i1> %[[ACTIVE_LANE_MASK]])
-; CHECK-TF-NOREC: %[[MASKED_LOAD:.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0({{.*}} <vscale x 2 x i1> %reverse
+; CHECK-NOTF-LABEL: define void @reverse(
+; CHECK-NOTF-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]]) #[[ATTR0]] {
+; CHECK-NOTF-NEXT: [[ENTRY:.*]]:
+; CHECK-NOTF-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NOTF-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NOTF-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
+; CHECK-NOTF-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NOTF: [[VECTOR_PH]]:
+; CHECK-NOTF-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 1
+; CHECK-NOTF-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; CHECK-NOTF-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
+; CHECK-NOTF-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-NOTF-NEXT: [[TMP4:%.*]] = sub i64 1023, [[N_VEC]]
+; CHECK-NOTF-NEXT: [[TMP5:%.*]] = shl i64 [[N_VEC]], 3
+; CHECK-NOTF-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP5]]
+; CHECK-NOTF-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-NOTF: [[VECTOR_BODY]]:
+; CHECK-NOTF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NOTF-NEXT: [[TMP7:%.*]] = sub i64 1023, [[INDEX]]
+; CHECK-NOTF-NEXT: [[TMP8:%.*]] = shl i64 [[INDEX]], 3
+; CHECK-NOTF-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP8]]
+; CHECK-NOTF-NEXT: [[TMP9:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[TMP7]]
+; CHECK-NOTF-NEXT: [[TMP10:%.*]] = sub nuw nsw i64 [[TMP2]], 1
+; CHECK-NOTF-NEXT: [[TMP11:%.*]] = sub i64 0, [[TMP10]]
+; CHECK-NOTF-NEXT: [[TMP12:%.*]] = getelementptr inbounds double, ptr [[TMP9]], i64 [[TMP11]]
+; CHECK-NOTF-NEXT: [[TMP13:%.*]] = sub i64 [[TMP11]], [[TMP2]]
+; CHECK-NOTF-NEXT: [[TMP14:%.*]] = getelementptr inbounds double, ptr [[TMP9]], i64 [[TMP13]]
+; CHECK-NOTF-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 2 x double>, ptr [[TMP12]], align 8
+; CHECK-NOTF-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 2 x double>, ptr [[TMP14]], align 8
+; CHECK-NOTF-NEXT: [[TMP15:%.*]] = fadd <vscale x 2 x double> [[WIDE_LOAD]], splat (double 1.000000e+00)
+; CHECK-NOTF-NEXT: [[TMP16:%.*]] = fadd <vscale x 2 x double> [[WIDE_LOAD1]], splat (double 1.000000e+00)
+; CHECK-NOTF-NEXT: [[REVERSE:%.*]] = call <vscale x 2 x double> @llvm.vector.reverse.nxv2f64(<vscale x 2 x double> [[TMP15]])
+; CHECK-NOTF-NEXT: [[REVERSE2:%.*]] = call <vscale x 2 x double> @llvm.vector.reverse.nxv2f64(<vscale x 2 x double> [[TMP16]])
+; CHECK-NOTF-NEXT: [[TMP17:%.*]] = getelementptr double, ptr [[NEXT_GEP]], i64 [[TMP2]]
+; CHECK-NOTF-NEXT: store <vscale x 2 x double> [[REVERSE]], ptr [[NEXT_GEP]], align 8
+; CHECK-NOTF-NEXT: store <vscale x 2 x double> [[REVERSE2]], ptr [[TMP17]], align 8
+; CHECK-NOTF-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-NOTF-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NOTF-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-NOTF: [[MIDDLE_BLOCK]]:
+; CHECK-NOTF-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-NOTF-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-NOTF: [[SCALAR_PH]]:
+; CHECK-NOTF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP4]], %[[MIDDLE_BLOCK]] ], [ 1023, %[[ENTRY]] ]
+; CHECK-NOTF-NEXT: [[BC_RESUME_VAL3:%.*]] = phi ptr [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ [[DST]], %[[ENTRY]] ]
+; CHECK-NOTF-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-NOTF: [[FOR_BODY]]:
+; CHECK-NOTF-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NOTF-NEXT: [[INDVARS_PTR:%.*]] = phi ptr [ [[BC_RESUME_VAL3]], %[[SCALAR_PH]] ], [ [[INDVARS_PTR_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NOTF-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[INDVARS_IV]]
+; CHECK-NOTF-NEXT: [[TMP19:%.*]] = load double, ptr [[ARRAYIDX]], align 8
+; CHECK-NOTF-NEXT: [[ADD:%.*]] = fadd double [[TMP19]], 1.000000e+00
+; CHECK-NOTF-NEXT: store double [[ADD]], ptr [[INDVARS_PTR]], align 8
+; CHECK-NOTF-NEXT: [[INDVARS_IV_NEXT]] = add nsw i64 [[INDVARS_IV]], -1
+; CHECK-NOTF-NEXT: [[INDVARS_PTR_NEXT]] = getelementptr inbounds double, ptr [[INDVARS_PTR]], i64 1
+; CHECK-NOTF-NEXT: [[CMP_NOT:%.*]] = icmp eq i64 [[INDVARS_IV]], 0
+; CHECK-NOTF-NEXT: br i1 [[CMP_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-NOTF: [[FOR_END]]:
+; CHECK-NOTF-NEXT: ret void
+;
+; CHECK-TF-ALL-LABEL: define void @reverse(
+; CHECK-TF-ALL-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]]) #[[ATTR0]] {
+; CHECK-TF-ALL-NEXT: [[ENTRY:.*:]]
+; CHECK-TF-ALL-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-TF-ALL: [[VECTOR_PH]]:
+; CHECK-TF-ALL-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-ALL-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; CHECK-TF-ALL-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1024)
+; CHECK-TF-ALL-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-ALL: [[VECTOR_BODY]]:
+; CHECK-TF-ALL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-ALL-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-ALL-NEXT: [[TMP2:%.*]] = sub i64 1023, [[INDEX]]
+; CHECK-TF-ALL-NEXT: [[TMP3:%.*]] = shl i64 [[INDEX]], 3
+; CHECK-TF-ALL-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP3]]
+; CHECK-TF-ALL-NEXT: [[TMP4:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[TMP2]]
+; CHECK-TF-ALL-NEXT: [[TMP5:%.*]] = sub nuw nsw i64 [[TMP1]], 1
+; CHECK-TF-ALL-NEXT: [[TMP6:%.*]] = sub i64 0, [[TMP5]]
+; CHECK-TF-ALL-NEXT: [[TMP7:%.*]] = getelementptr double, ptr [[TMP4]], i64 [[TMP6]]
+; CHECK-TF-ALL-NEXT: [[REVERSE:%.*]] = call <vscale x 2 x i1> @llvm.vector.reverse.nxv2i1(<vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-ALL-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP7]], <vscale x 2 x i1> [[REVERSE]], <vscale x 2 x double> poison)
+; CHECK-TF-ALL-NEXT: [[TMP8:%.*]] = fadd <vscale x 2 x double> [[WIDE_MASKED_LOAD]], splat (double 1.000000e+00)
+; CHECK-TF-ALL-NEXT: [[REVERSE1:%.*]] = call <vscale x 2 x double> @llvm.vector.reverse.nxv2f64(<vscale x 2 x double> [[TMP8]])
+; CHECK-TF-ALL-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[REVERSE1]], ptr align 8 [[NEXT_GEP]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-ALL-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-TF-ALL-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1024)
+; CHECK-TF-ALL-NEXT: [[TMP9:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-ALL-NEXT: [[TMP10:%.*]] = xor i1 [[TMP9]], true
+; CHECK-TF-ALL-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-TF-ALL: [[MIDDLE_BLOCK]]:
+; CHECK-TF-ALL-NEXT: br label %[[FOR_END:.*]]
+; CHECK-TF-ALL: [[FOR_END]]:
+; CHECK-TF-ALL-NEXT: ret void
+;
+; CHECK-TF-LABEL: define void @reverse(
+; CHECK-TF-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]]) #[[ATTR0]] {
+; CHECK-TF-NEXT: [[ENTRY:.*:]]
+; CHECK-TF-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-TF: [[VECTOR_PH]]:
+; CHECK-TF-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1024)
+; CHECK-TF-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF: [[VECTOR_BODY]]:
+; CHECK-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NEXT: [[TMP2:%.*]] = sub i64 1023, [[INDEX]]
+; CHECK-TF-NEXT: [[TMP3:%.*]] = shl i64 [[INDEX]], 3
+; CHECK-TF-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP3]]
+; CHECK-TF-NEXT: [[TMP4:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[TMP2]]
+; CHECK-TF-NEXT: [[TMP5:%.*]] = sub nuw nsw i64 [[TMP1]], 1
+; CHECK-TF-NEXT: [[TMP6:%.*]] = sub i64 0, [[TMP5]]
+; CHECK-TF-NEXT: [[TMP7:%.*]] = getelementptr double, ptr [[TMP4]], i64 [[TMP6]]
+; CHECK-TF-NEXT: [[REVERSE:%.*]] = call <vscale x 2 x i1> @llvm.vector.reverse.nxv2i1(<vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP7]], <vscale x 2 x i1> [[REVERSE]], <vscale x 2 x double> poison)
+; CHECK-TF-NEXT: [[TMP8:%.*]] = fadd <vscale x 2 x double> [[WIDE_MASKED_LOAD]], splat (double 1.000000e+00)
+; CHECK-TF-NEXT: [[REVERSE1:%.*]] = call <vscale x 2 x double> @llvm.vector.reverse.nxv2f64(<vscale x 2 x double> [[TMP8]])
+; CHECK-TF-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[REVERSE1]], ptr align 8 [[NEXT_GEP]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1024)
+; CHECK-TF-NEXT: [[TMP9:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-NEXT: [[TMP10:%.*]] = xor i1 [[TMP9]], true
+; CHECK-TF-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-TF: [[MIDDLE_BLOCK]]:
+; CHECK-TF-NEXT: br label %[[FOR_END:.*]]
+; CHECK-TF: [[FOR_END]]:
+; CHECK-TF-NEXT: ret void
+;
+; CHECK-TF-DEFAULT-LABEL: define void @reverse(
+; CHECK-TF-DEFAULT-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]]) #[[ATTR0]] {
+; CHECK-TF-DEFAULT-NEXT: [[ENTRY:.*:]]
+; CHECK-TF-DEFAULT-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-TF-DEFAULT: [[VECTOR_PH]]:
+; CHECK-TF-DEFAULT-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-DEFAULT-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; CHECK-TF-DEFAULT-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1024)
+; CHECK-TF-DEFAULT-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-DEFAULT: [[VECTOR_BODY]]:
+; CHECK-TF-DEFAULT-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-DEFAULT-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-DEFAULT-NEXT: [[TMP2:%.*]] = sub i64 1023, [[INDEX]]
+; CHECK-TF-DEFAULT-NEXT: [[TMP3:%.*]] = shl i64 [[INDEX]], 3
+; CHECK-TF-DEFAULT-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP3]]
+; CHECK-TF-DEFAULT-NEXT: [[TMP4:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[TMP2]]
+; CHECK-TF-DEFAULT-NEXT: [[TMP5:%.*]] = sub nuw nsw i64 [[TMP1]], 1
+; CHECK-TF-DEFAULT-NEXT: [[TMP6:%.*]] = sub i64 0, [[TMP5]]
+; CHECK-TF-DEFAULT-NEXT: [[TMP7:%.*]] = getelementptr double, ptr [[TMP4]], i64 [[TMP6]]
+; CHECK-TF-DEFAULT-NEXT: [[REVERSE:%.*]] = call <vscale x 2 x i1> @llvm.vector.reverse.nxv2i1(<vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-DEFAULT-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP7]], <vscale x 2 x i1> [[REVERSE]], <vscale x 2 x double> poison)
+; CHECK-TF-DEFAULT-NEXT: [[TMP8:%.*]] = fadd <vscale x 2 x double> [[WIDE_MASKED_LOAD]], splat (double 1.000000e+00)
+; CHECK-TF-DEFAULT-NEXT: [[REVERSE1:%.*]] = call <vscale x 2 x double> @llvm.vector.reverse.nxv2f64(<vscale x 2 x double> [[TMP8]])
+; CHECK-TF-DEFAULT-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[REVERSE1]], ptr align 8 [[NEXT_GEP]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-DEFAULT-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-TF-DEFAULT-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1024)
+; CHECK-TF-DEFAULT-NEXT: [[TMP9:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-DEFAULT-NEXT: [[TMP10:%.*]] = xor i1 [[TMP9]], true
+; CHECK-TF-DEFAULT-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-TF-DEFAULT: [[MIDDLE_BLOCK]]:
+; CHECK-TF-DEFAULT-NEXT: br label %[[FOR_END:.*]]
+; CHECK-TF-DEFAULT: [[FOR_END]]:
+; CHECK-TF-DEFAULT-NEXT: ret void
+;
+; CHECK-TF-NORED-LABEL: define void @reverse(
+; CHECK-TF-NORED-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]]) #[[ATTR0]] {
+; CHECK-TF-NORED-NEXT: [[ENTRY:.*:]]
+; CHECK-TF-NORED-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-TF-NORED: [[VECTOR_PH]]:
+; CHECK-TF-NORED-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NORED-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; CHECK-TF-NORED-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1024)
+; CHECK-TF-NORED-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-NORED: [[VECTOR_BODY]]:
+; CHECK-TF-NORED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NORED-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NORED-NEXT: [[TMP2:%.*]] = sub i64 1023, [[INDEX]]
+; CHECK-TF-NORED-NEXT: [[TMP3:%.*]] = shl i64 [[INDEX]], 3
+; CHECK-TF-NORED-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP3]]
+; CHECK-TF-NORED-NEXT: [[TMP4:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[TMP2]]
+; CHECK-TF-NORED-NEXT: [[TMP5:%.*]] = sub nuw nsw i64 [[TMP1]], 1
+; CHECK-TF-NORED-NEXT: [[TMP6:%.*]] = sub i64 0, [[TMP5]]
+; CHECK-TF-NORED-NEXT: [[TMP7:%.*]] = getelementptr double, ptr [[TMP4]], i64 [[TMP6]]
+; CHECK-TF-NORED-NEXT: [[REVERSE:%.*]] = call <vscale x 2 x i1> @llvm.vector.reverse.nxv2i1(<vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-NORED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP7]], <vscale x 2 x i1> [[REVERSE]], <vscale x 2 x double> poison)
+; CHECK-TF-NORED-NEXT: [[TMP8:%.*]] = fadd <vscale x 2 x double> [[WIDE_MASKED_LOAD]], splat (double 1.000000e+00)
+; CHECK-TF-NORED-NEXT: [[REVERSE1:%.*]] = call <vscale x 2 x double> @llvm.vector.reverse.nxv2f64(<vscale x 2 x double> [[TMP8]])
+; CHECK-TF-NORED-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[REVERSE1]], ptr align 8 [[NEXT_GEP]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-NORED-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-TF-NORED-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1024)
+; CHECK-TF-NORED-NEXT: [[TMP9:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-NORED-NEXT: [[TMP10:%.*]] = xor i1 [[TMP9]], true
+; CHECK-TF-NORED-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-TF-NORED: [[MIDDLE_BLOCK]]:
+; CHECK-TF-NORED-NEXT: br label %[[FOR_END:.*]]
+; CHECK-TF-NORED: [[FOR_END]]:
+; CHECK-TF-NORED-NEXT: ret void
+;
+; CHECK-TF-NOREC-LABEL: define void @reverse(
+; CHECK-TF-NOREC-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]]) #[[ATTR0]] {
+; CHECK-TF-NOREC-NEXT: [[ENTRY:.*:]]
+; CHECK-TF-NOREC-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-TF-NOREC: [[VECTOR_PH]]:
+; CHECK-TF-NOREC-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NOREC-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; CHECK-TF-NOREC-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1024)
+; CHECK-TF-NOREC-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-NOREC: [[VECTOR_BODY]]:
+; CHECK-TF-NOREC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NOREC-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NOREC-NEXT: [[TMP2:%.*]] = sub i64 1023, [[INDEX]]
+; CHECK-TF-NOREC-NEXT: [[TMP3:%.*]] = shl i64 [[INDEX]], 3
+; CHECK-TF-NOREC-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP3]]
+; CHECK-TF-NOREC-NEXT: [[TMP4:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[TMP2]]
+; CHECK-TF-NOREC-NEXT: [[TMP5:%.*]] = sub nuw nsw i64 [[TMP1]], 1
+; CHECK-TF-NOREC-NEXT: [[TMP6:%.*]] = sub i64 0, [[TMP5]]
+; CHECK-TF-NOREC-NEXT: [[TMP7:%.*]] = getelementptr double, ptr [[TMP4]], i64 [[TMP6]]
+; CHECK-TF-NOREC-NEXT: [[REVERSE:%.*]] = call <vscale x 2 x i1> @llvm.vector.reverse.nxv2i1(<vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-NOREC-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr align 8 [[TMP7]], <vscale x 2 x i1> [[REVERSE]], <vscale x 2 x double> poison)
+; CHECK-TF-NOREC-NEXT: [[TMP8:%.*]] = fadd <vscale x 2 x double> [[WIDE_MASKED_LOAD]], splat (double 1.000000e+00)
+; CHECK-TF-NOREC-NEXT: [[REVERSE1:%.*]] = call <vscale x 2 x double> @llvm.vector.reverse.nxv2f64(<vscale x 2 x double> [[TMP8]])
+; CHECK-TF-NOREC-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[REVERSE1]], ptr align 8 [[NEXT_GEP]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-TF-NOREC-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-TF-NOREC-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1024)
+; CHECK-TF-NOREC-NEXT: [[TMP9:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TF-NOREC-NEXT: [[TMP10:%.*]] = xor i1 [[TMP9]], true
+; CHECK-TF-NOREC-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-TF-NOREC: [[MIDDLE_BLOCK]]:
+; CHECK-TF-NOREC-NEXT: br label %[[FOR_END:.*]]
+; CHECK-TF-NOREC: [[FOR_END]]:
+; CHECK-TF-NOREC-NEXT: ret void
+;
+; CHECK-TF-NOREV-LABEL: define void @reverse(
+; CHECK-TF-NOREV-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]]) #[[ATTR0]] {
+; CHECK-TF-NOREV-NEXT: [[ENTRY:.*]]:
+; CHECK-TF-NOREV-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-NOREV-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-NOREV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
+; CHECK-TF-NOREV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-TF-NOREV: [[VECTOR_PH]]:
+; CHECK-TF-NOREV-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 1
+; CHECK-TF-NOREV-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; CHECK-TF-NOREV-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
+; CHECK-TF-NOREV-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-TF-NOREV-NEXT: [[TMP4:%.*]] = sub i64 1023, [[N_VEC]]
+; CHECK-TF-NOREV-NEXT: [[TMP5:%.*]] = shl i64 [[N_VEC]], 3
+; CHECK-TF-NOREV-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP5]]
+; CHECK-TF-NOREV-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-NOREV: [[VECTOR_BODY]]:
+; CHECK-TF-NOREV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-NOREV-NEXT: [[TMP7:%.*]] = sub i64 1023, [[INDEX]]
+; CHECK-TF-NOREV-NEXT: [[TMP8:%.*]] = shl i64 [[INDEX]], 3
+; CHECK-TF-NOREV-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP8]]
+; CHECK-TF-NOREV-NEXT: [[TMP9:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[TMP7]]
+; CHECK-TF-NOREV-NEXT: [[TMP10:%.*]] = sub nuw nsw i64 [[TMP2]], 1
+; CHECK-TF-NOREV-NEXT: [[TMP11:%.*]] = sub i64 0, [[TMP10]]
+; CHECK-TF-NOREV-NEXT: [[TMP12:%.*]] = getelementptr inbounds double, ptr [[TMP9]], i64 [[TMP11]]
+; CHECK-TF-NOREV-NEXT: [[TMP13:%.*]] = sub i64 [[TMP11]], [[TMP2]]
+; CHECK-TF-NOREV-NEXT: [[TMP14:%.*]] = getelementptr inbounds double, ptr [[TMP9]], i64 [[TMP13]]
+; CHECK-TF-NOREV-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 2 x double>, ptr [[TMP12]], align 8
+; CHECK-TF-NOREV-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 2 x double>, ptr [[TMP14]], align 8
+; CHECK-TF-NOREV-NEXT: [[TMP15:%.*]] = fadd <vscale x 2 x double> [[WIDE_LOAD]], splat (double 1.000000e+00)
+; CHECK-TF-NOREV-NEXT: [[TMP16:%.*]] = fadd <vscale x 2 x double> [[WIDE_LOAD1]], splat (double 1.000000e+00)
+; CHECK-TF-NOREV-NEXT: [[REVERSE:%.*]] = call <vscale x 2 x double> @llvm.vector.reverse.nxv2f64(<vscale x 2 x double> [[TMP15]])
+; CHECK-TF-NOREV-NEXT: [[REVERSE2:%.*]] = call <vscale x 2 x double> @llvm.vector.reverse.nxv2f64(<vscale x 2 x double> [[TMP16]])
+; CHECK-TF-NOREV-NEXT: [[TMP17:%.*]] = getelementptr double, ptr [[NEXT_GEP]], i64 [[TMP2]]
+; CHECK-TF-NOREV-NEXT: store <vscale x 2 x double> [[REVERSE]], ptr [[NEXT_GEP]], align 8
+; CHECK-TF-NOREV-NEXT: store <vscale x 2 x double> [[REVERSE2]], ptr [[TMP17]], align 8
+; CHECK-TF-NOREV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-TF-NOREV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-TF-NOREV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-TF-NOREV: [[MIDDLE_BLOCK]]:
+; CHECK-TF-NOREV-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-TF-NOREV-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-TF-NOREV: [[SCALAR_PH]]:
+; CHECK-TF-NOREV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP4]], %[[MIDDLE_BLOCK]] ], [ 1023, %[[ENTRY]] ]
+; CHECK-TF-NOREV-NEXT: [[BC_RESUME_VAL3:%.*]] = phi ptr [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ [[DST]], %[[ENTRY]] ]
+; CHECK-TF-NOREV-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-TF-NOREV: [[FOR_BODY]]:
+; CHECK-TF-NOREV-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-TF-NOREV-NEXT: [[INDVARS_PTR:%.*]] = phi ptr [ [[BC_RESUME_VAL3]], %[[SCALAR_PH]] ], [ [[INDVARS_PTR_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-TF-NOREV-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[INDVARS_IV]]
+; CHECK-TF-NOREV-NEXT: [[TMP19:%.*]] = load double, ptr [[ARRAYIDX]], align 8
+; CHECK-TF-NOREV-NEXT: [[ADD:%.*]] = fadd double [[TMP19]], 1.000000e+00
+; CHECK-TF-NOREV-NEXT: store double [[ADD]], ptr [[INDVARS_PTR]], align 8
+; CHECK-TF-NOREV-NEXT: [[INDVARS_IV_NEXT]] = add nsw i64 [[INDVARS_IV]], -1
+; CHECK-TF-NOREV-NEXT: [[INDVARS_PTR_NEXT]] = getelementptr inbounds double, ptr [[INDVARS_PTR]], i64 1
+; CHECK-TF-NOREV-NEXT: [[CMP_NOT:%.*]] = icmp eq i64 [[INDVARS_IV]], 0
+; CHECK-TF-NOREV-NEXT: br i1 [[CMP_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-TF-NOREV: [[FOR_END]]:
+; CHECK-TF-NOREV-NEXT: ret void
+;
+; CHECK-TF-ONLYRED-LABEL: define void @reverse(
+; CHECK-TF-ONLYRED-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]]) #[[ATTR0]] {
+; CHECK-TF-ONLYRED-NEXT: [[ENTRY:.*]]:
+; CHECK-TF-ONLYRED-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TF-ONLYRED-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-TF-ONLYRED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
+; CHECK-TF-ONLYRED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-TF-ONLYRED: [[VECTOR_PH]]:
+; CHECK-TF-ONLYRED-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 1
+; CHECK-TF-ONLYRED-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; CHECK-TF-ONLYRED-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
+; CHECK-TF-ONLYRED-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-TF-ONLYRED-NEXT: [[TMP4:%.*]] = sub i64 1023, [[N_VEC]]
+; CHECK-TF-ONLYRED-NEXT: [[TMP5:%.*]] = shl i64 [[N_VEC]], 3
+; CHECK-TF-ONLYRED-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP5]]
+; CHECK-TF-ONLYRED-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TF-ONLYRED: [[VECTOR_BODY]]:
+; CHECK-TF-ONLYRED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TF-ONLYRED-NEXT: [[TMP7:%.*]] = sub i64 1023, [[INDEX]]
+; CHECK-TF-ONLYRED-NEXT: [[TMP8:%.*]] = shl i64 [[INDEX]], 3
+; CHECK-TF-ONLYRED-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP8]]
+; CHECK-TF-ONLYRED-NEXT: [[TMP9:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[TMP7]]
+; CHECK-TF-ONLYRED-NEXT: [[TMP10:%.*]] = sub nuw nsw i64 [[TMP2]], 1
+; CHECK-TF-ONLYRED-NEXT: [[TMP11:%.*]] = sub i64 0, [[TMP10]]
+; CHECK-TF-ONLYRED-NEXT: [[TMP12:%.*]] = getelementptr inbounds double, ptr [[TMP9]], i64 [[TMP11]]
+; CHECK-TF-ONLYRED-NEXT: [[TMP13:%.*]] = sub i64 [[TMP11]], [[TMP2]]
+; CHECK-TF-ONLYRED-NEXT: [[TMP14:%.*]] = getelementptr inbounds double, ptr [[TMP9]], i64 [[TMP13]]
+; CHECK-TF-ONLYRED-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 2 x double>, ptr [[TMP12]], align 8
+; CHECK-TF-ONLYRED-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 2 x double>, ptr [[TMP14]], align 8
+; CHECK-TF-ONLYRED-NEXT: [[TMP15:%.*]] = fadd <vscale x 2 x double> [[WIDE_LOAD]], splat (double 1.000000e+00)
+; CHECK-TF-ONLYRED-NEXT: [[TMP16:%.*]] = fadd <vscale x 2 x double> [[WIDE_LOAD1]], splat (double 1.000000e+00)
+; CHECK-TF-ONLYRED-NEXT: [[REVERSE:%.*]] = call <vscale x 2 x double> @llvm.vector.reverse.nxv2f64(<vscale x 2 x double> [[TMP15]])
+; CHECK-TF-ONLYRED-NEXT: [[REVERSE2:%.*]] = call <vscale x 2 x double> @llvm.vector.reverse.nxv2f64(<vscale x 2 x double> [[TMP16]])
+; CHECK-TF-ONLYRED-NEXT: [[TMP17:%.*]] = getelementptr double, ptr [[NEXT_GEP]], i64 [[TMP2]]
+; CHECK-TF-ONLYRED-NEXT: store <vscale x 2 x double> [[REVERSE]], ptr [[NEXT_GEP]], align 8
+; CHECK-TF-ONLYRED-NEXT: store <vscale x 2 x double> [[REVERSE2]], ptr [[TMP17]], align 8
+; CHECK-TF-ONLYRED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-TF-ONLYRED-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-TF-ONLYRED-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-TF-ONLYRED: [[MIDDLE_BLOCK]]:
+; CHECK-TF-ONLYRED-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-TF-ONLYRED-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-TF-ONLYRED: [[SCALAR_PH]]:
+; CHECK-TF-ONLYRED-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP4]], %[[MIDDLE_BLOCK]] ], [ 1023, %[[ENTRY]] ]
+; CHECK-TF-ONLYRED-NEXT: [[BC_RESUME_VAL3:%.*]] = phi ptr [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ [[DST]], %[[ENTRY]] ]
+; CHECK-TF-ONLYRED-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-TF-ONLYRED: [[FOR_BODY]]:
+; CHECK-TF-ONLYRED-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-TF-ONLYRED-NEXT: [[INDVARS_PTR:%.*]] = phi ptr [ [[BC_RESUME_VAL3]], %[[SCALAR_PH]] ], [ [[INDVARS_PTR_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-TF-ONLYRED-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[INDVARS_IV]]
+; CHECK-TF-ONLYRED-NEXT: [[TMP19:%.*]] = load double, ptr [[ARRAYIDX]], align 8
+; CHECK-TF-ONLYRED-NEXT: [[ADD:%.*]] = fadd double [[TMP19]], 1.000000e+00
+; CHECK-TF-ONLYRED-NEXT: store double [[ADD]], ptr [[INDVARS_PTR]], align 8
+; CHECK-TF-ONLYRED-NEXT: [[INDVARS_IV_NEXT]] = add nsw i64 [[INDVARS_IV]], -1
+; CHECK-TF-ONLYRED-NEXT: [[INDVARS_PTR_NEXT]] = getelementptr inbounds double, ptr [[INDVARS_PTR]], i64 1
+; CHECK-TF-ONLYRED-NEXT: [[CMP_NOT:%.*]] = icmp eq i64 [[INDVARS_IV]], 0
+; CHECK-TF-ONLYRED-NEXT: br i1 [[CMP_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-TF-ONLYRED: [[FOR_END]]:
+; CHECK-TF-ONLYRED-NEXT: ret void
+;
+; CHECK-NEOVERSE-V1-LABEL: define void @reverse(
+; CHECK-NEOVERSE-V1-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]]) #[[ATTR0]] {
+; CHECK-NEOVERSE-V1-NEXT: [[ENTRY:.*]]:
+; CHECK-NEOVERSE-V1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEOVERSE-V1-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NEOVERSE-V1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
+; CHECK-NEOVERSE-V1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEOVERSE-V1: [[VECTOR_PH]]:
+; CHECK-NEOVERSE-V1-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 1
+; CHECK-NEOVERSE-V1-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; CHECK-NEOVERSE-V1-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
+; CHECK-NEOVERSE-V1-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-NEOVERSE-V1-NEXT: [[TMP4:%.*]] = sub i64 1023, [[N_VEC]]
+; CHECK-NEOVERSE-V1-NEXT: [[TMP5:%.*]] = shl i64 [[N_VEC]], 3
+; CHECK-NEOVERSE-V1-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP5]]
+; CHECK-NEOVERSE-V1-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-NEOVERSE-V1: [[VECTOR_BODY]]:
+; CHECK-NEOVERSE-V1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEOVERSE-V1-NEXT: [[TMP7:%.*]] = sub i64 1023, [[INDEX]]
+; CHECK-NEOVERSE-V1-NEXT: [[TMP8:%.*]] = shl i64 [[INDEX]], 3
+; CHECK-NEOVERSE-V1-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP8]]
+; CHECK-NEOVERSE-V1-NEXT: [[TMP9:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[TMP7]]
+; CHECK-NEOVERSE-V1-NEXT: [[TMP10:%.*]] = sub nuw nsw i64 [[TMP2]], 1
+; CHECK-NEOVERSE-V1-NEXT: [[TMP11:%.*]] = sub i64 0, [[TMP10]]
+; CHECK-NEOVERSE-V1-NEXT: [[TMP12:%.*]] = getelementptr inbounds double, ptr [[TMP9]], i64 [[TMP11]]
+; CHECK-NEOVERSE-V1-NEXT: [[TMP13:%.*]] = sub i64 [[TMP11]], [[TMP2]]
+; CHECK-NEOVERSE-V1-NEXT: [[TMP14:%.*]] = getelementptr inbounds double, ptr [[TMP9]], i64 [[TMP13]]
+; CHECK-NEOVERSE-V1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 2 x double>, ptr [[TMP12]], align 8
+; CHECK-NEOVERSE-V1-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 2 x double>, ptr [[TMP14]], align 8
+; CHECK-NEOVERSE-V1-NEXT: [[TMP15:%.*]] = fadd <vscale x 2 x double> [[WIDE_LOAD]], splat (double 1.000000e+00)
+; CHECK-NEOVERSE-V1-NEXT: [[TMP16:%.*]] = fadd <vscale x 2 x double> [[WIDE_LOAD1]], splat (double 1.000000e+00)
+; CHECK-NEOVERSE-V1-NEXT: [[REVERSE:%.*]] = call <vscale x 2 x double> @llvm.vector.reverse.nxv2f64(<vscale x 2 x double> [[TMP15]])
+; CHECK-NEOVERSE-V1-NEXT: [[REVERSE2:%.*]] = call <vscale x 2 x double> @llvm.vector.reverse.nxv2f64(<vscale x 2 x double> [[TMP16]])
+; CHECK-NEOVERSE-V1-NEXT: [[TMP17:%.*]] = getelementptr double, ptr [[NEXT_GEP]], i64 [[TMP2]]
+; CHECK-NEOVERSE-V1-NEXT: store <vscale x 2 x double> [[REVERSE]], ptr [[NEXT_GEP]], align 8
+; CHECK-NEOVERSE-V1-NEXT: store <vscale x 2 x double> [[REVERSE2]], ptr [[TMP17]], align 8
+; CHECK-NEOVERSE-V1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-NEOVERSE-V1-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEOVERSE-V1-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-NEOVERSE-V1: [[MIDDLE_BLOCK]]:
+; CHECK-NEOVERSE-V1-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-NEOVERSE-V1-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-NEOVERSE-V1: [[SCALAR_PH]]:
+; CHECK-NEOVERSE-V1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP4]], %[[MIDDLE_BLOCK]] ], [ 1023, %[[ENTRY]] ]
+; CHECK-NEOVERSE-V1-NEXT: [[BC_RESUME_VAL3:%.*]] = phi ptr [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ [[DST]], %[[ENTRY]] ]
+; CHECK-NEOVERSE-V1-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-NEOVERSE-V1: [[FOR_BODY]]:
+; CHECK-NEOVERSE-V1-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEOVERSE-V1-NEXT: [[INDVARS_PTR:%.*]] = phi ptr [ [[BC_RESUME_VAL3]], %[[SCALAR_PH]] ], [ [[INDVARS_PTR_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEOVERSE-V1-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds double, ptr [[SRC]], i64 [[INDVARS_IV]]
+; CHECK-NEOVERSE-V1-NEXT: [[TMP19:%.*]] = load double, ptr [[ARRAYIDX]], align 8
+; CHECK-NEOVERSE-V1-NEXT: [[ADD:%.*]] = fadd double [[TMP19]], 1.000000e+00
+; CHECK-NEOVERSE-V1-NEXT: store double [[ADD]], ptr [[INDVARS_PTR]], align 8
+; CHECK-NEOVERSE-V1-NEXT: [[INDVARS_IV_NEXT]] = add nsw i64 [[INDVARS_IV]], -1
+; CHECK-NEOVERSE-V1-NEXT: [[INDVARS_PTR_NEXT]] = getelementptr inbounds double, ptr [[INDVARS_PTR]], i64 1
+; CHECK-NEOVERSE-V1-NEXT: [[CMP_NOT:%.*]] = icmp eq i64 [[INDVARS_IV]], 0
+; CHECK-NEOVERSE-V1-NEXT: br i1 [[CMP_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-NEOVERSE-V1: [[FOR_END]]:
+; CHECK-NEOVERSE-V1-NEXT: ret void
+;
+
+
+
+
entry:
br label %for.body
@@ -402,3 +2220,102 @@ attributes #0 = { "target-features"="+sve" }
!2 = !{!"llvm.loop.vectorize.scalable.enable", i1 true}
!3 = !{!"llvm.loop.interleave.count", i32 1}
!4 = !{!"llvm.loop.vectorize.enable", i1 true}
+;.
+; CHECK-NOTF: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK-NOTF: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK-NOTF: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK-NOTF: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
+; CHECK-NOTF: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
+; CHECK-NOTF: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]}
+; CHECK-NOTF: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]}
+; CHECK-NOTF: [[LOOP7]] = distinct !{[[LOOP7]], [[META2]], [[META1]]}
+; CHECK-NOTF: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META2]]}
+; CHECK-NOTF: [[LOOP9]] = distinct !{[[LOOP9]], [[META2]], [[META1]]}
+; CHECK-NOTF: [[LOOP10]] = distinct !{[[LOOP10]], [[META1]], [[META2]]}
+; CHECK-NOTF: [[LOOP11]] = distinct !{[[LOOP11]], [[META2]], [[META1]]}
+;.
+; CHECK-TF-ALL: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK-TF-ALL: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK-TF-ALL: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK-TF-ALL: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]], [[META2]]}
+; CHECK-TF-ALL: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
+; CHECK-TF-ALL: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]], [[META2]]}
+; CHECK-TF-ALL: [[LOOP6]] = distinct !{[[LOOP6]], [[META2]], [[META1]]}
+; CHECK-TF-ALL: [[LOOP7]] = distinct !{[[LOOP7]], [[META1]], [[META2]]}
+;.
+; CHECK-TF: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK-TF: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK-TF: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK-TF: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]], [[META2]]}
+; CHECK-TF: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
+; CHECK-TF: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]], [[META2]]}
+; CHECK-TF: [[LOOP6]] = distinct !{[[LOOP6]], [[META2]], [[META1]]}
+; CHECK-TF: [[LOOP7]] = distinct !{[[LOOP7]], [[META1]], [[META2]]}
+;.
+; CHECK-TF-DEFAULT: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK-TF-DEFAULT: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK-TF-DEFAULT: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK-TF-DEFAULT: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]], [[META2]]}
+; CHECK-TF-DEFAULT: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
+; CHECK-TF-DEFAULT: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]], [[META2]]}
+; CHECK-TF-DEFAULT: [[PROF6]] = !{!"branch_weights", i32 8, i32 8}
+; CHECK-TF-DEFAULT: [[LOOP7]] = distinct !{[[LOOP7]], [[META1]], [[META2]]}
+; CHECK-TF-DEFAULT: [[LOOP8]] = distinct !{[[LOOP8]], [[META2]], [[META1]]}
+; CHECK-TF-DEFAULT: [[LOOP9]] = distinct !{[[LOOP9]], [[META1]], [[META2]]}
+;.
+; CHECK-TF-NORED: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK-TF-NORED: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK-TF-NORED: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK-TF-NORED: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]], [[META2]]}
+; CHECK-TF-NORED: [[LOOP4]] = distinct !{[[LOOP4]], [[META2]], [[META1]]}
+; CHECK-TF-NORED: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]], [[META2]]}
+; CHECK-TF-NORED: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]}
+; CHECK-TF-NORED: [[LOOP7]] = distinct !{[[LOOP7]], [[META2]], [[META1]]}
+; CHECK-TF-NORED: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META2]]}
+;.
+; CHECK-TF-NOREC: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK-TF-NOREC: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK-TF-NOREC: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK-TF-NOREC: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]], [[META2]]}
+; CHECK-TF-NOREC: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
+; CHECK-TF-NOREC: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]}
+; CHECK-TF-NOREC: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]}
+; CHECK-TF-NOREC: [[LOOP7]] = distinct !{[[LOOP7]], [[META2]], [[META1]]}
+; CHECK-TF-NOREC: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META2]]}
+;.
+; CHECK-TF-NOREV: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK-TF-NOREV: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK-TF-NOREV: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK-TF-NOREV: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]], [[META2]]}
+; CHECK-TF-NOREV: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
+; CHECK-TF-NOREV: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]], [[META2]]}
+; CHECK-TF-NOREV: [[LOOP6]] = distinct !{[[LOOP6]], [[META2]], [[META1]]}
+; CHECK-TF-NOREV: [[LOOP7]] = distinct !{[[LOOP7]], [[META1]], [[META2]]}
+; CHECK-TF-NOREV: [[LOOP8]] = distinct !{[[LOOP8]], [[META2]], [[META1]]}
+;.
+; CHECK-TF-ONLYRED: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK-TF-ONLYRED: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK-TF-ONLYRED: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK-TF-ONLYRED: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
+; CHECK-TF-ONLYRED: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
+; CHECK-TF-ONLYRED: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]], [[META2]]}
+; CHECK-TF-ONLYRED: [[LOOP6]] = distinct !{[[LOOP6]], [[META2]], [[META1]]}
+; CHECK-TF-ONLYRED: [[LOOP7]] = distinct !{[[LOOP7]], [[META1]], [[META2]]}
+; CHECK-TF-ONLYRED: [[LOOP8]] = distinct !{[[LOOP8]], [[META2]], [[META1]]}
+; CHECK-TF-ONLYRED: [[LOOP9]] = distinct !{[[LOOP9]], [[META1]], [[META2]]}
+; CHECK-TF-ONLYRED: [[LOOP10]] = distinct !{[[LOOP10]], [[META2]], [[META1]]}
+;.
+; CHECK-NEOVERSE-V1: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK-NEOVERSE-V1: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK-NEOVERSE-V1: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK-NEOVERSE-V1: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]], [[META2]]}
+; CHECK-NEOVERSE-V1: [[LOOP4]] = distinct !{[[LOOP4]], [[META2]], [[META1]]}
+; CHECK-NEOVERSE-V1: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]], [[META2]]}
+; CHECK-NEOVERSE-V1: [[LOOP6]] = distinct !{[[LOOP6]], [[META2]], [[META1]]}
+; CHECK-NEOVERSE-V1: [[LOOP7]] = distinct !{[[LOOP7]], [[META1]], [[META2]]}
+; CHECK-NEOVERSE-V1: [[PROF8]] = !{!"branch_weights", i32 8, i32 8}
+; CHECK-NEOVERSE-V1: [[LOOP9]] = distinct !{[[LOOP9]], [[META1]], [[META2]]}
+; CHECK-NEOVERSE-V1: [[LOOP10]] = distinct !{[[LOOP10]], [[META2]], [[META1]]}
+; CHECK-NEOVERSE-V1: [[LOOP11]] = distinct !{[[LOOP11]], [[META1]], [[META2]]}
+; CHECK-NEOVERSE-V1: [[LOOP12]] = distinct !{[[LOOP12]], [[META2]], [[META1]]}
+;.
More information about the llvm-commits
mailing list